3 bup_python="$(dirname "$0")/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 from __future__ import absolute_import, print_function
9 import os, re, stat, sys, textwrap, time
10 from collections import namedtuple
11 from functools import partial
12 from stat import S_ISDIR
14 from bup import git, client, helpers, vfs
15 from bup.compat import wrap_main
16 from bup.git import get_cat_data, parse_commit, walk_object
17 from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors
18 from bup.helpers import hostname, shstr, tty_width, userfullname, username
19 from bup.repo import LocalRepo, RemoteRepo
22 "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...",
24 """Transfer data from a source repository to a destination repository
25 according to the methods specified (--ff, --ff:, --append, etc.).
26 Both repositories default to BUP_DIR. A remote destination may be
27 specified with -r, and data may be pulled from a remote repository
28 with the related "bup on HOST get ..." command.""",
30 ('optional arguments:',
31 (('-h, --help', 'show this help message and exit'),
33 'increase log output (can be specified more than once)'),
34 ('-q, --quiet', "don't show progress meter"),
35 ('-s SOURCE, --source SOURCE',
36 'path to the source repository (defaults to BUP_DIR)'),
37 ('-r REMOTE, --remote REMOTE',
38 'hostname:/path/to/repo of remote destination repository'),
39 ('-t --print-trees', 'output a tree id for each ref set'),
40 ('-c, --print-commits', 'output a commit id for each ref set'),
41 ('--print-tags', 'output an id for each tag'),
42 ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'),
43 ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL',
44 'set compression LEVEL (default: 1)'))),
47 (('--ff REF, --ff: REF DEST',
48 'fast-forward dest REF (or DEST) to match source REF'),
49 ('--append REF, --append: REF DEST',
50 'append REF (treeish or committish) to dest REF (or DEST)'),
51 ('--pick REF, --pick: REF DEST',
52 'append single source REF commit to dest REF (or DEST)'),
53 ('--force-pick REF, --force-pick: REF DEST',
54 '--pick, overwriting REF (or DEST)'),
55 ('--new-tag REF, --new-tag: REF DEST',
56 'tag source ref REF as REF (or DEST) in dest unless it already exists'),
57 ('--replace, --replace: REF DEST',
58 'overwrite REF (or DEST) in dest with source REF'),
60 'fetch REF anonymously (without destination ref)'))))
62 def render_opts(opts, width=None):
66 for args, desc in opts:
67 result.append(textwrap.fill(args, width=width,
68 initial_indent=(' ' * 2),
69 subsequent_indent=(' ' * 4)))
71 result.append(textwrap.fill(desc, width=width,
72 initial_indent=(' ' * 6),
73 subsequent_indent=(' ' * 6)))
77 def usage(argspec, width=None):
80 usage, preamble, groups = argspec[0], argspec[1], argspec[2:]
82 msg.append(textwrap.fill(usage, width=width, subsequent_indent=' '))
84 msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width))
86 for group_name, group_args in groups:
87 msg.extend(['\n', group_name, '\n'])
88 msg.extend(render_opts(group_args, width=width))
91 def misuse(message=None):
92 sys.stderr.write(usage(argspec))
94 sys.stderr.write('\nerror: ')
95 sys.stderr.write(message)
96 sys.stderr.write('\n')
99 def require_n_args_or_die(n, args):
100 if len(args) < n + 1:
101 misuse('%s argument requires %d %s'
102 % (n, 'values' if n == 1 else 'value'))
103 result = args[1:1+n], args[1+n:]
104 assert len(result[0]) == n
107 def parse_args(args):
108 Spec = namedtuple('Spec', ['argopt', 'argval', 'src', 'dest', 'method'])
115 opt.print_commits = opt.print_trees = opt.print_tags = False
118 opt.source = opt.remote = None
119 opt.target_specs = []
121 remaining = args[1:] # Skip argv[0]
124 if arg in ('-h', '--help'):
125 sys.stdout.write(usage(argspec))
127 elif arg in ('-v', '--verbose'):
129 remaining = remaining[1:]
130 elif arg in ('--ff', '--append', '--pick', '--force-pick',
131 '--new-tag', '--replace', '--unnamed'):
132 (ref,), remaining = require_n_args_or_die(1, remaining)
133 opt.target_specs.append(Spec(argopt=arg,
134 argval=shstr((ref,)),
137 elif arg in ('--ff:', '--append:', '--pick:', '--force-pick:',
138 '--new-tag:', '--replace:'):
139 (ref, dest), remaining = require_n_args_or_die(2, remaining)
140 opt.target_specs.append(Spec(argopt=arg,
141 argval=shstr((ref, dest)),
144 elif arg in ('-s', '--source'):
145 (opt.source,), remaining = require_n_args_or_die(1, remaining)
146 elif arg in ('-r', '--remote'):
147 (opt.remote,), remaining = require_n_args_or_die(1, remaining)
148 elif arg in ('-c', '--print-commits'):
149 opt.print_commits, remaining = True, remaining[1:]
150 elif arg in ('-t', '--print-trees'):
151 opt.print_trees, remaining = True, remaining[1:]
152 elif arg == '--print-tags':
153 opt.print_tags, remaining = True, remaining[1:]
154 elif arg in ('-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'):
155 opt.compress = int(arg[1:])
156 remaining = remaining[1:]
157 elif arg == '--compress':
158 (opt.compress,), remaining = require_n_args_or_die(1, remaining)
159 opt.compress = int(opt.compress)
160 elif arg == '--bwlimit':
161 (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining)
162 opt.bwlimit = long(opt.bwlimit)
163 elif arg.startswith('-') and len(arg) > 2 and arg[1] != '-':
164 # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z".
165 # We do this last so that --foo -bar is valid if --foo
167 remaining[0:1] = ('-' + c for c in arg[1:])
174 # FIXME: client error handling (remote exceptions, etc.)
176 # FIXME: walk_object in in git.py doesn't support opt.verbose. Do we
177 # need to adjust for that here?
178 def get_random_item(name, hash, repo, writer, opt):
179 def already_seen(id):
180 return writer.exists(id.decode('hex'))
181 for item in walk_object(repo.cat, hash, stop_at=already_seen,
183 # already_seen ensures that writer.exists(id) is false.
184 # Otherwise, just_write() would fail.
185 writer.just_write(item.oid, item.type, item.data)
188 def append_commit(name, hash, parent, src_repo, writer, opt):
190 items = parse_commit(get_cat_data(src_repo.cat(hash), 'commit'))
191 tree = items.tree.decode('hex')
192 author = '%s <%s>' % (items.author_name, items.author_mail)
193 author_time = (items.author_sec, items.author_offset)
194 committer = '%s <%s@%s>' % (userfullname(), username(), hostname())
195 get_random_item(name, tree.encode('hex'), src_repo, writer, opt)
196 c = writer.new_commit(tree, parent,
197 author, items.author_sec, items.author_offset,
198 committer, now, None,
203 def append_commits(commits, src_name, dest_hash, src_repo, writer, opt):
204 last_c, tree = dest_hash, None
205 for commit in commits:
206 last_c, tree = append_commit(src_name, commit, last_c,
207 src_repo, writer, opt)
208 assert(tree is not None)
211 Loc = namedtuple('Loc', ['type', 'hash', 'path'])
212 default_loc = Loc(None, None, None)
214 def find_vfs_item(name, repo):
215 res = repo.resolve(name, follow=False, want_meta=False)
216 leaf_name, leaf_item = res[-1]
219 kind = type(leaf_item)
222 elif kind == vfs.Tags:
224 elif kind == vfs.RevList:
226 elif kind == vfs.Commit:
227 if len(res) > 1 and type(res[-2][1]) == vfs.RevList:
231 elif kind == vfs.Item:
232 if S_ISDIR(vfs.item_mode(leaf_item)):
236 elif kind == vfs.Chunky:
238 elif kind == vfs.FakeLink:
239 # Don't have to worry about ELOOP, excepting malicious
240 # remotes, since "latest" is the only FakeLink.
241 assert leaf_name == 'latest'
242 res = repo.resolve(leaf_item.target, parent=res[:-1],
243 follow=False, want_meta=False)
244 leaf_name, leaf_item = res[-1]
246 assert type(leaf_item) == vfs.Commit
247 name = '/'.join(x[0] for x in res)
250 raise Exception('unexpected resolution for %r: %r' % (name, res))
251 path = '/'.join(name for name, item in res)
252 if hasattr(leaf_item, 'coid'):
253 result = Loc(type=kind, hash=leaf_item.coid, path=path)
254 elif hasattr(leaf_item, 'oid'):
255 result = Loc(type=kind, hash=leaf_item.oid, path=path)
257 result = Loc(type=kind, hash=None, path=path)
261 Target = namedtuple('Target', ['spec', 'src', 'dest'])
265 loc = loc._replace(hash=loc.hash.encode('hex'))
269 # FIXME: see if resolve() means we can drop the vfs path cleanup
271 def cleanup_vfs_path(p):
272 result = os.path.normpath(p)
273 if result.startswith('/'):
278 def validate_vfs_path(p):
279 if p.startswith('/.') \
280 and not p.startswith('/.tag/'):
281 spec_args = '%s %s' % (spec.argopt, spec.argval)
282 misuse('unsupported destination path %r in %r' % (dest.path, spec_args))
286 def resolve_src(spec, src_repo):
287 src = find_vfs_item(spec.src, src_repo)
288 spec_args = '%s %s' % (spec.argopt, spec.argval)
290 misuse('cannot find source for %r' % spec_args)
291 if src.type == 'root':
292 misuse('cannot fetch entire repository for %r' % spec_args)
293 if src.type == 'tags':
294 misuse('cannot fetch entire /.tag directory for %r' % spec_args)
295 debug1('src: %s\n' % loc_desc(src))
299 def get_save_branch(repo, path):
300 res = repo.resolve(path, follow=False, want_meta=False)
301 leaf_name, leaf_item = res[-1]
303 misuse('error: cannot access %r in %r' % (leaf_name, path))
305 res_path = '/'.join(name for name, item in res[:-1])
309 def resolve_branch_dest(spec, src, src_repo, dest_repo):
310 # Resulting dest must be treeish, or not exist.
312 # Pick a default dest.
313 if src.type == 'branch':
314 spec = spec._replace(dest=spec.src)
315 elif src.type == 'save':
316 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
317 elif src.path.startswith('/.tag/'): # Dest defaults to the same.
318 spec = spec._replace(dest=spec.src)
320 spec_args = '%s %s' % (spec.argopt, spec.argval)
322 misuse('no destination (implicit or explicit) for %r', spec_args)
324 dest = find_vfs_item(spec.dest, dest_repo)
326 if dest.type == 'commit':
327 misuse('destination for %r is a tagged commit, not a branch'
329 if dest.type != 'branch':
330 misuse('destination for %r is a %s, not a branch'
331 % (spec_args, dest.type))
333 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
335 if dest.path.startswith('/.'):
336 misuse('destination for %r must be a valid branch name' % spec_args)
338 debug1('dest: %s\n' % loc_desc(dest))
342 def resolve_ff(spec, src_repo, dest_repo):
343 src = resolve_src(spec, src_repo)
344 spec_args = '%s %s' % (spec.argopt, spec.argval)
345 if src.type == 'tree':
346 misuse('%r is impossible; can only --append a tree to a branch'
348 if src.type not in ('branch', 'save', 'commit'):
349 misuse('source for %r must be a branch, save, or commit, not %s'
350 % (spec_args, src.type))
351 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
352 return Target(spec=spec, src=src, dest=dest)
355 def handle_ff(item, src_repo, writer, opt):
356 assert item.spec.method == 'ff'
357 assert item.src.type in ('branch', 'save', 'commit')
358 src_oidx = item.src.hash.encode('hex')
359 dest_oidx = item.dest.hash.encode('hex') if item.dest.hash else None
360 if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
362 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
363 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit'))
364 return item.src.hash, commit_items.tree.decode('hex')
365 spec_args = '%s %s' % (item.spec.argopt, item.spec.argval)
366 misuse('destination is not an ancestor of source for %r' % spec_args)
369 def resolve_append(spec, src_repo, dest_repo):
370 src = resolve_src(spec, src_repo)
371 if src.type not in ('branch', 'save', 'commit', 'tree'):
372 spec_args = '%s %s' % (spec.argopt, spec.argval)
373 misuse('source for %r must be a branch, save, commit, or tree, not %s'
374 % (spec_args, src.type))
375 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
376 return Target(spec=spec, src=src, dest=dest)
379 def handle_append(item, src_repo, writer, opt):
380 assert item.spec.method == 'append'
381 assert item.src.type in ('branch', 'save', 'commit', 'tree')
382 assert item.dest.type == 'branch' or not item.dest.type
383 src_oidx = item.src.hash.encode('hex')
384 if item.src.type == 'tree':
385 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
386 parent = item.dest.hash
387 msg = 'bup save\n\nGenerated by command:\n%r\n' % sys.argv
388 userline = '%s <%s@%s>' % (userfullname(), username(), hostname())
390 commit = writer.new_commit(item.src.hash, parent,
392 userline, now, None, msg)
393 return commit, item.src.hash
394 commits = list(src_repo.rev_list(src_oidx))
396 return append_commits(commits, item.spec.src, item.dest.hash,
397 src_repo, writer, opt)
400 def resolve_pick(spec, src_repo, dest_repo):
401 src = resolve_src(spec, src_repo)
402 spec_args = '%s %s' % (spec.argopt, spec.argval)
403 if src.type == 'tree':
404 misuse('%r is impossible; can only --append a tree' % spec_args)
405 if src.type not in ('commit', 'save'):
406 misuse('%r impossible; can only pick a commit or save, not %s'
407 % (spec_args, src.type))
409 if src.path.startswith('/.tag/'):
410 spec = spec._replace(dest=spec.src)
411 elif src.type == 'save':
412 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
414 misuse('no destination provided for %r', spec_args)
415 dest = find_vfs_item(spec.dest, dest_repo)
417 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
418 dest = default_loc._replace(path=cp)
420 if not dest.type == 'branch' and not dest.path.startswith('/.tag/'):
421 misuse('%r destination is not a tag or branch' % spec_args)
422 if spec.method == 'pick' \
423 and dest.hash and dest.path.startswith('/.tag/'):
424 misuse('cannot overwrite existing tag for %r (requires --force-pick)'
426 return Target(spec=spec, src=src, dest=dest)
429 def handle_pick(item, src_repo, writer, opt):
430 assert item.spec.method in ('pick', 'force-pick')
431 assert item.src.type in ('save', 'commit')
432 src_oidx = item.src.hash.encode('hex')
434 return append_commit(item.spec.src, src_oidx, item.dest.hash,
435 src_repo, writer, opt)
436 return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt)
439 def resolve_new_tag(spec, src_repo, dest_repo):
440 src = resolve_src(spec, src_repo)
441 spec_args = '%s %s' % (spec.argopt, spec.argval)
442 if not spec.dest and src.path.startswith('/.tag/'):
443 spec = spec._replace(dest=src.path)
445 misuse('no destination (implicit or explicit) for %r', spec_args)
446 dest = find_vfs_item(spec.dest, dest_repo)
448 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
449 if not dest.path.startswith('/.tag/'):
450 misuse('destination for %r must be a VFS tag' % spec_args)
452 misuse('cannot overwrite existing tag for %r (requires --replace)'
454 return Target(spec=spec, src=src, dest=dest)
457 def handle_new_tag(item, src_repo, writer, opt):
458 assert item.spec.method == 'new-tag'
459 assert item.dest.path.startswith('/.tag/')
460 get_random_item(item.spec.src, item.src.hash.encode('hex'),
461 src_repo, writer, opt)
462 return (item.src.hash,)
465 def resolve_replace(spec, src_repo, dest_repo):
466 src = resolve_src(spec, src_repo)
467 spec_args = '%s %s' % (spec.argopt, spec.argval)
469 if src.path.startswith('/.tag/') or src.type == 'branch':
470 spec = spec._replace(dest=spec.src)
472 misuse('no destination provided for %r', spec_args)
473 dest = find_vfs_item(spec.dest, dest_repo)
475 if not dest.type == 'branch' and not dest.path.startswith('/.tag/'):
476 misuse('%r impossible; can only overwrite branch or tag'
479 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
480 dest = default_loc._replace(path=cp)
481 if not dest.path.startswith('/.tag/') \
482 and not src.type in ('branch', 'save', 'commit'):
483 misuse('cannot overwrite branch with %s for %r' % (src.type, spec_args))
484 return Target(spec=spec, src=src, dest=dest)
487 def handle_replace(item, src_repo, writer, opt):
488 assert(item.spec.method == 'replace')
489 if item.dest.path.startswith('/.tag/'):
490 get_random_item(item.spec.src, item.src.hash.encode('hex'),
491 src_repo, writer, opt)
492 return (item.src.hash,)
493 assert(item.dest.type == 'branch' or not item.dest.type)
494 src_oidx = item.src.hash.encode('hex')
495 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
496 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit'))
497 return item.src.hash, commit_items.tree.decode('hex')
500 def resolve_unnamed(spec, src_repo, dest_repo):
502 spec_args = '%s %s' % (spec.argopt, spec.argval)
503 misuse('destination name given for %r' % spec_args)
504 src = resolve_src(spec, src_repo)
505 return Target(spec=spec, src=src, dest=None)
508 def handle_unnamed(item, src_repo, writer, opt):
509 get_random_item(item.spec.src, item.src.hash.encode('hex'),
510 src_repo, writer, opt)
514 def resolve_targets(specs, src_repo, dest_repo):
516 common_args = src_repo, dest_repo
518 debug1('initial-spec: %s\n' % str(spec))
519 if spec.method == 'ff':
520 resolved_items.append(resolve_ff(spec, *common_args))
521 elif spec.method == 'append':
522 resolved_items.append(resolve_append(spec, *common_args))
523 elif spec.method in ('pick', 'force-pick'):
524 resolved_items.append(resolve_pick(spec, *common_args))
525 elif spec.method == 'new-tag':
526 resolved_items.append(resolve_new_tag(spec, *common_args))
527 elif spec.method == 'replace':
528 resolved_items.append(resolve_replace(spec, *common_args))
529 elif spec.method == 'unnamed':
530 resolved_items.append(resolve_unnamed(spec, *common_args))
531 else: # Should be impossible -- prevented by the option parser.
534 # FIXME: check for prefix overlap? i.e.:
535 # bup get --ff foo --ff: baz foo/bar
536 # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar
538 # Now that we have all the items, check for duplicate tags.
539 tags_targeted = set()
540 for item in resolved_items:
541 dest_path = item.dest and item.dest.path
543 assert(dest_path.startswith('/'))
544 if dest_path.startswith('/.tag/'):
545 if dest_path in tags_targeted:
546 if item.spec.method not in ('replace', 'force-pick'):
547 spec_args = '%s %s' % (item.spec.argopt,
549 misuse('cannot overwrite tag %r via %r' \
550 % (dest_path, spec_args))
552 tags_targeted.add(dest_path)
553 return resolved_items
556 def log_item(name, type, opt, tree=None, commit=None, tag=None):
557 if tag and opt.print_tags:
558 print(tag.encode('hex'))
559 if tree and opt.print_trees:
560 print(tree.encode('hex'))
561 if commit and opt.print_commits:
562 print(commit.encode('hex'))
565 if type in ('root', 'branch', 'save', 'commit', 'tree'):
566 if not name.endswith('/'):
568 log('%s%s\n' % (name, last))
572 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
573 opt = parse_args(sys.argv)
574 git.check_repo_or_die()
575 src_dir = opt.source or git.repo()
577 client.bwlimit = parse_num(opt.bwlimit)
578 if is_reverse and opt.remote:
579 misuse("don't use -r in reverse mode; it's automatic")
580 if opt.remote or is_reverse:
581 dest_repo = RemoteRepo(opt.remote)
583 dest_repo = LocalRepo()
585 with dest_repo as dest_repo:
586 with LocalRepo(repo_dir=src_dir) as src_repo:
587 with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
589 src_repo = LocalRepo(repo_dir=src_dir)
591 # Resolve and validate all sources and destinations,
592 # implicit or explicit, and do it up-front, so we can
593 # fail before we start writing (for any obviously
595 target_items = resolve_targets(opt.target_specs,
598 updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
599 no_ref_info = (None, None)
601 handlers = {'ff': handle_ff,
602 'append': handle_append,
603 'force-pick': handle_pick,
605 'new-tag': handle_new_tag,
606 'replace': handle_replace,
607 'unnamed': handle_unnamed}
609 for item in target_items:
610 debug1('get-spec: %s\n' % str(item.spec))
611 debug1('get-src: %s\n' % loc_desc(item.src))
612 debug1('get-dest: %s\n' % loc_desc(item.dest))
613 dest_path = item.dest and item.dest.path
615 if dest_path.startswith('/.tag/'):
616 dest_ref = 'refs/tags/%s' % dest_path[6:]
618 dest_ref = 'refs/heads/%s' % dest_path[1:]
622 dest_hash = item.dest and item.dest.hash
623 orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
624 orig_ref = orig_ref or dest_hash
625 cur_ref = cur_ref or dest_hash
627 handler = handlers[item.spec.method]
628 item_result = handler(item, src_repo, writer, opt)
629 if len(item_result) > 1:
630 new_id, tree = item_result
632 new_id = item_result[0]
635 log_item(item.spec.src, item.src.type, opt)
637 updated_refs[dest_ref] = (orig_ref, new_id)
638 if dest_ref.startswith('refs/tags/'):
639 log_item(item.spec.src, item.src.type, opt, tag=new_id)
641 log_item(item.spec.src, item.src.type, opt,
642 tree=tree, commit=new_id)
644 # Only update the refs at the very end, once the writer is
645 # closed, so that if something goes wrong above, the old refs
646 # will be undisturbed.
647 for ref_name, info in updated_refs.iteritems():
648 orig_ref, new_ref = info
650 dest_repo.update_ref(ref_name, new_ref, orig_ref)
652 new_hex = new_ref.encode('hex')
654 orig_hex = orig_ref.encode('hex')
655 log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
657 log('updated %r (%s)\n' % (ref_name, new_hex))
658 except (git.GitError, client.ClientError), ex:
659 add_error('unable to update ref %r: %s' % (ref_name, ex))
662 log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))