3 bup_python="$(dirname "$0")/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 from __future__ import absolute_import, print_function
9 import os, re, stat, sys, textwrap, time
10 from collections import namedtuple
11 from functools import partial
12 from stat import S_ISDIR
14 from bup import git, client, helpers, vfs
15 from bup.compat import hexstr, wrap_main
16 from bup.git import get_cat_data, parse_commit, walk_object
17 from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors
18 from bup.helpers import hostname, shstr, tty_width
19 from bup.pwdgrp import userfullname, username
20 from bup.repo import LocalRepo, RemoteRepo
23 "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...",
25 """Transfer data from a source repository to a destination repository
26 according to the methods specified (--ff, --ff:, --append, etc.).
27 Both repositories default to BUP_DIR. A remote destination may be
28 specified with -r, and data may be pulled from a remote repository
29 with the related "bup on HOST get ..." command.""",
31 ('optional arguments:',
32 (('-h, --help', 'show this help message and exit'),
34 'increase log output (can be specified more than once)'),
35 ('-q, --quiet', "don't show progress meter"),
36 ('-s SOURCE, --source SOURCE',
37 'path to the source repository (defaults to BUP_DIR)'),
38 ('-r REMOTE, --remote REMOTE',
39 'hostname:/path/to/repo of remote destination repository'),
40 ('-t --print-trees', 'output a tree id for each ref set'),
41 ('-c, --print-commits', 'output a commit id for each ref set'),
42 ('--print-tags', 'output an id for each tag'),
43 ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'),
44 ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL',
45 'set compression LEVEL (default: 1)'))),
48 (('--ff REF, --ff: REF DEST',
49 'fast-forward dest REF (or DEST) to match source REF'),
50 ('--append REF, --append: REF DEST',
51 'append REF (treeish or committish) to dest REF (or DEST)'),
52 ('--pick REF, --pick: REF DEST',
53 'append single source REF commit to dest REF (or DEST)'),
54 ('--force-pick REF, --force-pick: REF DEST',
55 '--pick, overwriting REF (or DEST)'),
56 ('--new-tag REF, --new-tag: REF DEST',
57 'tag source ref REF as REF (or DEST) in dest unless it already exists'),
58 ('--replace, --replace: REF DEST',
59 'overwrite REF (or DEST) in dest with source REF'),
61 'fetch REF anonymously (without destination ref)'))))
63 def render_opts(opts, width=None):
67 for args, desc in opts:
68 result.append(textwrap.fill(args, width=width,
69 initial_indent=(' ' * 2),
70 subsequent_indent=(' ' * 4)))
72 result.append(textwrap.fill(desc, width=width,
73 initial_indent=(' ' * 6),
74 subsequent_indent=(' ' * 6)))
78 def usage(argspec, width=None):
81 usage, preamble, groups = argspec[0], argspec[1], argspec[2:]
83 msg.append(textwrap.fill(usage, width=width, subsequent_indent=' '))
85 msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width))
87 for group_name, group_args in groups:
88 msg.extend(['\n', group_name, '\n'])
89 msg.extend(render_opts(group_args, width=width))
92 def misuse(message=None):
93 sys.stderr.write(usage(argspec))
95 sys.stderr.write('\nerror: ')
96 sys.stderr.write(message)
97 sys.stderr.write('\n')
100 def require_n_args_or_die(n, args):
101 if len(args) < n + 1:
102 misuse('%s argument requires %d %s'
103 % (n, 'values' if n == 1 else 'value'))
104 result = args[1:1+n], args[1+n:]
105 assert len(result[0]) == n
108 def parse_args(args):
109 Spec = namedtuple('Spec', ['argopt', 'argval', 'src', 'dest', 'method'])
116 opt.print_commits = opt.print_trees = opt.print_tags = False
119 opt.source = opt.remote = None
120 opt.target_specs = []
122 remaining = args[1:] # Skip argv[0]
125 if arg in ('-h', '--help'):
126 sys.stdout.write(usage(argspec))
128 elif arg in ('-v', '--verbose'):
130 remaining = remaining[1:]
131 elif arg in ('--ff', '--append', '--pick', '--force-pick',
132 '--new-tag', '--replace', '--unnamed'):
133 (ref,), remaining = require_n_args_or_die(1, remaining)
134 opt.target_specs.append(Spec(argopt=arg,
135 argval=shstr((ref,)),
138 elif arg in ('--ff:', '--append:', '--pick:', '--force-pick:',
139 '--new-tag:', '--replace:'):
140 (ref, dest), remaining = require_n_args_or_die(2, remaining)
141 opt.target_specs.append(Spec(argopt=arg,
142 argval=shstr((ref, dest)),
145 elif arg in ('-s', '--source'):
146 (opt.source,), remaining = require_n_args_or_die(1, remaining)
147 elif arg in ('-r', '--remote'):
148 (opt.remote,), remaining = require_n_args_or_die(1, remaining)
149 elif arg in ('-c', '--print-commits'):
150 opt.print_commits, remaining = True, remaining[1:]
151 elif arg in ('-t', '--print-trees'):
152 opt.print_trees, remaining = True, remaining[1:]
153 elif arg == '--print-tags':
154 opt.print_tags, remaining = True, remaining[1:]
155 elif arg in ('-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'):
156 opt.compress = int(arg[1:])
157 remaining = remaining[1:]
158 elif arg == '--compress':
159 (opt.compress,), remaining = require_n_args_or_die(1, remaining)
160 opt.compress = int(opt.compress)
161 elif arg == '--bwlimit':
162 (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining)
163 opt.bwlimit = long(opt.bwlimit)
164 elif arg.startswith('-') and len(arg) > 2 and arg[1] != '-':
165 # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z".
166 # We do this last so that --foo -bar is valid if --foo
168 remaining[0:1] = ('-' + c for c in arg[1:])
175 # FIXME: client error handling (remote exceptions, etc.)
177 # FIXME: walk_object in in git.py doesn't support opt.verbose. Do we
178 # need to adjust for that here?
179 def get_random_item(name, hash, repo, writer, opt):
180 def already_seen(id):
181 return writer.exists(id.decode('hex'))
182 for item in walk_object(repo.cat, hash, stop_at=already_seen,
184 # already_seen ensures that writer.exists(id) is false.
185 # Otherwise, just_write() would fail.
186 writer.just_write(item.oid, item.type, item.data)
189 def append_commit(name, hash, parent, src_repo, writer, opt):
191 items = parse_commit(get_cat_data(src_repo.cat(hash), 'commit'))
192 tree = items.tree.decode('hex')
193 author = '%s <%s>' % (items.author_name, items.author_mail)
194 author_time = (items.author_sec, items.author_offset)
195 committer = '%s <%s@%s>' % (userfullname(), username(), hostname())
196 get_random_item(name, tree.encode('hex'), src_repo, writer, opt)
197 c = writer.new_commit(tree, parent,
198 author, items.author_sec, items.author_offset,
199 committer, now, None,
204 def append_commits(commits, src_name, dest_hash, src_repo, writer, opt):
205 last_c, tree = dest_hash, None
206 for commit in commits:
207 last_c, tree = append_commit(src_name, commit, last_c,
208 src_repo, writer, opt)
209 assert(tree is not None)
212 Loc = namedtuple('Loc', ['type', 'hash', 'path'])
213 default_loc = Loc(None, None, None)
215 def find_vfs_item(name, repo):
216 res = repo.resolve(name, follow=False, want_meta=False)
217 leaf_name, leaf_item = res[-1]
220 kind = type(leaf_item)
223 elif kind == vfs.Tags:
225 elif kind == vfs.RevList:
227 elif kind == vfs.Commit:
228 if len(res) > 1 and type(res[-2][1]) == vfs.RevList:
232 elif kind == vfs.Item:
233 if S_ISDIR(vfs.item_mode(leaf_item)):
237 elif kind == vfs.Chunky:
239 elif kind == vfs.FakeLink:
240 # Don't have to worry about ELOOP, excepting malicious
241 # remotes, since "latest" is the only FakeLink.
242 assert leaf_name == 'latest'
243 res = repo.resolve(leaf_item.target, parent=res[:-1],
244 follow=False, want_meta=False)
245 leaf_name, leaf_item = res[-1]
247 assert type(leaf_item) == vfs.Commit
248 name = '/'.join(x[0] for x in res)
251 raise Exception('unexpected resolution for %r: %r' % (name, res))
252 path = '/'.join(name for name, item in res)
253 if hasattr(leaf_item, 'coid'):
254 result = Loc(type=kind, hash=leaf_item.coid, path=path)
255 elif hasattr(leaf_item, 'oid'):
256 result = Loc(type=kind, hash=leaf_item.oid, path=path)
258 result = Loc(type=kind, hash=None, path=path)
262 Target = namedtuple('Target', ['spec', 'src', 'dest'])
266 loc = loc._replace(hash=loc.hash.encode('hex'))
270 # FIXME: see if resolve() means we can drop the vfs path cleanup
272 def cleanup_vfs_path(p):
273 result = os.path.normpath(p)
274 if result.startswith('/'):
279 def validate_vfs_path(p):
280 if p.startswith('/.') \
281 and not p.startswith('/.tag/'):
282 spec_args = '%s %s' % (spec.argopt, spec.argval)
283 misuse('unsupported destination path %r in %r' % (dest.path, spec_args))
287 def resolve_src(spec, src_repo):
288 src = find_vfs_item(spec.src, src_repo)
289 spec_args = '%s %s' % (spec.argopt, spec.argval)
291 misuse('cannot find source for %r' % spec_args)
292 if src.type == 'root':
293 misuse('cannot fetch entire repository for %r' % spec_args)
294 if src.type == 'tags':
295 misuse('cannot fetch entire /.tag directory for %r' % spec_args)
296 debug1('src: %s\n' % loc_desc(src))
300 def get_save_branch(repo, path):
301 res = repo.resolve(path, follow=False, want_meta=False)
302 leaf_name, leaf_item = res[-1]
304 misuse('error: cannot access %r in %r' % (leaf_name, path))
306 res_path = '/'.join(name for name, item in res[:-1])
310 def resolve_branch_dest(spec, src, src_repo, dest_repo):
311 # Resulting dest must be treeish, or not exist.
313 # Pick a default dest.
314 if src.type == 'branch':
315 spec = spec._replace(dest=spec.src)
316 elif src.type == 'save':
317 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
318 elif src.path.startswith('/.tag/'): # Dest defaults to the same.
319 spec = spec._replace(dest=spec.src)
321 spec_args = '%s %s' % (spec.argopt, spec.argval)
323 misuse('no destination (implicit or explicit) for %r', spec_args)
325 dest = find_vfs_item(spec.dest, dest_repo)
327 if dest.type == 'commit':
328 misuse('destination for %r is a tagged commit, not a branch'
330 if dest.type != 'branch':
331 misuse('destination for %r is a %s, not a branch'
332 % (spec_args, dest.type))
334 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
336 if dest.path.startswith('/.'):
337 misuse('destination for %r must be a valid branch name' % spec_args)
339 debug1('dest: %s\n' % loc_desc(dest))
343 def resolve_ff(spec, src_repo, dest_repo):
344 src = resolve_src(spec, src_repo)
345 spec_args = '%s %s' % (spec.argopt, spec.argval)
346 if src.type == 'tree':
347 misuse('%r is impossible; can only --append a tree to a branch'
349 if src.type not in ('branch', 'save', 'commit'):
350 misuse('source for %r must be a branch, save, or commit, not %s'
351 % (spec_args, src.type))
352 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
353 return Target(spec=spec, src=src, dest=dest)
356 def handle_ff(item, src_repo, writer, opt):
357 assert item.spec.method == 'ff'
358 assert item.src.type in ('branch', 'save', 'commit')
359 src_oidx = item.src.hash.encode('hex')
360 dest_oidx = item.dest.hash.encode('hex') if item.dest.hash else None
361 if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
363 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
364 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit'))
365 return item.src.hash, commit_items.tree.decode('hex')
366 spec_args = '%s %s' % (item.spec.argopt, item.spec.argval)
367 misuse('destination is not an ancestor of source for %r' % spec_args)
370 def resolve_append(spec, src_repo, dest_repo):
371 src = resolve_src(spec, src_repo)
372 if src.type not in ('branch', 'save', 'commit', 'tree'):
373 spec_args = '%s %s' % (spec.argopt, spec.argval)
374 misuse('source for %r must be a branch, save, commit, or tree, not %s'
375 % (spec_args, src.type))
376 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
377 return Target(spec=spec, src=src, dest=dest)
380 def handle_append(item, src_repo, writer, opt):
381 assert item.spec.method == 'append'
382 assert item.src.type in ('branch', 'save', 'commit', 'tree')
383 assert item.dest.type == 'branch' or not item.dest.type
384 src_oidx = item.src.hash.encode('hex')
385 if item.src.type == 'tree':
386 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
387 parent = item.dest.hash
388 msg = 'bup save\n\nGenerated by command:\n%r\n' % sys.argv
389 userline = '%s <%s@%s>' % (userfullname(), username(), hostname())
391 commit = writer.new_commit(item.src.hash, parent,
393 userline, now, None, msg)
394 return commit, item.src.hash
395 commits = list(src_repo.rev_list(src_oidx))
397 return append_commits(commits, item.spec.src, item.dest.hash,
398 src_repo, writer, opt)
401 def resolve_pick(spec, src_repo, dest_repo):
402 src = resolve_src(spec, src_repo)
403 spec_args = '%s %s' % (spec.argopt, spec.argval)
404 if src.type == 'tree':
405 misuse('%r is impossible; can only --append a tree' % spec_args)
406 if src.type not in ('commit', 'save'):
407 misuse('%r impossible; can only pick a commit or save, not %s'
408 % (spec_args, src.type))
410 if src.path.startswith('/.tag/'):
411 spec = spec._replace(dest=spec.src)
412 elif src.type == 'save':
413 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
415 misuse('no destination provided for %r', spec_args)
416 dest = find_vfs_item(spec.dest, dest_repo)
418 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
419 dest = default_loc._replace(path=cp)
421 if not dest.type == 'branch' and not dest.path.startswith('/.tag/'):
422 misuse('%r destination is not a tag or branch' % spec_args)
423 if spec.method == 'pick' \
424 and dest.hash and dest.path.startswith('/.tag/'):
425 misuse('cannot overwrite existing tag for %r (requires --force-pick)'
427 return Target(spec=spec, src=src, dest=dest)
430 def handle_pick(item, src_repo, writer, opt):
431 assert item.spec.method in ('pick', 'force-pick')
432 assert item.src.type in ('save', 'commit')
433 src_oidx = item.src.hash.encode('hex')
435 return append_commit(item.spec.src, src_oidx, item.dest.hash,
436 src_repo, writer, opt)
437 return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt)
440 def resolve_new_tag(spec, src_repo, dest_repo):
441 src = resolve_src(spec, src_repo)
442 spec_args = '%s %s' % (spec.argopt, spec.argval)
443 if not spec.dest and src.path.startswith('/.tag/'):
444 spec = spec._replace(dest=src.path)
446 misuse('no destination (implicit or explicit) for %r', spec_args)
447 dest = find_vfs_item(spec.dest, dest_repo)
449 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
450 if not dest.path.startswith('/.tag/'):
451 misuse('destination for %r must be a VFS tag' % spec_args)
453 misuse('cannot overwrite existing tag for %r (requires --replace)'
455 return Target(spec=spec, src=src, dest=dest)
458 def handle_new_tag(item, src_repo, writer, opt):
459 assert item.spec.method == 'new-tag'
460 assert item.dest.path.startswith('/.tag/')
461 get_random_item(item.spec.src, item.src.hash.encode('hex'),
462 src_repo, writer, opt)
463 return (item.src.hash,)
466 def resolve_replace(spec, src_repo, dest_repo):
467 src = resolve_src(spec, src_repo)
468 spec_args = '%s %s' % (spec.argopt, spec.argval)
470 if src.path.startswith('/.tag/') or src.type == 'branch':
471 spec = spec._replace(dest=spec.src)
473 misuse('no destination provided for %r', spec_args)
474 dest = find_vfs_item(spec.dest, dest_repo)
476 if not dest.type == 'branch' and not dest.path.startswith('/.tag/'):
477 misuse('%r impossible; can only overwrite branch or tag'
480 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
481 dest = default_loc._replace(path=cp)
482 if not dest.path.startswith('/.tag/') \
483 and not src.type in ('branch', 'save', 'commit'):
484 misuse('cannot overwrite branch with %s for %r' % (src.type, spec_args))
485 return Target(spec=spec, src=src, dest=dest)
488 def handle_replace(item, src_repo, writer, opt):
489 assert(item.spec.method == 'replace')
490 if item.dest.path.startswith('/.tag/'):
491 get_random_item(item.spec.src, item.src.hash.encode('hex'),
492 src_repo, writer, opt)
493 return (item.src.hash,)
494 assert(item.dest.type == 'branch' or not item.dest.type)
495 src_oidx = item.src.hash.encode('hex')
496 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
497 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit'))
498 return item.src.hash, commit_items.tree.decode('hex')
501 def resolve_unnamed(spec, src_repo, dest_repo):
503 spec_args = '%s %s' % (spec.argopt, spec.argval)
504 misuse('destination name given for %r' % spec_args)
505 src = resolve_src(spec, src_repo)
506 return Target(spec=spec, src=src, dest=None)
509 def handle_unnamed(item, src_repo, writer, opt):
510 get_random_item(item.spec.src, item.src.hash.encode('hex'),
511 src_repo, writer, opt)
515 def resolve_targets(specs, src_repo, dest_repo):
517 common_args = src_repo, dest_repo
519 debug1('initial-spec: %s\n' % str(spec))
520 if spec.method == 'ff':
521 resolved_items.append(resolve_ff(spec, *common_args))
522 elif spec.method == 'append':
523 resolved_items.append(resolve_append(spec, *common_args))
524 elif spec.method in ('pick', 'force-pick'):
525 resolved_items.append(resolve_pick(spec, *common_args))
526 elif spec.method == 'new-tag':
527 resolved_items.append(resolve_new_tag(spec, *common_args))
528 elif spec.method == 'replace':
529 resolved_items.append(resolve_replace(spec, *common_args))
530 elif spec.method == 'unnamed':
531 resolved_items.append(resolve_unnamed(spec, *common_args))
532 else: # Should be impossible -- prevented by the option parser.
535 # FIXME: check for prefix overlap? i.e.:
536 # bup get --ff foo --ff: baz foo/bar
537 # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar
539 # Now that we have all the items, check for duplicate tags.
540 tags_targeted = set()
541 for item in resolved_items:
542 dest_path = item.dest and item.dest.path
544 assert(dest_path.startswith('/'))
545 if dest_path.startswith('/.tag/'):
546 if dest_path in tags_targeted:
547 if item.spec.method not in ('replace', 'force-pick'):
548 spec_args = '%s %s' % (item.spec.argopt,
550 misuse('cannot overwrite tag %r via %r' \
551 % (dest_path, spec_args))
553 tags_targeted.add(dest_path)
554 return resolved_items
557 def log_item(name, type, opt, tree=None, commit=None, tag=None):
558 if tag and opt.print_tags:
560 if tree and opt.print_trees:
562 if commit and opt.print_commits:
563 print(hexstr(commit))
566 if type in ('root', 'branch', 'save', 'commit', 'tree'):
567 if not name.endswith('/'):
569 log('%s%s\n' % (name, last))
573 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
574 opt = parse_args(sys.argv)
575 git.check_repo_or_die()
576 src_dir = opt.source or git.repo()
578 client.bwlimit = parse_num(opt.bwlimit)
579 if is_reverse and opt.remote:
580 misuse("don't use -r in reverse mode; it's automatic")
582 opt.remote = argv_bytes(opt.remote)
583 if opt.remote or is_reverse:
584 dest_repo = RemoteRepo(opt.remote)
586 dest_repo = LocalRepo()
588 with dest_repo as dest_repo:
589 with LocalRepo(repo_dir=src_dir) as src_repo:
590 with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
591 # Resolve and validate all sources and destinations,
592 # implicit or explicit, and do it up-front, so we can
593 # fail before we start writing (for any obviously
595 target_items = resolve_targets(opt.target_specs,
598 updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
599 no_ref_info = (None, None)
601 handlers = {'ff': handle_ff,
602 'append': handle_append,
603 'force-pick': handle_pick,
605 'new-tag': handle_new_tag,
606 'replace': handle_replace,
607 'unnamed': handle_unnamed}
609 for item in target_items:
610 debug1('get-spec: %s\n' % str(item.spec))
611 debug1('get-src: %s\n' % loc_desc(item.src))
612 debug1('get-dest: %s\n' % loc_desc(item.dest))
613 dest_path = item.dest and item.dest.path
615 if dest_path.startswith('/.tag/'):
616 dest_ref = 'refs/tags/%s' % dest_path[6:]
618 dest_ref = 'refs/heads/%s' % dest_path[1:]
622 dest_hash = item.dest and item.dest.hash
623 orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
624 orig_ref = orig_ref or dest_hash
625 cur_ref = cur_ref or dest_hash
627 handler = handlers[item.spec.method]
628 item_result = handler(item, src_repo, writer, opt)
629 if len(item_result) > 1:
630 new_id, tree = item_result
632 new_id = item_result[0]
635 log_item(item.spec.src, item.src.type, opt)
637 updated_refs[dest_ref] = (orig_ref, new_id)
638 if dest_ref.startswith('refs/tags/'):
639 log_item(item.spec.src, item.src.type, opt, tag=new_id)
641 log_item(item.spec.src, item.src.type, opt,
642 tree=tree, commit=new_id)
644 # Only update the refs at the very end, once the writer is
645 # closed, so that if something goes wrong above, the old refs
646 # will be undisturbed.
647 for ref_name, info in updated_refs.iteritems():
648 orig_ref, new_ref = info
650 dest_repo.update_ref(ref_name, new_ref, orig_ref)
652 new_hex = new_ref.encode('hex')
654 orig_hex = orig_ref.encode('hex')
655 log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
657 log('updated %r (%s)\n' % (ref_name, new_hex))
658 except (git.GitError, client.ClientError), ex:
659 add_error('unable to update ref %r: %s' % (ref_name, ex))
662 log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))