3 # https://sourceware.org/bugzilla/show_bug.cgi?id=26034
4 export "BUP_ARGV_0"="$0"
7 export "BUP_ARGV_${arg_i}"="$arg"
11 # Here to end of preamble replaced during install
12 bup_python="$(dirname "$0")/bup-python" || exit $?
13 exec "$bup_python" "$0"
17 from __future__ import absolute_import, print_function
18 import os, re, stat, sys, textwrap, time
19 from binascii import hexlify, unhexlify
20 from collections import namedtuple
21 from functools import partial
22 from stat import S_ISDIR
24 sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..']
26 from bup import compat, git, client, helpers, vfs
27 from bup.compat import argv_bytes, environ, hexstr, items, wrap_main
28 from bup.git import get_cat_data, parse_commit, walk_object
29 from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors
30 from bup.helpers import hostname, shstr, tty_width
31 from bup.io import path_msg
32 from bup.pwdgrp import userfullname, username
33 from bup.repo import LocalRepo, RemoteRepo
36 "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...",
38 """Transfer data from a source repository to a destination repository
39 according to the methods specified (--ff, --ff:, --append, etc.).
40 Both repositories default to BUP_DIR. A remote destination may be
41 specified with -r, and data may be pulled from a remote repository
42 with the related "bup on HOST get ..." command.""",
44 ('optional arguments:',
45 (('-h, --help', 'show this help message and exit'),
47 'increase log output (can be specified more than once)'),
48 ('-q, --quiet', "don't show progress meter"),
49 ('-s SOURCE, --source SOURCE',
50 'path to the source repository (defaults to BUP_DIR)'),
51 ('-r REMOTE, --remote REMOTE',
52 'hostname:/path/to/repo of remote destination repository'),
53 ('-t --print-trees', 'output a tree id for each ref set'),
54 ('-c, --print-commits', 'output a commit id for each ref set'),
55 ('--print-tags', 'output an id for each tag'),
56 ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'),
57 ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL',
58 'set compression LEVEL (default: 1)'))),
61 (('--ff REF, --ff: REF DEST',
62 'fast-forward dest REF (or DEST) to match source REF'),
63 ('--append REF, --append: REF DEST',
64 'append REF (treeish or committish) to dest REF (or DEST)'),
65 ('--pick REF, --pick: REF DEST',
66 'append single source REF commit to dest REF (or DEST)'),
67 ('--force-pick REF, --force-pick: REF DEST',
68 '--pick, overwriting REF (or DEST)'),
69 ('--new-tag REF, --new-tag: REF DEST',
70 'tag source ref REF as REF (or DEST) in dest unless it already exists'),
71 ('--replace, --replace: REF DEST',
72 'overwrite REF (or DEST) in dest with source REF'),
74 'fetch REF anonymously (without destination ref)'))))
76 def render_opts(opts, width=None):
80 for args, desc in opts:
81 result.append(textwrap.fill(args, width=width,
82 initial_indent=(' ' * 2),
83 subsequent_indent=(' ' * 4)))
85 result.append(textwrap.fill(desc, width=width,
86 initial_indent=(' ' * 6),
87 subsequent_indent=(' ' * 6)))
91 def usage(argspec, width=None):
94 usage, preamble, groups = argspec[0], argspec[1], argspec[2:]
96 msg.append(textwrap.fill(usage, width=width, subsequent_indent=' '))
98 msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width))
100 for group_name, group_args in groups:
101 msg.extend(['\n', group_name, '\n'])
102 msg.extend(render_opts(group_args, width=width))
105 def misuse(message=None):
106 sys.stderr.write(usage(argspec))
108 sys.stderr.write('\nerror: ')
109 sys.stderr.write(message)
110 sys.stderr.write('\n')
113 def require_n_args_or_die(n, args):
114 if len(args) < n + 1:
115 misuse('%s argument requires %d %s'
116 % (n, 'values' if n == 1 else 'value'))
117 result = args[1:1+n], args[1+n:]
118 assert len(result[0]) == n
121 Spec = namedtuple('Spec', ('method', 'src', 'dest'))
125 return '--%s %s' % (s.method, path_msg(s.src))
126 return '--%s: %s %s' % (s.method, path_msg(s.src), path_msg(s.dest))
128 def parse_args(args):
135 opt.print_commits = opt.print_trees = opt.print_tags = False
138 opt.source = opt.remote = None
139 opt.target_specs = []
141 remaining = args[1:] # Skip argv[0]
144 if arg in ('-h', '--help'):
145 sys.stdout.write(usage(argspec))
147 elif arg in ('-v', '--verbose'):
149 remaining = remaining[1:]
150 elif arg in ('--ff', '--append', '--pick', '--force-pick',
151 '--new-tag', '--replace', '--unnamed'):
152 (ref,), remaining = require_n_args_or_die(1, remaining)
153 ref = argv_bytes(ref)
154 opt.target_specs.append(Spec(method=arg[2:], src=ref, dest=None))
155 elif arg in ('--ff:', '--append:', '--pick:', '--force-pick:',
156 '--new-tag:', '--replace:'):
157 (ref, dest), remaining = require_n_args_or_die(2, remaining)
158 ref, dest = argv_bytes(ref), argv_bytes(dest)
159 opt.target_specs.append(Spec(method=arg[2:-1], src=ref, dest=dest))
160 elif arg in ('-s', '--source'):
161 (opt.source,), remaining = require_n_args_or_die(1, remaining)
162 elif arg in ('-r', '--remote'):
163 (opt.remote,), remaining = require_n_args_or_die(1, remaining)
164 elif arg in ('-c', '--print-commits'):
165 opt.print_commits, remaining = True, remaining[1:]
166 elif arg in ('-t', '--print-trees'):
167 opt.print_trees, remaining = True, remaining[1:]
168 elif arg == '--print-tags':
169 opt.print_tags, remaining = True, remaining[1:]
170 elif arg in ('-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'):
171 opt.compress = int(arg[1:])
172 remaining = remaining[1:]
173 elif arg == '--compress':
174 (opt.compress,), remaining = require_n_args_or_die(1, remaining)
175 opt.compress = int(opt.compress)
176 elif arg == '--bwlimit':
177 (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining)
178 opt.bwlimit = long(opt.bwlimit)
179 elif arg.startswith('-') and len(arg) > 2 and arg[1] != '-':
180 # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z".
181 # We do this last so that --foo -bar is valid if --foo
183 remaining[0:1] = ('-' + c for c in arg[1:])
190 # FIXME: client error handling (remote exceptions, etc.)
192 # FIXME: walk_object in in git.py doesn't support opt.verbose. Do we
193 # need to adjust for that here?
194 def get_random_item(name, hash, repo, writer, opt):
195 def already_seen(oid):
196 return writer.exists(unhexlify(oid))
197 for item in walk_object(repo.cat, hash, stop_at=already_seen,
199 # already_seen ensures that writer.exists(id) is false.
200 # Otherwise, just_write() would fail.
201 writer.just_write(item.oid, item.type, item.data)
204 def append_commit(name, hash, parent, src_repo, writer, opt):
206 items = parse_commit(get_cat_data(src_repo.cat(hash), b'commit'))
207 tree = unhexlify(items.tree)
208 author = b'%s <%s>' % (items.author_name, items.author_mail)
209 author_time = (items.author_sec, items.author_offset)
210 committer = b'%s <%s@%s>' % (userfullname(), username(), hostname())
211 get_random_item(name, hexlify(tree), src_repo, writer, opt)
212 c = writer.new_commit(tree, parent,
213 author, items.author_sec, items.author_offset,
214 committer, now, None,
219 def append_commits(commits, src_name, dest_hash, src_repo, writer, opt):
220 last_c, tree = dest_hash, None
221 for commit in commits:
222 last_c, tree = append_commit(src_name, commit, last_c,
223 src_repo, writer, opt)
224 assert(tree is not None)
227 Loc = namedtuple('Loc', ['type', 'hash', 'path'])
228 default_loc = Loc(None, None, None)
230 def find_vfs_item(name, repo):
231 res = repo.resolve(name, follow=False, want_meta=False)
232 leaf_name, leaf_item = res[-1]
235 kind = type(leaf_item)
238 elif kind == vfs.Tags:
240 elif kind == vfs.RevList:
242 elif kind == vfs.Commit:
243 if len(res) > 1 and type(res[-2][1]) == vfs.RevList:
247 elif kind == vfs.Item:
248 if S_ISDIR(vfs.item_mode(leaf_item)):
252 elif kind == vfs.Chunky:
254 elif kind == vfs.FakeLink:
255 # Don't have to worry about ELOOP, excepting malicious
256 # remotes, since "latest" is the only FakeLink.
257 assert leaf_name == b'latest'
258 res = repo.resolve(leaf_item.target, parent=res[:-1],
259 follow=False, want_meta=False)
260 leaf_name, leaf_item = res[-1]
262 assert type(leaf_item) == vfs.Commit
263 name = b'/'.join(x[0] for x in res)
266 raise Exception('unexpected resolution for %s: %r'
267 % (path_msg(name), res))
268 path = b'/'.join(name for name, item in res)
269 if hasattr(leaf_item, 'coid'):
270 result = Loc(type=kind, hash=leaf_item.coid, path=path)
271 elif hasattr(leaf_item, 'oid'):
272 result = Loc(type=kind, hash=leaf_item.oid, path=path)
274 result = Loc(type=kind, hash=None, path=path)
278 Target = namedtuple('Target', ['spec', 'src', 'dest'])
282 loc = loc._replace(hash=hexlify(loc.hash))
286 # FIXME: see if resolve() means we can drop the vfs path cleanup
288 def cleanup_vfs_path(p):
289 result = os.path.normpath(p)
290 if result.startswith(b'/'):
295 def validate_vfs_path(p):
296 if p.startswith(b'/.') \
297 and not p.startswith(b'/.tag/'):
298 misuse('unsupported destination path %s in %s'
299 % (path_msg(dest.path), spec_msg(spec)))
303 def resolve_src(spec, src_repo):
304 src = find_vfs_item(spec.src, src_repo)
305 spec_args = spec_msg(spec)
307 misuse('cannot find source for %s' % spec_args)
308 if src.type == 'root':
309 misuse('cannot fetch entire repository for %s' % spec_args)
310 if src.type == 'tags':
311 misuse('cannot fetch entire /.tag directory for %s' % spec_args)
312 debug1('src: %s\n' % loc_desc(src))
316 def get_save_branch(repo, path):
317 res = repo.resolve(path, follow=False, want_meta=False)
318 leaf_name, leaf_item = res[-1]
320 misuse('error: cannot access %r in %r' % (leaf_name, path))
322 res_path = b'/'.join(name for name, item in res[:-1])
326 def resolve_branch_dest(spec, src, src_repo, dest_repo):
327 # Resulting dest must be treeish, or not exist.
329 # Pick a default dest.
330 if src.type == 'branch':
331 spec = spec._replace(dest=spec.src)
332 elif src.type == 'save':
333 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
334 elif src.path.startswith(b'/.tag/'): # Dest defaults to the same.
335 spec = spec._replace(dest=spec.src)
337 spec_args = spec_msg(spec)
339 misuse('no destination (implicit or explicit) for %s', spec_args)
341 dest = find_vfs_item(spec.dest, dest_repo)
343 if dest.type == 'commit':
344 misuse('destination for %s is a tagged commit, not a branch'
346 if dest.type != 'branch':
347 misuse('destination for %s is a %s, not a branch'
348 % (spec_args, dest.type))
350 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
352 if dest.path.startswith(b'/.'):
353 misuse('destination for %s must be a valid branch name' % spec_args)
355 debug1('dest: %s\n' % loc_desc(dest))
359 def resolve_ff(spec, src_repo, dest_repo):
360 src = resolve_src(spec, src_repo)
361 spec_args = spec_msg(spec)
362 if src.type == 'tree':
363 misuse('%s is impossible; can only --append a tree to a branch'
365 if src.type not in ('branch', 'save', 'commit'):
366 misuse('source for %s must be a branch, save, or commit, not %s'
367 % (spec_args, src.type))
368 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
369 return Target(spec=spec, src=src, dest=dest)
372 def handle_ff(item, src_repo, writer, opt):
373 assert item.spec.method == 'ff'
374 assert item.src.type in ('branch', 'save', 'commit')
375 src_oidx = hexlify(item.src.hash)
376 dest_oidx = hexlify(item.dest.hash) if item.dest.hash else None
377 if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
379 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
380 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
381 return item.src.hash, unhexlify(commit_items.tree)
382 misuse('destination is not an ancestor of source for %s'
383 % spec_msg(item.spec))
386 def resolve_append(spec, src_repo, dest_repo):
387 src = resolve_src(spec, src_repo)
388 if src.type not in ('branch', 'save', 'commit', 'tree'):
389 misuse('source for %s must be a branch, save, commit, or tree, not %s'
390 % (spec_msg(spec), src.type))
391 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
392 return Target(spec=spec, src=src, dest=dest)
395 def handle_append(item, src_repo, writer, opt):
396 assert item.spec.method == 'append'
397 assert item.src.type in ('branch', 'save', 'commit', 'tree')
398 assert item.dest.type == 'branch' or not item.dest.type
399 src_oidx = hexlify(item.src.hash)
400 if item.src.type == 'tree':
401 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
402 parent = item.dest.hash
403 msg = b'bup save\n\nGenerated by command:\n%r\n' % compat.argvb
404 userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
406 commit = writer.new_commit(item.src.hash, parent,
408 userline, now, None, msg)
409 return commit, item.src.hash
410 commits = list(src_repo.rev_list(src_oidx))
412 return append_commits(commits, item.spec.src, item.dest.hash,
413 src_repo, writer, opt)
416 def resolve_pick(spec, src_repo, dest_repo):
417 src = resolve_src(spec, src_repo)
418 spec_args = spec_msg(spec)
419 if src.type == 'tree':
420 misuse('%s is impossible; can only --append a tree' % spec_args)
421 if src.type not in ('commit', 'save'):
422 misuse('%s impossible; can only pick a commit or save, not %s'
423 % (spec_args, src.type))
425 if src.path.startswith(b'/.tag/'):
426 spec = spec._replace(dest=spec.src)
427 elif src.type == 'save':
428 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
430 misuse('no destination provided for %s', spec_args)
431 dest = find_vfs_item(spec.dest, dest_repo)
433 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
434 dest = default_loc._replace(path=cp)
436 if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
437 misuse('%s destination is not a tag or branch' % spec_args)
438 if spec.method == 'pick' \
439 and dest.hash and dest.path.startswith(b'/.tag/'):
440 misuse('cannot overwrite existing tag for %s (requires --force-pick)'
442 return Target(spec=spec, src=src, dest=dest)
445 def handle_pick(item, src_repo, writer, opt):
446 assert item.spec.method in ('pick', 'force-pick')
447 assert item.src.type in ('save', 'commit')
448 src_oidx = hexlify(item.src.hash)
450 return append_commit(item.spec.src, src_oidx, item.dest.hash,
451 src_repo, writer, opt)
452 return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt)
455 def resolve_new_tag(spec, src_repo, dest_repo):
456 src = resolve_src(spec, src_repo)
457 spec_args = spec_msg(spec)
458 if not spec.dest and src.path.startswith(b'/.tag/'):
459 spec = spec._replace(dest=src.path)
461 misuse('no destination (implicit or explicit) for %s', spec_args)
462 dest = find_vfs_item(spec.dest, dest_repo)
464 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
465 if not dest.path.startswith(b'/.tag/'):
466 misuse('destination for %s must be a VFS tag' % spec_args)
468 misuse('cannot overwrite existing tag for %s (requires --replace)'
470 return Target(spec=spec, src=src, dest=dest)
473 def handle_new_tag(item, src_repo, writer, opt):
474 assert item.spec.method == 'new-tag'
475 assert item.dest.path.startswith(b'/.tag/')
476 get_random_item(item.spec.src, hexlify(item.src.hash),
477 src_repo, writer, opt)
478 return (item.src.hash,)
481 def resolve_replace(spec, src_repo, dest_repo):
482 src = resolve_src(spec, src_repo)
483 spec_args = spec_msg(spec)
485 if src.path.startswith(b'/.tag/') or src.type == 'branch':
486 spec = spec._replace(dest=spec.src)
488 misuse('no destination provided for %s', spec_args)
489 dest = find_vfs_item(spec.dest, dest_repo)
491 if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
492 misuse('%s impossible; can only overwrite branch or tag'
495 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
496 dest = default_loc._replace(path=cp)
497 if not dest.path.startswith(b'/.tag/') \
498 and not src.type in ('branch', 'save', 'commit'):
499 misuse('cannot overwrite branch with %s for %s' % (src.type, spec_args))
500 return Target(spec=spec, src=src, dest=dest)
503 def handle_replace(item, src_repo, writer, opt):
504 assert(item.spec.method == 'replace')
505 if item.dest.path.startswith(b'/.tag/'):
506 get_random_item(item.spec.src, hexlify(item.src.hash),
507 src_repo, writer, opt)
508 return (item.src.hash,)
509 assert(item.dest.type == 'branch' or not item.dest.type)
510 src_oidx = hexlify(item.src.hash)
511 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
512 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
513 return item.src.hash, unhexlify(commit_items.tree)
516 def resolve_unnamed(spec, src_repo, dest_repo):
518 misuse('destination name given for %s' % spec_msg(spec))
519 src = resolve_src(spec, src_repo)
520 return Target(spec=spec, src=src, dest=None)
523 def handle_unnamed(item, src_repo, writer, opt):
524 get_random_item(item.spec.src, hexlify(item.src.hash),
525 src_repo, writer, opt)
529 def resolve_targets(specs, src_repo, dest_repo):
531 common_args = src_repo, dest_repo
533 debug1('initial-spec: %r\n' % (spec,))
534 if spec.method == 'ff':
535 resolved_items.append(resolve_ff(spec, *common_args))
536 elif spec.method == 'append':
537 resolved_items.append(resolve_append(spec, *common_args))
538 elif spec.method in ('pick', 'force-pick'):
539 resolved_items.append(resolve_pick(spec, *common_args))
540 elif spec.method == 'new-tag':
541 resolved_items.append(resolve_new_tag(spec, *common_args))
542 elif spec.method == 'replace':
543 resolved_items.append(resolve_replace(spec, *common_args))
544 elif spec.method == 'unnamed':
545 resolved_items.append(resolve_unnamed(spec, *common_args))
546 else: # Should be impossible -- prevented by the option parser.
549 # FIXME: check for prefix overlap? i.e.:
550 # bup get --ff foo --ff: baz foo/bar
551 # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar
553 # Now that we have all the items, check for duplicate tags.
554 tags_targeted = set()
555 for item in resolved_items:
556 dest_path = item.dest and item.dest.path
558 assert(dest_path.startswith(b'/'))
559 if dest_path.startswith(b'/.tag/'):
560 if dest_path in tags_targeted:
561 if item.spec.method not in ('replace', 'force-pick'):
562 misuse('cannot overwrite tag %s via %s' \
563 % (path_msg(dest_path), spec_msg(item.spec)))
565 tags_targeted.add(dest_path)
566 return resolved_items
569 def log_item(name, type, opt, tree=None, commit=None, tag=None):
570 if tag and opt.print_tags:
572 if tree and opt.print_trees:
574 if commit and opt.print_commits:
575 print(hexstr(commit))
578 if type in ('root', 'branch', 'save', 'commit', 'tree'):
579 if not name.endswith(b'/'):
581 log('%s%s\n' % (path_msg(name), last))
585 is_reverse = environ.get(b'BUP_SERVER_REVERSE')
586 opt = parse_args(compat.argv)
587 git.check_repo_or_die()
589 opt.source = argv_bytes(opt.source)
590 src_dir = opt.source or git.repo()
592 client.bwlimit = parse_num(opt.bwlimit)
593 if is_reverse and opt.remote:
594 misuse("don't use -r in reverse mode; it's automatic")
596 opt.remote = argv_bytes(opt.remote)
597 if opt.remote or is_reverse:
598 dest_repo = RemoteRepo(opt.remote)
600 dest_repo = LocalRepo()
602 with dest_repo as dest_repo:
603 with LocalRepo(repo_dir=src_dir) as src_repo:
604 with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
605 # Resolve and validate all sources and destinations,
606 # implicit or explicit, and do it up-front, so we can
607 # fail before we start writing (for any obviously
609 target_items = resolve_targets(opt.target_specs,
612 updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
613 no_ref_info = (None, None)
615 handlers = {'ff': handle_ff,
616 'append': handle_append,
617 'force-pick': handle_pick,
619 'new-tag': handle_new_tag,
620 'replace': handle_replace,
621 'unnamed': handle_unnamed}
623 for item in target_items:
624 debug1('get-spec: %r\n' % (item.spec,))
625 debug1('get-src: %s\n' % loc_desc(item.src))
626 debug1('get-dest: %s\n' % loc_desc(item.dest))
627 dest_path = item.dest and item.dest.path
629 if dest_path.startswith(b'/.tag/'):
630 dest_ref = b'refs/tags/%s' % dest_path[6:]
632 dest_ref = b'refs/heads/%s' % dest_path[1:]
636 dest_hash = item.dest and item.dest.hash
637 orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
638 orig_ref = orig_ref or dest_hash
639 cur_ref = cur_ref or dest_hash
641 handler = handlers[item.spec.method]
642 item_result = handler(item, src_repo, writer, opt)
643 if len(item_result) > 1:
644 new_id, tree = item_result
646 new_id = item_result[0]
649 log_item(item.spec.src, item.src.type, opt)
651 updated_refs[dest_ref] = (orig_ref, new_id)
652 if dest_ref.startswith(b'refs/tags/'):
653 log_item(item.spec.src, item.src.type, opt, tag=new_id)
655 log_item(item.spec.src, item.src.type, opt,
656 tree=tree, commit=new_id)
658 # Only update the refs at the very end, once the writer is
659 # closed, so that if something goes wrong above, the old refs
660 # will be undisturbed.
661 for ref_name, info in items(updated_refs):
662 orig_ref, new_ref = info
664 dest_repo.update_ref(ref_name, new_ref, orig_ref)
666 new_hex = hexlify(new_ref)
668 orig_hex = hexlify(orig_ref)
669 log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
671 log('updated %r (%s)\n' % (ref_name, new_hex))
672 except (git.GitError, client.ClientError) as ex:
673 add_error('unable to update ref %r: %s' % (ref_name, ex))
676 log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))