2 from __future__ import absolute_import, print_function
3 from binascii import hexlify, unhexlify
4 from collections import namedtuple
5 from functools import partial
6 from stat import S_ISDIR
7 import os, sys, textwrap, time
9 from bup import compat, git, client, helpers, vfs
10 from bup.compat import (
18 from bup.git import get_cat_data, parse_commit, walk_object
19 from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors
20 from bup.helpers import hostname, shstr, tty_width
21 from bup.io import path_msg
22 from bup.pwdgrp import userfullname, username
23 from bup.repo import LocalRepo, RemoteRepo
26 "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...",
28 """Transfer data from a source repository to a destination repository
29 according to the methods specified (--ff, --ff:, --append, etc.).
30 Both repositories default to BUP_DIR. A remote destination may be
31 specified with -r, and data may be pulled from a remote repository
32 with the related "bup on HOST get ..." command.""",
34 ('optional arguments:',
35 (('-h, --help', 'show this help message and exit'),
37 'increase log output (can be specified more than once)'),
38 ('-q, --quiet', "don't show progress meter"),
39 ('-s SOURCE, --source SOURCE',
40 'path to the source repository (defaults to BUP_DIR)'),
41 ('-r REMOTE, --remote REMOTE',
42 'hostname:/path/to/repo of remote destination repository'),
43 ('-t --print-trees', 'output a tree id for each ref set'),
44 ('-c, --print-commits', 'output a commit id for each ref set'),
45 ('--print-tags', 'output an id for each tag'),
46 ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'),
47 ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL',
48 'set compression LEVEL (default: 1)'))),
51 (('--ff REF, --ff: REF DEST',
52 'fast-forward dest REF (or DEST) to match source REF'),
53 ('--append REF, --append: REF DEST',
54 'append REF (treeish or committish) to dest REF (or DEST)'),
55 ('--pick REF, --pick: REF DEST',
56 'append single source REF commit to dest REF (or DEST)'),
57 ('--force-pick REF, --force-pick: REF DEST',
58 '--pick, overwriting REF (or DEST)'),
59 ('--new-tag REF, --new-tag: REF DEST',
60 'tag source ref REF as REF (or DEST) in dest unless it already exists'),
61 ('--replace, --replace: REF DEST',
62 'overwrite REF (or DEST) in dest with source REF'),
64 'fetch REF anonymously (without destination ref)'))))
66 def render_opts(opts, width=None):
70 for args, desc in opts:
71 result.append(textwrap.fill(args, width=width,
72 initial_indent=(' ' * 2),
73 subsequent_indent=(' ' * 4)))
75 result.append(textwrap.fill(desc, width=width,
76 initial_indent=(' ' * 6),
77 subsequent_indent=(' ' * 6)))
81 def usage(argspec, width=None):
84 usage, preamble, groups = argspec[0], argspec[1], argspec[2:]
86 msg.append(textwrap.fill(usage, width=width, subsequent_indent=' '))
88 msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width))
90 for group_name, group_args in groups:
91 msg.extend(['\n', group_name, '\n'])
92 msg.extend(render_opts(group_args, width=width))
95 def misuse(message=None):
96 sys.stderr.write(usage(argspec))
98 sys.stderr.write('\nerror: ')
99 sys.stderr.write(message)
100 sys.stderr.write('\n')
103 def require_n_args_or_die(n, args):
104 if len(args) < n + 1:
105 misuse('%s argument requires %d %s'
106 % (n, 'values' if n == 1 else 'value'))
107 result = args[1:1+n], args[1+n:]
108 assert len(result[0]) == n
111 Spec = namedtuple('Spec', ('method', 'src', 'dest'))
115 return '--%s %s' % (s.method, path_msg(s.src))
116 return '--%s: %s %s' % (s.method, path_msg(s.src), path_msg(s.dest))
118 def parse_args(args):
125 opt.print_commits = opt.print_trees = opt.print_tags = False
128 opt.source = opt.remote = None
129 opt.target_specs = []
131 remaining = args[1:] # Skip argv[0]
134 if arg in (b'-h', b'--help'):
135 sys.stdout.write(usage(argspec))
137 elif arg in (b'-v', b'--verbose'):
139 remaining = remaining[1:]
140 elif arg in (b'--ff', b'--append', b'--pick', b'--force-pick',
141 b'--new-tag', b'--replace', b'--unnamed'):
142 (ref,), remaining = require_n_args_or_die(1, remaining)
143 opt.target_specs.append(Spec(method=arg[2:].decode('ascii'),
145 elif arg in (b'--ff:', b'--append:', b'--pick:', b'--force-pick:',
146 b'--new-tag:', b'--replace:'):
147 (ref, dest), remaining = require_n_args_or_die(2, remaining)
148 opt.target_specs.append(Spec(method=arg[2:-1].decode('ascii'),
150 elif arg in (b'-s', b'--source'):
151 (opt.source,), remaining = require_n_args_or_die(1, remaining)
152 elif arg in (b'-r', b'--remote'):
153 (opt.remote,), remaining = require_n_args_or_die(1, remaining)
154 elif arg in (b'-c', b'--print-commits'):
155 opt.print_commits, remaining = True, remaining[1:]
156 elif arg in (b'-t', b'--print-trees'):
157 opt.print_trees, remaining = True, remaining[1:]
158 elif arg == b'--print-tags':
159 opt.print_tags, remaining = True, remaining[1:]
160 elif arg in (b'-0', b'-1', b'-2', b'-3', b'-4', b'-5', b'-6', b'-7',
162 opt.compress = int(arg[1:])
163 remaining = remaining[1:]
164 elif arg == b'--compress':
165 (opt.compress,), remaining = require_n_args_or_die(1, remaining)
166 opt.compress = int(opt.compress)
167 elif arg == b'--bwlimit':
168 (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining)
169 opt.bwlimit = long(opt.bwlimit)
170 elif arg.startswith(b'-') and len(arg) > 2 and arg[1] != b'-':
171 # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z".
172 # We do this last so that --foo -bar is valid if --foo
174 remaining[0:1] = (b'-' + bytes_from_byte(c) for c in arg[1:])
181 # FIXME: client error handling (remote exceptions, etc.)
183 # FIXME: walk_object in in git.py doesn't support opt.verbose. Do we
184 # need to adjust for that here?
185 def get_random_item(name, hash, repo, writer, opt):
186 def already_seen(oid):
187 return writer.exists(unhexlify(oid))
188 for item in walk_object(repo.cat, hash, stop_at=already_seen,
190 # already_seen ensures that writer.exists(id) is false.
191 # Otherwise, just_write() would fail.
192 writer.just_write(item.oid, item.type, item.data)
195 def append_commit(name, hash, parent, src_repo, writer, opt):
197 items = parse_commit(get_cat_data(src_repo.cat(hash), b'commit'))
198 tree = unhexlify(items.tree)
199 author = b'%s <%s>' % (items.author_name, items.author_mail)
200 author_time = (items.author_sec, items.author_offset)
201 committer = b'%s <%s@%s>' % (userfullname(), username(), hostname())
202 get_random_item(name, hexlify(tree), src_repo, writer, opt)
203 c = writer.new_commit(tree, parent,
204 author, items.author_sec, items.author_offset,
205 committer, now, None,
210 def append_commits(commits, src_name, dest_hash, src_repo, writer, opt):
211 last_c, tree = dest_hash, None
212 for commit in commits:
213 last_c, tree = append_commit(src_name, commit, last_c,
214 src_repo, writer, opt)
215 assert(tree is not None)
218 Loc = namedtuple('Loc', ['type', 'hash', 'path'])
219 default_loc = Loc(None, None, None)
221 def find_vfs_item(name, repo):
222 res = repo.resolve(name, follow=False, want_meta=False)
223 leaf_name, leaf_item = res[-1]
226 kind = type(leaf_item)
229 elif kind == vfs.Tags:
231 elif kind == vfs.RevList:
233 elif kind == vfs.Commit:
234 if len(res) > 1 and type(res[-2][1]) == vfs.RevList:
238 elif kind == vfs.Item:
239 if S_ISDIR(vfs.item_mode(leaf_item)):
243 elif kind == vfs.Chunky:
245 elif kind == vfs.FakeLink:
246 # Don't have to worry about ELOOP, excepting malicious
247 # remotes, since "latest" is the only FakeLink.
248 assert leaf_name == b'latest'
249 res = repo.resolve(leaf_item.target, parent=res[:-1],
250 follow=False, want_meta=False)
251 leaf_name, leaf_item = res[-1]
253 assert type(leaf_item) == vfs.Commit
254 name = b'/'.join(x[0] for x in res)
257 raise Exception('unexpected resolution for %s: %r'
258 % (path_msg(name), res))
259 path = b'/'.join(name for name, item in res)
260 if hasattr(leaf_item, 'coid'):
261 result = Loc(type=kind, hash=leaf_item.coid, path=path)
262 elif hasattr(leaf_item, 'oid'):
263 result = Loc(type=kind, hash=leaf_item.oid, path=path)
265 result = Loc(type=kind, hash=None, path=path)
269 Target = namedtuple('Target', ['spec', 'src', 'dest'])
273 loc = loc._replace(hash=hexlify(loc.hash))
277 # FIXME: see if resolve() means we can drop the vfs path cleanup
279 def cleanup_vfs_path(p):
280 result = os.path.normpath(p)
281 if result.startswith(b'/'):
286 def validate_vfs_path(p):
287 if p.startswith(b'/.') \
288 and not p.startswith(b'/.tag/'):
289 misuse('unsupported destination path %s in %s'
290 % (path_msg(dest.path), spec_msg(spec)))
294 def resolve_src(spec, src_repo):
295 src = find_vfs_item(spec.src, src_repo)
296 spec_args = spec_msg(spec)
298 misuse('cannot find source for %s' % spec_args)
299 if src.type == 'root':
300 misuse('cannot fetch entire repository for %s' % spec_args)
301 if src.type == 'tags':
302 misuse('cannot fetch entire /.tag directory for %s' % spec_args)
303 debug1('src: %s\n' % loc_desc(src))
307 def get_save_branch(repo, path):
308 res = repo.resolve(path, follow=False, want_meta=False)
309 leaf_name, leaf_item = res[-1]
311 misuse('error: cannot access %r in %r' % (leaf_name, path))
313 res_path = b'/'.join(name for name, item in res[:-1])
317 def resolve_branch_dest(spec, src, src_repo, dest_repo):
318 # Resulting dest must be treeish, or not exist.
320 # Pick a default dest.
321 if src.type == 'branch':
322 spec = spec._replace(dest=spec.src)
323 elif src.type == 'save':
324 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
325 elif src.path.startswith(b'/.tag/'): # Dest defaults to the same.
326 spec = spec._replace(dest=spec.src)
328 spec_args = spec_msg(spec)
330 misuse('no destination (implicit or explicit) for %s', spec_args)
332 dest = find_vfs_item(spec.dest, dest_repo)
334 if dest.type == 'commit':
335 misuse('destination for %s is a tagged commit, not a branch'
337 if dest.type != 'branch':
338 misuse('destination for %s is a %s, not a branch'
339 % (spec_args, dest.type))
341 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
343 if dest.path.startswith(b'/.'):
344 misuse('destination for %s must be a valid branch name' % spec_args)
346 debug1('dest: %s\n' % loc_desc(dest))
350 def resolve_ff(spec, src_repo, dest_repo):
351 src = resolve_src(spec, src_repo)
352 spec_args = spec_msg(spec)
353 if src.type == 'tree':
354 misuse('%s is impossible; can only --append a tree to a branch'
356 if src.type not in ('branch', 'save', 'commit'):
357 misuse('source for %s must be a branch, save, or commit, not %s'
358 % (spec_args, src.type))
359 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
360 return Target(spec=spec, src=src, dest=dest)
363 def handle_ff(item, src_repo, writer, opt):
364 assert item.spec.method == 'ff'
365 assert item.src.type in ('branch', 'save', 'commit')
366 src_oidx = hexlify(item.src.hash)
367 dest_oidx = hexlify(item.dest.hash) if item.dest.hash else None
368 if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
370 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
371 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
372 return item.src.hash, unhexlify(commit_items.tree)
373 misuse('destination is not an ancestor of source for %s'
374 % spec_msg(item.spec))
377 def resolve_append(spec, src_repo, dest_repo):
378 src = resolve_src(spec, src_repo)
379 if src.type not in ('branch', 'save', 'commit', 'tree'):
380 misuse('source for %s must be a branch, save, commit, or tree, not %s'
381 % (spec_msg(spec), src.type))
382 spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
383 return Target(spec=spec, src=src, dest=dest)
386 def handle_append(item, src_repo, writer, opt):
387 assert item.spec.method == 'append'
388 assert item.src.type in ('branch', 'save', 'commit', 'tree')
389 assert item.dest.type == 'branch' or not item.dest.type
390 src_oidx = hexlify(item.src.hash)
391 if item.src.type == 'tree':
392 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
393 parent = item.dest.hash
394 msg = b'bup save\n\nGenerated by command:\n%r\n' % compat.get_argvb()
395 userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
397 commit = writer.new_commit(item.src.hash, parent,
399 userline, now, None, msg)
400 return commit, item.src.hash
401 commits = list(src_repo.rev_list(src_oidx))
403 return append_commits(commits, item.spec.src, item.dest.hash,
404 src_repo, writer, opt)
407 def resolve_pick(spec, src_repo, dest_repo):
408 src = resolve_src(spec, src_repo)
409 spec_args = spec_msg(spec)
410 if src.type == 'tree':
411 misuse('%s is impossible; can only --append a tree' % spec_args)
412 if src.type not in ('commit', 'save'):
413 misuse('%s impossible; can only pick a commit or save, not %s'
414 % (spec_args, src.type))
416 if src.path.startswith(b'/.tag/'):
417 spec = spec._replace(dest=spec.src)
418 elif src.type == 'save':
419 spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
421 misuse('no destination provided for %s', spec_args)
422 dest = find_vfs_item(spec.dest, dest_repo)
424 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
425 dest = default_loc._replace(path=cp)
427 if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
428 misuse('%s destination is not a tag or branch' % spec_args)
429 if spec.method == 'pick' \
430 and dest.hash and dest.path.startswith(b'/.tag/'):
431 misuse('cannot overwrite existing tag for %s (requires --force-pick)'
433 return Target(spec=spec, src=src, dest=dest)
436 def handle_pick(item, src_repo, writer, opt):
437 assert item.spec.method in ('pick', 'force-pick')
438 assert item.src.type in ('save', 'commit')
439 src_oidx = hexlify(item.src.hash)
441 return append_commit(item.spec.src, src_oidx, item.dest.hash,
442 src_repo, writer, opt)
443 return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt)
446 def resolve_new_tag(spec, src_repo, dest_repo):
447 src = resolve_src(spec, src_repo)
448 spec_args = spec_msg(spec)
449 if not spec.dest and src.path.startswith(b'/.tag/'):
450 spec = spec._replace(dest=src.path)
452 misuse('no destination (implicit or explicit) for %s', spec_args)
453 dest = find_vfs_item(spec.dest, dest_repo)
455 dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
456 if not dest.path.startswith(b'/.tag/'):
457 misuse('destination for %s must be a VFS tag' % spec_args)
459 misuse('cannot overwrite existing tag for %s (requires --replace)'
461 return Target(spec=spec, src=src, dest=dest)
464 def handle_new_tag(item, src_repo, writer, opt):
465 assert item.spec.method == 'new-tag'
466 assert item.dest.path.startswith(b'/.tag/')
467 get_random_item(item.spec.src, hexlify(item.src.hash),
468 src_repo, writer, opt)
469 return (item.src.hash,)
472 def resolve_replace(spec, src_repo, dest_repo):
473 src = resolve_src(spec, src_repo)
474 spec_args = spec_msg(spec)
476 if src.path.startswith(b'/.tag/') or src.type == 'branch':
477 spec = spec._replace(dest=spec.src)
479 misuse('no destination provided for %s', spec_args)
480 dest = find_vfs_item(spec.dest, dest_repo)
482 if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
483 misuse('%s impossible; can only overwrite branch or tag'
486 cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
487 dest = default_loc._replace(path=cp)
488 if not dest.path.startswith(b'/.tag/') \
489 and not src.type in ('branch', 'save', 'commit'):
490 misuse('cannot overwrite branch with %s for %s' % (src.type, spec_args))
491 return Target(spec=spec, src=src, dest=dest)
494 def handle_replace(item, src_repo, writer, opt):
495 assert(item.spec.method == 'replace')
496 if item.dest.path.startswith(b'/.tag/'):
497 get_random_item(item.spec.src, hexlify(item.src.hash),
498 src_repo, writer, opt)
499 return (item.src.hash,)
500 assert(item.dest.type == 'branch' or not item.dest.type)
501 src_oidx = hexlify(item.src.hash)
502 get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
503 commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
504 return item.src.hash, unhexlify(commit_items.tree)
507 def resolve_unnamed(spec, src_repo, dest_repo):
509 misuse('destination name given for %s' % spec_msg(spec))
510 src = resolve_src(spec, src_repo)
511 return Target(spec=spec, src=src, dest=None)
514 def handle_unnamed(item, src_repo, writer, opt):
515 get_random_item(item.spec.src, hexlify(item.src.hash),
516 src_repo, writer, opt)
520 def resolve_targets(specs, src_repo, dest_repo):
522 common_args = src_repo, dest_repo
524 debug1('initial-spec: %r\n' % (spec,))
525 if spec.method == 'ff':
526 resolved_items.append(resolve_ff(spec, *common_args))
527 elif spec.method == 'append':
528 resolved_items.append(resolve_append(spec, *common_args))
529 elif spec.method in ('pick', 'force-pick'):
530 resolved_items.append(resolve_pick(spec, *common_args))
531 elif spec.method == 'new-tag':
532 resolved_items.append(resolve_new_tag(spec, *common_args))
533 elif spec.method == 'replace':
534 resolved_items.append(resolve_replace(spec, *common_args))
535 elif spec.method == 'unnamed':
536 resolved_items.append(resolve_unnamed(spec, *common_args))
537 else: # Should be impossible -- prevented by the option parser.
540 # FIXME: check for prefix overlap? i.e.:
541 # bup get --ff foo --ff: baz foo/bar
542 # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar
544 # Now that we have all the items, check for duplicate tags.
545 tags_targeted = set()
546 for item in resolved_items:
547 dest_path = item.dest and item.dest.path
549 assert(dest_path.startswith(b'/'))
550 if dest_path.startswith(b'/.tag/'):
551 if dest_path in tags_targeted:
552 if item.spec.method not in ('replace', 'force-pick'):
553 misuse('cannot overwrite tag %s via %s' \
554 % (path_msg(dest_path), spec_msg(item.spec)))
556 tags_targeted.add(dest_path)
557 return resolved_items
560 def log_item(name, type, opt, tree=None, commit=None, tag=None):
561 if tag and opt.print_tags:
563 if tree and opt.print_trees:
565 if commit and opt.print_commits:
566 print(hexstr(commit))
569 if type in ('root', 'branch', 'save', 'commit', 'tree'):
570 if not name.endswith(b'/'):
572 log('%s%s\n' % (path_msg(name), last))
575 is_reverse = environ.get(b'BUP_SERVER_REVERSE')
576 opt = parse_args(argv)
577 git.check_repo_or_die()
579 opt.source = argv_bytes(opt.source)
581 client.bwlimit = parse_num(opt.bwlimit)
582 if is_reverse and opt.remote:
583 misuse("don't use -r in reverse mode; it's automatic")
584 if opt.remote or is_reverse:
585 dest_repo = RemoteRepo(opt.remote)
587 dest_repo = LocalRepo()
589 with dest_repo as dest_repo:
590 with LocalRepo(repo_dir=opt.source) as src_repo:
591 with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
592 # Resolve and validate all sources and destinations,
593 # implicit or explicit, and do it up-front, so we can
594 # fail before we start writing (for any obviously
596 target_items = resolve_targets(opt.target_specs,
599 updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
600 no_ref_info = (None, None)
602 handlers = {'ff': handle_ff,
603 'append': handle_append,
604 'force-pick': handle_pick,
606 'new-tag': handle_new_tag,
607 'replace': handle_replace,
608 'unnamed': handle_unnamed}
610 for item in target_items:
611 debug1('get-spec: %r\n' % (item.spec,))
612 debug1('get-src: %s\n' % loc_desc(item.src))
613 debug1('get-dest: %s\n' % loc_desc(item.dest))
614 dest_path = item.dest and item.dest.path
616 if dest_path.startswith(b'/.tag/'):
617 dest_ref = b'refs/tags/%s' % dest_path[6:]
619 dest_ref = b'refs/heads/%s' % dest_path[1:]
623 dest_hash = item.dest and item.dest.hash
624 orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
625 orig_ref = orig_ref or dest_hash
626 cur_ref = cur_ref or dest_hash
628 handler = handlers[item.spec.method]
629 item_result = handler(item, src_repo, writer, opt)
630 if len(item_result) > 1:
631 new_id, tree = item_result
633 new_id = item_result[0]
636 log_item(item.spec.src, item.src.type, opt)
638 updated_refs[dest_ref] = (orig_ref, new_id)
639 if dest_ref.startswith(b'refs/tags/'):
640 log_item(item.spec.src, item.src.type, opt, tag=new_id)
642 log_item(item.spec.src, item.src.type, opt,
643 tree=tree, commit=new_id)
645 # Only update the refs at the very end, once the writer is
646 # closed, so that if something goes wrong above, the old refs
647 # will be undisturbed.
648 for ref_name, info in items(updated_refs):
649 orig_ref, new_ref = info
651 dest_repo.update_ref(ref_name, new_ref, orig_ref)
653 new_hex = hexlify(new_ref)
655 orig_hex = hexlify(orig_ref)
656 log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
658 log('updated %r (%s)\n' % (ref_name, new_hex))
659 except (git.GitError, client.ClientError) as ex:
660 add_error('unable to update ref %r: %s' % (ref_name, ex))
663 log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))