From 2e4e894f4544bc16cda9511b84155b3d5643d5a8 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Sun, 23 Mar 2014 12:41:06 -0500 Subject: [PATCH] Add bup get; see the documentation for further information WARNING: this is a new EXPERIMENTAL command that can (intentionally) modify your data in destructive ways. Treat with caution. Thanks to Karl Kiniger for helping track down various bugs in earlier versions, and for noting that we might want --verbose to be a little more effusive. And thanks to Patryck Rouleau for suggesting improvements to the documentation. Signed-off-by: Rob Browning Tested-by: Rob Browning --- Documentation/bup-get.md | 190 ++++++++ Documentation/bup-on.md | 3 +- Makefile | 21 +- README.md | 6 +- buptest.py | 2 +- cmd/get-cmd.py | 665 ++++++++++++++++++++++++++ cmd/on-cmd.py | 1 + lib/bup/git.py | 13 +- t/git-cat-tree | 22 +- t/test-get | 984 +++++++++++++++++++++++++++++++++++++++ wvtest.py | 8 + 11 files changed, 1899 insertions(+), 16 deletions(-) create mode 100644 Documentation/bup-get.md create mode 100755 cmd/get-cmd.py create mode 100755 t/test-get diff --git a/Documentation/bup-get.md b/Documentation/bup-get.md new file mode 100644 index 0000000..f40c753 --- /dev/null +++ b/Documentation/bup-get.md @@ -0,0 +1,190 @@ +% bup-get(1) Bup %BUP_VERSION% +% Rob Browning +% %BUP_DATE% + +# NAME + +bup-get - copy repository items (CAUTION: EXPERIMENTAL) + +# SYNOPSIS + +bup get \[-s *source-path*\] \[-r *host*:*path*\] OPTIONS \<(METHOD *ref* [*dest*])\>... + +# DESCRIPTION + +`bup get` copies the indicated *ref*s from the source repository to +the destination repository (respecting `--bup-dir` and `BUP_DIR`), +according to the specified METHOD, which may be one of `--ff`, +`--ff:`, `--append`, `--append:`, `--pick`, `--pick:`, `--force-pick`, +`--force-pick:`, `--new-tag`, `--new-tag:`, `--replace`, `--replace:`, +or `--unnamed`. See the EXAMPLES below for a quick introduction. + +The *ref* is the source repository reference of the object to be +fetched, and the *dest* is the optional destination reference. A +*dest* may only be specified for a METHOD whose name ends in a colon. +For example: + + bup get -s /source/repo --ff foo + bup get -s /source/repo --ff: foo/latest bar + bup get -s /source/repo --pick: foo/2010-10-10-101010 .tag/bar + +As a special case, if *ref* names the "latest" save symlink, then bup +will act exactly as if the save that "latest" points to had been +specified, rather than the "latest" symlink itself, so `bup get +foo/latest` will actually be interpreted as something like `bup get +foo/2013-01-01-030405`. + +In some situations `bup get` will evaluate a branch operation +according to whether or not it will be a "fast-forward" (which +requires that any existing destination branch be an ancestor of the +source). + +An existing destination tag can only be overwritten by a `--replace` +or `--force-pick`. + +When a new commit is created (i.e. via `--append`, `--pick`, etc.), it +will have the same author, author date, and message as the original, +but a committer and committer date corresponding to the current user +and time. + +If requested by the appropriate options, bup will print the commit, +tree, or tag hash for each destination reference updated. When +relevant, the tree hash will be printed before the commit hash. + +Local *ref*s can be pushed to a remote repository with the `--remote` +option, and remote *ref*s can be pulled into a local repository via +"bup on HOST get ...". See `bup-on`(1) and the EXAMPLES below for +further information. + +WARNING: This is one of the few bup commands that can modify your +archives in intentionally destructive ways. Though if an attempt to +join or restore the data you still care about succeeds after you've +run this command, then that's a fairly encouraging sign that it worked +correctly. (The t/compare-trees command in the source tree can be +used to help test before/after results.) + +# METHODS + +--ff *ref*, --ff: *ref* *dest* +: fast-forward *dest* to match *ref*. If *dest* is not specified + and *ref* names a save, set *dest* to the save's branch. If + *dest* is not specified and *ref* names a branch or a tag, use the + same name for *dest*. + +--append *ref*, --append: *ref* *dest* +: append all of the commits represented by *ref* to *dest* as new + commits. If *ref* names a directory/tree, append a new commit for + that tree. If *dest* is not specified and *ref* names a save or + branch, set *dest* to the *ref* branch name. If *dest* is not + specified and *ref* names a tag, use the same name for *dest*. + +--pick *ref*, --pick: *ref* *dest* +: append the single commit named by *ref* to *dest* as a new commit. + If *dest* is not specified and *ref* names a save, set *dest* to + the *ref* branch name. If *dest* is not specified and *ref* names + a tag, use the same name for *dest*. + +--force-pick *ref*, --force-pick: *ref* *dest* +: do the same thing as `--pick`, but don't refuse to overwrite an + existing tag. + +--new-tag *ref*, --new-tag: *ref* *dest* +: create a *dest* tag for *ref*, but refuse to overwrite an existing + tag. If *dest* is not specified and *ref* names a tag, use the + same name for *dest*. + +--replace *ref*, --replace: *ref* *dest* +: clobber *dest* with *ref*, overwriting any existing tag, or + replacing any existing branch. If *dest* is not specified and + *ref* names a branch or tag, use the same name for *dest*. + +--unnamed *ref* +: copy *ref* into the destination repository, without any name, + leaving a potentially dangling reference until/unless the object + named by *ref* is referred to some other way (cf. `bup tag`). + +# OPTIONS + +-s, --source=*path* +: use *path* as the source repository, instead of the default. + +-r, --remote=*host*:*path* +: store the indicated items on the given remote server. If *path* + is omitted, uses the default path on the remote server (you still + need to include the ':'). The connection to the remote server is + made with SSH. If you'd like to specify which port, user or + private key to use for the SSH connection, we recommend you use + the `~/.ssh/config` file. + +-c, --print-commits +: for each updated branch, print the new git commit id. + +-t, --print-trees +: for each updated branch, print the new git tree id of the + filesystem root. + +--print-tags +: for each updated tag, print the new git id. + +-v, --verbose +: increase verbosity (can be used more than once). With + `-v`, print the name of every item fetched, with `-vv` add + directory names, and with `-vvv` add every filename. + +--bwlimit=*bytes/sec* +: don't transmit more than *bytes/sec* bytes per second to the + server. This can help avoid sucking up all your network + bandwidth. Use a suffix like k, M, or G to specify multiples of + 1024, 1024\*1024, 1024\*1024\*1024 respectively. + +-*#*, --compress=*#* +: set the compression level to # (a value from 0-9, where + 9 is the highest and 0 is no compression). The default + is 1 (fast, loose compression) + +# EXAMPLES + + # Update or copy the archives branch in src-repo to the local repository. + $ bup get -s src-repo --ff archives + + # Append a particular archives save to the pruned-archives branch. + $ bup get -s src-repo --pick: archives/2013-01-01-030405 pruned-archives + + # Update or copy the archives branch on remotehost to the local + # repository. + $ bup on remotehost get --ff archives + + # Update or copy the local branch archives to remotehost. + $ bup get -r remotehost: --ff archives + + # Update or copy the archives branch in src-repo to remotehost. + $ bup get -s src-repo -r remotehost: --ff archives + + # Update the archives-2 branch on remotehost to match archives. + # If archives-2 exists and is not an ancestor of archives, bup + # will refuse. + $ bup get -r remotehost: --ff: archives archives-2 + + # Replace the contents of branch y with those of x. + $ bup get --replace: x y + + # Copy the latest local save from the archives branch to the + # remote tag foo. + $ bup get -r remotehost: --pick: archives/latest .tag/foo + + # Or if foo already exists: + $ bup get -r remotehost: --force-pick: archives/latest .tag/foo + + # Append foo (from above) to the local other-archives branch. + $ bup on remotehost get --append: .tag/foo other-archives + + # Append only the /home directory from archives/latest to only-home. + $ bup get -s "$BUP_DIR" --append: archives/latest/home only-home + +# SEE ALSO + +`bup-on`(1), `bup-tag`(1), `ssh_config`(5) + +# BUP + +Part of the `bup`(1) suite. diff --git a/Documentation/bup-on.md b/Documentation/bup-on.md index ac9247e..7f91577 100644 --- a/Documentation/bup-on.md +++ b/Documentation/bup-on.md @@ -14,6 +14,7 @@ bup on \ save ... bup on \ split ... +bup on \ get ... # DESCRIPTION @@ -77,7 +78,7 @@ basement. # SEE ALSO -`bup-index`(1), `bup-save`(1), `bup-split`(1) +`bup-index`(1), `bup-save`(1), `bup-split`(1), `bup-get`(1) # BUP diff --git a/Makefile b/Makefile index 7635f90..38e0ee4 100644 --- a/Makefile +++ b/Makefile @@ -189,6 +189,19 @@ cmdline_tests := \ t/test-xdev.sh \ t/test.sh +tmp-target-run-test-get-%: all t/tmp + $(pf); cd $$(pwd -P); TMPDIR="$(test_tmp)" \ + t/test-get $* 2>&1 | tee -a t/tmp/test-log/$$$$.log + +test_get_targets := \ + tmp-target-run-test-get-replace \ + tmp-target-run-test-get-universal \ + tmp-target-run-test-get-ff \ + tmp-target-run-test-get-append \ + tmp-target-run-test-get-pick \ + tmp-target-run-test-get-new-tag \ + tmp-target-run-test-get-unnamed + # For parallel runs. # The "pwd -P" here may not be appropriate in the long run, but we # need it until we settle the relevant drecurse/exclusion questions: @@ -197,7 +210,7 @@ tmp-target-run-test%: all t/tmp $(pf); cd $$(pwd -P); TMPDIR="$(test_tmp)" \ t/test$* 2>&1 | tee -a t/tmp/test-log/$$$$.log -runtests-cmdline: $(subst t/test,tmp-target-run-test,$(cmdline_tests)) +runtests-cmdline: $(test_get_targets) $(subst t/test,tmp-target-run-test,$(cmdline_tests)) stupid: PATH=/bin:/usr/bin $(MAKE) test @@ -220,6 +233,12 @@ cmd/python-cmd.sh: config/config.vars Makefile chmod +x cmd/python-cmd.sh.$$PPID.tmp mv cmd/python-cmd.sh.$$PPID.tmp cmd/python-cmd.sh +long-test: export BUP_TEST_LEVEL=11 +long-test: test + +long-check: export BUP_TEST_LEVEL=11 +long-check: check + cmd/bup-%: cmd/%-cmd.py rm -f $@ ln -s $*-cmd.py $@ diff --git a/README.md b/README.md index ff18630..aabd4b3 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,11 @@ From source - Run the tests: - make test + make long-check + + or if you're in a bit more of a hurry: + + make check The tests should pass. If they don't pass for you, stop here and send an email to bup-list@googlegroups.com. Though if there are diff --git a/buptest.py b/buptest.py index dd145ee..22523b7 100644 --- a/buptest.py +++ b/buptest.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, print_function from collections import namedtuple from contextlib import contextmanager -from os.path import basename, dirname, realpath +from os.path import abspath, basename, dirname, realpath from pipes import quote from subprocess import PIPE, Popen from traceback import extract_stack diff --git a/cmd/get-cmd.py b/cmd/get-cmd.py new file mode 100755 index 0000000..efeb78f --- /dev/null +++ b/cmd/get-cmd.py @@ -0,0 +1,665 @@ +#!/bin/sh +"""": # -*-python-*- +bup_python="$(dirname "$0")/bup-python" || exit $? +exec "$bup_python" "$0" ${1+"$@"} +""" +# end of bup preamble + +from __future__ import absolute_import, print_function +import os, re, stat, sys, textwrap, time +from collections import namedtuple +from functools import partial +from stat import S_ISDIR + +from bup import git, client, helpers, vfs +from bup.compat import wrap_main +from bup.git import get_cat_data, parse_commit, walk_object +from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors +from bup.helpers import hostname, shstr, tty_width, userfullname, username +from bup.repo import LocalRepo, RemoteRepo + +argspec = ( + "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...", + + """Transfer data from a source repository to a destination repository + according to the methods specified (--ff, --ff:, --append, etc.). + Both repositories default to BUP_DIR. A remote destination may be + specified with -r, and data may be pulled from a remote repository + with the related "bup on HOST get ..." command.""", + + ('optional arguments:', + (('-h, --help', 'show this help message and exit'), + ('-v, --verbose', + 'increase log output (can be specified more than once)'), + ('-q, --quiet', "don't show progress meter"), + ('-s SOURCE, --source SOURCE', + 'path to the source repository (defaults to BUP_DIR)'), + ('-r REMOTE, --remote REMOTE', + 'hostname:/path/to/repo of remote destination repository'), + ('-t --print-trees', 'output a tree id for each ref set'), + ('-c, --print-commits', 'output a commit id for each ref set'), + ('--print-tags', 'output an id for each tag'), + ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'), + ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL', + 'set compression LEVEL (default: 1)'))), + + ('transfer methods:', + (('--ff REF, --ff: REF DEST', + 'fast-forward dest REF (or DEST) to match source REF'), + ('--append REF, --append: REF DEST', + 'append REF (treeish or committish) to dest REF (or DEST)'), + ('--pick REF, --pick: REF DEST', + 'append single source REF commit to dest REF (or DEST)'), + ('--force-pick REF, --force-pick: REF DEST', + '--pick, overwriting REF (or DEST)'), + ('--new-tag REF, --new-tag: REF DEST', + 'tag source ref REF as REF (or DEST) in dest unless it already exists'), + ('--replace, --replace: REF DEST', + 'overwrite REF (or DEST) in dest with source REF'), + ('--unnamed REF', + 'fetch REF anonymously (without destination ref)')))) + +def render_opts(opts, width=None): + if not width: + width = tty_width() + result = [] + for args, desc in opts: + result.append(textwrap.fill(args, width=width, + initial_indent=(' ' * 2), + subsequent_indent=(' ' * 4))) + result.append('\n') + result.append(textwrap.fill(desc, width=width, + initial_indent=(' ' * 6), + subsequent_indent=(' ' * 6))) + result.append('\n') + return result + +def usage(argspec, width=None): + if not width: + width = tty_width() + usage, preamble, groups = argspec[0], argspec[1], argspec[2:] + msg = [] + msg.append(textwrap.fill(usage, width=width, subsequent_indent=' ')) + msg.append('\n\n') + msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width)) + msg.append('\n') + for group_name, group_args in groups: + msg.extend(['\n', group_name, '\n']) + msg.extend(render_opts(group_args, width=width)) + return ''.join(msg) + +def misuse(message=None): + sys.stderr.write(usage(argspec)) + if message: + sys.stderr.write('\nerror: ') + sys.stderr.write(message) + sys.stderr.write('\n') + sys.exit(1) + +def require_n_args_or_die(n, args): + if len(args) < n + 1: + misuse('%s argument requires %d %s' + % (n, 'values' if n == 1 else 'value')) + result = args[1:1+n], args[1+n:] + assert len(result[0]) == n + return result + +def parse_args(args): + Spec = namedtuple('Spec', ['argopt', 'argval', 'src', 'dest', 'method']) + class GetOpts: + pass + opt = GetOpts() + opt.help = False + opt.verbose = 0 + opt.quiet = False + opt.print_commits = opt.print_trees = opt.print_tags = False + opt.bwlimit = None + opt.compress = 1 + opt.source = opt.remote = None + opt.target_specs = [] + + remaining = args[1:] # Skip argv[0] + while remaining: + arg = remaining[0] + if arg in ('-h', '--help'): + sys.stdout.write(usage(argspec)) + sys.exit(0) + elif arg in ('-v', '--verbose'): + opt.verbose += 1 + remaining = remaining[1:] + elif arg in ('--ff', '--append', '--pick', '--force-pick', + '--new-tag', '--replace', '--unnamed'): + (ref,), remaining = require_n_args_or_die(1, remaining) + opt.target_specs.append(Spec(argopt=arg, + argval=shstr((ref,)), + src=ref, dest=None, + method=arg[2:])) + elif arg in ('--ff:', '--append:', '--pick:', '--force-pick:', + '--new-tag:', '--replace:'): + (ref, dest), remaining = require_n_args_or_die(2, remaining) + opt.target_specs.append(Spec(argopt=arg, + argval=shstr((ref, dest)), + src=ref, dest=dest, + method=arg[2:-1])) + elif arg in ('-s', '--source'): + (opt.source,), remaining = require_n_args_or_die(1, remaining) + elif arg in ('-r', '--remote'): + (opt.remote,), remaining = require_n_args_or_die(1, remaining) + elif arg in ('-c', '--print-commits'): + opt.print_commits, remaining = True, remaining[1:] + elif arg in ('-t', '--print-trees'): + opt.print_trees, remaining = True, remaining[1:] + elif arg == '--print-tags': + opt.print_tags, remaining = True, remaining[1:] + elif arg in ('-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'): + opt.compress = int(arg[1:]) + remaining = remaining[1:] + elif arg == '--compress': + (opt.compress,), remaining = require_n_args_or_die(1, remaining) + opt.compress = int(opt.compress) + elif arg == '--bwlimit': + (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining) + opt.bwlimit = long(opt.bwlimit) + elif arg.startswith('-') and len(arg) > 2 and arg[1] != '-': + # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z". + # We do this last so that --foo -bar is valid if --foo + # requires a value. + remaining[0:1] = ('-' + c for c in arg[1:]) + # FIXME + continue + else: + misuse() + return opt + +# FIXME: client error handling (remote exceptions, etc.) + +# FIXME: walk_object in in git.py doesn't support opt.verbose. Do we +# need to adjust for that here? +def get_random_item(name, hash, repo, writer, opt): + def already_seen(id): + return writer.exists(id.decode('hex')) + for item in walk_object(repo.cat, hash, stop_at=already_seen, + include_data=True): + # already_seen ensures that writer.exists(id) is false. + # Otherwise, just_write() would fail. + writer.just_write(item.oid, item.type, item.data) + + +def append_commit(name, hash, parent, src_repo, writer, opt): + now = time.time() + items = parse_commit(get_cat_data(src_repo.cat(hash), 'commit')) + tree = items.tree.decode('hex') + author = '%s <%s>' % (items.author_name, items.author_mail) + author_time = (items.author_sec, items.author_offset) + committer = '%s <%s@%s>' % (userfullname(), username(), hostname()) + get_random_item(name, tree.encode('hex'), src_repo, writer, opt) + c = writer.new_commit(tree, parent, + author, items.author_sec, items.author_offset, + committer, now, None, + items.message) + return c, tree + + +def append_commits(commits, src_name, dest_hash, src_repo, writer, opt): + last_c, tree = dest_hash, None + for commit in commits: + last_c, tree = append_commit(src_name, commit, last_c, + src_repo, writer, opt) + assert(tree is not None) + return last_c, tree + +Loc = namedtuple('Loc', ['type', 'hash', 'path']) +default_loc = Loc(None, None, None) + +def find_vfs_item(name, repo): + res = repo.resolve(name, follow=False, want_meta=False) + leaf_name, leaf_item = res[-1] + if not leaf_item: + return None + kind = type(leaf_item) + if kind == vfs.Root: + kind = 'root' + elif kind == vfs.Tags: + kind = 'tags' + elif kind == vfs.RevList: + kind = 'branch' + elif kind == vfs.Commit: + if len(res) > 1 and type(res[-2][1]) == vfs.RevList: + kind = 'save' + else: + kind = 'commit' + elif kind == vfs.Item: + if S_ISDIR(vfs.item_mode(leaf_item)): + kind = 'tree' + else: + kind = 'blob' + elif kind == vfs.Chunky: + kind = 'tree' + elif kind == vfs.FakeLink: + # Don't have to worry about ELOOP, excepting malicious + # remotes, since "latest" is the only FakeLink. + assert leaf_name == 'latest' + res = repo.resolve(leaf_item.target, parent=res[:-1], + follow=False, want_meta=False) + leaf_name, leaf_item = res[-1] + assert leaf_item + assert type(leaf_item) == vfs.Commit + name = '/'.join(x[0] for x in res) + kind = 'save' + else: + raise Exception('unexpected resolution for %r: %r' % (name, res)) + path = '/'.join(name for name, item in res) + if hasattr(leaf_item, 'coid'): + result = Loc(type=kind, hash=leaf_item.coid, path=path) + elif hasattr(leaf_item, 'oid'): + result = Loc(type=kind, hash=leaf_item.oid, path=path) + else: + result = Loc(type=kind, hash=None, path=path) + return result + + +Target = namedtuple('Target', ['spec', 'src', 'dest']) + +def loc_desc(loc): + if loc and loc.hash: + loc = loc._replace(hash=loc.hash.encode('hex')) + return str(loc) + + +# FIXME: see if resolve() means we can drop the vfs path cleanup + +def cleanup_vfs_path(p): + result = os.path.normpath(p) + if result.startswith('/'): + return result + return '/' + result + + +def validate_vfs_path(p): + if p.startswith('/.') \ + and not p.startswith('/.tag/'): + spec_args = '%s %s' % (spec.argopt, spec.argval) + misuse('unsupported destination path %r in %r' % (dest.path, spec_args)) + return p + + +def resolve_src(spec, src_repo): + src = find_vfs_item(spec.src, src_repo) + spec_args = '%s %s' % (spec.argopt, spec.argval) + if not src: + misuse('cannot find source for %r' % spec_args) + if src.type == 'root': + misuse('cannot fetch entire repository for %r' % spec_args) + if src.type == 'tags': + misuse('cannot fetch entire /.tag directory for %r' % spec_args) + debug1('src: %s\n' % loc_desc(src)) + return src + + +def get_save_branch(repo, path): + res = repo.resolve(path, follow=False, want_meta=False) + leaf_name, leaf_item = res[-1] + if not leaf_item: + misuse('error: cannot access %r in %r' % (leaf_name, path)) + assert len(res) == 3 + res_path = '/'.join(name for name, item in res[:-1]) + return res_path + + +def resolve_branch_dest(spec, src, src_repo, dest_repo): + # Resulting dest must be treeish, or not exist. + if not spec.dest: + # Pick a default dest. + if src.type == 'branch': + spec = spec._replace(dest=spec.src) + elif src.type == 'save': + spec = spec._replace(dest=get_save_branch(src_repo, spec.src)) + elif src.path.startswith('/.tag/'): # Dest defaults to the same. + spec = spec._replace(dest=spec.src) + + spec_args = '%s %s' % (spec.argopt, spec.argval) + if not spec.dest: + misuse('no destination (implicit or explicit) for %r', spec_args) + + dest = find_vfs_item(spec.dest, dest_repo) + if dest: + if dest.type == 'commit': + misuse('destination for %r is a tagged commit, not a branch' + % spec_args) + if dest.type != 'branch': + misuse('destination for %r is a %s, not a branch' + % (spec_args, dest.type)) + else: + dest = default_loc._replace(path=cleanup_vfs_path(spec.dest)) + + if dest.path.startswith('/.'): + misuse('destination for %r must be a valid branch name' % spec_args) + + debug1('dest: %s\n' % loc_desc(dest)) + return spec, dest + + +def resolve_ff(spec, src_repo, dest_repo): + src = resolve_src(spec, src_repo) + spec_args = '%s %s' % (spec.argopt, spec.argval) + if src.type == 'tree': + misuse('%r is impossible; can only --append a tree to a branch' + % spec_args) + if src.type not in ('branch', 'save', 'commit'): + misuse('source for %r must be a branch, save, or commit, not %s' + % (spec_args, src.type)) + spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo) + return Target(spec=spec, src=src, dest=dest) + + +def handle_ff(item, src_repo, writer, opt): + assert item.spec.method == 'ff' + assert item.src.type in ('branch', 'save', 'commit') + src_oidx = item.src.hash.encode('hex') + dest_oidx = item.dest.hash.encode('hex') if item.dest.hash else None + if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx): + # Can fast forward. + get_random_item(item.spec.src, src_oidx, src_repo, writer, opt) + commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit')) + return item.src.hash, commit_items.tree.decode('hex') + spec_args = '%s %s' % (item.spec.argopt, item.spec.argval) + misuse('destination is not an ancestor of source for %r' % spec_args) + + +def resolve_append(spec, src_repo, dest_repo): + src = resolve_src(spec, src_repo) + if src.type not in ('branch', 'save', 'commit', 'tree'): + spec_args = '%s %s' % (spec.argopt, spec.argval) + misuse('source for %r must be a branch, save, commit, or tree, not %s' + % (spec_args, src.type)) + spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo) + return Target(spec=spec, src=src, dest=dest) + + +def handle_append(item, src_repo, writer, opt): + assert item.spec.method == 'append' + assert item.src.type in ('branch', 'save', 'commit', 'tree') + assert item.dest.type == 'branch' or not item.dest.type + src_oidx = item.src.hash.encode('hex') + if item.src.type == 'tree': + get_random_item(item.spec.src, src_oidx, src_repo, writer, opt) + parent = item.dest.hash + msg = 'bup save\n\nGenerated by command:\n%r\n' % sys.argv + userline = '%s <%s@%s>' % (userfullname(), username(), hostname()) + now = time.time() + commit = writer.new_commit(item.src.hash, parent, + userline, now, None, + userline, now, None, msg) + return commit, item.src.hash + commits = list(src_repo.rev_list(src_oidx)) + commits.reverse() + return append_commits(commits, item.spec.src, item.dest.hash, + src_repo, writer, opt) + + +def resolve_pick(spec, src_repo, dest_repo): + src = resolve_src(spec, src_repo) + spec_args = '%s %s' % (spec.argopt, spec.argval) + if src.type == 'tree': + misuse('%r is impossible; can only --append a tree' % spec_args) + if src.type not in ('commit', 'save'): + misuse('%r impossible; can only pick a commit or save, not %s' + % (spec_args, src.type)) + if not spec.dest: + if src.path.startswith('/.tag/'): + spec = spec._replace(dest=spec.src) + elif src.type == 'save': + spec = spec._replace(dest=get_save_branch(src_repo, spec.src)) + if not spec.dest: + misuse('no destination provided for %r', spec_args) + dest = find_vfs_item(spec.dest, dest_repo) + if not dest: + cp = validate_vfs_path(cleanup_vfs_path(spec.dest)) + dest = default_loc._replace(path=cp) + else: + if not dest.type == 'branch' and not dest.path.startswith('/.tag/'): + misuse('%r destination is not a tag or branch' % spec_args) + if spec.method == 'pick' \ + and dest.hash and dest.path.startswith('/.tag/'): + misuse('cannot overwrite existing tag for %r (requires --force-pick)' + % spec_args) + return Target(spec=spec, src=src, dest=dest) + + +def handle_pick(item, src_repo, writer, opt): + assert item.spec.method in ('pick', 'force-pick') + assert item.src.type in ('save', 'commit') + src_oidx = item.src.hash.encode('hex') + if item.dest.hash: + return append_commit(item.spec.src, src_oidx, item.dest.hash, + src_repo, writer, opt) + return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt) + + +def resolve_new_tag(spec, src_repo, dest_repo): + src = resolve_src(spec, src_repo) + spec_args = '%s %s' % (spec.argopt, spec.argval) + if not spec.dest and src.path.startswith('/.tag/'): + spec = spec._replace(dest=src.path) + if not spec.dest: + misuse('no destination (implicit or explicit) for %r', spec_args) + dest = find_vfs_item(spec.dest, dest_repo) + if not dest: + dest = default_loc._replace(path=cleanup_vfs_path(spec.dest)) + if not dest.path.startswith('/.tag/'): + misuse('destination for %r must be a VFS tag' % spec_args) + if dest.hash: + misuse('cannot overwrite existing tag for %r (requires --replace)' + % spec_args) + return Target(spec=spec, src=src, dest=dest) + + +def handle_new_tag(item, src_repo, writer, opt): + assert item.spec.method == 'new-tag' + assert item.dest.path.startswith('/.tag/') + get_random_item(item.spec.src, item.src.hash.encode('hex'), + src_repo, writer, opt) + return (item.src.hash,) + + +def resolve_replace(spec, src_repo, dest_repo): + src = resolve_src(spec, src_repo) + spec_args = '%s %s' % (spec.argopt, spec.argval) + if not spec.dest: + if src.path.startswith('/.tag/') or src.type == 'branch': + spec = spec._replace(dest=spec.src) + if not spec.dest: + misuse('no destination provided for %r', spec_args) + dest = find_vfs_item(spec.dest, dest_repo) + if dest: + if not dest.type == 'branch' and not dest.path.startswith('/.tag/'): + misuse('%r impossible; can only overwrite branch or tag' + % spec_args) + else: + cp = validate_vfs_path(cleanup_vfs_path(spec.dest)) + dest = default_loc._replace(path=cp) + if not dest.path.startswith('/.tag/') \ + and not src.type in ('branch', 'save', 'commit'): + misuse('cannot overwrite branch with %s for %r' % (src.type, spec_args)) + return Target(spec=spec, src=src, dest=dest) + + +def handle_replace(item, src_repo, writer, opt): + assert(item.spec.method == 'replace') + if item.dest.path.startswith('/.tag/'): + get_random_item(item.spec.src, item.src.hash.encode('hex'), + src_repo, writer, opt) + return (item.src.hash,) + assert(item.dest.type == 'branch' or not item.dest.type) + src_oidx = item.src.hash.encode('hex') + get_random_item(item.spec.src, src_oidx, src_repo, writer, opt) + commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), 'commit')) + return item.src.hash, commit_items.tree.decode('hex') + + +def resolve_unnamed(spec, src_repo, dest_repo): + if spec.dest: + spec_args = '%s %s' % (spec.argopt, spec.argval) + misuse('destination name given for %r' % spec_args) + src = resolve_src(spec, src_repo) + return Target(spec=spec, src=src, dest=None) + + +def handle_unnamed(item, src_repo, writer, opt): + get_random_item(item.spec.src, item.src.hash.encode('hex'), + src_repo, writer, opt) + return (None,) + + +def resolve_targets(specs, src_repo, dest_repo): + resolved_items = [] + common_args = src_repo, dest_repo + for spec in specs: + debug1('initial-spec: %s\n' % str(spec)) + if spec.method == 'ff': + resolved_items.append(resolve_ff(spec, *common_args)) + elif spec.method == 'append': + resolved_items.append(resolve_append(spec, *common_args)) + elif spec.method in ('pick', 'force-pick'): + resolved_items.append(resolve_pick(spec, *common_args)) + elif spec.method == 'new-tag': + resolved_items.append(resolve_new_tag(spec, *common_args)) + elif spec.method == 'replace': + resolved_items.append(resolve_replace(spec, *common_args)) + elif spec.method == 'unnamed': + resolved_items.append(resolve_unnamed(spec, *common_args)) + else: # Should be impossible -- prevented by the option parser. + assert(False) + + # FIXME: check for prefix overlap? i.e.: + # bup get --ff foo --ff: baz foo/bar + # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar + + # Now that we have all the items, check for duplicate tags. + tags_targeted = set() + for item in resolved_items: + dest_path = item.dest and item.dest.path + if dest_path: + assert(dest_path.startswith('/')) + if dest_path.startswith('/.tag/'): + if dest_path in tags_targeted: + if item.spec.method not in ('replace', 'force-pick'): + spec_args = '%s %s' % (item.spec.argopt, + item.spec.argval) + misuse('cannot overwrite tag %r via %r' \ + % (dest_path, spec_args)) + else: + tags_targeted.add(dest_path) + return resolved_items + + +def log_item(name, type, opt, tree=None, commit=None, tag=None): + if tag and opt.print_tags: + print(tag.encode('hex')) + if tree and opt.print_trees: + print(tree.encode('hex')) + if commit and opt.print_commits: + print(commit.encode('hex')) + if opt.verbose: + last = '' + if type in ('root', 'branch', 'save', 'commit', 'tree'): + if not name.endswith('/'): + last = '/' + log('%s%s\n' % (name, last)) + +def main(): + handle_ctrl_c() + is_reverse = os.environ.get('BUP_SERVER_REVERSE') + opt = parse_args(sys.argv) + git.check_repo_or_die() + src_dir = opt.source or git.repo() + if opt.bwlimit: + client.bwlimit = parse_num(opt.bwlimit) + if is_reverse and opt.remote: + misuse("don't use -r in reverse mode; it's automatic") + if opt.remote or is_reverse: + dest_repo = RemoteRepo(opt.remote) + else: + dest_repo = LocalRepo() + + with dest_repo as dest_repo: + with LocalRepo(repo_dir=src_dir) as src_repo: + with dest_repo.new_packwriter(compression_level=opt.compress) as writer: + + src_repo = LocalRepo(repo_dir=src_dir) + + # Resolve and validate all sources and destinations, + # implicit or explicit, and do it up-front, so we can + # fail before we start writing (for any obviously + # broken cases). + target_items = resolve_targets(opt.target_specs, + src_repo, dest_repo) + + updated_refs = {} # ref_name -> (original_ref, tip_commit(bin)) + no_ref_info = (None, None) + + handlers = {'ff': handle_ff, + 'append': handle_append, + 'force-pick': handle_pick, + 'pick': handle_pick, + 'new-tag': handle_new_tag, + 'replace': handle_replace, + 'unnamed': handle_unnamed} + + for item in target_items: + debug1('get-spec: %s\n' % str(item.spec)) + debug1('get-src: %s\n' % loc_desc(item.src)) + debug1('get-dest: %s\n' % loc_desc(item.dest)) + dest_path = item.dest and item.dest.path + if dest_path: + if dest_path.startswith('/.tag/'): + dest_ref = 'refs/tags/%s' % dest_path[6:] + else: + dest_ref = 'refs/heads/%s' % dest_path[1:] + else: + dest_ref = None + + dest_hash = item.dest and item.dest.hash + orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info) + orig_ref = orig_ref or dest_hash + cur_ref = cur_ref or dest_hash + + handler = handlers[item.spec.method] + item_result = handler(item, src_repo, writer, opt) + if len(item_result) > 1: + new_id, tree = item_result + else: + new_id = item_result[0] + + if not dest_ref: + log_item(item.spec.src, item.src.type, opt) + else: + updated_refs[dest_ref] = (orig_ref, new_id) + if dest_ref.startswith('refs/tags/'): + log_item(item.spec.src, item.src.type, opt, tag=new_id) + else: + log_item(item.spec.src, item.src.type, opt, + tree=tree, commit=new_id) + + # Only update the refs at the very end, once the writer is + # closed, so that if something goes wrong above, the old refs + # will be undisturbed. + for ref_name, info in updated_refs.iteritems(): + orig_ref, new_ref = info + try: + dest_repo.update_ref(ref_name, new_ref, orig_ref) + if opt.verbose: + new_hex = new_ref.encode('hex') + if orig_ref: + orig_hex = orig_ref.encode('hex') + log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex)) + else: + log('updated %r (%s)\n' % (ref_name, new_hex)) + except (git.GitError, client.ClientError), ex: + add_error('unable to update ref %r: %s' % (ref_name, ex)) + + if saved_errors: + log('WARNING: %d errors encountered while saving.\n' % len(saved_errors)) + sys.exit(1) + +wrap_main(main) diff --git a/cmd/on-cmd.py b/cmd/on-cmd.py index e4f660c..0643ef8 100755 --- a/cmd/on-cmd.py +++ b/cmd/on-cmd.py @@ -17,6 +17,7 @@ optspec = """ bup on index ... bup on save ... bup on split ... +bup on get ... """ o = options.Options(optspec, optfunc=getopt.getopt) (opt, flags, extra) = o.parse(sys.argv[1:]) diff --git a/lib/bup/git.py b/lib/bup/git.py index 45fab04..73dce19 100644 --- a/lib/bup/git.py +++ b/lib/bup/git.py @@ -112,13 +112,14 @@ def parse_commit(content): message=matches['message']) -def get_commit_items(id, cp): - commit_it = cp.get(id) - _, typ, _ = next(commit_it) - assert(typ == 'commit') - commit_content = ''.join(commit_it) - return parse_commit(commit_content) +def get_cat_data(cat_iterator, expected_type): + _, kind, _ = next(cat_iterator) + if kind != expected_type: + raise Exception('expected %r, saw %r' % (expected_type, kind)) + return ''.join(cat_iterator) +def get_commit_items(id, cp): + return parse_commit(get_cat_data(cp.get(id), 'commit')) def _local_git_date_str(epoch_sec): return '%d %s' % (epoch_sec, utc_offset_str(epoch_sec)) diff --git a/t/git-cat-tree b/t/git-cat-tree index bbbfa6b..3a12f4d 100755 --- a/t/git-cat-tree +++ b/t/git-cat-tree @@ -6,7 +6,7 @@ set -o pipefail usage() { cat <&2 - exit 1 -fi +case $# in + 1) ;; + 3) + if test "$1" != --git-dir; then + usage 1>&2 + exit 1 + fi + export GIT_DIR="$2" + shift 2 + ;; + *) + usage 1>&2 + exit 1 + ;; +esac top="$1" type=$(git cat-file -t "$top") || exit $? diff --git a/t/test-get b/t/test-get new file mode 100755 index 0000000..9d3381e --- /dev/null +++ b/t/test-get @@ -0,0 +1,984 @@ +#!/bin/sh +"""": # -*-python-*- +bup_python="$(dirname "$0")/../cmd/bup-python" || exit $? +exec "$bup_python" "$0" ${1+"$@"} +""" +# end of bup preamble + +from __future__ import print_function +from errno import ENOENT +from os import chdir, environ, getcwd, mkdir, rename +from os.path import abspath, dirname +from pipes import quote +from shutil import rmtree +from subprocess import PIPE +import re, sys + +script_home = abspath(dirname(sys.argv[0] or '.')) +sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')] + +from bup import compat +from buptest import ex, exo, test_tempdir +from wvtest import wvcheck, wvfail, wvmsg, wvpass, wvpasseq, wvpassne, wvstart + +# FIXME: per-test function +environ['GIT_AUTHOR_NAME'] = 'bup test-get' +environ['GIT_COMMITTER_NAME'] = 'bup test-get' +environ['GIT_AUTHOR_EMAIL'] = 'bup@85430dcca2b611e4b2c3-8f5691723476' +environ['GIT_COMMITTER_EMAIL'] = 'bup@85430dcca2b611e4b2c3-8f5691723476' + +# The clean-repo test can probably be applied more broadly. It was +# initially just applied to test-pick to catch a bug. + +top = getcwd() +bup_cmd = top + '/bup' + +def rmrf(path): + err = [] # because python's scoping mess... + def onerror(function, path, excinfo): + err.append((function, path, excinfo)) + rmtree(path, onerror=onerror) + if err: + function, path, excinfo = err[0] + ex_type, ex, traceback = excinfo + if (not isinstance(ex, OSError)) or ex.errno != ENOENT: + raise ex + +def verify_trees_match(path1, path2): + global top + exr = exo((top + '/t/compare-trees', '-c', path1, path2), check=False) + print(exr.out) + sys.stdout.flush() + wvcheck(exr.rc == 0, 'process exit %d == 0' % exr.rc) + +def verify_rcz(cmd, **kwargs): + assert not kwargs.get('check') + kwargs['check'] = False + result = exo(cmd, **kwargs) + print(result.out) + rc = result.proc.returncode + wvcheck(rc == 0, 'process exit %d == 0' % rc) + return result + +# FIXME: multline, or allow opts generally? + +def verify_rx(rx, string): + wvcheck(re.search(rx, string), 'rx %r matches %r' % (rx, string)) + +def verify_nrx(rx, string): + wvcheck(not re.search(rx, string), "rx %r doesn't match %r" % (rx, string)) + +def validate_clean_repo(): + out = verify_rcz(('git', '--git-dir', 'get-dest', 'fsck')).out + verify_nrx(r'dangling|mismatch|missing|unreachable', out) + +def validate_blob(src_id, dest_id): + global top + rmrf('restore-src') + rmrf('restore-dest') + cat_tree = top + '/t/git-cat-tree' + src_blob = verify_rcz((cat_tree, '--git-dir', 'get-src', src_id)).out + dest_blob = verify_rcz((cat_tree, '--git-dir', 'get-src', src_id)).out + wvpasseq(src_blob, dest_blob) + +def validate_tree(src_id, dest_id): + + def set_committer_date(): + environ['GIT_COMMITTER_DATE'] = "2014-01-01 01:01" + + rmrf('restore-src') + rmrf('restore-dest') + mkdir('restore-src') + mkdir('restore-dest') + + # Create a commit so the archive contents will have matching timestamps. + src_c = exo(('git', '--git-dir', 'get-src', + 'commit-tree', '-m', 'foo', src_id), + preexec_fn=set_committer_date).out.strip() + dest_c = exo(('git', '--git-dir', 'get-dest', + 'commit-tree', '-m', 'foo', dest_id), + preexec_fn=set_committer_date).out.strip() + exr = verify_rcz('git --git-dir get-src archive %s | tar xvf - -C restore-src' + % quote(src_c), + shell=True) + if exr.rc != 0: return False + ex('cd restore-src && ls --full-time -aR .', shell=True) + exr = verify_rcz('git --git-dir get-dest archive %s | tar xvf - -C restore-dest' + % quote(dest_c), + shell=True) + if exr.rc != 0: return False + + # git archive doesn't include an entry for ./. + ex(('touch', '-r', 'restore-src', 'restore-dest')) + verify_trees_match('restore-src/', 'restore-dest/') + rmrf('restore-src') + rmrf('restore-dest') + +def validate_commit(src_id, dest_id): + exr = verify_rcz(('git', '--git-dir', 'get-src', 'cat-file', 'commit', src_id)) + if exr.rc != 0: return False + src_cat = exr.out + exr = verify_rcz(('git', '--git-dir', 'get-dest', 'cat-file', 'commit', dest_id)) + if exr.rc != 0: return False + dest_cat = exr.out + wvpasseq(src_cat, dest_cat) + if src_cat != dest_cat: return False + + rmrf('restore-src') + rmrf('restore-dest') + mkdir('restore-src') + mkdir('restore-dest') + qsrc = quote(src_id) + qdest = quote(dest_id) + exr = verify_rcz(('git --git-dir get-src archive ' + qsrc + + ' | tar xf - -C restore-src'), + shell=True) + if exr.rc != 0: return False + exr = verify_rcz(('git --git-dir get-dest archive ' + qdest + + ' | tar xf - -C restore-dest'), + shell=True) + if exr.rc != 0: return False + + # git archive doesn't include an entry for ./. + ex(('touch', '-r', 'restore-src', 'restore-dest')) + verify_trees_match('restore-src/', 'restore-dest/') + rmrf('restore-src') + rmrf('restore-dest') + +def _validate_save(orig_dir, save_path, commit_id, tree_id): + global bup_cmd + rmrf('restore') + exr = verify_rcz((bup_cmd, '-d', 'get-dest', + 'restore', '-C', 'restore', save_path + '/.')) + if exr.rc: return False + verify_trees_match(orig_dir + '/', 'restore/') + if tree_id: + # FIXME: double check that get-dest is correct + exr = verify_rcz(('git', '--git-dir', 'get-dest', 'ls-tree', tree_id)) + if exr.rc: return False + cat = verify_rcz(('git', '--git-dir', 'get-dest', + 'cat-file', 'commit', commit_id)) + if cat.rc: return False + wvpasseq('tree ' + tree_id, cat.out.splitlines()[0]) + +# FIXME: re-merge save and new_save? + +def validate_save(dest_name, restore_subpath, commit_id, tree_id, orig_value, + get_out): + out = get_out.splitlines() + wvpasseq(2, len(out)) + get_tree_id = out[0] + get_commit_id = out[1] + wvpasseq(tree_id, get_tree_id) + wvpasseq(commit_id, get_commit_id) + _validate_save(orig_value, dest_name + restore_subpath, commit_id, tree_id) + +def validate_new_save(dest_name, restore_subpath, commit_id, tree_id, orig_value, + get_out): + out = get_out.splitlines() + wvpasseq(2, len(out)) + get_tree_id = out[0] + get_commit_id = out[1] + wvpasseq(tree_id, get_tree_id) + wvpassne(commit_id, get_commit_id) + _validate_save(orig_value, dest_name + restore_subpath, get_commit_id, tree_id) + +def validate_tagged_save(tag_name, restore_subpath, + commit_id, tree_id, orig_value, get_out): + out = get_out.splitlines() + wvpasseq(1, len(out)) + get_tag_id = out[0] + wvpasseq(commit_id, get_tag_id) + # Make sure tmp doesn't already exist. + exr = exo(('git', '--git-dir', 'get-dest', 'show-ref', 'tmp-branch-for-tag'), + check=False) + wvpasseq(1, exr.rc) + + ex(('git', '--git-dir', 'get-dest', 'branch', 'tmp-branch-for-tag', + 'refs/tags/' + tag_name)) + _validate_save(orig_value, 'tmp-branch-for-tag/latest' + restore_subpath, + commit_id, tree_id) + ex(('git', '--git-dir', 'get-dest', 'branch', '-D', 'tmp-branch-for-tag')) + +def validate_new_tagged_commit(tag_name, commit_id, tree_id, get_out): + out = get_out.splitlines() + wvpasseq(1, len(out)) + get_tag_id = out[0] + wvpassne(commit_id, get_tag_id) + validate_tree(tree_id, tag_name + ':') + + +get_cases_tested = 0 + + +def _run_get(disposition, method, what): + global bup_cmd + + if disposition == 'get': + get_cmd = (bup_cmd, '-d', 'get-dest', + 'get', '-vvct', '--print-tags', '-s', 'get-src') + elif disposition == 'get-on': + get_cmd = (bup_cmd, '-d', 'get-dest', + 'on', '-', 'get', '-vvct', '--print-tags', '-s', 'get-src') + elif disposition == 'get-to': + get_cmd = (bup_cmd, '-d', 'get-dest', + 'get', '-vvct', '--print-tags', '-s', 'get-src', + '-r', '-:' + getcwd() + '/get-dest') + else: + raise Exception('error: unexpected get disposition ' + disposition) + + global get_cases_tested + if isinstance(what, compat.str_type): + cmd = get_cmd + (method, what) + else: + if method in ('--ff', '--append', '--pick', '--force-pick', '--new-tag', + '--replace'): + method += ':' + src, dest = what + cmd = get_cmd + (method, src, dest) + result = exo(cmd, check=False, stderr=PIPE) + get_cases_tested += 1 + return result + +def run_get(disposition, method, what=None, given=None): + global bup_cmd + rmrf('get-dest') + ex((bup_cmd, '-d', 'get-dest', 'init')) + + if given: + # FIXME: replace bup-get with independent commands as is feasible + exr = _run_get(disposition, '--replace', given) + assert not exr.rc + return _run_get(disposition, method, what) + +def test_universal_behaviors(get_disposition): + methods = ('--ff', '--append', '--pick', '--force-pick', '--new-tag', + '--replace', '--unnamed') + for method in methods: + wvstart(get_disposition + ' ' + method + ', missing source, fails') + exr = run_get(get_disposition, method, 'not-there') + wvpassne(0, exr.rc) + verify_rx(r'cannot find source', exr.err) + for method in methods: + wvstart(get_disposition + ' ' + method + ' / fails') + exr = run_get(get_disposition, method, '/') + wvpassne(0, exr.rc) + verify_rx('cannot fetch entire repository', exr.err) + +def verify_only_refs(**kwargs): + for kind, refs in kwargs.iteritems(): + if kind == 'heads': + abs_refs = ['refs/heads/' + ref for ref in refs] + karg = '--heads' + elif kind == 'tags': + abs_refs = ['refs/tags/' + ref for ref in refs] + karg = '--tags' + else: + raise TypeError('unexpected keyword argument %r' % kind) + if abs_refs: + verify_rcz(['git', '--git-dir', 'get-dest', + 'show-ref', '--verify', karg] + abs_refs) + exr = exo(('git', '--git-dir', 'get-dest', 'show-ref', karg), + check=False) + wvpasseq(0, exr.rc) + expected_refs = sorted(abs_refs) + repo_refs = sorted([x.split()[1] for x in exr.out.splitlines()]) + wvpasseq(expected_refs, repo_refs) + else: + # FIXME: can we just check "git show-ref --heads == ''"? + exr = exo(('git', '--git-dir', 'get-dest', 'show-ref', karg), + check=False) + wvpasseq(1, exr.rc) + wvpasseq('', exr.out.strip()) + +def test_replace(get_disposition, src_info): + + wvstart(get_disposition + ' --replace to root fails') + for item in ('.tag/tinyfile', + 'src/latest' + src_info['tinyfile-path'], + '.tag/subtree', + 'src/latest' + src_info['subtree-vfs-path'], + '.tag/commit-1', + 'src/latest', + 'src'): + exr = run_get(get_disposition, '--replace', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'impossible; can only overwrite branch or tag', exr.err) + + tinyfile_id = src_info['tinyfile-id'] + tinyfile_path = src_info['tinyfile-path'] + subtree_vfs_path = src_info['subtree-vfs-path'] + subtree_id = src_info['subtree-id'] + commit_2_id = src_info['commit-2-id'] + tree_2_id = src_info['tree-2-id'] + + # Anything to tag + existing_items = {'nothing' : None, + 'blob' : ('.tag/tinyfile', '.tag/obj'), + 'tree' : ('.tag/tree-1', '.tag/obj'), + 'commit': ('.tag/commit-1', '.tag/obj')} + for ex_type, ex_ref in existing_items.iteritems(): + wvstart(get_disposition + ' --replace ' + ex_type + ' with blob tag') + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + exr = run_get(get_disposition, '--replace', (item ,'.tag/obj'), + given=ex_ref) + wvpasseq(0, exr.rc) + validate_blob(tinyfile_id, tinyfile_id) + verify_only_refs(heads=[], tags=('obj',)) + wvstart(get_disposition + ' --replace ' + ex_type + ' with tree tag') + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + exr = run_get(get_disposition, '--replace', (item, '.tag/obj'), + given=ex_ref) + validate_tree(subtree_id, subtree_id) + verify_only_refs(heads=[], tags=('obj',)) + wvstart(get_disposition + ' --replace ' + ex_type + ' with commitish tag') + for item in ('.tag/commit-2', 'src/latest', 'src'): + exr = run_get(get_disposition, '--replace', (item, '.tag/obj'), + given=ex_ref) + validate_tagged_save('obj', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=[], tags=('obj',)) + + # Committish to branch. + existing_items = (('nothing', None), + ('branch', ('.tag/commit-1', 'obj'))) + for ex_type, ex_ref in existing_items: + for item_type, item in (('commit', '.tag/commit-2'), + ('save', 'src/latest'), + ('branch', 'src')): + wvstart(get_disposition + ' --replace ' + + ex_type + ' with ' + item_type) + exr = run_get(get_disposition, '--replace', (item, 'obj'), + given=ex_ref) + validate_save('obj/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + # Not committish to branch + existing_items = (('nothing', None), + ('branch', ('.tag/commit-1', 'obj'))) + for ex_type, ex_ref in existing_items: + for item_type, item in (('blob', '.tag/tinyfile'), + ('blob', 'src/latest' + tinyfile_path), + ('tree', '.tag/subtree'), + ('tree', 'src/latest' + subtree_vfs_path)): + wvstart(get_disposition + ' --replace branch with ' + + item_type + ' given ' + ex_type + ' fails') + + exr = run_get(get_disposition, '--replace', (item, 'obj'), + given=ex_ref) + wvpassne(0, exr.rc) + verify_rx(r'cannot overwrite branch with .+ for', exr.err) + + wvstart(get_disposition + ' --replace, implicit destinations') + + exr = run_get(get_disposition, '--replace', 'src') + validate_save('src/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('src',), tags=[]) + + exr = run_get(get_disposition, '--replace', '.tag/commit-2') + validate_tagged_save('commit-2', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=[], tags=('commit-2',)) + +def test_ff(get_disposition, src_info): + + wvstart(get_disposition + ' --ff to root fails') + tinyfile_path = src_info['tinyfile-path'] + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + exr = run_get(get_disposition, '--ff', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'source for .+ must be a branch, save, or commit', exr.err) + subtree_vfs_path = src_info['subtree-vfs-path'] + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + exr = run_get(get_disposition, '--ff', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'is impossible; can only --append a tree to a branch', + exr.err) + for item in ('.tag/commit-1', 'src/latest', 'src'): + exr = run_get(get_disposition, '--ff', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'destination for .+ is a root, not a branch', exr.err) + + wvstart(get_disposition + ' --ff of not-committish fails') + for src in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + # FIXME: use get_item elsewhere? + for given, get_item in ((None, (src, 'obj')), + (None, (src, '.tag/obj')), + (('.tag/tinyfile', '.tag/obj'), (src, '.tag/obj')), + (('.tag/tree-1', '.tag/obj'), (src, '.tag/obj')), + (('.tag/commit-1', '.tag/obj'), (src, '.tag/obj')), + (('.tag/commit-1', 'obj'), (src, 'obj'))): + exr = run_get(get_disposition, '--ff', get_item, given=given) + wvpassne(0, exr.rc) + verify_rx(r'must be a branch, save, or commit', exr.err) + for src in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + for given, get_item in ((None, (src, 'obj')), + (None, (src, '.tag/obj')), + (('.tag/tinyfile', '.tag/obj'), (src, '.tag/obj')), + (('.tag/tree-1', '.tag/obj'), (src, '.tag/obj')), + (('.tag/commit-1', '.tag/obj'), (src, '.tag/obj')), + (('.tag/commit-1', 'obj'), (src, 'obj'))): + exr = run_get(get_disposition, '--ff', get_item, given=given) + wvpassne(0, exr.rc) + verify_rx(r'can only --append a tree to a branch', exr.err) + + wvstart(get_disposition + ' --ff committish, ff possible') + save_2 = src_info['save-2'] + for src in ('.tag/commit-2', 'src/' + save_2, 'src'): + for given, get_item, complaint in \ + ((None, (src, '.tag/obj'), + r'destination .+ must be a valid branch name'), + (('.tag/tinyfile', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a blob, not a branch'), + (('.tag/tree-1', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a tree, not a branch'), + (('.tag/commit-1', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a tagged commit, not a branch'), + (('.tag/commit-2', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a tagged commit, not a branch')): + exr = run_get(get_disposition, '--ff', get_item, given=given) + wvpassne(0, exr.rc) + verify_rx(complaint, exr.err) + # FIXME: use src or item and given or existing consistently in loops... + commit_2_id = src_info['commit-2-id'] + tree_2_id = src_info['tree-2-id'] + for src in ('.tag/commit-2', 'src/' + save_2, 'src'): + for given in (None, ('.tag/commit-1', 'obj'), ('.tag/commit-2', 'obj')): + exr = run_get(get_disposition, '--ff', (src, 'obj'), given=given) + wvpasseq(0, exr.rc) + validate_save('obj/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + wvstart(get_disposition + ' --ff, implicit destinations') + for item in ('src', 'src/latest'): + exr = run_get(get_disposition, '--ff', item) + wvpasseq(0, exr.rc) + + ex(('find', 'get-dest/refs')) + ex((bup_cmd, '-d', 'get-dest', 'ls')) + + validate_save('src/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + #verify_only_refs(heads=('src',), tags=[]) + + wvstart(get_disposition + ' --ff, ff impossible') + for given, get_item in ((('unrelated-branch', 'src'), 'src'), + (('.tag/commit-2', 'src'), ('.tag/commit-1', 'src'))): + exr = run_get(get_disposition, '--ff', get_item, given=given) + wvpassne(0, exr.rc) + verify_rx(r'destination is not an ancestor of source', exr.err) + +def test_append(get_disposition, src_info): + tinyfile_path = src_info['tinyfile-path'] + subtree_vfs_path = src_info['subtree-vfs-path'] + + wvstart(get_disposition + ' --append to root fails') + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + exr = run_get(get_disposition, '--append', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'source for .+ must be a branch, save, commit, or tree', + exr.err) + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path, + '.tag/commit-1', 'src/latest', 'src'): + exr = run_get(get_disposition, '--append', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'destination for .+ is a root, not a branch', exr.err) + + wvstart(get_disposition + ' --append of not-treeish fails') + for src in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + for given, item in ((None, (src, 'obj')), + (None, (src, '.tag/obj')), + (('.tag/tinyfile', '.tag/obj'), (src, '.tag/obj')), + (('.tag/tree-1', '.tag/obj'), (src, '.tag/obj')), + (('.tag/commit-1', '.tag/obj'), (src, '.tag/obj')), + (('.tag/commit-1', 'obj'), (src, 'obj'))): + exr = run_get(get_disposition, '--append', item, given=given) + wvpassne(0, exr.rc) + verify_rx(r'must be a branch, save, commit, or tree', exr.err) + + wvstart(get_disposition + ' --append committish failure cases') + save_2 = src_info['save-2'] + for src in ('.tag/subtree', 'src/latest' + subtree_vfs_path, + '.tag/commit-2', 'src/' + save_2, 'src'): + for given, item, complaint in \ + ((None, (src, '.tag/obj'), + r'destination .+ must be a valid branch name'), + (('.tag/tinyfile', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a blob, not a branch'), + (('.tag/tree-1', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a tree, not a branch'), + (('.tag/commit-1', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a tagged commit, not a branch'), + (('.tag/commit-2', '.tag/obj'), (src, '.tag/obj'), + r'destination .+ is a tagged commit, not a branch')): + exr = run_get(get_disposition, '--append', item, given=given) + wvpassne(0, exr.rc) + verify_rx(complaint, exr.err) + + wvstart(get_disposition + ' --append committish') + commit_2_id = src_info['commit-2-id'] + tree_2_id = src_info['tree-2-id'] + for item in ('.tag/commit-2', 'src/' + save_2, 'src'): + for existing in (None, ('.tag/commit-1', 'obj'), + ('.tag/commit-2', 'obj'), + ('unrelated-branch', 'obj')): + exr = run_get(get_disposition, '--append', (item, 'obj'), + given=existing) + wvpasseq(0, exr.rc) + validate_new_save('obj/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + # Append ancestor + save_1 = src_info['save-1'] + commit_1_id = src_info['commit-1-id'] + tree_1_id = src_info['tree-1-id'] + for item in ('.tag/commit-1', 'src/' + save_1, 'src-1'): + exr = run_get(get_disposition, '--append', (item, 'obj'), + given=('.tag/commit-2', 'obj')) + wvpasseq(0, exr.rc) + validate_new_save('obj/latest', getcwd() + '/src', + commit_1_id, tree_1_id, 'src-1', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + wvstart(get_disposition + ' --append tree') + subtree_path = src_info['subtree-path'] + subtree_id = src_info['subtree-id'] + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + for existing in (None, ('.tag/commit-1', 'obj'), ('.tag/commit-2','obj')): + exr = run_get(get_disposition, '--append', (item, 'obj'), + given=existing) + wvpasseq(0, exr.rc) + validate_new_save('obj/latest', '/', None, subtree_id, subtree_path, + exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + wvstart(get_disposition + ' --append, implicit destinations') + + for item in ('src', 'src/latest'): + exr = run_get(get_disposition, '--append', item) + wvpasseq(0, exr.rc) + validate_new_save('src/latest', getcwd() + '/src', commit_2_id, tree_2_id, + 'src-2', exr.out) + verify_only_refs(heads=('src',), tags=[]) + +def test_pick(get_disposition, src_info, force=False): + flavor = '--force-pick' if force else '--pick' + tinyfile_path = src_info['tinyfile-path'] + subtree_vfs_path = src_info['subtree-vfs-path'] + + wvstart(get_disposition + ' ' + flavor + ' to root fails') + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path, 'src'): + exr = run_get(get_disposition, flavor, (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'can only pick a commit or save', exr.err) + for item in ('.tag/commit-1', 'src/latest'): + exr = run_get(get_disposition, flavor, (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'destination is not a tag or branch', exr.err) + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + exr = run_get(get_disposition, flavor, (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'is impossible; can only --append a tree', exr.err) + + wvstart(get_disposition + ' ' + flavor + ' of blob or branch fails') + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path, 'src'): + for given, get_item in ((None, (item, 'obj')), + (None, (item, '.tag/obj')), + (('.tag/tinyfile', '.tag/obj'), (item, '.tag/obj')), + (('.tag/tree-1', '.tag/obj'), (item, '.tag/obj')), + (('.tag/commit-1', '.tag/obj'), (item, '.tag/obj')), + (('.tag/commit-1', 'obj'), (item, 'obj'))): + exr = run_get(get_disposition, flavor, get_item, given=given) + wvpassne(0, exr.rc) + verify_rx(r'impossible; can only pick a commit or save', exr.err) + + wvstart(get_disposition + ' ' + flavor + ' of tree fails') + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + for given, get_item in ((None, (item, 'obj')), + (None, (item, '.tag/obj')), + (('.tag/tinyfile', '.tag/obj'), (item, '.tag/obj')), + (('.tag/tree-1', '.tag/obj'), (item, '.tag/obj')), + (('.tag/commit-1', '.tag/obj'), (item, '.tag/obj')), + (('.tag/commit-1', 'obj'), (item, 'obj'))): + exr = run_get(get_disposition, flavor, get_item, given=given) + wvpassne(0, exr.rc) + verify_rx(r'impossible; can only --append a tree', exr.err) + + save_2 = src_info['save-2'] + commit_2_id = src_info['commit-2-id'] + tree_2_id = src_info['tree-2-id'] + # FIXME: these two wvstart texts? + if force: + wvstart(get_disposition + ' ' + flavor + ' commit/save to existing tag') + for item in ('.tag/commit-2', 'src/' + save_2): + for given in (('.tag/tinyfile', '.tag/obj'), + ('.tag/tree-1', '.tag/obj'), + ('.tag/commit-1', '.tag/obj')): + exr = run_get(get_disposition, flavor, (item, '.tag/obj'), + given=given) + wvpasseq(0, exr.rc) + validate_new_tagged_commit('obj', commit_2_id, tree_2_id, + exr.out) + verify_only_refs(heads=[], tags=('obj',)) + else: # --pick + wvstart(get_disposition + ' ' + flavor + + ' commit/save to existing tag fails') + for item in ('.tag/commit-2', 'src/' + save_2): + for given in (('.tag/tinyfile', '.tag/obj'), + ('.tag/tree-1', '.tag/obj'), + ('.tag/commit-1', '.tag/obj')): + exr = run_get(get_disposition, flavor, (item, '.tag/obj'), given=given) + wvpassne(0, exr.rc) + verify_rx(r'cannot overwrite existing tag', exr.err) + + wvstart(get_disposition + ' ' + flavor + ' commit/save to tag') + for item in ('.tag/commit-2', 'src/' + save_2): + exr = run_get(get_disposition, flavor, (item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_clean_repo() + validate_new_tagged_commit('obj', commit_2_id, tree_2_id, exr.out) + verify_only_refs(heads=[], tags=('obj',)) + + wvstart(get_disposition + ' ' + flavor + ' commit/save to branch') + for item in ('.tag/commit-2', 'src/' + save_2): + for given in (None, ('.tag/commit-1', 'obj'), ('.tag/commit-2', 'obj')): + exr = run_get(get_disposition, flavor, (item, 'obj'), given=given) + wvpasseq(0, exr.rc) + validate_clean_repo() + validate_new_save('obj/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + wvstart(get_disposition + ' ' + flavor + + ' commit/save unrelated commit to branch') + for item in('.tag/commit-2', 'src/' + save_2): + exr = run_get(get_disposition, flavor, (item, 'obj'), + given=('unrelated-branch', 'obj')) + wvpasseq(0, exr.rc) + validate_clean_repo() + validate_new_save('obj/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + wvstart(get_disposition + ' ' + flavor + ' commit/save ancestor to branch') + save_1 = src_info['save-1'] + commit_1_id = src_info['commit-1-id'] + tree_1_id = src_info['tree-1-id'] + for item in ('.tag/commit-1', 'src/' + save_1): + exr = run_get(get_disposition, flavor, (item, 'obj'), + given=('.tag/commit-2', 'obj')) + wvpasseq(0, exr.rc) + validate_clean_repo() + validate_new_save('obj/latest', getcwd() + '/src', + commit_1_id, tree_1_id, 'src-1', exr.out) + verify_only_refs(heads=('obj',), tags=[]) + + + wvstart(get_disposition + ' ' + flavor + ', implicit destinations') + exr = run_get(get_disposition, flavor, '.tag/commit-2') + wvpasseq(0, exr.rc) + validate_clean_repo() + validate_new_tagged_commit('commit-2', commit_2_id, tree_2_id, exr.out) + verify_only_refs(heads=[], tags=('commit-2',)) + + exr = run_get(get_disposition, flavor, 'src/latest') + wvpasseq(0, exr.rc) + validate_clean_repo() + validate_new_save('src/latest', getcwd() + '/src', + commit_2_id, tree_2_id, 'src-2', exr.out) + verify_only_refs(heads=('src',), tags=[]) + +def test_new_tag(get_disposition, src_info): + tinyfile_id = src_info['tinyfile-id'] + tinyfile_path = src_info['tinyfile-path'] + commit_2_id = src_info['commit-2-id'] + tree_2_id = src_info['tree-2-id'] + subtree_id = src_info['subtree-id'] + subtree_vfs_path = src_info['subtree-vfs-path'] + + wvstart(get_disposition + ' --new-tag to root fails') + for item in ('.tag/tinyfile', + 'src/latest' + tinyfile_path, + '.tag/subtree', + 'src/latest' + subtree_vfs_path, + '.tag/commit-1', + 'src/latest', + 'src'): + exr = run_get(get_disposition, '--new-tag', (item, '/')) + wvpassne(0, exr.rc) + verify_rx(r'destination for .+ must be a VFS tag', exr.err) + + # Anything to new tag. + wvstart(get_disposition + ' --new-tag, blob tag') + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + exr = run_get(get_disposition, '--new-tag', (item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_blob(tinyfile_id, tinyfile_id) + verify_only_refs(heads=[], tags=('obj',)) + + wvstart(get_disposition + ' --new-tag, tree tag') + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + exr = run_get(get_disposition, '--new-tag', (item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_tree(subtree_id, subtree_id) + verify_only_refs(heads=[], tags=('obj',)) + + wvstart(get_disposition + ' --new-tag, committish tag') + for item in ('.tag/commit-2', 'src/latest', 'src'): + exr = run_get(get_disposition, '--new-tag', (item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_tagged_save('obj', getcwd() + '/src/', commit_2_id, tree_2_id, + 'src-2', exr.out) + verify_only_refs(heads=[], tags=('obj',)) + + # Anything to existing tag (fails). + for ex_type, ex_tag in (('blob', ('.tag/tinyfile', '.tag/obj')), + ('tree', ('.tag/tree-1', '.tag/obj')), + ('commit', ('.tag/commit-1', '.tag/obj'))): + for item_type, item in (('blob tag', '.tag/tinyfile'), + ('blob path', 'src/latest' + tinyfile_path), + ('tree tag', '.tag/subtree'), + ('tree path', 'src/latest' + subtree_vfs_path), + ('commit tag', '.tag/commit-2'), + ('save', 'src/latest'), + ('branch', 'src')): + wvstart(get_disposition + ' --new-tag of ' + item_type + + ', given existing ' + ex_type + ' tag, fails') + exr = run_get(get_disposition, '--new-tag', (item, '.tag/obj'), + given=ex_tag) + wvpassne(0, exr.rc) + verify_rx(r'cannot overwrite existing tag .* \(requires --replace\)', + exr.err) + + # Anything to branch (fails). + for ex_type, ex_tag in (('nothing', None), + ('blob', ('.tag/tinyfile', '.tag/obj')), + ('tree', ('.tag/tree-1', '.tag/obj')), + ('commit', ('.tag/commit-1', '.tag/obj'))): + for item_type, item in (('blob tag', '.tag/tinyfile'), + ('blob path', 'src/latest' + tinyfile_path), + ('tree tag', '.tag/subtree'), + ('tree path', 'src/latest' + subtree_vfs_path), + ('commit tag', '.tag/commit-2'), + ('save', 'src/latest'), + ('branch', 'src')): + wvstart(get_disposition + ' --new-tag to branch of ' + item_type + + ', given existing ' + ex_type + ' tag, fails') + exr = run_get(get_disposition, '--new-tag', (item, 'obj'), + given=ex_tag) + wvpassne(0, exr.rc) + verify_rx(r'destination for .+ must be a VFS tag', exr.err) + + wvstart(get_disposition + ' --new-tag, implicit destinations') + exr = run_get(get_disposition, '--new-tag', '.tag/commit-2') + wvpasseq(0, exr.rc) + validate_tagged_save('commit-2', getcwd() + '/src/', commit_2_id, tree_2_id, + 'src-2', exr.out) + verify_only_refs(heads=[], tags=('commit-2',)) + +def test_unnamed(get_disposition, src_info): + tinyfile_id = src_info['tinyfile-id'] + tinyfile_path = src_info['tinyfile-path'] + subtree_vfs_path = src_info['subtree-vfs-path'] + wvstart(get_disposition + ' --unnamed to root fails') + for item in ('.tag/tinyfile', + 'src/latest' + tinyfile_path, + '.tag/subtree', + 'src/latest' + subtree_vfs_path, + '.tag/commit-1', + 'src/latest', + 'src'): + for ex_ref in (None, (item, '.tag/obj')): + exr = run_get(get_disposition, '--unnamed', (item, '/'), + given=ex_ref) + wvpassne(0, exr.rc) + verify_rx(r'usage: bup get ', exr.err) + + wvstart(get_disposition + ' --unnamed file') + for item in ('.tag/tinyfile', 'src/latest' + tinyfile_path): + exr = run_get(get_disposition, '--unnamed', item) + wvpasseq(0, exr.rc) + validate_blob(tinyfile_id, tinyfile_id) + verify_only_refs(heads=[], tags=[]) + + exr = run_get(get_disposition, '--unnamed', item, + given=(item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_blob(tinyfile_id, tinyfile_id) + verify_only_refs(heads=[], tags=('obj',)) + + wvstart(get_disposition + ' --unnamed tree') + subtree_id = src_info['subtree-id'] + for item in ('.tag/subtree', 'src/latest' + subtree_vfs_path): + exr = run_get(get_disposition, '--unnamed', item) + wvpasseq(0, exr.rc) + validate_tree(subtree_id, subtree_id) + verify_only_refs(heads=[], tags=[]) + + exr = run_get(get_disposition, '--unnamed', item, + given=(item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_tree(subtree_id, subtree_id) + verify_only_refs(heads=[], tags=('obj',)) + + wvstart(get_disposition + ' --unnamed committish') + save_2 = src_info['save-2'] + commit_2_id = src_info['commit-2-id'] + for item in ('.tag/commit-2', 'src/' + save_2, 'src'): + exr = run_get(get_disposition, '--unnamed', item) + wvpasseq(0, exr.rc) + validate_commit(commit_2_id, commit_2_id) + verify_only_refs(heads=[], tags=[]) + + exr = run_get(get_disposition, '--unnamed', item, + given=(item, '.tag/obj')) + wvpasseq(0, exr.rc) + validate_commit(commit_2_id, commit_2_id) + verify_only_refs(heads=[], tags=('obj',)) + +def create_get_src(): + global bup_cmd, src_info + wvstart('preparing') + ex((bup_cmd, '-d', 'get-src', 'init')) + + mkdir('src') + open('src/unrelated', 'a').close() + ex((bup_cmd, '-d', 'get-src', 'index', 'src')) + ex((bup_cmd, '-d', 'get-src', 'save', '-tcn', 'unrelated-branch', 'src')) + + ex((bup_cmd, '-d', 'get-src', 'index', '--clear')) + rmrf('src') + mkdir('src') + open('src/zero', 'a').close() + ex((bup_cmd, '-d', 'get-src', 'index', 'src')) + exr = exo((bup_cmd, '-d', 'get-src', 'save', '-tcn', 'src', 'src')) + out = exr.out.splitlines() + tree_0_id = out[0] + commit_0_id = out[-1] + exr = exo((bup_cmd, '-d', 'get-src', 'ls', 'src')) + save_0 = exr.out.splitlines()[0] + ex(('git', '--git-dir', 'get-src', 'branch', 'src-0', 'src')) + ex(('cp', '-a', 'src', 'src-0')) + + rmrf('src') + mkdir('src') + mkdir('src/x') + mkdir('src/x/y') + ex((bup_cmd + ' -d get-src random 1k > src/1'), shell=True) + ex((bup_cmd + ' -d get-src random 1k > src/x/2'), shell=True) + ex((bup_cmd, '-d', 'get-src', 'index', 'src')) + exr = exo((bup_cmd, '-d', 'get-src', 'save', '-tcn', 'src', 'src')) + out = exr.out.splitlines() + tree_1_id = out[0] + commit_1_id = out[-1] + exr = exo((bup_cmd, '-d', 'get-src', 'ls', 'src')) + save_1 = exr.out.splitlines()[1] + ex(('git', '--git-dir', 'get-src', 'branch', 'src-1', 'src')) + ex(('cp', '-a', 'src', 'src-1')) + + # Make a copy the current state of src so we'll have an ancestor. + ex(('cp', '-a', + 'get-src/refs/heads/src', 'get-src/refs/heads/src-ancestor')) + + with open('src/tiny-file', 'a') as f: f.write('xyzzy') + ex((bup_cmd, '-d', 'get-src', 'index', 'src')) + ex((bup_cmd, '-d', 'get-src', 'tick')) # Ensure the save names differ + exr = exo((bup_cmd, '-d', 'get-src', 'save', '-tcn', 'src', 'src')) + out = exr.out.splitlines() + tree_2_id = out[0] + commit_2_id = out[-1] + exr = exo((bup_cmd, '-d', 'get-src', 'ls', 'src')) + save_2 = exr.out.splitlines()[2] + rename('src', 'src-2') + + src_root = getcwd() + '/src' + + subtree_path = 'src-2/x' + subtree_vfs_path = src_root + '/x' + + # No support for "ls -d", so grep... + exr = exo((bup_cmd, '-d', 'get-src', 'ls', '-s', 'src/latest' + src_root)) + out = exr.out.splitlines() + subtree_id = None + for line in out: + if 'x' in line: + subtree_id = line.split()[0] + assert(subtree_id) + + # With a tiny file, we'll get a single blob, not a chunked tree + tinyfile_path = src_root + '/tiny-file' + exr = exo((bup_cmd, '-d', 'get-src', 'ls', '-s', 'src/latest' + tinyfile_path)) + tinyfile_id = exr.out.splitlines()[0].split()[0] + + ex((bup_cmd, '-d', 'get-src', 'tag', 'tinyfile', tinyfile_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'subtree', subtree_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'tree-0', tree_0_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'tree-1', tree_1_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'tree-2', tree_2_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'commit-0', commit_0_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'commit-1', commit_1_id)) + ex((bup_cmd, '-d', 'get-src', 'tag', 'commit-2', commit_2_id)) + ex(('git', '--git-dir', 'get-src', 'branch', 'commit-1', commit_1_id)) + ex(('git', '--git-dir', 'get-src', 'branch', 'commit-2', commit_2_id)) + + return {'tinyfile-path' : tinyfile_path, + 'tinyfile-id' : tinyfile_id, + 'subtree-id' : subtree_id, + 'tree-0-id' : tree_0_id, + 'tree-1-id' : tree_1_id, + 'tree-2-id' : tree_2_id, + 'commit-0-id' : commit_0_id, + 'commit-1-id' : commit_1_id, + 'commit-2-id' : commit_2_id, + 'save-1' : save_1, + 'save-2' : save_2, + 'subtree-path' : subtree_path, + 'subtree-vfs-path' : subtree_vfs_path} + +# FIXME: this fails in a strange way: +# WVPASS given nothing get --ff not-there + +dispositions_to_test = ('get',) + +if int(environ.get('BUP_TEST_LEVEL', '0')) >= 11: + dispositions_to_test += ('get-on', 'get-to') + +if len(sys.argv) == 1: + categories = ('replace', 'universal', 'ff', 'append', 'pick', 'new-tag', + 'unnamed') +else: + categories = sys.argv[1:] + +with test_tempdir('get-') as tmpdir: + chdir(tmpdir) + try: + src_info = create_get_src() + for category in categories: + for disposition in dispositions_to_test: + # given=FOO depends on --replace, so test it early + if category == 'replace': + test_replace(disposition, src_info) + elif category == 'universal': + test_universal_behaviors(disposition) + elif category == 'ff': + test_ff(disposition, src_info) + elif category == 'append': + test_append(disposition, src_info) + elif category == 'pick': + test_pick(disposition, src_info, force=False) + test_pick(disposition, src_info, force=True) + elif category == 'new-tag': + test_new_tag(disposition, src_info) + elif category == 'unnamed': + test_unnamed(disposition, src_info) + else: + raise Exception('unrecognized get test category') + except Exception, ex: + chdir(top) + raise + chdir(top) + +wvmsg('checked %d cases' % get_cases_tested) diff --git a/wvtest.py b/wvtest.py index 2f48ef6..8b5ab64 100755 --- a/wvtest.py +++ b/wvtest.py @@ -86,6 +86,14 @@ if __name__ != '__main__': # we're imported as a module _result(msg, tb, 'FAILED') return cond + def wvcheck(cond, msg, tb = None): + if tb == None: tb = _caller_stack(2) + if cond: + _result(msg, tb, 'ok') + else: + _result(msg, tb, 'FAILED') + return cond + _code_rx = re.compile(r'^\w+\((.*)\)(\s*#.*)?$') def _code(): text = _caller_stack(2)[3] -- 2.39.2