From 2fc209306bb7cafab77a482f1a2f88ff263ba93e Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Sat, 31 Mar 2018 16:25:34 -0500 Subject: [PATCH] repo: add VFS resolve(); test resolve() via local and remote repos Add resolve() to the repositories. While it's likely to be generally useful, we need it more immediately to support the forthcoming bup-get command, Signed-off-by: Rob Browning Tested-by: Rob Browning --- cmd/server-cmd.py | 28 ++- lib/bup/client.py | 29 ++- lib/bup/compat.py | 2 + lib/bup/repo.py | 14 +- lib/bup/t/tresolve.py | 509 ++++++++++++++++++++++-------------------- lib/bup/t/tvfs.py | 10 +- lib/bup/vfs.py | 119 +++++++++- lib/bup/vint.py | 18 +- 8 files changed, 470 insertions(+), 259 deletions(-) diff --git a/cmd/server-cmd.py b/cmd/server-cmd.py index 853ab7f..c55c46e 100755 --- a/cmd/server-cmd.py +++ b/cmd/server-cmd.py @@ -8,7 +8,7 @@ exec "$bup_python" "$0" ${1+"$@"} from __future__ import absolute_import import os, sys, struct, subprocess -from bup import options, git +from bup import options, git, vfs, vint from bup.git import MissingObject from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel, log) @@ -233,6 +233,29 @@ def rev_list(conn, _): raise GitError(msg) conn.ok() +def resolve(conn, args): + _init_session() + (flags,) = args.split() + flags = int(flags) + want_meta = bool(flags & 1) + follow = bool(flags & 2) + have_parent = bool(flags & 4) + parent = vfs.read_resolution(conn) if have_parent else None + path = vint.read_bvec(conn) + if not len(path): + raise Exception('Empty resolve path') + try: + res = list(vfs.resolve(repo, path, parent=parent, want_meta=want_meta, + follow=follow)) + except vfs.IOError as ex: + res = ex + if isinstance(res, vfs.IOError): + conn.write(b'\0') # error + vfs.write_ioerror(conn, res) + else: + conn.write(b'\1') # success + vfs.write_resolution(conn, res) + conn.ok() optspec = """ bup server @@ -259,7 +282,8 @@ commands = { 'cat': join, # apocryphal alias 'cat-batch' : cat_batch, 'refs': refs, - 'rev-list': rev_list + 'rev-list': rev_list, + 'resolve': resolve } # FIXME: this protocol is totally lame and not at all future-proof. diff --git a/lib/bup/client.py b/lib/bup/client.py index cad6e28..f02c6b5 100644 --- a/lib/bup/client.py +++ b/lib/bup/client.py @@ -2,11 +2,12 @@ from __future__ import absolute_import import errno, os, re, struct, sys, time, zlib -from bup import git, ssh +from bup import git, ssh, vfs from bup.compat import range from bup.helpers import (Conn, atomically_replaced_file, chunkyreader, debug1, debug2, linereader, lines_until_sentinel, mkdirp, progress, qprogress) +from bup.vint import read_bvec, read_vuint, write_bvec bwlimit = None @@ -436,6 +437,32 @@ class Client: raise not_ok self._not_busy() + def resolve(self, path, parent=None, want_meta=True, follow=False): + self._require_command('resolve') + self.check_busy() + self._busy = 'resolve' + conn = self.conn + conn.write('resolve %d\n' % ((1 if want_meta else 0) + | (2 if follow else 0) + | (4 if parent else 0))) + if parent: + vfs.write_resolution(conn, parent) + write_bvec(conn, path) + success = ord(conn.read(1)) + assert success in (0, 1) + if success: + result = vfs.read_resolution(conn) + else: + result = vfs.read_ioerror(conn) + # FIXME: confusing + not_ok = self.check_ok() + if not_ok: + raise not_ok + self._not_busy() + if isinstance(result, vfs.IOError): + raise result + return result + class PackWriter_Remote(git.PackWriter): def __init__(self, conn, objcache_maker, suggest_packs, diff --git a/lib/bup/compat.py b/lib/bup/compat.py index 895a84a..6f2e6d2 100644 --- a/lib/bup/compat.py +++ b/lib/bup/compat.py @@ -11,6 +11,7 @@ py3 = py_maj >= 3 if py3: + from shlex import quote range = range str_type = str @@ -27,6 +28,7 @@ if py3: else: # Python 2 + from pipes import quote range = xrange str_type = basestring diff --git a/lib/bup/repo.py b/lib/bup/repo.py index 0bfccc3..3b17825 100644 --- a/lib/bup/repo.py +++ b/lib/bup/repo.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from os.path import realpath from functools import partial -from bup import client, git +from bup import client, git, vfs _next_repo_id = 0 @@ -70,6 +70,13 @@ class LocalRepo: repo_dir=self.repo_dir): yield ref + ## Of course, the vfs better not call this... + def resolve(self, path, parent=None, want_meta=True, follow=True): + ## FIXME: mode_only=? + return vfs.resolve(self, path, + parent=parent, want_meta=want_meta, follow=follow) + + class RemoteRepo: def __init__(self, address): self.address = address @@ -126,3 +133,8 @@ class RemoteRepo: limit_to_heads=limit_to_heads, limit_to_tags=limit_to_tags): yield ref + + def resolve(self, path, parent=None, want_meta=True, follow=True): + ## FIXME: mode_only=? + return self.client.resolve(path, parent=parent, want_meta=want_meta, + follow=follow) diff --git a/lib/bup/t/tresolve.py b/lib/bup/t/tresolve.py index 533bf87..646d78a 100644 --- a/lib/bup/t/tresolve.py +++ b/lib/bup/t/tresolve.py @@ -10,7 +10,7 @@ from wvtest import * from bup import git, vfs from bup.metadata import Metadata -from bup.repo import LocalRepo +from bup.repo import LocalRepo, RemoteRepo from bup.test.vfs import tree_dict from buptest import ex, exo, no_lingering_errors, test_tempdir @@ -25,279 +25,286 @@ start_dir = os.getcwd() ## be promoted from a mode to a Metadata instance once the tree it ## refers to is traversed. -@wvtest -def test_resolve(): +def prep_and_test_repo(name, create_repo, test_repo): with no_lingering_errors(): - with test_tempdir('bup-tvfs-') as tmpdir: - resolve = vfs.resolve + with test_tempdir('bup-t' + name) as tmpdir: bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir - git.repodir = bup_dir - data_path = tmpdir + '/src' - save_time = 100000 - save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time)) - os.mkdir(data_path) - os.mkdir(data_path + '/dir') - with open(data_path + '/file', 'w+') as tmpfile: - print('canary', file=tmpfile) - symlink('file', data_path + '/file-symlink') - symlink('dir', data_path + '/dir-symlink') - symlink('not-there', data_path + '/bad-symlink') + environ['BUP_MAIN_EXE'] = bup_path ex((bup_path, 'init')) - ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test', - '--strip', data_path)) - ex((bup_path, 'tag', 'test-tag', 'test')) - repo = LocalRepo() + git.repodir = bup_dir + with create_repo(bup_dir) as repo: + test_repo(repo, tmpdir) - tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0] - tip_oidx = tip_hash.strip().split()[0] - tip_oid = tip_oidx.decode('hex') - tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1', - tip_oidx))[0].strip() - tip_tree_oid = tip_tree_oidx.decode('hex') - tip_tree = tree_dict(repo, tip_tree_oid) - test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta, - oid=tip_oid) - expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755, - oid=tip_tree_oid, - coid=tip_oid) - expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, - oid=tip_tree_oid, - coid=tip_oid) - expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode, - target=save_time_str) - expected_test_tag_item = expected_latest_item +# Currently, we just test through the repos since LocalRepo resolve is +# just a straight redirection to vfs.resolve. - wvstart('resolve: /') - vfs.clear_cache() - res = resolve(repo, '/') - wvpasseq(1, len(res)) - wvpasseq((('', vfs._root),), res) - ignore, root_item = res[0] - root_content = frozenset(vfs.contents(repo, root_item)) - wvpasseq(frozenset([('.', root_item), - ('.tag', vfs._tags), - ('test', test_revlist_w_meta)]), - root_content) - for path in ('//', '/.', '/./', '/..', '/../', - '/test/latest/dir/../../..', - '/test/latest/dir/../../../', - '/test/latest/dir/../../../.', - '/test/latest/dir/../../..//', - '/test//latest/dir/../../..', - '/test/./latest/dir/../../..', - '/test/././latest/dir/../../..', - '/test/.//./latest/dir/../../..', - '/test//.//.//latest/dir/../../..' - '/test//./latest/dir/../../..'): - wvstart('resolve: ' + path) - vfs.clear_cache() - res = resolve(repo, path) - wvpasseq((('', vfs._root),), res) +def test_resolve(repo, tmpdir): + data_path = tmpdir + '/src' + resolve = repo.resolve + save_time = 100000 + save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time)) + os.mkdir(data_path) + os.mkdir(data_path + '/dir') + with open(data_path + '/file', 'w+') as tmpfile: + print('canary', file=tmpfile) + symlink('file', data_path + '/file-symlink') + symlink('dir', data_path + '/dir-symlink') + symlink('not-there', data_path + '/bad-symlink') + ex((bup_path, 'index', '-v', data_path)) + ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test', + '--strip', data_path)) + ex((bup_path, 'tag', 'test-tag', 'test')) - wvstart('resolve: /.tag') - vfs.clear_cache() - res = resolve(repo, '/.tag') - wvpasseq(2, len(res)) - wvpasseq((('', vfs._root), ('.tag', vfs._tags)), - res) - ignore, tag_item = res[1] - tag_content = frozenset(vfs.contents(repo, tag_item)) - wvpasseq(frozenset([('.', tag_item), - ('test-tag', expected_test_tag_item)]), - tag_content) + tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0] + tip_oidx = tip_hash.strip().split()[0] + tip_oid = tip_oidx.decode('hex') + tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1', + tip_oidx))[0].strip() + tip_tree_oid = tip_tree_oidx.decode('hex') + tip_tree = tree_dict(repo, tip_tree_oid) + test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta, + oid=tip_oid) + expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755, + oid=tip_tree_oid, + coid=tip_oid) + expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, + oid=tip_tree_oid, + coid=tip_oid) + expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode, + target=save_time_str) + expected_test_tag_item = expected_latest_item - wvstart('resolve: /test') + wvstart('resolve: /') + vfs.clear_cache() + res = resolve('/') + wvpasseq(1, len(res)) + wvpasseq((('', vfs._root),), res) + ignore, root_item = res[0] + root_content = frozenset(vfs.contents(repo, root_item)) + wvpasseq(frozenset([('.', root_item), + ('.tag', vfs._tags), + ('test', test_revlist_w_meta)]), + root_content) + for path in ('//', '/.', '/./', '/..', '/../', + '/test/latest/dir/../../..', + '/test/latest/dir/../../../', + '/test/latest/dir/../../../.', + '/test/latest/dir/../../..//', + '/test//latest/dir/../../..', + '/test/./latest/dir/../../..', + '/test/././latest/dir/../../..', + '/test/.//./latest/dir/../../..', + '/test//.//.//latest/dir/../../..' + '/test//./latest/dir/../../..'): + wvstart('resolve: ' + path) vfs.clear_cache() - res = resolve(repo, '/test') - wvpasseq(2, len(res)) - wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res) - ignore, test_item = res[1] - test_content = frozenset(vfs.contents(repo, test_item)) - # latest has metadata here due to caching - wvpasseq(frozenset([('.', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('latest', expected_latest_link)]), - test_content) + res = resolve(path) + wvpasseq((('', vfs._root),), res) - wvstart('resolve: /test/latest') - vfs.clear_cache() - res = resolve(repo, '/test/latest') - wvpasseq(3, len(res)) - expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, - oid=tip_tree_oid, - coid=tip_oid) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta)) - wvpasseq(expected, res) - ignore, latest_item = res[2] - latest_content = frozenset(vfs.contents(repo, latest_item)) - expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta)) - for x in (tip_tree[name] - for name in ('.', - 'bad-symlink', - 'dir', - 'dir-symlink', - 'file', - 'file-symlink'))) - wvpasseq(expected, latest_content) + wvstart('resolve: /.tag') + vfs.clear_cache() + res = resolve('/.tag') + wvpasseq(2, len(res)) + wvpasseq((('', vfs._root), ('.tag', vfs._tags)), + res) + ignore, tag_item = res[1] + tag_content = frozenset(vfs.contents(repo, tag_item)) + wvpasseq(frozenset([('.', tag_item), + ('test-tag', expected_test_tag_item)]), + tag_content) - wvstart('resolve: /test/latest/file') - vfs.clear_cache() - res = resolve(repo, '/test/latest/file') - wvpasseq(4, len(res)) - expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta, - oid=tip_tree['file'].oid) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('file', expected_file_item_w_meta)) - wvpasseq(expected, res) + wvstart('resolve: /test') + vfs.clear_cache() + res = resolve('/test') + wvpasseq(2, len(res)) + wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res) + ignore, test_item = res[1] + test_content = frozenset(vfs.contents(repo, test_item)) + # latest has metadata here due to caching + wvpasseq(frozenset([('.', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('latest', expected_latest_link)]), + test_content) - wvstart('resolve: /test/latest/bad-symlink') - vfs.clear_cache() - res = resolve(repo, '/test/latest/bad-symlink') - wvpasseq(4, len(res)) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('not-there', None)) - wvpasseq(expected, res) + wvstart('resolve: /test/latest') + vfs.clear_cache() + res = resolve('/test/latest') + wvpasseq(3, len(res)) + expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, + oid=tip_tree_oid, + coid=tip_oid) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta)) + wvpasseq(expected, res) + ignore, latest_item = res[2] + latest_content = frozenset(vfs.contents(repo, latest_item)) + expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta)) + for x in (tip_tree[name] + for name in ('.', + 'bad-symlink', + 'dir', + 'dir-symlink', + 'file', + 'file-symlink'))) + wvpasseq(expected, latest_content) - wvstart('resolve nofollow: /test/latest/bad-symlink') - vfs.clear_cache() - res = resolve(repo, '/test/latest/bad-symlink', follow=False) - wvpasseq(4, len(res)) - bad_symlink_value = tip_tree['bad-symlink'] - expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta, - oid=bad_symlink_value.oid) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('bad-symlink', expected_bad_symlink_item_w_meta)) - wvpasseq(expected, res) + wvstart('resolve: /test/latest/file') + vfs.clear_cache() + res = resolve('/test/latest/file') + wvpasseq(4, len(res)) + expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta, + oid=tip_tree['file'].oid) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('file', expected_file_item_w_meta)) + wvpasseq(expected, res) - wvstart('resolve: /test/latest/file-symlink') - vfs.clear_cache() - res = resolve(repo, '/test/latest/file-symlink') - wvpasseq(4, len(res)) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('file', expected_file_item_w_meta)) - wvpasseq(expected, res) + wvstart('resolve: /test/latest/bad-symlink') + vfs.clear_cache() + res = resolve('/test/latest/bad-symlink') + wvpasseq(4, len(res)) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('not-there', None)) + wvpasseq(expected, res) - wvstart('resolve nofollow: /test/latest/file-symlink') - vfs.clear_cache() - res = resolve(repo, '/test/latest/file-symlink', follow=False) - wvpasseq(4, len(res)) - file_symlink_value = tip_tree['file-symlink'] - expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta, - oid=file_symlink_value.oid) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('file-symlink', expected_file_symlink_item_w_meta)) - wvpasseq(expected, res) + wvstart('resolve nofollow: /test/latest/bad-symlink') + vfs.clear_cache() + res = resolve('/test/latest/bad-symlink', follow=False) + wvpasseq(4, len(res)) + bad_symlink_value = tip_tree['bad-symlink'] + expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta, + oid=bad_symlink_value.oid) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('bad-symlink', expected_bad_symlink_item_w_meta)) + wvpasseq(expected, res) - wvstart('resolve: /test/latest/missing') - vfs.clear_cache() - res = resolve(repo, '/test/latest/missing') - wvpasseq(4, len(res)) - name, item = res[-1] - wvpasseq('missing', name) - wvpass(item is None) + wvstart('resolve: /test/latest/file-symlink') + vfs.clear_cache() + res = resolve('/test/latest/file-symlink') + wvpasseq(4, len(res)) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('file', expected_file_item_w_meta)) + wvpasseq(expected, res) - for path in ('/test/latest/file/', - '/test/latest/file/.', - '/test/latest/file/..', - '/test/latest/file/../', - '/test/latest/file/../.', - '/test/latest/file/../..', - '/test/latest/file/foo'): - wvstart('resolve: ' + path) - vfs.clear_cache() - try: - resolve(repo, path) - except vfs.IOError as res_ex: - wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', save_time_str, 'file'], - [name for name, item in res_ex.terminus]) + wvstart('resolve nofollow: /test/latest/file-symlink') + vfs.clear_cache() + res = resolve('/test/latest/file-symlink', follow=False) + wvpasseq(4, len(res)) + file_symlink_value = tip_tree['file-symlink'] + expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta, + oid=file_symlink_value.oid) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('file-symlink', expected_file_symlink_item_w_meta)) + wvpasseq(expected, res) - for path in ('/test/latest/file-symlink/', - '/test/latest/file-symlink/.', - '/test/latest/file-symlink/..', - '/test/latest/file-symlink/../', - '/test/latest/file-symlink/../.', - '/test/latest/file-symlink/../..'): - wvstart('resolve nofollow: ' + path) - vfs.clear_cache() - try: - resolve(repo, path, follow=False) - except vfs.IOError as res_ex: - wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', save_time_str, 'file'], - [name for name, item in res_ex.terminus]) + wvstart('resolve: /test/latest/missing') + vfs.clear_cache() + res = resolve('/test/latest/missing') + wvpasseq(4, len(res)) + name, item = res[-1] + wvpasseq('missing', name) + wvpass(item is None) - wvstart('resolve: non-directory parent') + for path in ('/test/latest/file/', + '/test/latest/file/.', + '/test/latest/file/..', + '/test/latest/file/../', + '/test/latest/file/../.', + '/test/latest/file/../..', + '/test/latest/file/foo'): + wvstart('resolve: ' + path) vfs.clear_cache() - file_res = resolve(repo, '/test/latest/file') try: - resolve(repo, 'foo', parent=file_res) + resolve(path) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(None, res_ex.terminus) + wvpasseq(['', 'test', save_time_str, 'file'], + [name for name, item in res_ex.terminus]) - wvstart('resolve nofollow: /test/latest/dir-symlink') + for path in ('/test/latest/file-symlink/', + '/test/latest/file-symlink/.', + '/test/latest/file-symlink/..', + '/test/latest/file-symlink/../', + '/test/latest/file-symlink/../.', + '/test/latest/file-symlink/../..'): + wvstart('resolve nofollow: ' + path) vfs.clear_cache() - res = resolve(repo, '/test/latest/dir-symlink', follow=False) - wvpasseq(4, len(res)) - dir_symlink_value = tip_tree['dir-symlink'] - expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta, - oid=dir_symlink_value.oid) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('dir-symlink', expected_dir_symlink_item_w_meta)) - wvpasseq(expected, res) + try: + resolve(path, follow=False) + except vfs.IOError as res_ex: + wvpasseq(ENOTDIR, res_ex.errno) + wvpasseq(['', 'test', save_time_str, 'file'], + [name for name, item in res_ex.terminus]) - dir_value = tip_tree['dir'] - expected_dir_item = vfs.Item(oid=dir_value.oid, - meta=tree_dict(repo, dir_value.oid)['.'].meta) - expected = (('', vfs._root), - ('test', test_revlist_w_meta), - (save_time_str, expected_latest_item_w_meta), - ('dir', expected_dir_item)) - def lresolve(*args, **keys): - return resolve(*args, **dict(keys, follow=False)) - for resname, resolver in (('resolve', resolve), - ('resolve nofollow', lresolve)): - for path in ('/test/latest/dir-symlink/', - '/test/latest/dir-symlink/.'): - wvstart(resname + ': ' + path) - vfs.clear_cache() - res = resolver(repo, path) - wvpasseq(4, len(res)) - wvpasseq(expected, res) - wvstart('resolve: /test/latest/dir-symlink') - vfs.clear_cache() - res = resolve(repo, path) - wvpasseq(4, len(res)) - wvpasseq(expected, res) + wvstart('resolve: non-directory parent') + vfs.clear_cache() + file_res = resolve('/test/latest/file') + try: + resolve('foo', parent=file_res) + except vfs.IOError as res_ex: + wvpasseq(ENOTDIR, res_ex.errno) + wvpasseq(None, res_ex.terminus) + + wvstart('resolve nofollow: /test/latest/dir-symlink') + vfs.clear_cache() + res = resolve('/test/latest/dir-symlink', follow=False) + wvpasseq(4, len(res)) + dir_symlink_value = tip_tree['dir-symlink'] + expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta, + oid=dir_symlink_value.oid) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('dir-symlink', expected_dir_symlink_item_w_meta)) + wvpasseq(expected, res) + + dir_value = tip_tree['dir'] + expected_dir_item = vfs.Item(oid=dir_value.oid, + meta=tree_dict(repo, dir_value.oid)['.'].meta) + expected = (('', vfs._root), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), + ('dir', expected_dir_item)) + def lresolve(*args, **keys): + return resolve(*args, **dict(keys, follow=False)) + for resname, resolver in (('resolve', resolve), + ('resolve nofollow', lresolve)): + for path in ('/test/latest/dir-symlink/', + '/test/latest/dir-symlink/.'): + wvstart(resname + ': ' + path) + vfs.clear_cache() + res = resolver(path) + wvpasseq(4, len(res)) + wvpasseq(expected, res) + wvstart('resolve: /test/latest/dir-symlink') + vfs.clear_cache() + res = resolve(path) + wvpasseq(4, len(res)) + wvpasseq(expected, res) @wvtest -def test_resolve_loop(): - with no_lingering_errors(): - with test_tempdir('bup-tvfs-resloop-') as tmpdir: - resolve = vfs.resolve - bup_dir = tmpdir + '/bup' - environ['GIT_DIR'] = bup_dir - environ['BUP_DIR'] = bup_dir - git.repodir = bup_dir - repo = LocalRepo() +def test_local_resolve(): + prep_and_test_repo('local-vfs-resolve', + lambda x: LocalRepo(repo_dir=x), test_resolve) + +@wvtest +def test_remote_resolve(): + prep_and_test_repo('remote-vfs-resolve', + lambda x: RemoteRepo(x), test_resolve) + +def test_resolve_loop(repo, tmpdir): data_path = tmpdir + '/src' os.mkdir(data_path) symlink('loop', data_path + '/loop') @@ -309,10 +316,20 @@ def test_resolve_loop(): save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) try: wvpasseq('this call should never return', - resolve(repo, '/test/%s/loop' % save_name)) + repo.resolve('/test/%s/loop' % save_name)) except vfs.IOError as res_ex: wvpasseq(ELOOP, res_ex.errno) wvpasseq(['', 'test', save_name, 'loop'], [name for name, item in res_ex.terminus]) +@wvtest +def test_local_resolve_loop(): + prep_and_test_repo('local-vfs-resolve-loop', + lambda x: LocalRepo(x), test_resolve_loop) + +@wvtest +def test_remote_resolve_loop(): + prep_and_test_repo('remote-vfs-resolve-loop', + lambda x: RemoteRepo(x), test_resolve_loop) + # FIXME: add tests for the want_meta=False cases. diff --git a/lib/bup/t/tvfs.py b/lib/bup/t/tvfs.py index d0470c0..dcd862b 100644 --- a/lib/bup/t/tvfs.py +++ b/lib/bup/t/tvfs.py @@ -383,4 +383,12 @@ def test_duplicate_save_dates(): 'latest'), tuple(sorted(x[0] for x in vfs.contents(repo, revlist)))) -# FIXME: add tests for the want_meta=False cases. +@wvtest +def test_item_read_write(): + with no_lingering_errors(): + x = vfs.Root(meta=13) + stream = BytesIO() + vfs.write_item(stream, x) + print('stream:', repr(stream.getvalue()), stream.tell(), file=sys.stderr) + stream.seek(0) + wvpasseq(x, vfs.read_item(stream)) diff --git a/lib/bup/vfs.py b/lib/bup/vfs.py index e3ea16f..3ab0e04 100644 --- a/lib/bup/vfs.py +++ b/lib/bup/vfs.py @@ -55,18 +55,44 @@ from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISLNK, S_ISREG from time import localtime, strftime import exceptions, re, sys -from bup import client, git, metadata +from bup import git, metadata, vint from bup.compat import range from bup.git import BUP_CHUNKED, cp, get_commit_items, parse_commit, tree_decode from bup.helpers import debug2, last from bup.metadata import Metadata +from bup.vint import read_bvec, write_bvec +from bup.vint import read_vint, write_vint +from bup.vint import read_vuint, write_vuint +# We currently assume that it's always appropriate to just forward IOErrors +# to a remote client. class IOError(exceptions.IOError): def __init__(self, errno, message, terminus=None): exceptions.IOError.__init__(self, errno, message) self.terminus = terminus +def write_ioerror(port, ex): + assert isinstance(ex, IOError) + write_vuint(port, + (1 if ex.errno is not None else 0) + | (2 if ex.message is not None else 0) + | (4 if ex.terminus is not None else 0)) + if ex.errno is not None: + write_vint(port, ex.errno) + if ex.message is not None: + write_bvec(port, ex.message.encode('utf-8')) + if ex.terminus is not None: + write_resolution(port, ex.terminus) + +def read_ioerror(port): + mask = read_vuint(port) + no = read_vint(port) if 1 & mask else None + msg = read_bvec(port).decode('utf-8') if 2 & mask else None + term = read_resolution(port) if 4 & mask else None + return IOError(errno=no, message=msg, terminus=term) + + default_file_mode = S_IFREG | 0o644 default_dir_mode = S_IFDIR | 0o755 default_symlink_mode = S_IFLNK | 0o755 @@ -244,6 +270,93 @@ Commit = namedtuple('Commit', ('meta', 'oid', 'coid')) item_types = frozenset((Item, Chunky, Root, Tags, RevList, Commit)) real_tree_types = frozenset((Item, Commit)) +def write_item(port, item): + kind = type(item) + name = bytes(kind.__name__) + meta = item.meta + has_meta = 1 if isinstance(meta, Metadata) else 0 + if kind in (Item, Chunky, RevList): + assert len(item.oid) == 20 + if has_meta: + vint.send(port, 'sVs', name, has_meta, item.oid) + Metadata.write(meta, port, include_path=False) + else: + vint.send(port, 'sVsV', name, has_meta, item.oid, item.meta) + elif kind in (Root, Tags): + if has_meta: + vint.send(port, 'sV', name, has_meta) + Metadata.write(meta, port, include_path=False) + else: + vint.send(port, 'sVV', name, has_meta, item.meta) + elif kind == Commit: + assert len(item.oid) == 20 + assert len(item.coid) == 20 + if has_meta: + vint.send(port, 'sVss', name, has_meta, item.oid, item.coid) + Metadata.write(meta, port, include_path=False) + else: + vint.send(port, 'sVssV', name, has_meta, item.oid, item.coid, + item.meta) + elif kind == FakeLink: + if has_meta: + vint.send(port, 'sVs', name, has_meta, item.target) + Metadata.write(meta, port, include_path=False) + else: + vint.send(port, 'sVsV', name, has_meta, item.target, item.meta) + else: + assert False + +def read_item(port): + def read_m(port, has_meta): + if has_meta: + m = Metadata.read(port) + return m + return read_vuint(port) + kind, has_meta = vint.recv(port, 'sV') + if kind == b'Item': + oid, meta = read_bvec(port), read_m(port, has_meta) + return Item(oid=oid, meta=meta) + if kind == b'Chunky': + oid, meta = read_bvec(port), read_m(port, has_meta) + return Chunky(oid=oid, meta=meta) + if kind == b'RevList': + oid, meta = read_bvec(port), read_m(port, has_meta) + return RevList(oid=oid, meta=meta) + if kind == b'Root': + return Root(meta=read_m(port, has_meta)) + if kind == b'Tags': + return Tags(meta=read_m(port, has_meta)) + if kind == b'Commit': + oid, coid = vint.recv(port, 'ss') + meta = read_m(port, has_meta) + return Commit(oid=oid, coid=coid, meta=meta) + if kind == b'FakeLink': + target, meta = read_bvec(port), read_m(port, has_meta) + return FakeLink(target=target, meta=meta) + assert False + +def write_resolution(port, resolution): + write_vuint(port, len(resolution)) + for name, item in resolution: + write_bvec(port, name) + if item: + port.write(b'\1') + write_item(port, item) + else: + port.write(b'\0') + +def read_resolution(port): + n = read_vuint(port) + result = [] + for i in range(n): + name = read_bvec(port) + have_item = ord(port.read(1)) + assert have_item in (0, 1) + item = read_item(port) if have_item else None + result.append((name, item)) + return tuple(result) + + _root = Root(meta=default_dir_mode) _tags = Tags(meta=default_dir_mode) @@ -940,6 +1053,10 @@ def resolve(repo, path, parent=None, want_meta=True, follow=True): needed, make a copy via item.meta.copy() and modify that instead. """ + if repo.is_remote(): + # Redirect to the more efficient remote version + return repo.resolve(path, parent=parent, want_meta=want_meta, + follow=follow) result = _resolve_path(repo, path, parent=parent, want_meta=want_meta, follow=follow) _, leaf_item = result[-1] diff --git a/lib/bup/vint.py b/lib/bup/vint.py index 70c2dce..cd729ce 100644 --- a/lib/bup/vint.py +++ b/lib/bup/vint.py @@ -110,11 +110,9 @@ def read_bvec(port): def skip_bvec(port): port.read(read_vuint(port)) - -def pack(types, *args): +def send(port, types, *args): if len(types) != len(args): raise Exception('number of arguments does not match format string') - port = BytesIO() for (type, value) in zip(types, args): if type == 'V': write_vuint(port, value) @@ -124,12 +122,9 @@ def pack(types, *args): write_bvec(port, value) else: raise Exception('unknown xpack format string item "' + type + '"') - return port.getvalue() - -def unpack(types, data): +def recv(port, types): result = [] - port = BytesIO(data) for type in types: if type == 'V': result.append(read_vuint(port)) @@ -140,3 +135,12 @@ def unpack(types, data): else: raise Exception('unknown xunpack format string item "' + type + '"') return result + +def pack(types, *args): + port = BytesIO() + send(port, types, *args) + return port.getvalue() + +def unpack(types, data): + port = BytesIO(data) + return recv(port, types) -- 2.39.2