X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=lib%2Fbup%2Ft%2Ftvfs.py;h=86a038b6fe1e94f11ce624a0c76eb15c70ccba4c;hb=06da068a793cd12629f8be5d105bb3af281e61dd;hp=d8670a71b774f5d35ff0e8eb4f6f24e26c759f7d;hpb=8c3db8e73da813181213b71eff278fd0122c9b42;p=bup.git diff --git a/lib/bup/t/tvfs.py b/lib/bup/t/tvfs.py index d8670a7..86a038b 100644 --- a/lib/bup/t/tvfs.py +++ b/lib/bup/t/tvfs.py @@ -4,12 +4,14 @@ from collections import namedtuple from errno import ELOOP, ENOTDIR from io import BytesIO from os import environ, symlink -from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG +from random import Random, randint +from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG from sys import stderr from time import localtime, strftime from wvtest import * +from bup._helpers import write_random from bup import git, metadata, vfs from bup.git import BUP_CHUNKED from bup.helpers import exc, exo, shstr @@ -26,6 +28,12 @@ def ex(cmd, **kwargs): print(shstr(cmd), file=stderr) return exc(cmd, **kwargs) +@wvtest +def test_default_modes(): + wvpasseq(S_IFREG | 0o644, vfs.default_file_mode) + wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode) + wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode) + @wvtest def test_cache_behavior(): orig_max = vfs._cache_max_items @@ -35,10 +43,10 @@ def test_cache_behavior(): wvpasseq({}, vfs._cache) wvpasseq([], vfs._cache_keys) wvfail(vfs._cache_keys) - wvexcept(AssertionError, vfs.cache_notice, 'x', 1) - key_0 = b'\0' * 20 - key_1 = b'\1' * 20 - key_2 = b'\2' * 20 + wvexcept(Exception, vfs.cache_notice, 'x', 1) + key_0 = 'itm:' + b'\0' * 20 + key_1 = 'itm:' + b'\1' * 20 + key_2 = 'itm:' + b'\2' * 20 vfs.cache_notice(key_0, 'something') wvpasseq({key_0 : 'something'}, vfs._cache) wvpasseq([key_0], vfs._cache_keys) @@ -81,7 +89,7 @@ def tree_items(repo, oid): try: maybe_meta = lambda : Metadata.read(bupm) if bupm else None m = maybe_meta() - if m: + if m and m.size is None: m.size = 0 yield TreeDictValue(name='.', oid=oid, meta=m) tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True) @@ -109,7 +117,7 @@ def run_augment_item_meta_tests(repo, file_path, file_size, link_path, link_target): _, file_item = vfs.resolve(repo, file_path)[-1] - _, link_item = vfs.lresolve(repo, link_path)[-1] + _, link_item = vfs.resolve(repo, link_path, follow=False)[-1] wvpass(isinstance(file_item.meta, Metadata)) wvpass(isinstance(link_item.meta, Metadata)) # Note: normally, modifying item.meta values is forbidden @@ -245,7 +253,6 @@ def test_resolve(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: resolve = vfs.resolve - lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir @@ -282,6 +289,8 @@ def test_resolve(): expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, oid=tip_tree_oid, coid=tip_oid) + expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode, + target=save_time_str) expected_test_tag_item = expected_latest_item wvstart('resolve: /') @@ -333,7 +342,7 @@ def test_resolve(): # latest has metadata here due to caching wvpasseq(frozenset([('.', test_revlist_w_meta), (save_time_str, expected_latest_item_w_meta), - ('latest', expected_latest_item_w_meta)]), + ('latest', expected_latest_link)]), test_content) wvstart('resolve: /test/latest') @@ -345,7 +354,7 @@ def test_resolve(): coid=tip_oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta)) + (save_time_str, expected_latest_item_w_meta)) wvpasseq(expected, res) ignore, latest_item = res[2] latest_content = frozenset(vfs.contents(repo, latest_item)) @@ -367,7 +376,7 @@ def test_resolve(): oid=tip_tree['file'].oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) @@ -377,20 +386,20 @@ def test_resolve(): wvpasseq(4, len(res)) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('not-there', None)) wvpasseq(expected, res) - wvstart('lresolve: /test/latest/bad-symlink') + wvstart('resolve nofollow: /test/latest/bad-symlink') vfs.clear_cache() - res = lresolve(repo, '/test/latest/bad-symlink') + res = resolve(repo, '/test/latest/bad-symlink', follow=False) wvpasseq(4, len(res)) bad_symlink_value = tip_tree['bad-symlink'] expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta, oid=bad_symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('bad-symlink', expected_bad_symlink_item_w_meta)) wvpasseq(expected, res) @@ -400,20 +409,20 @@ def test_resolve(): wvpasseq(4, len(res)) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) - wvstart('lresolve: /test/latest/file-symlink') + wvstart('resolve nofollow: /test/latest/file-symlink') vfs.clear_cache() - res = lresolve(repo, '/test/latest/file-symlink') + res = resolve(repo, '/test/latest/file-symlink', follow=False) wvpasseq(4, len(res)) file_symlink_value = tip_tree['file-symlink'] expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta, oid=file_symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file-symlink', expected_file_symlink_item_w_meta)) wvpasseq(expected, res) @@ -438,7 +447,7 @@ def test_resolve(): resolve(repo, path) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'file'], + wvpasseq(['', 'test', save_time_str, 'file'], [name for name, item in res_ex.terminus]) for path in ('/test/latest/file-symlink/', @@ -447,13 +456,13 @@ def test_resolve(): '/test/latest/file-symlink/../', '/test/latest/file-symlink/../.', '/test/latest/file-symlink/../..'): - wvstart('lresolve: ' + path) + wvstart('resolve nofollow: ' + path) vfs.clear_cache() try: - lresolve(repo, path) + resolve(repo, path, follow=False) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'file'], + wvpasseq(['', 'test', save_time_str, 'file'], [name for name, item in res_ex.terminus]) wvstart('resolve: non-directory parent') @@ -465,16 +474,16 @@ def test_resolve(): wvpasseq(ENOTDIR, res_ex.errno) wvpasseq(None, res_ex.terminus) - wvstart('lresolve: /test/latest/dir-symlink') + wvstart('resolve nofollow: /test/latest/dir-symlink') vfs.clear_cache() - res = lresolve(repo, '/test/latest/dir-symlink') + res = resolve(repo, '/test/latest/dir-symlink', follow=False) wvpasseq(4, len(res)) dir_symlink_value = tip_tree['dir-symlink'] expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta, oid=dir_symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('dir-symlink', expected_dir_symlink_item_w_meta)) wvpasseq(expected, res) @@ -483,10 +492,12 @@ def test_resolve(): meta=tree_dict(repo, dir_value.oid)['.'].meta) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('dir', expected_dir_item)) + def lresolve(*args, **keys): + return resolve(*args, **dict(keys, follow=False)) for resname, resolver in (('resolve', resolve), - ('lresolve', lresolve)): + ('resolve nofollow', lresolve)): for path in ('/test/latest/dir-symlink/', '/test/latest/dir-symlink/.'): wvstart(resname + ': ' + path) @@ -500,12 +511,107 @@ def test_resolve(): wvpasseq(4, len(res)) wvpasseq(expected, res) +def write_sized_random_content(parent_dir, size, seed): + verbose = 0 + with open('%s/%d' % (parent_dir, size), 'wb') as f: + write_random(f.fileno(), size, seed, verbose) + +def validate_vfs_streaming_read(repo, item, expected_path, read_sizes): + for read_size in read_sizes: + with open(expected_path, 'rb') as expected: + with vfs.fopen(repo, item) as actual: + ex_buf = expected.read(read_size) + act_buf = actual.read(read_size) + while ex_buf and act_buf: + wvpassge(read_size, len(ex_buf)) + wvpassge(read_size, len(act_buf)) + wvpasseq(len(ex_buf), len(act_buf)) + wvpass(ex_buf == act_buf) + ex_buf = expected.read(read_size) + act_buf = actual.read(read_size) + wvpasseq('', ex_buf) + wvpasseq('', act_buf) + +def validate_vfs_seeking_read(repo, item, expected_path, read_sizes): + def read_act(act_pos): + with vfs.fopen(repo, item) as actual: + actual.seek(act_pos) + wvpasseq(act_pos, actual.tell()) + act_buf = actual.read(read_size) + act_pos += len(act_buf) + wvpasseq(act_pos, actual.tell()) + return act_pos, act_buf + + for read_size in read_sizes: + with open(expected_path, 'rb') as expected: + ex_buf = expected.read(read_size) + act_buf = None + act_pos = 0 + while ex_buf: + act_pos, act_buf = read_act(act_pos) + wvpassge(read_size, len(ex_buf)) + wvpassge(read_size, len(act_buf)) + wvpasseq(len(ex_buf), len(act_buf)) + wvpass(ex_buf == act_buf) + if not act_buf: + break + ex_buf = expected.read(read_size) + else: # hit expected eof first + act_pos, act_buf = read_act(act_pos) + wvpasseq('', ex_buf) + wvpasseq('', act_buf) + +@wvtest +def test_read_and_seek(): + # Write a set of randomly sized files containing random data whose + # names are their sizes, and then verify that what we get back + # from the vfs when seeking and reading with various block sizes + # matches the original content. + with no_lingering_errors(): + with test_tempdir('bup-tvfs-read-') as tmpdir: + resolve = vfs.resolve + bup_dir = tmpdir + '/bup' + environ['GIT_DIR'] = bup_dir + environ['BUP_DIR'] = bup_dir + git.repodir = bup_dir + repo = LocalRepo() + data_path = tmpdir + '/src' + os.mkdir(data_path) + seed = randint(-(1 << 31), (1 << 31) - 1) + rand = Random() + rand.seed(seed) + print('test_read seed:', seed, file=sys.stderr) + max_size = 2 * 1024 * 1024 + sizes = set((rand.randint(1, max_size) for _ in xrange(5))) + sizes.add(1) + sizes.add(max_size) + for size in sizes: + write_sized_random_content(data_path, size, seed) + ex((bup_path, 'init')) + ex((bup_path, 'index', '-v', data_path)) + ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', + data_path)) + read_sizes = set((rand.randint(1, max_size) for _ in xrange(10))) + sizes.add(1) + sizes.add(max_size) + print('test_read src sizes:', sizes, file=sys.stderr) + print('test_read read sizes:', read_sizes, file=sys.stderr) + for size in sizes: + res = resolve(repo, '/test/latest/' + str(size)) + _, item = res[-1] + wvpasseq(size, vfs.item_size(repo, res[-1][1])) + validate_vfs_streaming_read(repo, item, + '%s/%d' % (data_path, size), + read_sizes) + validate_vfs_seeking_read(repo, item, + '%s/%d' % (data_path, size), + read_sizes) + @wvtest def test_resolve_loop(): with no_lingering_errors(): with test_tempdir('bup-tvfs-resloop-') as tmpdir: resolve = vfs.resolve - lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir @@ -516,13 +622,16 @@ def test_resolve_loop(): symlink('loop', data_path + '/loop') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', + save_utc = 100000 + ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip', data_path)) + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) try: - resolve(repo, '/test/latest/loop') + wvpasseq('this call should never return', + resolve(repo, '/test/%s/loop' % save_name)) except vfs.IOError as res_ex: wvpasseq(ELOOP, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'loop'], + wvpasseq(['', 'test', save_name, 'loop'], [name for name, item in res_ex.terminus]) @wvtest @@ -540,8 +649,10 @@ def test_contents_with_mismatched_bupm_git_ordering(): tmpfile.write(b'canary\n') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-tvvn', 'test', '--strip', - data_path)) + save_utc = 100000 + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) + ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc), + '--strip', data_path)) repo = LocalRepo() tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0] tip_oidx = tip_sref.strip().split()[0] @@ -551,7 +662,7 @@ def test_contents_with_mismatched_bupm_git_ordering(): tip_tree = tree_dict(repo, tip_tree_oid) name, item = vfs.resolve(repo, '/test/latest')[2] - wvpasseq('latest', name) + wvpasseq(save_name, name) expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta)) for x in (tip_tree[name] for name in ('.', 'foo', 'foo.')))