X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=lib%2Fbup%2Ft%2Ftvfs.py;h=86a038b6fe1e94f11ce624a0c76eb15c70ccba4c;hb=06da068a793cd12629f8be5d105bb3af281e61dd;hp=1a18b6fd650eb6fea9e40e0a9d6fcf35e49040a2;hpb=dda87026d58e8386d593c0fada01f15f325ed8c3;p=bup.git diff --git a/lib/bup/t/tvfs.py b/lib/bup/t/tvfs.py index 1a18b6f..86a038b 100644 --- a/lib/bup/t/tvfs.py +++ b/lib/bup/t/tvfs.py @@ -1,16 +1,18 @@ -from __future__ import print_function +from __future__ import absolute_import, print_function from collections import namedtuple from errno import ELOOP, ENOTDIR from io import BytesIO from os import environ, symlink -from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG +from random import Random, randint +from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG from sys import stderr from time import localtime, strftime from wvtest import * -from bup import git, metadata, vfs2 as vfs +from bup._helpers import write_random +from bup import git, metadata, vfs from bup.git import BUP_CHUNKED from bup.helpers import exc, exo, shstr from bup.metadata import Metadata @@ -22,16 +24,56 @@ bup_tmp = os.path.realpath('../../../t/tmp') bup_path = top_dir + '/bup' start_dir = os.getcwd() +def ex(cmd, **kwargs): + print(shstr(cmd), file=stderr) + return exc(cmd, **kwargs) + +@wvtest +def test_default_modes(): + wvpasseq(S_IFREG | 0o644, vfs.default_file_mode) + wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode) + wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode) + +@wvtest +def test_cache_behavior(): + orig_max = vfs._cache_max_items + try: + vfs._cache_max_items = 2 + vfs.clear_cache() + wvpasseq({}, vfs._cache) + wvpasseq([], vfs._cache_keys) + wvfail(vfs._cache_keys) + wvexcept(Exception, vfs.cache_notice, 'x', 1) + key_0 = 'itm:' + b'\0' * 20 + key_1 = 'itm:' + b'\1' * 20 + key_2 = 'itm:' + b'\2' * 20 + vfs.cache_notice(key_0, 'something') + wvpasseq({key_0 : 'something'}, vfs._cache) + wvpasseq([key_0], vfs._cache_keys) + vfs.cache_notice(key_1, 'something else') + wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache) + wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys)) + vfs.cache_notice(key_2, 'and also') + wvpasseq(2, len(vfs._cache)) + wvpass(frozenset(vfs._cache.iteritems()) + < frozenset({key_0 : 'something', + key_1 : 'something else', + key_2 : 'and also'}.iteritems())) + wvpasseq(2, len(vfs._cache_keys)) + wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2])) + vfs.clear_cache() + wvpasseq({}, vfs._cache) + wvpasseq([], vfs._cache_keys) + finally: + vfs._cache_max_items = orig_max + vfs.clear_cache() + ## The clear_cache() calls below are to make sure that the test starts ## from a known state since at the moment the cache entry for a given ## item (like a commit) can change. For example, its meta value might ## be promoted from a mode to a Metadata instance once the tree it ## refers to is traversed. -def ex(cmd, **kwargs): - print(shstr(cmd), file=stderr) - return exc(cmd, **kwargs) - TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta')) def tree_items(repo, oid): @@ -47,7 +89,7 @@ def tree_items(repo, oid): try: maybe_meta = lambda : Metadata.read(bupm) if bupm else None m = maybe_meta() - if m: + if m and m.size is None: m.size = 0 yield TreeDictValue(name='.', oid=oid, meta=m) tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True) @@ -75,7 +117,7 @@ def run_augment_item_meta_tests(repo, file_path, file_size, link_path, link_target): _, file_item = vfs.resolve(repo, file_path)[-1] - _, link_item = vfs.lresolve(repo, link_path)[-1] + _, link_item = vfs.resolve(repo, link_path, follow=False)[-1] wvpass(isinstance(file_item.meta, Metadata)) wvpass(isinstance(link_item.meta, Metadata)) # Note: normally, modifying item.meta values is forbidden @@ -211,7 +253,6 @@ def test_resolve(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: resolve = vfs.resolve - lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir @@ -248,6 +289,8 @@ def test_resolve(): expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, oid=tip_tree_oid, coid=tip_oid) + expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode, + target=save_time_str) expected_test_tag_item = expected_latest_item wvstart('resolve: /') @@ -299,7 +342,7 @@ def test_resolve(): # latest has metadata here due to caching wvpasseq(frozenset([('.', test_revlist_w_meta), (save_time_str, expected_latest_item_w_meta), - ('latest', expected_latest_item_w_meta)]), + ('latest', expected_latest_link)]), test_content) wvstart('resolve: /test/latest') @@ -311,7 +354,7 @@ def test_resolve(): coid=tip_oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta)) + (save_time_str, expected_latest_item_w_meta)) wvpasseq(expected, res) ignore, latest_item = res[2] latest_content = frozenset(vfs.contents(repo, latest_item)) @@ -333,7 +376,7 @@ def test_resolve(): oid=tip_tree['file'].oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) @@ -343,20 +386,20 @@ def test_resolve(): wvpasseq(4, len(res)) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('not-there', None)) wvpasseq(expected, res) - wvstart('lresolve: /test/latest/bad-symlink') + wvstart('resolve nofollow: /test/latest/bad-symlink') vfs.clear_cache() - res = lresolve(repo, '/test/latest/bad-symlink') + res = resolve(repo, '/test/latest/bad-symlink', follow=False) wvpasseq(4, len(res)) bad_symlink_value = tip_tree['bad-symlink'] expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta, oid=bad_symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('bad-symlink', expected_bad_symlink_item_w_meta)) wvpasseq(expected, res) @@ -366,20 +409,20 @@ def test_resolve(): wvpasseq(4, len(res)) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) - wvstart('lresolve: /test/latest/file-symlink') + wvstart('resolve nofollow: /test/latest/file-symlink') vfs.clear_cache() - res = lresolve(repo, '/test/latest/file-symlink') + res = resolve(repo, '/test/latest/file-symlink', follow=False) wvpasseq(4, len(res)) file_symlink_value = tip_tree['file-symlink'] expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta, oid=file_symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file-symlink', expected_file_symlink_item_w_meta)) wvpasseq(expected, res) @@ -404,7 +447,7 @@ def test_resolve(): resolve(repo, path) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'file'], + wvpasseq(['', 'test', save_time_str, 'file'], [name for name, item in res_ex.terminus]) for path in ('/test/latest/file-symlink/', @@ -413,13 +456,13 @@ def test_resolve(): '/test/latest/file-symlink/../', '/test/latest/file-symlink/../.', '/test/latest/file-symlink/../..'): - wvstart('lresolve: ' + path) + wvstart('resolve nofollow: ' + path) vfs.clear_cache() try: - lresolve(repo, path) + resolve(repo, path, follow=False) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'file'], + wvpasseq(['', 'test', save_time_str, 'file'], [name for name, item in res_ex.terminus]) wvstart('resolve: non-directory parent') @@ -431,16 +474,16 @@ def test_resolve(): wvpasseq(ENOTDIR, res_ex.errno) wvpasseq(None, res_ex.terminus) - wvstart('lresolve: /test/latest/dir-symlink') + wvstart('resolve nofollow: /test/latest/dir-symlink') vfs.clear_cache() - res = lresolve(repo, '/test/latest/dir-symlink') + res = resolve(repo, '/test/latest/dir-symlink', follow=False) wvpasseq(4, len(res)) dir_symlink_value = tip_tree['dir-symlink'] expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta, oid=dir_symlink_value.oid) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('dir-symlink', expected_dir_symlink_item_w_meta)) wvpasseq(expected, res) @@ -449,10 +492,12 @@ def test_resolve(): meta=tree_dict(repo, dir_value.oid)['.'].meta) expected = (('', vfs._root), ('test', test_revlist_w_meta), - ('latest', expected_latest_item_w_meta), + (save_time_str, expected_latest_item_w_meta), ('dir', expected_dir_item)) + def lresolve(*args, **keys): + return resolve(*args, **dict(keys, follow=False)) for resname, resolver in (('resolve', resolve), - ('lresolve', lresolve)): + ('resolve nofollow', lresolve)): for path in ('/test/latest/dir-symlink/', '/test/latest/dir-symlink/.'): wvstart(resname + ': ' + path) @@ -466,12 +511,107 @@ def test_resolve(): wvpasseq(4, len(res)) wvpasseq(expected, res) +def write_sized_random_content(parent_dir, size, seed): + verbose = 0 + with open('%s/%d' % (parent_dir, size), 'wb') as f: + write_random(f.fileno(), size, seed, verbose) + +def validate_vfs_streaming_read(repo, item, expected_path, read_sizes): + for read_size in read_sizes: + with open(expected_path, 'rb') as expected: + with vfs.fopen(repo, item) as actual: + ex_buf = expected.read(read_size) + act_buf = actual.read(read_size) + while ex_buf and act_buf: + wvpassge(read_size, len(ex_buf)) + wvpassge(read_size, len(act_buf)) + wvpasseq(len(ex_buf), len(act_buf)) + wvpass(ex_buf == act_buf) + ex_buf = expected.read(read_size) + act_buf = actual.read(read_size) + wvpasseq('', ex_buf) + wvpasseq('', act_buf) + +def validate_vfs_seeking_read(repo, item, expected_path, read_sizes): + def read_act(act_pos): + with vfs.fopen(repo, item) as actual: + actual.seek(act_pos) + wvpasseq(act_pos, actual.tell()) + act_buf = actual.read(read_size) + act_pos += len(act_buf) + wvpasseq(act_pos, actual.tell()) + return act_pos, act_buf + + for read_size in read_sizes: + with open(expected_path, 'rb') as expected: + ex_buf = expected.read(read_size) + act_buf = None + act_pos = 0 + while ex_buf: + act_pos, act_buf = read_act(act_pos) + wvpassge(read_size, len(ex_buf)) + wvpassge(read_size, len(act_buf)) + wvpasseq(len(ex_buf), len(act_buf)) + wvpass(ex_buf == act_buf) + if not act_buf: + break + ex_buf = expected.read(read_size) + else: # hit expected eof first + act_pos, act_buf = read_act(act_pos) + wvpasseq('', ex_buf) + wvpasseq('', act_buf) + +@wvtest +def test_read_and_seek(): + # Write a set of randomly sized files containing random data whose + # names are their sizes, and then verify that what we get back + # from the vfs when seeking and reading with various block sizes + # matches the original content. + with no_lingering_errors(): + with test_tempdir('bup-tvfs-read-') as tmpdir: + resolve = vfs.resolve + bup_dir = tmpdir + '/bup' + environ['GIT_DIR'] = bup_dir + environ['BUP_DIR'] = bup_dir + git.repodir = bup_dir + repo = LocalRepo() + data_path = tmpdir + '/src' + os.mkdir(data_path) + seed = randint(-(1 << 31), (1 << 31) - 1) + rand = Random() + rand.seed(seed) + print('test_read seed:', seed, file=sys.stderr) + max_size = 2 * 1024 * 1024 + sizes = set((rand.randint(1, max_size) for _ in xrange(5))) + sizes.add(1) + sizes.add(max_size) + for size in sizes: + write_sized_random_content(data_path, size, seed) + ex((bup_path, 'init')) + ex((bup_path, 'index', '-v', data_path)) + ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', + data_path)) + read_sizes = set((rand.randint(1, max_size) for _ in xrange(10))) + sizes.add(1) + sizes.add(max_size) + print('test_read src sizes:', sizes, file=sys.stderr) + print('test_read read sizes:', read_sizes, file=sys.stderr) + for size in sizes: + res = resolve(repo, '/test/latest/' + str(size)) + _, item = res[-1] + wvpasseq(size, vfs.item_size(repo, res[-1][1])) + validate_vfs_streaming_read(repo, item, + '%s/%d' % (data_path, size), + read_sizes) + validate_vfs_seeking_read(repo, item, + '%s/%d' % (data_path, size), + read_sizes) + @wvtest def test_resolve_loop(): with no_lingering_errors(): with test_tempdir('bup-tvfs-resloop-') as tmpdir: resolve = vfs.resolve - lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir @@ -482,13 +622,16 @@ def test_resolve_loop(): symlink('loop', data_path + '/loop') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', + save_utc = 100000 + ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip', data_path)) + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) try: - resolve(repo, '/test/latest/loop') + wvpasseq('this call should never return', + resolve(repo, '/test/%s/loop' % save_name)) except vfs.IOError as res_ex: wvpasseq(ELOOP, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'loop'], + wvpasseq(['', 'test', save_name, 'loop'], [name for name, item in res_ex.terminus]) @wvtest @@ -506,8 +649,10 @@ def test_contents_with_mismatched_bupm_git_ordering(): tmpfile.write(b'canary\n') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-tvvn', 'test', '--strip', - data_path)) + save_utc = 100000 + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) + ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc), + '--strip', data_path)) repo = LocalRepo() tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0] tip_oidx = tip_sref.strip().split()[0] @@ -517,7 +662,7 @@ def test_contents_with_mismatched_bupm_git_ordering(): tip_tree = tree_dict(repo, tip_tree_oid) name, item = vfs.resolve(repo, '/test/latest')[2] - wvpasseq('latest', name) + wvpasseq(save_name, name) expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta)) for x in (tip_tree[name] for name in ('.', 'foo', 'foo.')))