X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=lib%2Fbup%2Ft%2Ftvfs.py;h=86a038b6fe1e94f11ce624a0c76eb15c70ccba4c;hb=06da068a793cd12629f8be5d105bb3af281e61dd;hp=7dbc6e8f555aa071a51ba29cae142463c8de2cb5;hpb=7884c51f475b5270a6b9b54110144348fe90ad38;p=bup.git diff --git a/lib/bup/t/tvfs.py b/lib/bup/t/tvfs.py index 7dbc6e8..86a038b 100644 --- a/lib/bup/t/tvfs.py +++ b/lib/bup/t/tvfs.py @@ -1,16 +1,18 @@ -from __future__ import print_function +from __future__ import absolute_import, print_function from collections import namedtuple from errno import ELOOP, ENOTDIR from io import BytesIO from os import environ, symlink -from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG +from random import Random, randint +from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG from sys import stderr from time import localtime, strftime from wvtest import * -from bup import git, metadata, vfs2 as vfs +from bup._helpers import write_random +from bup import git, metadata, vfs from bup.git import BUP_CHUNKED from bup.helpers import exc, exo, shstr from bup.metadata import Metadata @@ -26,6 +28,52 @@ def ex(cmd, **kwargs): print(shstr(cmd), file=stderr) return exc(cmd, **kwargs) +@wvtest +def test_default_modes(): + wvpasseq(S_IFREG | 0o644, vfs.default_file_mode) + wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode) + wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode) + +@wvtest +def test_cache_behavior(): + orig_max = vfs._cache_max_items + try: + vfs._cache_max_items = 2 + vfs.clear_cache() + wvpasseq({}, vfs._cache) + wvpasseq([], vfs._cache_keys) + wvfail(vfs._cache_keys) + wvexcept(Exception, vfs.cache_notice, 'x', 1) + key_0 = 'itm:' + b'\0' * 20 + key_1 = 'itm:' + b'\1' * 20 + key_2 = 'itm:' + b'\2' * 20 + vfs.cache_notice(key_0, 'something') + wvpasseq({key_0 : 'something'}, vfs._cache) + wvpasseq([key_0], vfs._cache_keys) + vfs.cache_notice(key_1, 'something else') + wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache) + wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys)) + vfs.cache_notice(key_2, 'and also') + wvpasseq(2, len(vfs._cache)) + wvpass(frozenset(vfs._cache.iteritems()) + < frozenset({key_0 : 'something', + key_1 : 'something else', + key_2 : 'and also'}.iteritems())) + wvpasseq(2, len(vfs._cache_keys)) + wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2])) + vfs.clear_cache() + wvpasseq({}, vfs._cache) + wvpasseq([], vfs._cache_keys) + finally: + vfs._cache_max_items = orig_max + vfs.clear_cache() + +## The clear_cache() calls below are to make sure that the test starts +## from a known state since at the moment the cache entry for a given +## item (like a commit) can change. For example, its meta value might +## be promoted from a mode to a Metadata instance once the tree it +## refers to is traversed. + TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta')) def tree_items(repo, oid): @@ -41,7 +89,7 @@ def tree_items(repo, oid): try: maybe_meta = lambda : Metadata.read(bupm) if bupm else None m = maybe_meta() - if m: + if m and m.size is None: m.size = 0 yield TreeDictValue(name='.', oid=oid, meta=m) tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True) @@ -69,7 +117,7 @@ def run_augment_item_meta_tests(repo, file_path, file_size, link_path, link_target): _, file_item = vfs.resolve(repo, file_path)[-1] - _, link_item = vfs.lresolve(repo, link_path)[-1] + _, link_item = vfs.resolve(repo, link_path, follow=False)[-1] wvpass(isinstance(file_item.meta, Metadata)) wvpass(isinstance(link_item.meta, Metadata)) # Note: normally, modifying item.meta values is forbidden @@ -205,7 +253,6 @@ def test_resolve(): with no_lingering_errors(): with test_tempdir('bup-tvfs-') as tmpdir: resolve = vfs.resolve - lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir @@ -234,15 +281,20 @@ def test_resolve(): tip_oidx))[0].strip() tip_tree_oid = tip_tree_oidx.decode('hex') tip_tree = tree_dict(repo, tip_tree_oid) - test_revlist = vfs.RevList(meta=S_IFDIR | 0o755, oid=tip_oid) test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta, oid=tip_oid) expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755, oid=tip_tree_oid, coid=tip_oid) + expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, + oid=tip_tree_oid, + coid=tip_oid) + expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode, + target=save_time_str) expected_test_tag_item = expected_latest_item wvstart('resolve: /') + vfs.clear_cache() res = resolve(repo, '/') wvpasseq(1, len(res)) wvpasseq((('', vfs._root),), res) @@ -250,7 +302,7 @@ def test_resolve(): root_content = frozenset(vfs.contents(repo, root_item)) wvpasseq(frozenset([('.', root_item), ('.tag', vfs._tags), - ('test', test_revlist)]), + ('test', test_revlist_w_meta)]), root_content) for path in ('//', '/.', '/./', '/..', '/../', '/test/latest/dir/../../..', @@ -264,10 +316,12 @@ def test_resolve(): '/test//.//.//latest/dir/../../..' '/test//./latest/dir/../../..'): wvstart('resolve: ' + path) + vfs.clear_cache() res = resolve(repo, path) wvpasseq((('', vfs._root),), res) wvstart('resolve: /.tag') + vfs.clear_cache() res = resolve(repo, '/.tag') wvpasseq(2, len(res)) wvpasseq((('', vfs._root), ('.tag', vfs._tags)), @@ -279,25 +333,28 @@ def test_resolve(): tag_content) wvstart('resolve: /test') + vfs.clear_cache() res = resolve(repo, '/test') wvpasseq(2, len(res)) - wvpasseq((('', vfs._root), ('test', test_revlist)), res) + wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res) ignore, test_item = res[1] test_content = frozenset(vfs.contents(repo, test_item)) + # latest has metadata here due to caching wvpasseq(frozenset([('.', test_revlist_w_meta), - (save_time_str, expected_latest_item), - ('latest', expected_latest_item)]), + (save_time_str, expected_latest_item_w_meta), + ('latest', expected_latest_link)]), test_content) wvstart('resolve: /test/latest') + vfs.clear_cache() res = resolve(repo, '/test/latest') wvpasseq(3, len(res)) expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta, oid=tip_tree_oid, coid=tip_oid) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta)) + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta)) wvpasseq(expected, res) ignore, latest_item = res[2] latest_content = frozenset(vfs.contents(repo, latest_item)) @@ -312,59 +369,65 @@ def test_resolve(): wvpasseq(expected, latest_content) wvstart('resolve: /test/latest/file') + vfs.clear_cache() res = resolve(repo, '/test/latest/file') wvpasseq(4, len(res)) expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta, oid=tip_tree['file'].oid) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) wvstart('resolve: /test/latest/bad-symlink') + vfs.clear_cache() res = resolve(repo, '/test/latest/bad-symlink') wvpasseq(4, len(res)) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('not-there', None)) wvpasseq(expected, res) - wvstart('lresolve: /test/latest/bad-symlink') - res = lresolve(repo, '/test/latest/bad-symlink') + wvstart('resolve nofollow: /test/latest/bad-symlink') + vfs.clear_cache() + res = resolve(repo, '/test/latest/bad-symlink', follow=False) wvpasseq(4, len(res)) bad_symlink_value = tip_tree['bad-symlink'] expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta, oid=bad_symlink_value.oid) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('bad-symlink', expected_bad_symlink_item_w_meta)) wvpasseq(expected, res) wvstart('resolve: /test/latest/file-symlink') + vfs.clear_cache() res = resolve(repo, '/test/latest/file-symlink') wvpasseq(4, len(res)) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file', expected_file_item_w_meta)) wvpasseq(expected, res) - wvstart('lresolve: /test/latest/file-symlink') - res = lresolve(repo, '/test/latest/file-symlink') + wvstart('resolve nofollow: /test/latest/file-symlink') + vfs.clear_cache() + res = resolve(repo, '/test/latest/file-symlink', follow=False) wvpasseq(4, len(res)) file_symlink_value = tip_tree['file-symlink'] expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta, oid=file_symlink_value.oid) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('file-symlink', expected_file_symlink_item_w_meta)) wvpasseq(expected, res) wvstart('resolve: /test/latest/missing') + vfs.clear_cache() res = resolve(repo, '/test/latest/missing') wvpasseq(4, len(res)) name, item = res[-1] @@ -379,11 +442,12 @@ def test_resolve(): '/test/latest/file/../..', '/test/latest/file/foo'): wvstart('resolve: ' + path) + vfs.clear_cache() try: resolve(repo, path) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'file'], + wvpasseq(['', 'test', save_time_str, 'file'], [name for name, item in res_ex.terminus]) for path in ('/test/latest/file-symlink/', @@ -392,15 +456,17 @@ def test_resolve(): '/test/latest/file-symlink/../', '/test/latest/file-symlink/../.', '/test/latest/file-symlink/../..'): - wvstart('lresolve: ' + path) + wvstart('resolve nofollow: ' + path) + vfs.clear_cache() try: - lresolve(repo, path) + resolve(repo, path, follow=False) except vfs.IOError as res_ex: wvpasseq(ENOTDIR, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'file'], + wvpasseq(['', 'test', save_time_str, 'file'], [name for name, item in res_ex.terminus]) wvstart('resolve: non-directory parent') + vfs.clear_cache() file_res = resolve(repo, '/test/latest/file') try: resolve(repo, 'foo', parent=file_res) @@ -408,15 +474,16 @@ def test_resolve(): wvpasseq(ENOTDIR, res_ex.errno) wvpasseq(None, res_ex.terminus) - wvstart('lresolve: /test/latest/dir-symlink') - res = lresolve(repo, '/test/latest/dir-symlink') + wvstart('resolve nofollow: /test/latest/dir-symlink') + vfs.clear_cache() + res = resolve(repo, '/test/latest/dir-symlink', follow=False) wvpasseq(4, len(res)) dir_symlink_value = tip_tree['dir-symlink'] expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta, oid=dir_symlink_value.oid) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('dir-symlink', expected_dir_symlink_item_w_meta)) wvpasseq(expected, res) @@ -424,28 +491,127 @@ def test_resolve(): expected_dir_item = vfs.Item(oid=dir_value.oid, meta=tree_dict(repo, dir_value.oid)['.'].meta) expected = (('', vfs._root), - ('test', test_revlist), - ('latest', expected_latest_item_w_meta), + ('test', test_revlist_w_meta), + (save_time_str, expected_latest_item_w_meta), ('dir', expected_dir_item)) + def lresolve(*args, **keys): + return resolve(*args, **dict(keys, follow=False)) for resname, resolver in (('resolve', resolve), - ('lresolve', lresolve)): + ('resolve nofollow', lresolve)): for path in ('/test/latest/dir-symlink/', '/test/latest/dir-symlink/.'): wvstart(resname + ': ' + path) + vfs.clear_cache() res = resolver(repo, path) wvpasseq(4, len(res)) wvpasseq(expected, res) wvstart('resolve: /test/latest/dir-symlink') + vfs.clear_cache() res = resolve(repo, path) wvpasseq(4, len(res)) wvpasseq(expected, res) +def write_sized_random_content(parent_dir, size, seed): + verbose = 0 + with open('%s/%d' % (parent_dir, size), 'wb') as f: + write_random(f.fileno(), size, seed, verbose) + +def validate_vfs_streaming_read(repo, item, expected_path, read_sizes): + for read_size in read_sizes: + with open(expected_path, 'rb') as expected: + with vfs.fopen(repo, item) as actual: + ex_buf = expected.read(read_size) + act_buf = actual.read(read_size) + while ex_buf and act_buf: + wvpassge(read_size, len(ex_buf)) + wvpassge(read_size, len(act_buf)) + wvpasseq(len(ex_buf), len(act_buf)) + wvpass(ex_buf == act_buf) + ex_buf = expected.read(read_size) + act_buf = actual.read(read_size) + wvpasseq('', ex_buf) + wvpasseq('', act_buf) + +def validate_vfs_seeking_read(repo, item, expected_path, read_sizes): + def read_act(act_pos): + with vfs.fopen(repo, item) as actual: + actual.seek(act_pos) + wvpasseq(act_pos, actual.tell()) + act_buf = actual.read(read_size) + act_pos += len(act_buf) + wvpasseq(act_pos, actual.tell()) + return act_pos, act_buf + + for read_size in read_sizes: + with open(expected_path, 'rb') as expected: + ex_buf = expected.read(read_size) + act_buf = None + act_pos = 0 + while ex_buf: + act_pos, act_buf = read_act(act_pos) + wvpassge(read_size, len(ex_buf)) + wvpassge(read_size, len(act_buf)) + wvpasseq(len(ex_buf), len(act_buf)) + wvpass(ex_buf == act_buf) + if not act_buf: + break + ex_buf = expected.read(read_size) + else: # hit expected eof first + act_pos, act_buf = read_act(act_pos) + wvpasseq('', ex_buf) + wvpasseq('', act_buf) + +@wvtest +def test_read_and_seek(): + # Write a set of randomly sized files containing random data whose + # names are their sizes, and then verify that what we get back + # from the vfs when seeking and reading with various block sizes + # matches the original content. + with no_lingering_errors(): + with test_tempdir('bup-tvfs-read-') as tmpdir: + resolve = vfs.resolve + bup_dir = tmpdir + '/bup' + environ['GIT_DIR'] = bup_dir + environ['BUP_DIR'] = bup_dir + git.repodir = bup_dir + repo = LocalRepo() + data_path = tmpdir + '/src' + os.mkdir(data_path) + seed = randint(-(1 << 31), (1 << 31) - 1) + rand = Random() + rand.seed(seed) + print('test_read seed:', seed, file=sys.stderr) + max_size = 2 * 1024 * 1024 + sizes = set((rand.randint(1, max_size) for _ in xrange(5))) + sizes.add(1) + sizes.add(max_size) + for size in sizes: + write_sized_random_content(data_path, size, seed) + ex((bup_path, 'init')) + ex((bup_path, 'index', '-v', data_path)) + ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', + data_path)) + read_sizes = set((rand.randint(1, max_size) for _ in xrange(10))) + sizes.add(1) + sizes.add(max_size) + print('test_read src sizes:', sizes, file=sys.stderr) + print('test_read read sizes:', read_sizes, file=sys.stderr) + for size in sizes: + res = resolve(repo, '/test/latest/' + str(size)) + _, item = res[-1] + wvpasseq(size, vfs.item_size(repo, res[-1][1])) + validate_vfs_streaming_read(repo, item, + '%s/%d' % (data_path, size), + read_sizes) + validate_vfs_seeking_read(repo, item, + '%s/%d' % (data_path, size), + read_sizes) + @wvtest def test_resolve_loop(): with no_lingering_errors(): with test_tempdir('bup-tvfs-resloop-') as tmpdir: resolve = vfs.resolve - lresolve = vfs.lresolve bup_dir = tmpdir + '/bup' environ['GIT_DIR'] = bup_dir environ['BUP_DIR'] = bup_dir @@ -456,13 +622,16 @@ def test_resolve_loop(): symlink('loop', data_path + '/loop') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', + save_utc = 100000 + ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip', data_path)) + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) try: - resolve(repo, '/test/latest/loop') + wvpasseq('this call should never return', + resolve(repo, '/test/%s/loop' % save_name)) except vfs.IOError as res_ex: wvpasseq(ELOOP, res_ex.errno) - wvpasseq(['', 'test', 'latest', 'loop'], + wvpasseq(['', 'test', save_name, 'loop'], [name for name, item in res_ex.terminus]) @wvtest @@ -480,8 +649,10 @@ def test_contents_with_mismatched_bupm_git_ordering(): tmpfile.write(b'canary\n') ex((bup_path, 'init')) ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-tvvn', 'test', '--strip', - data_path)) + save_utc = 100000 + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) + ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc), + '--strip', data_path)) repo = LocalRepo() tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0] tip_oidx = tip_sref.strip().split()[0] @@ -491,7 +662,7 @@ def test_contents_with_mismatched_bupm_git_ordering(): tip_tree = tree_dict(repo, tip_tree_oid) name, item = vfs.resolve(repo, '/test/latest')[2] - wvpasseq('latest', name) + wvpasseq(save_name, name) expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta)) for x in (tip_tree[name] for name in ('.', 'foo', 'foo.')))