X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=lib%2Fbup%2Ft%2Ftvfs.py;h=2563333a6481ccb1e213a35124799d8f9a4f1588;hb=f86e3d704af40bf98a557e58f634b1c917019821;hp=a35ad057fb0bd8ad5f4980843c8f7d1014889546;hpb=023917c97cbf9bf6dbfad7120b41850dc5d7c7cd;p=bup.git diff --git a/lib/bup/t/tvfs.py b/lib/bup/t/tvfs.py index a35ad05..2563333 100644 --- a/lib/bup/t/tvfs.py +++ b/lib/bup/t/tvfs.py @@ -1,18 +1,20 @@ from __future__ import absolute_import, print_function +from binascii import unhexlify from collections import namedtuple from errno import ELOOP, ENOTDIR from io import BytesIO -from os import environ, symlink +from os import symlink from random import Random, randint from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG from sys import stderr -from time import localtime, strftime +from time import localtime, strftime, tzset from wvtest import * from bup._helpers import write_random from bup import git, metadata, vfs +from bup.compat import environ, fsencode, items, range from bup.git import BUP_CHUNKED from bup.helpers import exc, shstr from bup.metadata import Metadata @@ -20,9 +22,9 @@ from bup.repo import LocalRepo from bup.test.vfs import tree_dict from buptest import ex, exo, no_lingering_errors, test_tempdir -top_dir = '../../..' -bup_tmp = os.path.realpath('../../../t/tmp') -bup_path = top_dir + '/bup' +top_dir = b'../../..' +bup_tmp = os.path.realpath(b'../../../t/tmp') +bup_path = top_dir + b'/bup' start_dir = os.getcwd() def ex(cmd, **kwargs): @@ -44,22 +46,22 @@ def test_cache_behavior(): wvpasseq({}, vfs._cache) wvpasseq([], vfs._cache_keys) wvfail(vfs._cache_keys) - wvexcept(Exception, vfs.cache_notice, 'x', 1) - key_0 = 'itm:' + b'\0' * 20 - key_1 = 'itm:' + b'\1' * 20 - key_2 = 'itm:' + b'\2' * 20 - vfs.cache_notice(key_0, 'something') - wvpasseq({key_0 : 'something'}, vfs._cache) + wvexcept(Exception, vfs.cache_notice, b'x', 1) + key_0 = b'itm:' + b'\0' * 20 + key_1 = b'itm:' + b'\1' * 20 + key_2 = b'itm:' + b'\2' * 20 + vfs.cache_notice(key_0, b'something') + wvpasseq({key_0 : b'something'}, vfs._cache) wvpasseq([key_0], vfs._cache_keys) - vfs.cache_notice(key_1, 'something else') - wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache) + vfs.cache_notice(key_1, b'something else') + wvpasseq({key_0 : b'something', key_1 : b'something else'}, vfs._cache) wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys)) - vfs.cache_notice(key_2, 'and also') + vfs.cache_notice(key_2, b'and also') wvpasseq(2, len(vfs._cache)) - wvpass(frozenset(vfs._cache.iteritems()) - < frozenset({key_0 : 'something', - key_1 : 'something else', - key_2 : 'and also'}.iteritems())) + wvpass(frozenset(items(vfs._cache)) + < frozenset(items({key_0 : b'something', + key_1 : b'something else', + key_2 : b'and also'}))) wvpasseq(2, len(vfs._cache_keys)) wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2])) vfs.clear_cache() @@ -112,7 +114,7 @@ def run_augment_item_meta_tests(repo, wvpass(item is not file_item) wvpass(isinstance(meta, Metadata)) wvpasseq(vfs.default_file_mode, meta.mode) - wvpasseq((0, 0, 0, 0, 0), + wvpasseq((None, None, 0, 0, 0), (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime)) wvpass(augmented.meta.size is None) wvpasseq(file_size, augmented_w_size.meta.size) @@ -135,75 +137,75 @@ def run_augment_item_meta_tests(repo, def test_item_mode(): with no_lingering_errors(): mode = S_IFDIR | 0o755 - meta = metadata.from_path('.') - oid = '\0' * 20 + meta = metadata.from_path(b'.') + oid = b'\0' * 20 wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode))) wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta))) @wvtest def test_reverse_suffix_duplicates(): suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x)) - wvpasseq(('x',), suffix(('x',))) - wvpasseq(('x', 'y'), suffix(('x', 'y'))) - wvpasseq(('x-1', 'x-0'), suffix(('x',) * 2)) - wvpasseq(['x-%02d' % n for n in reversed(range(11))], - list(suffix(('x',) * 11))) - wvpasseq(('x-1', 'x-0', 'y'), suffix(('x', 'x', 'y'))) - wvpasseq(('x', 'y-1', 'y-0'), suffix(('x', 'y', 'y'))) - wvpasseq(('x', 'y-1', 'y-0', 'z'), suffix(('x', 'y', 'y', 'z'))) + wvpasseq((b'x',), suffix((b'x',))) + wvpasseq((b'x', b'y'), suffix((b'x', b'y'))) + wvpasseq((b'x-1', b'x-0'), suffix((b'x',) * 2)) + wvpasseq([b'x-%02d' % n for n in reversed(range(11))], + list(suffix((b'x',) * 11))) + wvpasseq((b'x-1', b'x-0', b'y'), suffix((b'x', b'x', b'y'))) + wvpasseq((b'x', b'y-1', b'y-0'), suffix((b'x', b'y', b'y'))) + wvpasseq((b'x', b'y-1', b'y-0', b'z'), suffix((b'x', b'y', b'y', b'z'))) @wvtest def test_misc(): with no_lingering_errors(): - with test_tempdir('bup-tvfs-') as tmpdir: - bup_dir = tmpdir + '/bup' - environ['GIT_DIR'] = bup_dir - environ['BUP_DIR'] = bup_dir + with test_tempdir(b'bup-tvfs-') as tmpdir: + bup_dir = tmpdir + b'/bup' + environ[b'GIT_DIR'] = bup_dir + environ[b'BUP_DIR'] = bup_dir git.repodir = bup_dir - data_path = tmpdir + '/src' + data_path = tmpdir + b'/src' os.mkdir(data_path) - with open(data_path + '/file', 'w+') as tmpfile: + with open(data_path + b'/file', 'wb+') as tmpfile: tmpfile.write(b'canary\n') - symlink('file', data_path + '/symlink') - ex((bup_path, 'init')) - ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', - data_path)) + symlink(b'file', data_path + b'/symlink') + ex((bup_path, b'init')) + ex((bup_path, b'index', b'-v', data_path)) + ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test', + b'--strip', data_path)) repo = LocalRepo() wvstart('readlink') - ls_tree = exo(('git', 'ls-tree', 'test', 'symlink')).out + ls_tree = exo((b'git', b'ls-tree', b'test', b'symlink')).out mode, typ, oidx, name = ls_tree.strip().split(None, 3) - assert name == 'symlink' - link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8)) - wvpasseq('file', vfs.readlink(repo, link_item)) + assert name == b'symlink' + link_item = vfs.Item(oid=unhexlify(oidx), meta=int(mode, 8)) + wvpasseq(b'file', vfs.readlink(repo, link_item)) - ls_tree = exo(('git', 'ls-tree', 'test', 'file')).out + ls_tree = exo((b'git', b'ls-tree', b'test', b'file')).out mode, typ, oidx, name = ls_tree.strip().split(None, 3) - assert name == 'file' - file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8)) + assert name == b'file' + file_item = vfs.Item(oid=unhexlify(oidx), meta=int(mode, 8)) wvexcept(Exception, vfs.readlink, repo, file_item) wvstart('item_size') wvpasseq(4, vfs.item_size(repo, link_item)) wvpasseq(7, vfs.item_size(repo, file_item)) - meta = metadata.from_path(__file__) + meta = metadata.from_path(fsencode(__file__)) meta.size = 42 fake_item = file_item._replace(meta=meta) wvpasseq(42, vfs.item_size(repo, fake_item)) - _, fakelink_item = vfs.resolve(repo, '/test/latest', follow=False)[-1] + _, fakelink_item = vfs.resolve(repo, b'/test/latest', follow=False)[-1] wvpasseq(17, vfs.item_size(repo, fakelink_item)) wvstart('augment_item_meta') run_augment_item_meta_tests(repo, - '/test/latest/file', 7, - '/test/latest/symlink', 'file') + b'/test/latest/file', 7, + b'/test/latest/symlink', b'file') wvstart('copy_item') # FIXME: this caused StopIteration #_, file_item = vfs.resolve(repo, '/file')[-1] - _, file_item = vfs.resolve(repo, '/test/latest/file')[-1] + _, file_item = vfs.resolve(repo, b'/test/latest/file')[-1] file_copy = vfs.copy_item(file_item) wvpass(file_copy is not file_item) wvpass(file_copy.meta is not file_item.meta) @@ -215,7 +217,7 @@ def test_misc(): def write_sized_random_content(parent_dir, size, seed): verbose = 0 - with open('%s/%d' % (parent_dir, size), 'wb') as f: + with open(b'%s/%d' % (parent_dir, size), 'wb') as f: write_random(f.fileno(), size, seed, verbose) def validate_vfs_streaming_read(repo, item, expected_path, read_sizes): @@ -231,8 +233,8 @@ def validate_vfs_streaming_read(repo, item, expected_path, read_sizes): wvpass(ex_buf == act_buf) ex_buf = expected.read(read_size) act_buf = actual.read(read_size) - wvpasseq('', ex_buf) - wvpasseq('', act_buf) + wvpasseq(b'', ex_buf) + wvpasseq(b'', act_buf) def validate_vfs_seeking_read(repo, item, expected_path, read_sizes): def read_act(act_pos): @@ -260,8 +262,8 @@ def validate_vfs_seeking_read(repo, item, expected_path, read_sizes): ex_buf = expected.read(read_size) else: # hit expected eof first act_pos, act_buf = read_act(act_pos) - wvpasseq('', ex_buf) - wvpasseq('', act_buf) + wvpasseq(b'', ex_buf) + wvpasseq(b'', act_buf) @wvtest def test_read_and_seek(): @@ -270,120 +272,123 @@ def test_read_and_seek(): # from the vfs when seeking and reading with various block sizes # matches the original content. with no_lingering_errors(): - with test_tempdir('bup-tvfs-read-') as tmpdir: + with test_tempdir(b'bup-tvfs-read-') as tmpdir: resolve = vfs.resolve - bup_dir = tmpdir + '/bup' - environ['GIT_DIR'] = bup_dir - environ['BUP_DIR'] = bup_dir + bup_dir = tmpdir + b'/bup' + environ[b'GIT_DIR'] = bup_dir + environ[b'BUP_DIR'] = bup_dir git.repodir = bup_dir repo = LocalRepo() - data_path = tmpdir + '/src' + data_path = tmpdir + b'/src' os.mkdir(data_path) seed = randint(-(1 << 31), (1 << 31) - 1) rand = Random() rand.seed(seed) print('test_read seed:', seed, file=sys.stderr) max_size = 2 * 1024 * 1024 - sizes = set((rand.randint(1, max_size) for _ in xrange(5))) + sizes = set((rand.randint(1, max_size) for _ in range(5))) sizes.add(1) sizes.add(max_size) for size in sizes: write_sized_random_content(data_path, size, seed) - ex((bup_path, 'init')) - ex((bup_path, 'index', '-v', data_path)) - ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip', - data_path)) - read_sizes = set((rand.randint(1, max_size) for _ in xrange(10))) + ex((bup_path, b'init')) + ex((bup_path, b'index', b'-v', data_path)) + ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test', + b'--strip', data_path)) + read_sizes = set((rand.randint(1, max_size) for _ in range(10))) sizes.add(1) sizes.add(max_size) print('test_read src sizes:', sizes, file=sys.stderr) print('test_read read sizes:', read_sizes, file=sys.stderr) for size in sizes: - res = resolve(repo, '/test/latest/' + str(size)) + res = resolve(repo, b'/test/latest/' + str(size).encode('ascii')) _, item = res[-1] wvpasseq(size, vfs.item_size(repo, res[-1][1])) validate_vfs_streaming_read(repo, item, - '%s/%d' % (data_path, size), + b'%s/%d' % (data_path, size), read_sizes) validate_vfs_seeking_read(repo, item, - '%s/%d' % (data_path, size), + b'%s/%d' % (data_path, size), read_sizes) @wvtest def test_contents_with_mismatched_bupm_git_ordering(): with no_lingering_errors(): - with test_tempdir('bup-tvfs-') as tmpdir: - bup_dir = tmpdir + '/bup' - environ['GIT_DIR'] = bup_dir - environ['BUP_DIR'] = bup_dir + with test_tempdir(b'bup-tvfs-') as tmpdir: + bup_dir = tmpdir + b'/bup' + environ[b'GIT_DIR'] = bup_dir + environ[b'BUP_DIR'] = bup_dir git.repodir = bup_dir - data_path = tmpdir + '/src' + data_path = tmpdir + b'/src' os.mkdir(data_path) - os.mkdir(data_path + '/foo') - with open(data_path + '/foo.', 'w+') as tmpfile: + os.mkdir(data_path + b'/foo') + with open(data_path + b'/foo.', 'wb+') as tmpfile: tmpfile.write(b'canary\n') - ex((bup_path, 'init')) - ex((bup_path, 'index', '-v', data_path)) + ex((bup_path, b'init')) + ex((bup_path, b'index', b'-v', data_path)) save_utc = 100000 - save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) - ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc), - '--strip', data_path)) + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)).encode('ascii') + ex((bup_path, b'save', b'-tvvn', b'test', b'-d', b'%d' % save_utc, + b'--strip', data_path)) repo = LocalRepo() - tip_sref = exo(('git', 'show-ref', 'refs/heads/test')).out + tip_sref = exo((b'git', b'show-ref', b'refs/heads/test')).out tip_oidx = tip_sref.strip().split()[0] - tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1', + tip_tree_oidx = exo((b'git', b'log', b'--pretty=%T', b'-n1', tip_oidx)).out.strip() - tip_tree_oid = tip_tree_oidx.decode('hex') + tip_tree_oid = unhexlify(tip_tree_oidx) tip_tree = tree_dict(repo, tip_tree_oid) - name, item = vfs.resolve(repo, '/test/latest')[2] + name, item = vfs.resolve(repo, b'/test/latest')[2] wvpasseq(save_name, name) expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta)) for x in (tip_tree[name] - for name in ('.', 'foo', 'foo.'))) + for name in (b'.', b'foo', b'foo.'))) contents = tuple(vfs.contents(repo, item)) wvpasseq(expected, frozenset(contents)) # Spot check, in case tree_dict shares too much code with the vfs - name, item = next(((n, i) for n, i in contents if n == 'foo')) + name, item = next(((n, i) for n, i in contents if n == b'foo')) wvpass(S_ISDIR(item.meta)) - name, item = next(((n, i) for n, i in contents if n == 'foo.')) + name, item = next(((n, i) for n, i in contents if n == b'foo.')) wvpass(S_ISREG(item.meta.mode)) @wvtest def test_duplicate_save_dates(): with no_lingering_errors(): - with test_tempdir('bup-tvfs-') as tmpdir: - bup_dir = tmpdir + '/bup' - environ['GIT_DIR'] = bup_dir - environ['BUP_DIR'] = bup_dir - environ['TZ'] = 'UTC' + with test_tempdir(b'bup-tvfs-') as tmpdir: + bup_dir = tmpdir + b'/bup' + environ[b'GIT_DIR'] = bup_dir + environ[b'BUP_DIR'] = bup_dir + environ[b'TZ'] = b'UTC' + tzset() git.repodir = bup_dir - data_path = tmpdir + '/src' + data_path = tmpdir + b'/src' os.mkdir(data_path) - with open(data_path + '/file', 'w+') as tmpfile: + with open(data_path + b'/file', 'wb+') as tmpfile: tmpfile.write(b'canary\n') - ex((bup_path, 'init')) - ex((bup_path, 'index', '-v', data_path)) + ex((b'env',)) + ex((bup_path, b'init')) + ex((bup_path, b'index', b'-v', data_path)) for i in range(11): - ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path)) + ex((bup_path, b'save', b'-d', b'100000', b'-n', b'test', + data_path)) repo = LocalRepo() - res = vfs.resolve(repo, '/test') + res = vfs.resolve(repo, b'/test') wvpasseq(2, len(res)) name, revlist = res[-1] - wvpasseq('test', name) - wvpasseq(('.', - '1970-01-02-034640-00', - '1970-01-02-034640-01', - '1970-01-02-034640-02', - '1970-01-02-034640-03', - '1970-01-02-034640-04', - '1970-01-02-034640-05', - '1970-01-02-034640-06', - '1970-01-02-034640-07', - '1970-01-02-034640-08', - '1970-01-02-034640-09', - '1970-01-02-034640-10', - 'latest'), + wvpasseq(b'test', name) + wvpasseq((b'.', + b'1970-01-02-034640-00', + b'1970-01-02-034640-01', + b'1970-01-02-034640-02', + b'1970-01-02-034640-03', + b'1970-01-02-034640-04', + b'1970-01-02-034640-05', + b'1970-01-02-034640-06', + b'1970-01-02-034640-07', + b'1970-01-02-034640-08', + b'1970-01-02-034640-09', + b'1970-01-02-034640-10', + b'latest'), tuple(sorted(x[0] for x in vfs.contents(repo, revlist)))) @wvtest