from errno import ELOOP, ENOTDIR
from io import BytesIO
from os import environ, symlink
-from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG
+from random import Random, randint
+from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG
from sys import stderr
from time import localtime, strftime
from wvtest import *
+from bup._helpers import write_random
from bup import git, metadata, vfs
from bup.git import BUP_CHUNKED
from bup.helpers import exc, exo, shstr
print(shstr(cmd), file=stderr)
return exc(cmd, **kwargs)
+@wvtest
+def test_default_modes():
+ wvpasseq(S_IFREG | 0o644, vfs.default_file_mode)
+ wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode)
+ wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode)
+
@wvtest
def test_cache_behavior():
orig_max = vfs._cache_max_items
wvpasseq({}, vfs._cache)
wvpasseq([], vfs._cache_keys)
wvfail(vfs._cache_keys)
- wvexcept(AssertionError, vfs.cache_notice, 'x', 1)
- key_0 = b'\0' * 20
- key_1 = b'\1' * 20
- key_2 = b'\2' * 20
+ wvexcept(Exception, vfs.cache_notice, 'x', 1)
+ key_0 = 'itm:' + b'\0' * 20
+ key_1 = 'itm:' + b'\1' * 20
+ key_2 = 'itm:' + b'\2' * 20
vfs.cache_notice(key_0, 'something')
wvpasseq({key_0 : 'something'}, vfs._cache)
wvpasseq([key_0], vfs._cache_keys)
try:
maybe_meta = lambda : Metadata.read(bupm) if bupm else None
m = maybe_meta()
- if m:
+ if m and m.size is None:
m.size = 0
yield TreeDictValue(name='.', oid=oid, meta=m)
tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
file_path, file_size,
link_path, link_target):
_, file_item = vfs.resolve(repo, file_path)[-1]
- _, link_item = vfs.lresolve(repo, link_path)[-1]
+ _, link_item = vfs.resolve(repo, link_path, follow=False)[-1]
wvpass(isinstance(file_item.meta, Metadata))
wvpass(isinstance(link_item.meta, Metadata))
# Note: normally, modifying item.meta values is forbidden
with no_lingering_errors():
with test_tempdir('bup-tvfs-') as tmpdir:
resolve = vfs.resolve
- lresolve = vfs.lresolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
oid=tip_tree_oid,
coid=tip_oid)
+ expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode,
+ target=save_time_str)
expected_test_tag_item = expected_latest_item
wvstart('resolve: /')
# latest has metadata here due to caching
wvpasseq(frozenset([('.', test_revlist_w_meta),
(save_time_str, expected_latest_item_w_meta),
- ('latest', expected_latest_item_w_meta)]),
+ ('latest', expected_latest_link)]),
test_content)
wvstart('resolve: /test/latest')
coid=tip_oid)
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta))
+ (save_time_str, expected_latest_item_w_meta))
wvpasseq(expected, res)
ignore, latest_item = res[2]
latest_content = frozenset(vfs.contents(repo, latest_item))
oid=tip_tree['file'].oid)
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('file', expected_file_item_w_meta))
wvpasseq(expected, res)
wvpasseq(4, len(res))
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('not-there', None))
wvpasseq(expected, res)
- wvstart('lresolve: /test/latest/bad-symlink')
+ wvstart('resolve nofollow: /test/latest/bad-symlink')
vfs.clear_cache()
- res = lresolve(repo, '/test/latest/bad-symlink')
+ res = resolve(repo, '/test/latest/bad-symlink', follow=False)
wvpasseq(4, len(res))
bad_symlink_value = tip_tree['bad-symlink']
expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta,
oid=bad_symlink_value.oid)
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('bad-symlink', expected_bad_symlink_item_w_meta))
wvpasseq(expected, res)
wvpasseq(4, len(res))
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('file', expected_file_item_w_meta))
wvpasseq(expected, res)
- wvstart('lresolve: /test/latest/file-symlink')
+ wvstart('resolve nofollow: /test/latest/file-symlink')
vfs.clear_cache()
- res = lresolve(repo, '/test/latest/file-symlink')
+ res = resolve(repo, '/test/latest/file-symlink', follow=False)
wvpasseq(4, len(res))
file_symlink_value = tip_tree['file-symlink']
expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta,
oid=file_symlink_value.oid)
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('file-symlink', expected_file_symlink_item_w_meta))
wvpasseq(expected, res)
resolve(repo, path)
except vfs.IOError as res_ex:
wvpasseq(ENOTDIR, res_ex.errno)
- wvpasseq(['', 'test', 'latest', 'file'],
+ wvpasseq(['', 'test', save_time_str, 'file'],
[name for name, item in res_ex.terminus])
for path in ('/test/latest/file-symlink/',
'/test/latest/file-symlink/../',
'/test/latest/file-symlink/../.',
'/test/latest/file-symlink/../..'):
- wvstart('lresolve: ' + path)
+ wvstart('resolve nofollow: ' + path)
vfs.clear_cache()
try:
- lresolve(repo, path)
+ resolve(repo, path, follow=False)
except vfs.IOError as res_ex:
wvpasseq(ENOTDIR, res_ex.errno)
- wvpasseq(['', 'test', 'latest', 'file'],
+ wvpasseq(['', 'test', save_time_str, 'file'],
[name for name, item in res_ex.terminus])
wvstart('resolve: non-directory parent')
wvpasseq(ENOTDIR, res_ex.errno)
wvpasseq(None, res_ex.terminus)
- wvstart('lresolve: /test/latest/dir-symlink')
+ wvstart('resolve nofollow: /test/latest/dir-symlink')
vfs.clear_cache()
- res = lresolve(repo, '/test/latest/dir-symlink')
+ res = resolve(repo, '/test/latest/dir-symlink', follow=False)
wvpasseq(4, len(res))
dir_symlink_value = tip_tree['dir-symlink']
expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta,
oid=dir_symlink_value.oid)
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('dir-symlink', expected_dir_symlink_item_w_meta))
wvpasseq(expected, res)
meta=tree_dict(repo, dir_value.oid)['.'].meta)
expected = (('', vfs._root),
('test', test_revlist_w_meta),
- ('latest', expected_latest_item_w_meta),
+ (save_time_str, expected_latest_item_w_meta),
('dir', expected_dir_item))
+ def lresolve(*args, **keys):
+ return resolve(*args, **dict(keys, follow=False))
for resname, resolver in (('resolve', resolve),
- ('lresolve', lresolve)):
+ ('resolve nofollow', lresolve)):
for path in ('/test/latest/dir-symlink/',
'/test/latest/dir-symlink/.'):
wvstart(resname + ': ' + path)
wvpasseq(4, len(res))
wvpasseq(expected, res)
+def write_sized_random_content(parent_dir, size, seed):
+ verbose = 0
+ with open('%s/%d' % (parent_dir, size), 'wb') as f:
+ write_random(f.fileno(), size, seed, verbose)
+
+def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
+ for read_size in read_sizes:
+ with open(expected_path, 'rb') as expected:
+ with vfs.fopen(repo, item) as actual:
+ ex_buf = expected.read(read_size)
+ act_buf = actual.read(read_size)
+ while ex_buf and act_buf:
+ wvpassge(read_size, len(ex_buf))
+ wvpassge(read_size, len(act_buf))
+ wvpasseq(len(ex_buf), len(act_buf))
+ wvpass(ex_buf == act_buf)
+ ex_buf = expected.read(read_size)
+ act_buf = actual.read(read_size)
+ wvpasseq('', ex_buf)
+ wvpasseq('', act_buf)
+
+def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
+ def read_act(act_pos):
+ with vfs.fopen(repo, item) as actual:
+ actual.seek(act_pos)
+ wvpasseq(act_pos, actual.tell())
+ act_buf = actual.read(read_size)
+ act_pos += len(act_buf)
+ wvpasseq(act_pos, actual.tell())
+ return act_pos, act_buf
+
+ for read_size in read_sizes:
+ with open(expected_path, 'rb') as expected:
+ ex_buf = expected.read(read_size)
+ act_buf = None
+ act_pos = 0
+ while ex_buf:
+ act_pos, act_buf = read_act(act_pos)
+ wvpassge(read_size, len(ex_buf))
+ wvpassge(read_size, len(act_buf))
+ wvpasseq(len(ex_buf), len(act_buf))
+ wvpass(ex_buf == act_buf)
+ if not act_buf:
+ break
+ ex_buf = expected.read(read_size)
+ else: # hit expected eof first
+ act_pos, act_buf = read_act(act_pos)
+ wvpasseq('', ex_buf)
+ wvpasseq('', act_buf)
+
+@wvtest
+def test_read_and_seek():
+ # Write a set of randomly sized files containing random data whose
+ # names are their sizes, and then verify that what we get back
+ # from the vfs when seeking and reading with various block sizes
+ # matches the original content.
+ with no_lingering_errors():
+ with test_tempdir('bup-tvfs-read-') as tmpdir:
+ resolve = vfs.resolve
+ bup_dir = tmpdir + '/bup'
+ environ['GIT_DIR'] = bup_dir
+ environ['BUP_DIR'] = bup_dir
+ git.repodir = bup_dir
+ repo = LocalRepo()
+ data_path = tmpdir + '/src'
+ os.mkdir(data_path)
+ seed = randint(-(1 << 31), (1 << 31) - 1)
+ rand = Random()
+ rand.seed(seed)
+ print('test_read seed:', seed, file=sys.stderr)
+ max_size = 2 * 1024 * 1024
+ sizes = set((rand.randint(1, max_size) for _ in xrange(5)))
+ sizes.add(1)
+ sizes.add(max_size)
+ for size in sizes:
+ write_sized_random_content(data_path, size, seed)
+ ex((bup_path, 'init'))
+ ex((bup_path, 'index', '-v', data_path))
+ ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
+ data_path))
+ read_sizes = set((rand.randint(1, max_size) for _ in xrange(10)))
+ sizes.add(1)
+ sizes.add(max_size)
+ print('test_read src sizes:', sizes, file=sys.stderr)
+ print('test_read read sizes:', read_sizes, file=sys.stderr)
+ for size in sizes:
+ res = resolve(repo, '/test/latest/' + str(size))
+ _, item = res[-1]
+ wvpasseq(size, vfs.item_size(repo, res[-1][1]))
+ validate_vfs_streaming_read(repo, item,
+ '%s/%d' % (data_path, size),
+ read_sizes)
+ validate_vfs_seeking_read(repo, item,
+ '%s/%d' % (data_path, size),
+ read_sizes)
+
@wvtest
def test_resolve_loop():
with no_lingering_errors():
with test_tempdir('bup-tvfs-resloop-') as tmpdir:
resolve = vfs.resolve
- lresolve = vfs.lresolve
bup_dir = tmpdir + '/bup'
environ['GIT_DIR'] = bup_dir
environ['BUP_DIR'] = bup_dir
symlink('loop', data_path + '/loop')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
- ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
+ save_utc = 100000
+ ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip',
data_path))
+ save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
try:
- resolve(repo, '/test/latest/loop')
+ wvpasseq('this call should never return',
+ resolve(repo, '/test/%s/loop' % save_name))
except vfs.IOError as res_ex:
wvpasseq(ELOOP, res_ex.errno)
- wvpasseq(['', 'test', 'latest', 'loop'],
+ wvpasseq(['', 'test', save_name, 'loop'],
[name for name, item in res_ex.terminus])
@wvtest
tmpfile.write(b'canary\n')
ex((bup_path, 'init'))
ex((bup_path, 'index', '-v', data_path))
- ex((bup_path, 'save', '-tvvn', 'test', '--strip',
- data_path))
+ save_utc = 100000
+ save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
+ ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc),
+ '--strip', data_path))
repo = LocalRepo()
tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
tip_oidx = tip_sref.strip().split()[0]
tip_tree = tree_dict(repo, tip_tree_oid)
name, item = vfs.resolve(repo, '/test/latest')[2]
- wvpasseq('latest', name)
+ wvpasseq(save_name, name)
expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
for x in (tip_tree[name]
for name in ('.', 'foo', 'foo.')))
projects / bup.git / blobdiff