2 from __future__ import absolute_import, print_function
3 from binascii import unhexlify
6 from random import Random, randint
7 from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG
11 from time import localtime, strftime, tzset
13 from wvpytest import *
15 from bup._helpers import write_random
16 from bup import git, metadata, vfs
17 from bup.compat import environ, fsencode, items, range
18 from bup.helpers import exc, shstr
19 from bup.metadata import Metadata
20 from bup.repo import LocalRepo
21 from buptest import ex, exo
22 from buptest.vfs import tree_dict
24 lib_t_dir = os.path.dirname(fsencode(__file__))
25 top_dir = os.path.join(lib_t_dir, b'../..')
26 bup_path = top_dir + b'/bup'
28 def ex(cmd, **kwargs):
29 print(shstr(cmd), file=stderr)
30 return exc(cmd, **kwargs)
32 def test_default_modes():
33 wvpasseq(S_IFREG | 0o644, vfs.default_file_mode)
34 wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode)
35 wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode)
37 def test_cache_behavior():
38 orig_max = vfs._cache_max_items
40 vfs._cache_max_items = 2
42 wvpasseq({}, vfs._cache)
43 wvpasseq([], vfs._cache_keys)
44 wvfail(vfs._cache_keys)
45 wvexcept(Exception, vfs.cache_notice, b'x', 1)
46 key_0 = b'itm:' + b'\0' * 20
47 key_1 = b'itm:' + b'\1' * 20
48 key_2 = b'itm:' + b'\2' * 20
49 vfs.cache_notice(key_0, b'something')
50 wvpasseq({key_0 : b'something'}, vfs._cache)
51 wvpasseq([key_0], vfs._cache_keys)
52 vfs.cache_notice(key_1, b'something else')
53 wvpasseq({key_0 : b'something', key_1 : b'something else'}, vfs._cache)
54 wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
55 vfs.cache_notice(key_2, b'and also')
56 wvpasseq(2, len(vfs._cache))
57 wvpass(frozenset(items(vfs._cache))
58 < frozenset(items({key_0 : b'something',
59 key_1 : b'something else',
60 key_2 : b'and also'})))
61 wvpasseq(2, len(vfs._cache_keys))
62 wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
64 wvpasseq({}, vfs._cache)
65 wvpasseq([], vfs._cache_keys)
67 vfs._cache_max_items = orig_max
70 ## The clear_cache() calls below are to make sure that the test starts
71 ## from a known state since at the moment the cache entry for a given
72 ## item (like a commit) can change. For example, its meta value might
73 ## be promoted from a mode to a Metadata instance once the tree it
74 ## refers to is traversed.
76 def run_augment_item_meta_tests(repo,
78 link_path, link_target):
79 _, file_item = vfs.resolve(repo, file_path)[-1]
80 _, link_item = vfs.resolve(repo, link_path, follow=False)[-1]
81 wvpass(isinstance(file_item.meta, Metadata))
82 wvpass(isinstance(link_item.meta, Metadata))
83 # Note: normally, modifying item.meta values is forbidden
84 file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
85 link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
87 ## Ensure a fully populated item is left alone
88 augmented = vfs.augment_item_meta(repo, file_item)
89 wvpass(augmented is file_item)
90 wvpass(augmented.meta is file_item.meta)
91 augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
92 wvpass(augmented is file_item)
93 wvpass(augmented.meta is file_item.meta)
95 ## Ensure a missing size is handled poperly
96 file_item.meta.size = None
97 augmented = vfs.augment_item_meta(repo, file_item)
98 wvpass(augmented is file_item)
99 wvpass(augmented.meta is file_item.meta)
100 augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
101 wvpass(augmented is not file_item)
102 wvpasseq(file_size, augmented.meta.size)
104 ## Ensure a meta mode is handled properly
105 mode_item = file_item._replace(meta=vfs.default_file_mode)
106 augmented = vfs.augment_item_meta(repo, mode_item)
107 augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
108 for item in (augmented, augmented_w_size):
110 wvpass(item is not file_item)
111 wvpass(isinstance(meta, Metadata))
112 wvpasseq(vfs.default_file_mode, meta.mode)
113 wvpasseq((None, None, 0, 0, 0),
114 (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
115 wvpass(augmented.meta.size is None)
116 wvpasseq(file_size, augmented_w_size.meta.size)
118 ## Ensure symlinks are handled properly
119 mode_item = link_item._replace(meta=vfs.default_symlink_mode)
120 augmented = vfs.augment_item_meta(repo, mode_item)
121 wvpass(augmented is not mode_item)
122 wvpass(isinstance(augmented.meta, Metadata))
123 wvpasseq(link_target, augmented.meta.symlink_target)
124 wvpasseq(len(link_target), augmented.meta.size)
125 augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
126 wvpass(augmented is not mode_item)
127 wvpass(isinstance(augmented.meta, Metadata))
128 wvpasseq(link_target, augmented.meta.symlink_target)
129 wvpasseq(len(link_target), augmented.meta.size)
132 def test_item_mode():
133 mode = S_IFDIR | 0o755
134 meta = metadata.from_path(b'.')
136 wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
137 wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
139 def test_reverse_suffix_duplicates():
140 suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x))
141 wvpasseq((b'x',), suffix((b'x',)))
142 wvpasseq((b'x', b'y'), suffix((b'x', b'y')))
143 wvpasseq((b'x-1', b'x-0'), suffix((b'x',) * 2))
144 wvpasseq([b'x-%02d' % n for n in reversed(range(11))],
145 list(suffix((b'x',) * 11)))
146 wvpasseq((b'x-1', b'x-0', b'y'), suffix((b'x', b'x', b'y')))
147 wvpasseq((b'x', b'y-1', b'y-0'), suffix((b'x', b'y', b'y')))
148 wvpasseq((b'x', b'y-1', b'y-0', b'z'), suffix((b'x', b'y', b'y', b'z')))
150 def test_misc(tmpdir):
151 bup_dir = tmpdir + b'/bup'
152 environ[b'GIT_DIR'] = bup_dir
153 environ[b'BUP_DIR'] = bup_dir
154 git.repodir = bup_dir
155 data_path = tmpdir + b'/src'
157 with open(data_path + b'/file', 'wb+') as tmpfile:
158 tmpfile.write(b'canary\n')
159 symlink(b'file', data_path + b'/symlink')
160 ex((bup_path, b'init'))
161 ex((bup_path, b'index', b'-v', data_path))
162 ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test',
163 b'--strip', data_path))
166 ls_tree = exo((b'git', b'ls-tree', b'test', b'symlink')).out
167 mode, typ, oidx, name = ls_tree.strip().split(None, 3)
168 assert name == b'symlink'
169 link_item = vfs.Item(oid=unhexlify(oidx), meta=int(mode, 8))
170 wvpasseq(b'file', vfs.readlink(repo, link_item))
172 ls_tree = exo((b'git', b'ls-tree', b'test', b'file')).out
173 mode, typ, oidx, name = ls_tree.strip().split(None, 3)
174 assert name == b'file'
175 file_item = vfs.Item(oid=unhexlify(oidx), meta=int(mode, 8))
176 wvexcept(Exception, vfs.readlink, repo, file_item)
178 wvpasseq(4, vfs.item_size(repo, link_item))
179 wvpasseq(7, vfs.item_size(repo, file_item))
180 meta = metadata.from_path(fsencode(__file__))
182 fake_item = file_item._replace(meta=meta)
183 wvpasseq(42, vfs.item_size(repo, fake_item))
185 _, fakelink_item = vfs.resolve(repo, b'/test/latest', follow=False)[-1]
186 wvpasseq(17, vfs.item_size(repo, fakelink_item))
188 run_augment_item_meta_tests(repo,
189 b'/test/latest/file', 7,
190 b'/test/latest/symlink', b'file')
192 # FIXME: this caused StopIteration
193 #_, file_item = vfs.resolve(repo, '/file')[-1]
194 _, file_item = vfs.resolve(repo, b'/test/latest/file')[-1]
195 file_copy = vfs.copy_item(file_item)
196 wvpass(file_copy is not file_item)
197 wvpass(file_copy.meta is not file_item.meta)
198 wvpass(isinstance(file_copy, tuple))
199 wvpass(file_item.meta.user)
200 wvpass(file_copy.meta.user)
201 file_copy.meta.user = None
202 wvpass(file_item.meta.user)
204 def write_sized_random_content(parent_dir, size, seed):
206 with open(b'%s/%d' % (parent_dir, size), 'wb') as f:
207 write_random(f.fileno(), size, seed, verbose)
209 def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
210 for read_size in read_sizes:
211 with open(expected_path, 'rb') as expected:
212 with vfs.fopen(repo, item) as actual:
213 ex_buf = expected.read(read_size)
214 act_buf = actual.read(read_size)
215 while ex_buf and act_buf:
216 wvpassge(read_size, len(ex_buf))
217 wvpassge(read_size, len(act_buf))
218 wvpasseq(len(ex_buf), len(act_buf))
219 wvpass(ex_buf == act_buf)
220 ex_buf = expected.read(read_size)
221 act_buf = actual.read(read_size)
222 wvpasseq(b'', ex_buf)
223 wvpasseq(b'', act_buf)
225 def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
226 def read_act(act_pos):
227 with vfs.fopen(repo, item) as actual:
229 wvpasseq(act_pos, actual.tell())
230 act_buf = actual.read(read_size)
231 act_pos += len(act_buf)
232 wvpasseq(act_pos, actual.tell())
233 return act_pos, act_buf
235 for read_size in read_sizes:
236 with open(expected_path, 'rb') as expected:
237 ex_buf = expected.read(read_size)
241 act_pos, act_buf = read_act(act_pos)
242 wvpassge(read_size, len(ex_buf))
243 wvpassge(read_size, len(act_buf))
244 wvpasseq(len(ex_buf), len(act_buf))
245 wvpass(ex_buf == act_buf)
248 ex_buf = expected.read(read_size)
249 else: # hit expected eof first
250 act_pos, act_buf = read_act(act_pos)
251 wvpasseq(b'', ex_buf)
252 wvpasseq(b'', act_buf)
254 def test_read_and_seek(tmpdir):
255 # Write a set of randomly sized files containing random data whose
256 # names are their sizes, and then verify that what we get back
257 # from the vfs when seeking and reading with various block sizes
258 # matches the original content.
259 resolve = vfs.resolve
260 bup_dir = tmpdir + b'/bup'
261 environ[b'GIT_DIR'] = bup_dir
262 environ[b'BUP_DIR'] = bup_dir
263 git.repodir = bup_dir
265 data_path = tmpdir + b'/src'
267 seed = randint(-(1 << 31), (1 << 31) - 1)
270 print('test_read seed:', seed, file=sys.stderr)
271 max_size = 2 * 1024 * 1024
272 sizes = set((rand.randint(1, max_size) for _ in range(5)))
276 write_sized_random_content(data_path, size, seed)
277 ex((bup_path, b'init'))
278 ex((bup_path, b'index', b'-v', data_path))
279 ex((bup_path, b'save', b'-d', b'100000', b'-tvvn', b'test',
280 b'--strip', data_path))
281 read_sizes = set((rand.randint(1, max_size) for _ in range(10)))
284 print('test_read src sizes:', sizes, file=sys.stderr)
285 print('test_read read sizes:', read_sizes, file=sys.stderr)
287 res = resolve(repo, b'/test/latest/' + str(size).encode('ascii'))
289 wvpasseq(size, vfs.item_size(repo, res[-1][1]))
290 validate_vfs_streaming_read(repo, item,
291 b'%s/%d' % (data_path, size),
293 validate_vfs_seeking_read(repo, item,
294 b'%s/%d' % (data_path, size),
297 def test_contents_with_mismatched_bupm_git_ordering(tmpdir):
298 bup_dir = tmpdir + b'/bup'
299 environ[b'GIT_DIR'] = bup_dir
300 environ[b'BUP_DIR'] = bup_dir
301 git.repodir = bup_dir
302 data_path = tmpdir + b'/src'
304 os.mkdir(data_path + b'/foo')
305 with open(data_path + b'/foo.', 'wb+') as tmpfile:
306 tmpfile.write(b'canary\n')
307 ex((bup_path, b'init'))
308 ex((bup_path, b'index', b'-v', data_path))
310 save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)).encode('ascii')
311 ex((bup_path, b'save', b'-tvvn', b'test', b'-d', b'%d' % save_utc,
312 b'--strip', data_path))
314 tip_sref = exo((b'git', b'show-ref', b'refs/heads/test')).out
315 tip_oidx = tip_sref.strip().split()[0]
316 tip_tree_oidx = exo((b'git', b'log', b'--pretty=%T', b'-n1',
317 tip_oidx)).out.strip()
318 tip_tree_oid = unhexlify(tip_tree_oidx)
319 tip_tree = tree_dict(repo, tip_tree_oid)
321 name, item = vfs.resolve(repo, b'/test/latest')[2]
322 wvpasseq(save_name, name)
323 expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
324 for x in (tip_tree[name]
325 for name in (b'.', b'foo', b'foo.')))
326 contents = tuple(vfs.contents(repo, item))
327 wvpasseq(expected, frozenset(contents))
328 # Spot check, in case tree_dict shares too much code with the vfs
329 name, item = next(((n, i) for n, i in contents if n == b'foo'))
330 wvpass(S_ISDIR(item.meta))
331 name, item = next(((n, i) for n, i in contents if n == b'foo.'))
332 wvpass(S_ISREG(item.meta.mode))
334 def test_duplicate_save_dates(tmpdir):
335 bup_dir = tmpdir + b'/bup'
336 environ[b'GIT_DIR'] = bup_dir
337 environ[b'BUP_DIR'] = bup_dir
338 environ[b'TZ'] = b'UTC'
340 git.repodir = bup_dir
341 data_path = tmpdir + b'/src'
343 with open(data_path + b'/file', 'wb+') as tmpfile:
344 tmpfile.write(b'canary\n')
346 ex((bup_path, b'init'))
347 ex((bup_path, b'index', b'-v', data_path))
349 ex((bup_path, b'save', b'-d', b'100000', b'-n', b'test',
352 res = vfs.resolve(repo, b'/test')
353 wvpasseq(2, len(res))
354 name, revlist = res[-1]
355 wvpasseq(b'test', name)
357 b'1970-01-02-034640-00',
358 b'1970-01-02-034640-01',
359 b'1970-01-02-034640-02',
360 b'1970-01-02-034640-03',
361 b'1970-01-02-034640-04',
362 b'1970-01-02-034640-05',
363 b'1970-01-02-034640-06',
364 b'1970-01-02-034640-07',
365 b'1970-01-02-034640-08',
366 b'1970-01-02-034640-09',
367 b'1970-01-02-034640-10',
369 tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
371 def test_item_read_write():
372 x = vfs.Root(meta=13)
374 vfs.write_item(stream, x)
375 print('stream:', repr(stream.getvalue()), stream.tell(), file=sys.stderr)
377 wvpasseq(x, vfs.read_item(stream))