2 from __future__ import absolute_import, print_function
3 from collections import namedtuple
4 from errno import ELOOP, ENOTDIR
6 from os import environ, symlink
7 from random import Random, randint
8 from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG
10 from time import localtime, strftime
14 from bup._helpers import write_random
15 from bup import git, metadata, vfs
16 from bup.git import BUP_CHUNKED
17 from bup.helpers import exc, shstr
18 from bup.metadata import Metadata
19 from bup.repo import LocalRepo
20 from bup.test.vfs import tree_dict
21 from buptest import ex, exo, no_lingering_errors, test_tempdir
24 bup_tmp = os.path.realpath('../../../t/tmp')
25 bup_path = top_dir + '/bup'
26 start_dir = os.getcwd()
28 def ex(cmd, **kwargs):
29 print(shstr(cmd), file=stderr)
30 return exc(cmd, **kwargs)
33 def test_default_modes():
34 wvpasseq(S_IFREG | 0o644, vfs.default_file_mode)
35 wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode)
36 wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode)
39 def test_cache_behavior():
40 orig_max = vfs._cache_max_items
42 vfs._cache_max_items = 2
44 wvpasseq({}, vfs._cache)
45 wvpasseq([], vfs._cache_keys)
46 wvfail(vfs._cache_keys)
47 wvexcept(Exception, vfs.cache_notice, 'x', 1)
48 key_0 = 'itm:' + b'\0' * 20
49 key_1 = 'itm:' + b'\1' * 20
50 key_2 = 'itm:' + b'\2' * 20
51 vfs.cache_notice(key_0, 'something')
52 wvpasseq({key_0 : 'something'}, vfs._cache)
53 wvpasseq([key_0], vfs._cache_keys)
54 vfs.cache_notice(key_1, 'something else')
55 wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache)
56 wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
57 vfs.cache_notice(key_2, 'and also')
58 wvpasseq(2, len(vfs._cache))
59 wvpass(frozenset(vfs._cache.iteritems())
60 < frozenset({key_0 : 'something',
61 key_1 : 'something else',
62 key_2 : 'and also'}.iteritems()))
63 wvpasseq(2, len(vfs._cache_keys))
64 wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
66 wvpasseq({}, vfs._cache)
67 wvpasseq([], vfs._cache_keys)
69 vfs._cache_max_items = orig_max
72 ## The clear_cache() calls below are to make sure that the test starts
73 ## from a known state since at the moment the cache entry for a given
74 ## item (like a commit) can change. For example, its meta value might
75 ## be promoted from a mode to a Metadata instance once the tree it
76 ## refers to is traversed.
78 def run_augment_item_meta_tests(repo,
80 link_path, link_target):
81 _, file_item = vfs.resolve(repo, file_path)[-1]
82 _, link_item = vfs.resolve(repo, link_path, follow=False)[-1]
83 wvpass(isinstance(file_item.meta, Metadata))
84 wvpass(isinstance(link_item.meta, Metadata))
85 # Note: normally, modifying item.meta values is forbidden
86 file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
87 link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
89 ## Ensure a fully populated item is left alone
90 augmented = vfs.augment_item_meta(repo, file_item)
91 wvpass(augmented is file_item)
92 wvpass(augmented.meta is file_item.meta)
93 augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
94 wvpass(augmented is file_item)
95 wvpass(augmented.meta is file_item.meta)
97 ## Ensure a missing size is handled poperly
98 file_item.meta.size = None
99 augmented = vfs.augment_item_meta(repo, file_item)
100 wvpass(augmented is file_item)
101 wvpass(augmented.meta is file_item.meta)
102 augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
103 wvpass(augmented is not file_item)
104 wvpasseq(file_size, augmented.meta.size)
106 ## Ensure a meta mode is handled properly
107 mode_item = file_item._replace(meta=vfs.default_file_mode)
108 augmented = vfs.augment_item_meta(repo, mode_item)
109 augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
110 for item in (augmented, augmented_w_size):
112 wvpass(item is not file_item)
113 wvpass(isinstance(meta, Metadata))
114 wvpasseq(vfs.default_file_mode, meta.mode)
115 wvpasseq((0, 0, 0, 0, 0),
116 (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
117 wvpass(augmented.meta.size is None)
118 wvpasseq(file_size, augmented_w_size.meta.size)
120 ## Ensure symlinks are handled properly
121 mode_item = link_item._replace(meta=vfs.default_symlink_mode)
122 augmented = vfs.augment_item_meta(repo, mode_item)
123 wvpass(augmented is not mode_item)
124 wvpass(isinstance(augmented.meta, Metadata))
125 wvpasseq(link_target, augmented.meta.symlink_target)
126 wvpasseq(len(link_target), augmented.meta.size)
127 augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
128 wvpass(augmented is not mode_item)
129 wvpass(isinstance(augmented.meta, Metadata))
130 wvpasseq(link_target, augmented.meta.symlink_target)
131 wvpasseq(len(link_target), augmented.meta.size)
135 def test_item_mode():
136 with no_lingering_errors():
137 mode = S_IFDIR | 0o755
138 meta = metadata.from_path('.')
140 wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
141 wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
144 def test_reverse_suffix_duplicates():
145 suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x))
146 wvpasseq(('x',), suffix(('x',)))
147 wvpasseq(('x', 'y'), suffix(('x', 'y')))
148 wvpasseq(('x-1', 'x-0'), suffix(('x',) * 2))
149 wvpasseq(['x-%02d' % n for n in reversed(range(11))],
150 list(suffix(('x',) * 11)))
151 wvpasseq(('x-1', 'x-0', 'y'), suffix(('x', 'x', 'y')))
152 wvpasseq(('x', 'y-1', 'y-0'), suffix(('x', 'y', 'y')))
153 wvpasseq(('x', 'y-1', 'y-0', 'z'), suffix(('x', 'y', 'y', 'z')))
157 with no_lingering_errors():
158 with test_tempdir('bup-tvfs-') as tmpdir:
159 bup_dir = tmpdir + '/bup'
160 environ['GIT_DIR'] = bup_dir
161 environ['BUP_DIR'] = bup_dir
162 git.repodir = bup_dir
163 data_path = tmpdir + '/src'
165 with open(data_path + '/file', 'w+') as tmpfile:
166 tmpfile.write(b'canary\n')
167 symlink('file', data_path + '/symlink')
168 ex((bup_path, 'init'))
169 ex((bup_path, 'index', '-v', data_path))
170 ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
175 ls_tree = exo(('git', 'ls-tree', 'test', 'symlink')).out
176 mode, typ, oidx, name = ls_tree.strip().split(None, 3)
177 assert name == 'symlink'
178 link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
179 wvpasseq('file', vfs.readlink(repo, link_item))
181 ls_tree = exo(('git', 'ls-tree', 'test', 'file')).out
182 mode, typ, oidx, name = ls_tree.strip().split(None, 3)
183 assert name == 'file'
184 file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
185 wvexcept(Exception, vfs.readlink, repo, file_item)
188 wvpasseq(4, vfs.item_size(repo, link_item))
189 wvpasseq(7, vfs.item_size(repo, file_item))
190 meta = metadata.from_path(__file__)
192 fake_item = file_item._replace(meta=meta)
193 wvpasseq(42, vfs.item_size(repo, fake_item))
195 _, fakelink_item = vfs.resolve(repo, '/test/latest', follow=False)[-1]
196 wvpasseq(17, vfs.item_size(repo, fakelink_item))
198 wvstart('augment_item_meta')
199 run_augment_item_meta_tests(repo,
200 '/test/latest/file', 7,
201 '/test/latest/symlink', 'file')
204 # FIXME: this caused StopIteration
205 #_, file_item = vfs.resolve(repo, '/file')[-1]
206 _, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
207 file_copy = vfs.copy_item(file_item)
208 wvpass(file_copy is not file_item)
209 wvpass(file_copy.meta is not file_item.meta)
210 wvpass(isinstance(file_copy, tuple))
211 wvpass(file_item.meta.user)
212 wvpass(file_copy.meta.user)
213 file_copy.meta.user = None
214 wvpass(file_item.meta.user)
216 def write_sized_random_content(parent_dir, size, seed):
218 with open('%s/%d' % (parent_dir, size), 'wb') as f:
219 write_random(f.fileno(), size, seed, verbose)
221 def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
222 for read_size in read_sizes:
223 with open(expected_path, 'rb') as expected:
224 with vfs.fopen(repo, item) as actual:
225 ex_buf = expected.read(read_size)
226 act_buf = actual.read(read_size)
227 while ex_buf and act_buf:
228 wvpassge(read_size, len(ex_buf))
229 wvpassge(read_size, len(act_buf))
230 wvpasseq(len(ex_buf), len(act_buf))
231 wvpass(ex_buf == act_buf)
232 ex_buf = expected.read(read_size)
233 act_buf = actual.read(read_size)
235 wvpasseq('', act_buf)
237 def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
238 def read_act(act_pos):
239 with vfs.fopen(repo, item) as actual:
241 wvpasseq(act_pos, actual.tell())
242 act_buf = actual.read(read_size)
243 act_pos += len(act_buf)
244 wvpasseq(act_pos, actual.tell())
245 return act_pos, act_buf
247 for read_size in read_sizes:
248 with open(expected_path, 'rb') as expected:
249 ex_buf = expected.read(read_size)
253 act_pos, act_buf = read_act(act_pos)
254 wvpassge(read_size, len(ex_buf))
255 wvpassge(read_size, len(act_buf))
256 wvpasseq(len(ex_buf), len(act_buf))
257 wvpass(ex_buf == act_buf)
260 ex_buf = expected.read(read_size)
261 else: # hit expected eof first
262 act_pos, act_buf = read_act(act_pos)
264 wvpasseq('', act_buf)
267 def test_read_and_seek():
268 # Write a set of randomly sized files containing random data whose
269 # names are their sizes, and then verify that what we get back
270 # from the vfs when seeking and reading with various block sizes
271 # matches the original content.
272 with no_lingering_errors():
273 with test_tempdir('bup-tvfs-read-') as tmpdir:
274 resolve = vfs.resolve
275 bup_dir = tmpdir + '/bup'
276 environ['GIT_DIR'] = bup_dir
277 environ['BUP_DIR'] = bup_dir
278 git.repodir = bup_dir
280 data_path = tmpdir + '/src'
282 seed = randint(-(1 << 31), (1 << 31) - 1)
285 print('test_read seed:', seed, file=sys.stderr)
286 max_size = 2 * 1024 * 1024
287 sizes = set((rand.randint(1, max_size) for _ in xrange(5)))
291 write_sized_random_content(data_path, size, seed)
292 ex((bup_path, 'init'))
293 ex((bup_path, 'index', '-v', data_path))
294 ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
296 read_sizes = set((rand.randint(1, max_size) for _ in xrange(10)))
299 print('test_read src sizes:', sizes, file=sys.stderr)
300 print('test_read read sizes:', read_sizes, file=sys.stderr)
302 res = resolve(repo, '/test/latest/' + str(size))
304 wvpasseq(size, vfs.item_size(repo, res[-1][1]))
305 validate_vfs_streaming_read(repo, item,
306 '%s/%d' % (data_path, size),
308 validate_vfs_seeking_read(repo, item,
309 '%s/%d' % (data_path, size),
313 def test_contents_with_mismatched_bupm_git_ordering():
314 with no_lingering_errors():
315 with test_tempdir('bup-tvfs-') as tmpdir:
316 bup_dir = tmpdir + '/bup'
317 environ['GIT_DIR'] = bup_dir
318 environ['BUP_DIR'] = bup_dir
319 git.repodir = bup_dir
320 data_path = tmpdir + '/src'
322 os.mkdir(data_path + '/foo')
323 with open(data_path + '/foo.', 'w+') as tmpfile:
324 tmpfile.write(b'canary\n')
325 ex((bup_path, 'init'))
326 ex((bup_path, 'index', '-v', data_path))
328 save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
329 ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc),
330 '--strip', data_path))
332 tip_sref = exo(('git', 'show-ref', 'refs/heads/test')).out
333 tip_oidx = tip_sref.strip().split()[0]
334 tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
335 tip_oidx)).out.strip()
336 tip_tree_oid = tip_tree_oidx.decode('hex')
337 tip_tree = tree_dict(repo, tip_tree_oid)
339 name, item = vfs.resolve(repo, '/test/latest')[2]
340 wvpasseq(save_name, name)
341 expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
342 for x in (tip_tree[name]
343 for name in ('.', 'foo', 'foo.')))
344 contents = tuple(vfs.contents(repo, item))
345 wvpasseq(expected, frozenset(contents))
346 # Spot check, in case tree_dict shares too much code with the vfs
347 name, item = next(((n, i) for n, i in contents if n == 'foo'))
348 wvpass(S_ISDIR(item.meta))
349 name, item = next(((n, i) for n, i in contents if n == 'foo.'))
350 wvpass(S_ISREG(item.meta.mode))
353 def test_duplicate_save_dates():
354 with no_lingering_errors():
355 with test_tempdir('bup-tvfs-') as tmpdir:
356 bup_dir = tmpdir + '/bup'
357 environ['GIT_DIR'] = bup_dir
358 environ['BUP_DIR'] = bup_dir
359 environ['TZ'] = 'UTC'
360 git.repodir = bup_dir
361 data_path = tmpdir + '/src'
363 with open(data_path + '/file', 'w+') as tmpfile:
364 tmpfile.write(b'canary\n')
365 ex((bup_path, 'init'))
366 ex((bup_path, 'index', '-v', data_path))
368 ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
370 res = vfs.resolve(repo, '/test')
371 wvpasseq(2, len(res))
372 name, revlist = res[-1]
373 wvpasseq('test', name)
375 '1970-01-02-034640-00',
376 '1970-01-02-034640-01',
377 '1970-01-02-034640-02',
378 '1970-01-02-034640-03',
379 '1970-01-02-034640-04',
380 '1970-01-02-034640-05',
381 '1970-01-02-034640-06',
382 '1970-01-02-034640-07',
383 '1970-01-02-034640-08',
384 '1970-01-02-034640-09',
385 '1970-01-02-034640-10',
387 tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
390 def test_item_read_write():
391 with no_lingering_errors():
392 x = vfs.Root(meta=13)
394 vfs.write_item(stream, x)
395 print('stream:', repr(stream.getvalue()), stream.tell(), file=sys.stderr)
397 wvpasseq(x, vfs.read_item(stream))