2 from __future__ import absolute_import, print_function
3 from collections import namedtuple
4 from errno import ELOOP, ENOTDIR
6 from os import environ, symlink
7 from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG
9 from time import localtime, strftime
13 from bup import git, metadata, vfs
14 from bup.git import BUP_CHUNKED
15 from bup.helpers import exc, exo, shstr
16 from bup.metadata import Metadata
17 from bup.repo import LocalRepo
18 from buptest import no_lingering_errors, test_tempdir
21 bup_tmp = os.path.realpath('../../../t/tmp')
22 bup_path = top_dir + '/bup'
23 start_dir = os.getcwd()
25 ## The clear_cache() calls below are to make sure that the test starts
26 ## from a known state since at the moment the cache entry for a given
27 ## item (like a commit) can change. For example, its meta value might
28 ## be promoted from a mode to a Metadata instance once the tree it
29 ## refers to is traversed.
31 def ex(cmd, **kwargs):
32 print(shstr(cmd), file=stderr)
33 return exc(cmd, **kwargs)
35 TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
37 def tree_items(repo, oid):
38 """Yield (name, entry_oid, meta) for each entry in oid. meta will be
39 a Metadata object for any non-directories and for '.', otherwise
43 # This is a simpler approach than the one in the vfs, used to
44 # cross-check its behavior.
45 tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid)
46 bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
48 maybe_meta = lambda : Metadata.read(bupm) if bupm else None
52 yield TreeDictValue(name='.', oid=oid, meta=m)
53 tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
54 for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
55 if mangled_name == '.bupm':
59 if kind == BUP_CHUNKED:
60 yield TreeDictValue(name=name, oid=sub_oid,
63 yield TreeDictValue(name=name, oid=sub_oid,
64 meta=vfs.default_dir_mode)
66 yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
71 def tree_dict(repo, oid):
72 return dict((x.name, x) for x in tree_items(repo, oid))
74 def run_augment_item_meta_tests(repo,
76 link_path, link_target):
77 _, file_item = vfs.resolve(repo, file_path)[-1]
78 _, link_item = vfs.lresolve(repo, link_path)[-1]
79 wvpass(isinstance(file_item.meta, Metadata))
80 wvpass(isinstance(link_item.meta, Metadata))
81 # Note: normally, modifying item.meta values is forbidden
82 file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
83 link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
85 ## Ensure a fully populated item is left alone
86 augmented = vfs.augment_item_meta(repo, file_item)
87 wvpass(augmented is file_item)
88 wvpass(augmented.meta is file_item.meta)
89 augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
90 wvpass(augmented is file_item)
91 wvpass(augmented.meta is file_item.meta)
93 ## Ensure a missing size is handled poperly
94 file_item.meta.size = None
95 augmented = vfs.augment_item_meta(repo, file_item)
96 wvpass(augmented is file_item)
97 wvpass(augmented.meta is file_item.meta)
98 augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
99 wvpass(augmented is not file_item)
100 wvpasseq(file_size, augmented.meta.size)
102 ## Ensure a meta mode is handled properly
103 mode_item = file_item._replace(meta=vfs.default_file_mode)
104 augmented = vfs.augment_item_meta(repo, mode_item)
105 augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
106 for item in (augmented, augmented_w_size):
108 wvpass(item is not file_item)
109 wvpass(isinstance(meta, Metadata))
110 wvpasseq(vfs.default_file_mode, meta.mode)
111 wvpasseq((0, 0, 0, 0, 0),
112 (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
113 wvpass(augmented.meta.size is None)
114 wvpasseq(file_size, augmented_w_size.meta.size)
116 ## Ensure symlinks are handled properly
117 mode_item = link_item._replace(meta=vfs.default_symlink_mode)
118 augmented = vfs.augment_item_meta(repo, mode_item)
119 wvpass(augmented is not mode_item)
120 wvpass(isinstance(augmented.meta, Metadata))
121 wvpasseq(link_target, augmented.meta.symlink_target)
122 wvpasseq(len(link_target), augmented.meta.size)
123 augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
124 wvpass(augmented is not mode_item)
125 wvpass(isinstance(augmented.meta, Metadata))
126 wvpasseq(link_target, augmented.meta.symlink_target)
127 wvpasseq(len(link_target), augmented.meta.size)
131 def test_item_mode():
132 with no_lingering_errors():
133 mode = S_IFDIR | 0o755
134 meta = metadata.from_path('.')
136 wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
137 wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
140 def test_reverse_suffix_duplicates():
141 suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x))
142 wvpasseq(('x',), suffix(('x',)))
143 wvpasseq(('x', 'y'), suffix(('x', 'y')))
144 wvpasseq(('x-1', 'x-0'), suffix(('x',) * 2))
145 wvpasseq(['x-%02d' % n for n in reversed(range(11))],
146 list(suffix(('x',) * 11)))
147 wvpasseq(('x-1', 'x-0', 'y'), suffix(('x', 'x', 'y')))
148 wvpasseq(('x', 'y-1', 'y-0'), suffix(('x', 'y', 'y')))
149 wvpasseq(('x', 'y-1', 'y-0', 'z'), suffix(('x', 'y', 'y', 'z')))
153 with no_lingering_errors():
154 with test_tempdir('bup-tvfs-') as tmpdir:
155 bup_dir = tmpdir + '/bup'
156 environ['GIT_DIR'] = bup_dir
157 environ['BUP_DIR'] = bup_dir
158 git.repodir = bup_dir
159 data_path = tmpdir + '/src'
161 with open(data_path + '/file', 'w+') as tmpfile:
162 tmpfile.write(b'canary\n')
163 symlink('file', data_path + '/symlink')
164 ex((bup_path, 'init'))
165 ex((bup_path, 'index', '-v', data_path))
166 ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
171 ls_tree = exo(('git', 'ls-tree', 'test', 'symlink'))
172 mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
173 assert name == 'symlink'
174 link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
175 wvpasseq('file', vfs.readlink(repo, link_item))
177 ls_tree = exo(('git', 'ls-tree', 'test', 'file'))
178 mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
179 assert name == 'file'
180 file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
181 wvexcept(Exception, vfs.readlink, repo, file_item)
184 wvpasseq(4, vfs.item_size(repo, link_item))
185 wvpasseq(7, vfs.item_size(repo, file_item))
186 meta = metadata.from_path(__file__)
188 fake_item = file_item._replace(meta=meta)
189 wvpasseq(42, vfs.item_size(repo, fake_item))
191 wvstart('augment_item_meta')
192 run_augment_item_meta_tests(repo,
193 '/test/latest/file', 7,
194 '/test/latest/symlink', 'file')
197 # FIXME: this caused StopIteration
198 #_, file_item = vfs.resolve(repo, '/file')[-1]
199 _, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
200 file_copy = vfs.copy_item(file_item)
201 wvpass(file_copy is not file_item)
202 wvpass(file_copy.meta is not file_item.meta)
203 wvpass(isinstance(file_copy, tuple))
204 wvpass(file_item.meta.user)
205 wvpass(file_copy.meta.user)
206 file_copy.meta.user = None
207 wvpass(file_item.meta.user)
211 with no_lingering_errors():
212 with test_tempdir('bup-tvfs-') as tmpdir:
213 resolve = vfs.resolve
214 lresolve = vfs.lresolve
215 bup_dir = tmpdir + '/bup'
216 environ['GIT_DIR'] = bup_dir
217 environ['BUP_DIR'] = bup_dir
218 git.repodir = bup_dir
219 data_path = tmpdir + '/src'
221 save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time))
223 os.mkdir(data_path + '/dir')
224 with open(data_path + '/file', 'w+') as tmpfile:
225 print('canary', file=tmpfile)
226 symlink('file', data_path + '/file-symlink')
227 symlink('dir', data_path + '/dir-symlink')
228 symlink('not-there', data_path + '/bad-symlink')
229 ex((bup_path, 'init'))
230 ex((bup_path, 'index', '-v', data_path))
231 ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test',
232 '--strip', data_path))
233 ex((bup_path, 'tag', 'test-tag', 'test'))
236 tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0]
237 tip_oidx = tip_hash.strip().split()[0]
238 tip_oid = tip_oidx.decode('hex')
239 tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
240 tip_oidx))[0].strip()
241 tip_tree_oid = tip_tree_oidx.decode('hex')
242 tip_tree = tree_dict(repo, tip_tree_oid)
243 test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta,
245 expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755,
248 expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
251 expected_test_tag_item = expected_latest_item
253 wvstart('resolve: /')
255 res = resolve(repo, '/')
256 wvpasseq(1, len(res))
257 wvpasseq((('', vfs._root),), res)
258 ignore, root_item = res[0]
259 root_content = frozenset(vfs.contents(repo, root_item))
260 wvpasseq(frozenset([('.', root_item),
262 ('test', test_revlist_w_meta)]),
264 for path in ('//', '/.', '/./', '/..', '/../',
265 '/test/latest/dir/../../..',
266 '/test/latest/dir/../../../',
267 '/test/latest/dir/../../../.',
268 '/test/latest/dir/../../..//',
269 '/test//latest/dir/../../..',
270 '/test/./latest/dir/../../..',
271 '/test/././latest/dir/../../..',
272 '/test/.//./latest/dir/../../..',
273 '/test//.//.//latest/dir/../../..'
274 '/test//./latest/dir/../../..'):
275 wvstart('resolve: ' + path)
277 res = resolve(repo, path)
278 wvpasseq((('', vfs._root),), res)
280 wvstart('resolve: /.tag')
282 res = resolve(repo, '/.tag')
283 wvpasseq(2, len(res))
284 wvpasseq((('', vfs._root), ('.tag', vfs._tags)),
286 ignore, tag_item = res[1]
287 tag_content = frozenset(vfs.contents(repo, tag_item))
288 wvpasseq(frozenset([('.', tag_item),
289 ('test-tag', expected_test_tag_item)]),
292 wvstart('resolve: /test')
294 res = resolve(repo, '/test')
295 wvpasseq(2, len(res))
296 wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res)
297 ignore, test_item = res[1]
298 test_content = frozenset(vfs.contents(repo, test_item))
299 # latest has metadata here due to caching
300 wvpasseq(frozenset([('.', test_revlist_w_meta),
301 (save_time_str, expected_latest_item_w_meta),
302 ('latest', expected_latest_item_w_meta)]),
305 wvstart('resolve: /test/latest')
307 res = resolve(repo, '/test/latest')
308 wvpasseq(3, len(res))
309 expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
312 expected = (('', vfs._root),
313 ('test', test_revlist_w_meta),
314 ('latest', expected_latest_item_w_meta))
315 wvpasseq(expected, res)
316 ignore, latest_item = res[2]
317 latest_content = frozenset(vfs.contents(repo, latest_item))
318 expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
319 for x in (tip_tree[name]
326 wvpasseq(expected, latest_content)
328 wvstart('resolve: /test/latest/file')
330 res = resolve(repo, '/test/latest/file')
331 wvpasseq(4, len(res))
332 expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta,
333 oid=tip_tree['file'].oid)
334 expected = (('', vfs._root),
335 ('test', test_revlist_w_meta),
336 ('latest', expected_latest_item_w_meta),
337 ('file', expected_file_item_w_meta))
338 wvpasseq(expected, res)
340 wvstart('resolve: /test/latest/bad-symlink')
342 res = resolve(repo, '/test/latest/bad-symlink')
343 wvpasseq(4, len(res))
344 expected = (('', vfs._root),
345 ('test', test_revlist_w_meta),
346 ('latest', expected_latest_item_w_meta),
348 wvpasseq(expected, res)
350 wvstart('lresolve: /test/latest/bad-symlink')
352 res = lresolve(repo, '/test/latest/bad-symlink')
353 wvpasseq(4, len(res))
354 bad_symlink_value = tip_tree['bad-symlink']
355 expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta,
356 oid=bad_symlink_value.oid)
357 expected = (('', vfs._root),
358 ('test', test_revlist_w_meta),
359 ('latest', expected_latest_item_w_meta),
360 ('bad-symlink', expected_bad_symlink_item_w_meta))
361 wvpasseq(expected, res)
363 wvstart('resolve: /test/latest/file-symlink')
365 res = resolve(repo, '/test/latest/file-symlink')
366 wvpasseq(4, len(res))
367 expected = (('', vfs._root),
368 ('test', test_revlist_w_meta),
369 ('latest', expected_latest_item_w_meta),
370 ('file', expected_file_item_w_meta))
371 wvpasseq(expected, res)
373 wvstart('lresolve: /test/latest/file-symlink')
375 res = lresolve(repo, '/test/latest/file-symlink')
376 wvpasseq(4, len(res))
377 file_symlink_value = tip_tree['file-symlink']
378 expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta,
379 oid=file_symlink_value.oid)
380 expected = (('', vfs._root),
381 ('test', test_revlist_w_meta),
382 ('latest', expected_latest_item_w_meta),
383 ('file-symlink', expected_file_symlink_item_w_meta))
384 wvpasseq(expected, res)
386 wvstart('resolve: /test/latest/missing')
388 res = resolve(repo, '/test/latest/missing')
389 wvpasseq(4, len(res))
391 wvpasseq('missing', name)
394 for path in ('/test/latest/file/',
395 '/test/latest/file/.',
396 '/test/latest/file/..',
397 '/test/latest/file/../',
398 '/test/latest/file/../.',
399 '/test/latest/file/../..',
400 '/test/latest/file/foo'):
401 wvstart('resolve: ' + path)
405 except vfs.IOError as res_ex:
406 wvpasseq(ENOTDIR, res_ex.errno)
407 wvpasseq(['', 'test', 'latest', 'file'],
408 [name for name, item in res_ex.terminus])
410 for path in ('/test/latest/file-symlink/',
411 '/test/latest/file-symlink/.',
412 '/test/latest/file-symlink/..',
413 '/test/latest/file-symlink/../',
414 '/test/latest/file-symlink/../.',
415 '/test/latest/file-symlink/../..'):
416 wvstart('lresolve: ' + path)
420 except vfs.IOError as res_ex:
421 wvpasseq(ENOTDIR, res_ex.errno)
422 wvpasseq(['', 'test', 'latest', 'file'],
423 [name for name, item in res_ex.terminus])
425 wvstart('resolve: non-directory parent')
427 file_res = resolve(repo, '/test/latest/file')
429 resolve(repo, 'foo', parent=file_res)
430 except vfs.IOError as res_ex:
431 wvpasseq(ENOTDIR, res_ex.errno)
432 wvpasseq(None, res_ex.terminus)
434 wvstart('lresolve: /test/latest/dir-symlink')
436 res = lresolve(repo, '/test/latest/dir-symlink')
437 wvpasseq(4, len(res))
438 dir_symlink_value = tip_tree['dir-symlink']
439 expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta,
440 oid=dir_symlink_value.oid)
441 expected = (('', vfs._root),
442 ('test', test_revlist_w_meta),
443 ('latest', expected_latest_item_w_meta),
444 ('dir-symlink', expected_dir_symlink_item_w_meta))
445 wvpasseq(expected, res)
447 dir_value = tip_tree['dir']
448 expected_dir_item = vfs.Item(oid=dir_value.oid,
449 meta=tree_dict(repo, dir_value.oid)['.'].meta)
450 expected = (('', vfs._root),
451 ('test', test_revlist_w_meta),
452 ('latest', expected_latest_item_w_meta),
453 ('dir', expected_dir_item))
454 for resname, resolver in (('resolve', resolve),
455 ('lresolve', lresolve)):
456 for path in ('/test/latest/dir-symlink/',
457 '/test/latest/dir-symlink/.'):
458 wvstart(resname + ': ' + path)
460 res = resolver(repo, path)
461 wvpasseq(4, len(res))
462 wvpasseq(expected, res)
463 wvstart('resolve: /test/latest/dir-symlink')
465 res = resolve(repo, path)
466 wvpasseq(4, len(res))
467 wvpasseq(expected, res)
470 def test_resolve_loop():
471 with no_lingering_errors():
472 with test_tempdir('bup-tvfs-resloop-') as tmpdir:
473 resolve = vfs.resolve
474 lresolve = vfs.lresolve
475 bup_dir = tmpdir + '/bup'
476 environ['GIT_DIR'] = bup_dir
477 environ['BUP_DIR'] = bup_dir
478 git.repodir = bup_dir
480 data_path = tmpdir + '/src'
482 symlink('loop', data_path + '/loop')
483 ex((bup_path, 'init'))
484 ex((bup_path, 'index', '-v', data_path))
485 ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
488 resolve(repo, '/test/latest/loop')
489 except vfs.IOError as res_ex:
490 wvpasseq(ELOOP, res_ex.errno)
491 wvpasseq(['', 'test', 'latest', 'loop'],
492 [name for name, item in res_ex.terminus])
495 def test_contents_with_mismatched_bupm_git_ordering():
496 with no_lingering_errors():
497 with test_tempdir('bup-tvfs-') as tmpdir:
498 bup_dir = tmpdir + '/bup'
499 environ['GIT_DIR'] = bup_dir
500 environ['BUP_DIR'] = bup_dir
501 git.repodir = bup_dir
502 data_path = tmpdir + '/src'
504 os.mkdir(data_path + '/foo')
505 with open(data_path + '/foo.', 'w+') as tmpfile:
506 tmpfile.write(b'canary\n')
507 ex((bup_path, 'init'))
508 ex((bup_path, 'index', '-v', data_path))
509 ex((bup_path, 'save', '-tvvn', 'test', '--strip',
512 tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
513 tip_oidx = tip_sref.strip().split()[0]
514 tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
515 tip_oidx))[0].strip()
516 tip_tree_oid = tip_tree_oidx.decode('hex')
517 tip_tree = tree_dict(repo, tip_tree_oid)
519 name, item = vfs.resolve(repo, '/test/latest')[2]
520 wvpasseq('latest', name)
521 expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
522 for x in (tip_tree[name]
523 for name in ('.', 'foo', 'foo.')))
524 contents = tuple(vfs.contents(repo, item))
525 wvpasseq(expected, frozenset(contents))
526 # Spot check, in case tree_dict shares too much code with the vfs
527 name, item = next(((n, i) for n, i in contents if n == 'foo'))
528 wvpass(S_ISDIR(item.meta))
529 name, item = next(((n, i) for n, i in contents if n == 'foo.'))
530 wvpass(S_ISREG(item.meta.mode))
533 def test_duplicate_save_dates():
534 with no_lingering_errors():
535 with test_tempdir('bup-tvfs-') as tmpdir:
536 bup_dir = tmpdir + '/bup'
537 environ['GIT_DIR'] = bup_dir
538 environ['BUP_DIR'] = bup_dir
539 environ['TZ'] = 'UTC'
540 git.repodir = bup_dir
541 data_path = tmpdir + '/src'
543 with open(data_path + '/file', 'w+') as tmpfile:
544 tmpfile.write(b'canary\n')
545 ex((bup_path, 'init'))
546 ex((bup_path, 'index', '-v', data_path))
548 ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
550 res = vfs.resolve(repo, '/test')
551 wvpasseq(2, len(res))
552 name, revlist = res[-1]
553 wvpasseq('test', name)
555 '1970-01-02-034640-00',
556 '1970-01-02-034640-01',
557 '1970-01-02-034640-02',
558 '1970-01-02-034640-03',
559 '1970-01-02-034640-04',
560 '1970-01-02-034640-05',
561 '1970-01-02-034640-06',
562 '1970-01-02-034640-07',
563 '1970-01-02-034640-08',
564 '1970-01-02-034640-09',
565 '1970-01-02-034640-10',
567 tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
569 # FIXME: add tests for the want_meta=False cases.