lib/bup/t/tvfs.py

   1
   2 from __future__ import absolute_import, print_function
   3 from collections import namedtuple
   4 from errno import ELOOP, ENOTDIR
   5 from io import BytesIO
   6 from os import environ, symlink
   7 from random import Random, randint
   8 from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG
   9 from sys import stderr
  10 from time import localtime, strftime
  11
  12 from wvtest import *
  13
  14 from bup._helpers import write_random
  15 from bup import git, metadata, vfs
  16 from bup.git import BUP_CHUNKED
  17 from bup.helpers import exc, exo, shstr
  18 from bup.metadata import Metadata
  19 from bup.repo import LocalRepo
  20 from buptest import no_lingering_errors, test_tempdir
  21
  22 top_dir = '../../..'
  23 bup_tmp = os.path.realpath('../../../t/tmp')
  24 bup_path = top_dir + '/bup'
  25 start_dir = os.getcwd()
  26
  27 def ex(cmd, **kwargs):
  28     print(shstr(cmd), file=stderr)
  29     return exc(cmd, **kwargs)
  30
  31 @wvtest
  32 def test_default_modes():
  33     wvpasseq(S_IFREG | 0o644, vfs.default_file_mode)
  34     wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode)
  35     wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode)
  36
  37 @wvtest
  38 def test_cache_behavior():
  39     orig_max = vfs._cache_max_items
  40     try:
  41         vfs._cache_max_items = 2
  42         vfs.clear_cache()
  43         wvpasseq({}, vfs._cache)
  44         wvpasseq([], vfs._cache_keys)
  45         wvfail(vfs._cache_keys)
  46         wvexcept(Exception, vfs.cache_notice, 'x', 1)
  47         key_0 = 'itm:' + b'\0' * 20
  48         key_1 = 'itm:' + b'\1' * 20
  49         key_2 = 'itm:' + b'\2' * 20
  50         vfs.cache_notice(key_0, 'something')
  51         wvpasseq({key_0 : 'something'}, vfs._cache)
  52         wvpasseq([key_0], vfs._cache_keys)
  53         vfs.cache_notice(key_1, 'something else')
  54         wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache)
  55         wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
  56         vfs.cache_notice(key_2, 'and also')
  57         wvpasseq(2, len(vfs._cache))
  58         wvpass(frozenset(vfs._cache.iteritems())
  59                < frozenset({key_0 : 'something',
  60                             key_1 : 'something else',
  61                             key_2 : 'and also'}.iteritems()))
  62         wvpasseq(2, len(vfs._cache_keys))
  63         wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
  64         vfs.clear_cache()
  65         wvpasseq({}, vfs._cache)
  66         wvpasseq([], vfs._cache_keys)
  67     finally:
  68         vfs._cache_max_items = orig_max
  69         vfs.clear_cache()
  70
  71 ## The clear_cache() calls below are to make sure that the test starts
  72 ## from a known state since at the moment the cache entry for a given
  73 ## item (like a commit) can change.  For example, its meta value might
  74 ## be promoted from a mode to a Metadata instance once the tree it
  75 ## refers to is traversed.
  76
  77 TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
  78
  79 def tree_items(repo, oid):
  80     """Yield (name, entry_oid, meta) for each entry in oid.  meta will be
  81     a Metadata object for any non-directories and for '.', otherwise
  82     None.
  83
  84     """
  85     # This is a simpler approach than the one in the vfs, used to
  86     # cross-check its behavior.
  87     tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid)
  88     bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
  89     try:
  90         maybe_meta = lambda : Metadata.read(bupm) if bupm else None
  91         m = maybe_meta()
  92         if m:
  93             m.size = 0
  94         yield TreeDictValue(name='.', oid=oid, meta=m)
  95         tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
  96         for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
  97             if mangled_name == '.bupm':
  98                 continue
  99             assert name != '.'
 100             if S_ISDIR(gitmode):
 101                 if kind == BUP_CHUNKED:
 102                     yield TreeDictValue(name=name, oid=sub_oid,
 103                                         meta=maybe_meta())
 104                 else:
 105                     yield TreeDictValue(name=name, oid=sub_oid,
 106                                         meta=vfs.default_dir_mode)
 107             else:
 108                 yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
 109     finally:
 110         if bupm:
 111             bupm.close()
 112
 113 def tree_dict(repo, oid):
 114     return dict((x.name, x) for x in tree_items(repo, oid))
 115
 116 def run_augment_item_meta_tests(repo,
 117                                 file_path, file_size,
 118                                 link_path, link_target):
 119     _, file_item = vfs.resolve(repo, file_path)[-1]
 120     _, link_item = vfs.lresolve(repo, link_path)[-1]
 121     wvpass(isinstance(file_item.meta, Metadata))
 122     wvpass(isinstance(link_item.meta, Metadata))
 123     # Note: normally, modifying item.meta values is forbidden
 124     file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
 125     link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
 126
 127     ## Ensure a fully populated item is left alone
 128     augmented = vfs.augment_item_meta(repo, file_item)
 129     wvpass(augmented is file_item)
 130     wvpass(augmented.meta is file_item.meta)
 131     augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
 132     wvpass(augmented is file_item)
 133     wvpass(augmented.meta is file_item.meta)
 134
 135     ## Ensure a missing size is handled poperly
 136     file_item.meta.size = None
 137     augmented = vfs.augment_item_meta(repo, file_item)
 138     wvpass(augmented is file_item)
 139     wvpass(augmented.meta is file_item.meta)
 140     augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
 141     wvpass(augmented is not file_item)
 142     wvpasseq(file_size, augmented.meta.size)
 143
 144     ## Ensure a meta mode is handled properly
 145     mode_item = file_item._replace(meta=vfs.default_file_mode)
 146     augmented = vfs.augment_item_meta(repo, mode_item)
 147     augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
 148     for item in (augmented, augmented_w_size):
 149         meta = item.meta
 150         wvpass(item is not file_item)
 151         wvpass(isinstance(meta, Metadata))
 152         wvpasseq(vfs.default_file_mode, meta.mode)
 153         wvpasseq((0, 0, 0, 0, 0),
 154                  (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
 155     wvpass(augmented.meta.size is None)
 156     wvpasseq(file_size, augmented_w_size.meta.size)
 157
 158     ## Ensure symlinks are handled properly
 159     mode_item = link_item._replace(meta=vfs.default_symlink_mode)
 160     augmented = vfs.augment_item_meta(repo, mode_item)
 161     wvpass(augmented is not mode_item)
 162     wvpass(isinstance(augmented.meta, Metadata))
 163     wvpasseq(link_target, augmented.meta.symlink_target)
 164     wvpasseq(len(link_target), augmented.meta.size)
 165     augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
 166     wvpass(augmented is not mode_item)
 167     wvpass(isinstance(augmented.meta, Metadata))
 168     wvpasseq(link_target, augmented.meta.symlink_target)
 169     wvpasseq(len(link_target), augmented.meta.size)
 170
 171
 172 @wvtest
 173 def test_item_mode():
 174     with no_lingering_errors():
 175         mode = S_IFDIR | 0o755
 176         meta = metadata.from_path('.')
 177         oid = '\0' * 20
 178         wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
 179         wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
 180
 181 @wvtest
 182 def test_reverse_suffix_duplicates():
 183     suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x))
 184     wvpasseq(('x',), suffix(('x',)))
 185     wvpasseq(('x', 'y'), suffix(('x', 'y')))
 186     wvpasseq(('x-1', 'x-0'), suffix(('x',) * 2))
 187     wvpasseq(['x-%02d' % n for n in reversed(range(11))],
 188              list(suffix(('x',) * 11)))
 189     wvpasseq(('x-1', 'x-0', 'y'), suffix(('x', 'x', 'y')))
 190     wvpasseq(('x', 'y-1', 'y-0'), suffix(('x', 'y', 'y')))
 191     wvpasseq(('x', 'y-1', 'y-0', 'z'), suffix(('x', 'y', 'y', 'z')))
 192
 193 @wvtest
 194 def test_misc():
 195     with no_lingering_errors():
 196         with test_tempdir('bup-tvfs-') as tmpdir:
 197             bup_dir = tmpdir + '/bup'
 198             environ['GIT_DIR'] = bup_dir
 199             environ['BUP_DIR'] = bup_dir
 200             git.repodir = bup_dir
 201             data_path = tmpdir + '/src'
 202             os.mkdir(data_path)
 203             with open(data_path + '/file', 'w+') as tmpfile:
 204                 tmpfile.write(b'canary\n')
 205             symlink('file', data_path + '/symlink')
 206             ex((bup_path, 'init'))
 207             ex((bup_path, 'index', '-v', data_path))
 208             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 209                 data_path))
 210             repo = LocalRepo()
 211
 212             wvstart('readlink')
 213             ls_tree = exo(('git', 'ls-tree', 'test', 'symlink'))
 214             mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
 215             assert name == 'symlink'
 216             link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
 217             wvpasseq('file', vfs.readlink(repo, link_item))
 218
 219             ls_tree = exo(('git', 'ls-tree', 'test', 'file'))
 220             mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
 221             assert name == 'file'
 222             file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
 223             wvexcept(Exception, vfs.readlink, repo, file_item)
 224
 225             wvstart('item_size')
 226             wvpasseq(4, vfs.item_size(repo, link_item))
 227             wvpasseq(7, vfs.item_size(repo, file_item))
 228             meta = metadata.from_path(__file__)
 229             meta.size = 42
 230             fake_item = file_item._replace(meta=meta)
 231             wvpasseq(42, vfs.item_size(repo, fake_item))
 232
 233             wvstart('augment_item_meta')
 234             run_augment_item_meta_tests(repo,
 235                                         '/test/latest/file', 7,
 236                                         '/test/latest/symlink', 'file')
 237
 238             wvstart('copy_item')
 239             # FIXME: this caused StopIteration
 240             #_, file_item = vfs.resolve(repo, '/file')[-1]
 241             _, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
 242             file_copy = vfs.copy_item(file_item)
 243             wvpass(file_copy is not file_item)
 244             wvpass(file_copy.meta is not file_item.meta)
 245             wvpass(isinstance(file_copy, tuple))
 246             wvpass(file_item.meta.user)
 247             wvpass(file_copy.meta.user)
 248             file_copy.meta.user = None
 249             wvpass(file_item.meta.user)
 250
 251 @wvtest
 252 def test_resolve():
 253     with no_lingering_errors():
 254         with test_tempdir('bup-tvfs-') as tmpdir:
 255             resolve = vfs.resolve
 256             lresolve = vfs.lresolve
 257             bup_dir = tmpdir + '/bup'
 258             environ['GIT_DIR'] = bup_dir
 259             environ['BUP_DIR'] = bup_dir
 260             git.repodir = bup_dir
 261             data_path = tmpdir + '/src'
 262             save_time = 100000
 263             save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time))
 264             os.mkdir(data_path)
 265             os.mkdir(data_path + '/dir')
 266             with open(data_path + '/file', 'w+') as tmpfile:
 267                 print('canary', file=tmpfile)
 268             symlink('file', data_path + '/file-symlink')
 269             symlink('dir', data_path + '/dir-symlink')
 270             symlink('not-there', data_path + '/bad-symlink')
 271             ex((bup_path, 'init'))
 272             ex((bup_path, 'index', '-v', data_path))
 273             ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test',
 274                 '--strip', data_path))
 275             ex((bup_path, 'tag', 'test-tag', 'test'))
 276             repo = LocalRepo()
 277
 278             tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0]
 279             tip_oidx = tip_hash.strip().split()[0]
 280             tip_oid = tip_oidx.decode('hex')
 281             tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
 282                                  tip_oidx))[0].strip()
 283             tip_tree_oid = tip_tree_oidx.decode('hex')
 284             tip_tree = tree_dict(repo, tip_tree_oid)
 285             test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta,
 286                                               oid=tip_oid)
 287             expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755,
 288                                               oid=tip_tree_oid,
 289                                               coid=tip_oid)
 290             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
 291                                                      oid=tip_tree_oid,
 292                                                      coid=tip_oid)
 293             expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode,
 294                                                 target=save_time_str)
 295             expected_test_tag_item = expected_latest_item
 296
 297             wvstart('resolve: /')
 298             vfs.clear_cache()
 299             res = resolve(repo, '/')
 300             wvpasseq(1, len(res))
 301             wvpasseq((('', vfs._root),), res)
 302             ignore, root_item = res[0]
 303             root_content = frozenset(vfs.contents(repo, root_item))
 304             wvpasseq(frozenset([('.', root_item),
 305                                 ('.tag', vfs._tags),
 306                                 ('test', test_revlist_w_meta)]),
 307                      root_content)
 308             for path in ('//', '/.', '/./', '/..', '/../',
 309                          '/test/latest/dir/../../..',
 310                          '/test/latest/dir/../../../',
 311                          '/test/latest/dir/../../../.',
 312                          '/test/latest/dir/../../..//',
 313                          '/test//latest/dir/../../..',
 314                          '/test/./latest/dir/../../..',
 315                          '/test/././latest/dir/../../..',
 316                          '/test/.//./latest/dir/../../..',
 317                          '/test//.//.//latest/dir/../../..'
 318                          '/test//./latest/dir/../../..'):
 319                 wvstart('resolve: ' + path)
 320                 vfs.clear_cache()
 321                 res = resolve(repo, path)
 322                 wvpasseq((('', vfs._root),), res)
 323
 324             wvstart('resolve: /.tag')
 325             vfs.clear_cache()
 326             res = resolve(repo, '/.tag')
 327             wvpasseq(2, len(res))
 328             wvpasseq((('', vfs._root), ('.tag', vfs._tags)),
 329                      res)
 330             ignore, tag_item = res[1]
 331             tag_content = frozenset(vfs.contents(repo, tag_item))
 332             wvpasseq(frozenset([('.', tag_item),
 333                                 ('test-tag', expected_test_tag_item)]),
 334                      tag_content)
 335
 336             wvstart('resolve: /test')
 337             vfs.clear_cache()
 338             res = resolve(repo, '/test')
 339             wvpasseq(2, len(res))
 340             wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res)
 341             ignore, test_item = res[1]
 342             test_content = frozenset(vfs.contents(repo, test_item))
 343             # latest has metadata here due to caching
 344             wvpasseq(frozenset([('.', test_revlist_w_meta),
 345                                 (save_time_str, expected_latest_item_w_meta),
 346                                 ('latest', expected_latest_link)]),
 347                      test_content)
 348
 349             wvstart('resolve: /test/latest')
 350             vfs.clear_cache()
 351             res = resolve(repo, '/test/latest')
 352             wvpasseq(3, len(res))
 353             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
 354                                                      oid=tip_tree_oid,
 355                                                      coid=tip_oid)
 356             expected = (('', vfs._root),
 357                         ('test', test_revlist_w_meta),
 358                         (save_time_str, expected_latest_item_w_meta))
 359             wvpasseq(expected, res)
 360             ignore, latest_item = res[2]
 361             latest_content = frozenset(vfs.contents(repo, latest_item))
 362             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
 363                                  for x in (tip_tree[name]
 364                                            for name in ('.',
 365                                                         'bad-symlink',
 366                                                         'dir',
 367                                                         'dir-symlink',
 368                                                         'file',
 369                                                         'file-symlink')))
 370             wvpasseq(expected, latest_content)
 371
 372             wvstart('resolve: /test/latest/file')
 373             vfs.clear_cache()
 374             res = resolve(repo, '/test/latest/file')
 375             wvpasseq(4, len(res))
 376             expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta,
 377                                                  oid=tip_tree['file'].oid)
 378             expected = (('', vfs._root),
 379                         ('test', test_revlist_w_meta),
 380                         (save_time_str, expected_latest_item_w_meta),
 381                         ('file', expected_file_item_w_meta))
 382             wvpasseq(expected, res)
 383
 384             wvstart('resolve: /test/latest/bad-symlink')
 385             vfs.clear_cache()
 386             res = resolve(repo, '/test/latest/bad-symlink')
 387             wvpasseq(4, len(res))
 388             expected = (('', vfs._root),
 389                         ('test', test_revlist_w_meta),
 390                         (save_time_str, expected_latest_item_w_meta),
 391                         ('not-there', None))
 392             wvpasseq(expected, res)
 393
 394             wvstart('lresolve: /test/latest/bad-symlink')
 395             vfs.clear_cache()
 396             res = lresolve(repo, '/test/latest/bad-symlink')
 397             wvpasseq(4, len(res))
 398             bad_symlink_value = tip_tree['bad-symlink']
 399             expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta,
 400                                                         oid=bad_symlink_value.oid)
 401             expected = (('', vfs._root),
 402                         ('test', test_revlist_w_meta),
 403                         (save_time_str, expected_latest_item_w_meta),
 404                         ('bad-symlink', expected_bad_symlink_item_w_meta))
 405             wvpasseq(expected, res)
 406
 407             wvstart('resolve: /test/latest/file-symlink')
 408             vfs.clear_cache()
 409             res = resolve(repo, '/test/latest/file-symlink')
 410             wvpasseq(4, len(res))
 411             expected = (('', vfs._root),
 412                         ('test', test_revlist_w_meta),
 413                         (save_time_str, expected_latest_item_w_meta),
 414                         ('file', expected_file_item_w_meta))
 415             wvpasseq(expected, res)
 416
 417             wvstart('lresolve: /test/latest/file-symlink')
 418             vfs.clear_cache()
 419             res = lresolve(repo, '/test/latest/file-symlink')
 420             wvpasseq(4, len(res))
 421             file_symlink_value = tip_tree['file-symlink']
 422             expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta,
 423                                                          oid=file_symlink_value.oid)
 424             expected = (('', vfs._root),
 425                         ('test', test_revlist_w_meta),
 426                         (save_time_str, expected_latest_item_w_meta),
 427                         ('file-symlink', expected_file_symlink_item_w_meta))
 428             wvpasseq(expected, res)
 429
 430             wvstart('resolve: /test/latest/missing')
 431             vfs.clear_cache()
 432             res = resolve(repo, '/test/latest/missing')
 433             wvpasseq(4, len(res))
 434             name, item = res[-1]
 435             wvpasseq('missing', name)
 436             wvpass(item is None)
 437
 438             for path in ('/test/latest/file/',
 439                          '/test/latest/file/.',
 440                          '/test/latest/file/..',
 441                          '/test/latest/file/../',
 442                          '/test/latest/file/../.',
 443                          '/test/latest/file/../..',
 444                          '/test/latest/file/foo'):
 445                 wvstart('resolve: ' + path)
 446                 vfs.clear_cache()
 447                 try:
 448                     resolve(repo, path)
 449                 except vfs.IOError as res_ex:
 450                     wvpasseq(ENOTDIR, res_ex.errno)
 451                     wvpasseq(['', 'test', save_time_str, 'file'],
 452                              [name for name, item in res_ex.terminus])
 453
 454             for path in ('/test/latest/file-symlink/',
 455                          '/test/latest/file-symlink/.',
 456                          '/test/latest/file-symlink/..',
 457                          '/test/latest/file-symlink/../',
 458                          '/test/latest/file-symlink/../.',
 459                          '/test/latest/file-symlink/../..'):
 460                 wvstart('lresolve: ' + path)
 461                 vfs.clear_cache()
 462                 try:
 463                     lresolve(repo, path)
 464                 except vfs.IOError as res_ex:
 465                     wvpasseq(ENOTDIR, res_ex.errno)
 466                     wvpasseq(['', 'test', save_time_str, 'file'],
 467                              [name for name, item in res_ex.terminus])
 468
 469             wvstart('resolve: non-directory parent')
 470             vfs.clear_cache()
 471             file_res = resolve(repo, '/test/latest/file')
 472             try:
 473                 resolve(repo, 'foo', parent=file_res)
 474             except vfs.IOError as res_ex:
 475                 wvpasseq(ENOTDIR, res_ex.errno)
 476                 wvpasseq(None, res_ex.terminus)
 477
 478             wvstart('lresolve: /test/latest/dir-symlink')
 479             vfs.clear_cache()
 480             res = lresolve(repo, '/test/latest/dir-symlink')
 481             wvpasseq(4, len(res))
 482             dir_symlink_value = tip_tree['dir-symlink']
 483             expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta,
 484                                                          oid=dir_symlink_value.oid)
 485             expected = (('', vfs._root),
 486                         ('test', test_revlist_w_meta),
 487                         (save_time_str, expected_latest_item_w_meta),
 488                         ('dir-symlink', expected_dir_symlink_item_w_meta))
 489             wvpasseq(expected, res)
 490
 491             dir_value = tip_tree['dir']
 492             expected_dir_item = vfs.Item(oid=dir_value.oid,
 493                                          meta=tree_dict(repo, dir_value.oid)['.'].meta)
 494             expected = (('', vfs._root),
 495                         ('test', test_revlist_w_meta),
 496                         (save_time_str, expected_latest_item_w_meta),
 497                         ('dir', expected_dir_item))
 498             for resname, resolver in (('resolve', resolve),
 499                                       ('lresolve', lresolve)):
 500                 for path in ('/test/latest/dir-symlink/',
 501                              '/test/latest/dir-symlink/.'):
 502                     wvstart(resname + ': ' + path)
 503                     vfs.clear_cache()
 504                     res = resolver(repo, path)
 505                     wvpasseq(4, len(res))
 506                     wvpasseq(expected, res)
 507             wvstart('resolve: /test/latest/dir-symlink')
 508             vfs.clear_cache()
 509             res = resolve(repo, path)
 510             wvpasseq(4, len(res))
 511             wvpasseq(expected, res)
 512
 513 def write_sized_random_content(parent_dir, size, seed):
 514     verbose = 0
 515     with open('%s/%d' % (parent_dir, size), 'wb') as f:
 516         write_random(f.fileno(), size, seed, verbose)
 517
 518 def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
 519     for read_size in read_sizes:
 520         with open(expected_path, 'rb') as expected:
 521             with vfs.fopen(repo, item) as actual:
 522                 ex_buf = expected.read(read_size)
 523                 act_buf = actual.read(read_size)
 524                 while ex_buf and act_buf:
 525                     wvpassge(read_size, len(ex_buf))
 526                     wvpassge(read_size, len(act_buf))
 527                     wvpasseq(len(ex_buf), len(act_buf))
 528                     wvpass(ex_buf == act_buf)
 529                     ex_buf = expected.read(read_size)
 530                     act_buf = actual.read(read_size)
 531                 wvpasseq('', ex_buf)
 532                 wvpasseq('', act_buf)
 533
 534 def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
 535     def read_act(act_pos):
 536         with vfs.fopen(repo, item) as actual:
 537             actual.seek(act_pos)
 538             wvpasseq(act_pos, actual.tell())
 539             act_buf = actual.read(read_size)
 540             act_pos += len(act_buf)
 541             wvpasseq(act_pos, actual.tell())
 542             return act_pos, act_buf
 543
 544     for read_size in read_sizes:
 545         with open(expected_path, 'rb') as expected:
 546                 ex_buf = expected.read(read_size)
 547                 act_buf = None
 548                 act_pos = 0
 549                 while ex_buf:
 550                     act_pos, act_buf = read_act(act_pos)
 551                     wvpassge(read_size, len(ex_buf))
 552                     wvpassge(read_size, len(act_buf))
 553                     wvpasseq(len(ex_buf), len(act_buf))
 554                     wvpass(ex_buf == act_buf)
 555                     if not act_buf:
 556                         break
 557                     ex_buf = expected.read(read_size)
 558                 else:  # hit expected eof first
 559                     act_pos, act_buf = read_act(act_pos)
 560                 wvpasseq('', ex_buf)
 561                 wvpasseq('', act_buf)
 562
 563 @wvtest
 564 def test_read_and_seek():
 565     # Write a set of randomly sized files containing random data whose
 566     # names are their sizes, and then verify that what we get back
 567     # from the vfs when seeking and reading with various block sizes
 568     # matches the original content.
 569     with no_lingering_errors():
 570         with test_tempdir('bup-tvfs-read-') as tmpdir:
 571             resolve = vfs.resolve
 572             bup_dir = tmpdir + '/bup'
 573             environ['GIT_DIR'] = bup_dir
 574             environ['BUP_DIR'] = bup_dir
 575             git.repodir = bup_dir
 576             repo = LocalRepo()
 577             data_path = tmpdir + '/src'
 578             os.mkdir(data_path)
 579             seed = randint(-(1 << 31), (1 << 31) - 1)
 580             rand = Random()
 581             rand.seed(seed)
 582             print('test_read seed:', seed, file=sys.stderr)
 583             max_size = 2 * 1024 * 1024
 584             sizes = set((rand.randint(1, max_size) for _ in xrange(5)))
 585             sizes.add(1)
 586             sizes.add(max_size)
 587             for size in sizes:
 588                 write_sized_random_content(data_path, size, seed)
 589             ex((bup_path, 'init'))
 590             ex((bup_path, 'index', '-v', data_path))
 591             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 592                 data_path))
 593             read_sizes = set((rand.randint(1, max_size) for _ in xrange(10)))
 594             sizes.add(1)
 595             sizes.add(max_size)
 596             print('test_read src sizes:', sizes, file=sys.stderr)
 597             print('test_read read sizes:', read_sizes, file=sys.stderr)
 598             for size in sizes:
 599                 res = resolve(repo, '/test/latest/' + str(size))
 600                 _, item = res[-1]
 601                 wvpasseq(size, vfs.item_size(repo, res[-1][1]))
 602                 validate_vfs_streaming_read(repo, item,
 603                                             '%s/%d' % (data_path, size),
 604                                             read_sizes)
 605                 validate_vfs_seeking_read(repo, item,
 606                                           '%s/%d' % (data_path, size),
 607                                           read_sizes)
 608
 609 @wvtest
 610 def test_resolve_loop():
 611     with no_lingering_errors():
 612         with test_tempdir('bup-tvfs-resloop-') as tmpdir:
 613             resolve = vfs.resolve
 614             lresolve = vfs.lresolve
 615             bup_dir = tmpdir + '/bup'
 616             environ['GIT_DIR'] = bup_dir
 617             environ['BUP_DIR'] = bup_dir
 618             git.repodir = bup_dir
 619             repo = LocalRepo()
 620             data_path = tmpdir + '/src'
 621             os.mkdir(data_path)
 622             symlink('loop', data_path + '/loop')
 623             ex((bup_path, 'init'))
 624             ex((bup_path, 'index', '-v', data_path))
 625             save_utc = 100000
 626             ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip',
 627                 data_path))
 628             save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
 629             try:
 630                 resolve(repo, '/test/%s/loop' % save_utc)
 631             except vfs.IOError as res_ex:
 632                 wvpasseq(ELOOP, res_ex.errno)
 633                 wvpasseq(['', 'test', 'latest', 'loop'],
 634                          [name for name, item in res_ex.terminus])
 635
 636 @wvtest
 637 def test_contents_with_mismatched_bupm_git_ordering():
 638     with no_lingering_errors():
 639         with test_tempdir('bup-tvfs-') as tmpdir:
 640             bup_dir = tmpdir + '/bup'
 641             environ['GIT_DIR'] = bup_dir
 642             environ['BUP_DIR'] = bup_dir
 643             git.repodir = bup_dir
 644             data_path = tmpdir + '/src'
 645             os.mkdir(data_path)
 646             os.mkdir(data_path + '/foo')
 647             with open(data_path + '/foo.', 'w+') as tmpfile:
 648                 tmpfile.write(b'canary\n')
 649             ex((bup_path, 'init'))
 650             ex((bup_path, 'index', '-v', data_path))
 651             save_utc = 100000
 652             save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
 653             ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc),
 654                 '--strip', data_path))
 655             repo = LocalRepo()
 656             tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
 657             tip_oidx = tip_sref.strip().split()[0]
 658             tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
 659                                  tip_oidx))[0].strip()
 660             tip_tree_oid = tip_tree_oidx.decode('hex')
 661             tip_tree = tree_dict(repo, tip_tree_oid)
 662
 663             name, item = vfs.resolve(repo, '/test/latest')[2]
 664             wvpasseq(save_name, name)
 665             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
 666                                  for x in (tip_tree[name]
 667                                            for name in ('.', 'foo', 'foo.')))
 668             contents = tuple(vfs.contents(repo, item))
 669             wvpasseq(expected, frozenset(contents))
 670             # Spot check, in case tree_dict shares too much code with the vfs
 671             name, item = next(((n, i) for n, i in contents if n == 'foo'))
 672             wvpass(S_ISDIR(item.meta))
 673             name, item = next(((n, i) for n, i in contents if n == 'foo.'))
 674             wvpass(S_ISREG(item.meta.mode))
 675
 676 @wvtest
 677 def test_duplicate_save_dates():
 678     with no_lingering_errors():
 679         with test_tempdir('bup-tvfs-') as tmpdir:
 680             bup_dir = tmpdir + '/bup'
 681             environ['GIT_DIR'] = bup_dir
 682             environ['BUP_DIR'] = bup_dir
 683             environ['TZ'] = 'UTC'
 684             git.repodir = bup_dir
 685             data_path = tmpdir + '/src'
 686             os.mkdir(data_path)
 687             with open(data_path + '/file', 'w+') as tmpfile:
 688                 tmpfile.write(b'canary\n')
 689             ex((bup_path, 'init'))
 690             ex((bup_path, 'index', '-v', data_path))
 691             for i in range(11):
 692                 ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
 693             repo = LocalRepo()
 694             res = vfs.resolve(repo, '/test')
 695             wvpasseq(2, len(res))
 696             name, revlist = res[-1]
 697             wvpasseq('test', name)
 698             wvpasseq(('.',
 699                       '1970-01-02-034640-00',
 700                       '1970-01-02-034640-01',
 701                       '1970-01-02-034640-02',
 702                       '1970-01-02-034640-03',
 703                       '1970-01-02-034640-04',
 704                       '1970-01-02-034640-05',
 705                       '1970-01-02-034640-06',
 706                       '1970-01-02-034640-07',
 707                       '1970-01-02-034640-08',
 708                       '1970-01-02-034640-09',
 709                       '1970-01-02-034640-10',
 710                       'latest'),
 711                      tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
 712
 713 # FIXME: add tests for the want_meta=False cases.