lib/bup/t/tvfs.py

   1
   2 from __future__ import absolute_import, print_function
   3 from collections import namedtuple
   4 from errno import ELOOP, ENOTDIR
   5 from io import BytesIO
   6 from os import environ, symlink
   7 from random import Random, randint
   8 from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG
   9 from sys import stderr
  10 from time import localtime, strftime
  11
  12 from wvtest import *
  13
  14 from bup._helpers import write_random
  15 from bup import git, metadata, vfs
  16 from bup.git import BUP_CHUNKED
  17 from bup.helpers import exc, exo, shstr
  18 from bup.metadata import Metadata
  19 from bup.repo import LocalRepo
  20 from buptest import no_lingering_errors, test_tempdir
  21
  22 top_dir = '../../..'
  23 bup_tmp = os.path.realpath('../../../t/tmp')
  24 bup_path = top_dir + '/bup'
  25 start_dir = os.getcwd()
  26
  27 def ex(cmd, **kwargs):
  28     print(shstr(cmd), file=stderr)
  29     return exc(cmd, **kwargs)
  30
  31 @wvtest
  32 def test_cache_behavior():
  33     orig_max = vfs._cache_max_items
  34     try:
  35         vfs._cache_max_items = 2
  36         vfs.clear_cache()
  37         wvpasseq({}, vfs._cache)
  38         wvpasseq([], vfs._cache_keys)
  39         wvfail(vfs._cache_keys)
  40         wvexcept(AssertionError, vfs.cache_notice, 'x', 1)
  41         key_0 = b'\0' * 20
  42         key_1 = b'\1' * 20
  43         key_2 = b'\2' * 20
  44         vfs.cache_notice(key_0, 'something')
  45         wvpasseq({key_0 : 'something'}, vfs._cache)
  46         wvpasseq([key_0], vfs._cache_keys)
  47         vfs.cache_notice(key_1, 'something else')
  48         wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache)
  49         wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
  50         vfs.cache_notice(key_2, 'and also')
  51         wvpasseq(2, len(vfs._cache))
  52         wvpass(frozenset(vfs._cache.iteritems())
  53                < frozenset({key_0 : 'something',
  54                             key_1 : 'something else',
  55                             key_2 : 'and also'}.iteritems()))
  56         wvpasseq(2, len(vfs._cache_keys))
  57         wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
  58         vfs.clear_cache()
  59         wvpasseq({}, vfs._cache)
  60         wvpasseq([], vfs._cache_keys)
  61     finally:
  62         vfs._cache_max_items = orig_max
  63         vfs.clear_cache()
  64
  65 ## The clear_cache() calls below are to make sure that the test starts
  66 ## from a known state since at the moment the cache entry for a given
  67 ## item (like a commit) can change.  For example, its meta value might
  68 ## be promoted from a mode to a Metadata instance once the tree it
  69 ## refers to is traversed.
  70
  71 TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
  72
  73 def tree_items(repo, oid):
  74     """Yield (name, entry_oid, meta) for each entry in oid.  meta will be
  75     a Metadata object for any non-directories and for '.', otherwise
  76     None.
  77
  78     """
  79     # This is a simpler approach than the one in the vfs, used to
  80     # cross-check its behavior.
  81     tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid)
  82     bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
  83     try:
  84         maybe_meta = lambda : Metadata.read(bupm) if bupm else None
  85         m = maybe_meta()
  86         if m:
  87             m.size = 0
  88         yield TreeDictValue(name='.', oid=oid, meta=m)
  89         tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
  90         for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
  91             if mangled_name == '.bupm':
  92                 continue
  93             assert name != '.'
  94             if S_ISDIR(gitmode):
  95                 if kind == BUP_CHUNKED:
  96                     yield TreeDictValue(name=name, oid=sub_oid,
  97                                         meta=maybe_meta())
  98                 else:
  99                     yield TreeDictValue(name=name, oid=sub_oid,
 100                                         meta=vfs.default_dir_mode)
 101             else:
 102                 yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
 103     finally:
 104         if bupm:
 105             bupm.close()
 106
 107 def tree_dict(repo, oid):
 108     return dict((x.name, x) for x in tree_items(repo, oid))
 109
 110 def run_augment_item_meta_tests(repo,
 111                                 file_path, file_size,
 112                                 link_path, link_target):
 113     _, file_item = vfs.resolve(repo, file_path)[-1]
 114     _, link_item = vfs.lresolve(repo, link_path)[-1]
 115     wvpass(isinstance(file_item.meta, Metadata))
 116     wvpass(isinstance(link_item.meta, Metadata))
 117     # Note: normally, modifying item.meta values is forbidden
 118     file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
 119     link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
 120
 121     ## Ensure a fully populated item is left alone
 122     augmented = vfs.augment_item_meta(repo, file_item)
 123     wvpass(augmented is file_item)
 124     wvpass(augmented.meta is file_item.meta)
 125     augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
 126     wvpass(augmented is file_item)
 127     wvpass(augmented.meta is file_item.meta)
 128
 129     ## Ensure a missing size is handled poperly
 130     file_item.meta.size = None
 131     augmented = vfs.augment_item_meta(repo, file_item)
 132     wvpass(augmented is file_item)
 133     wvpass(augmented.meta is file_item.meta)
 134     augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
 135     wvpass(augmented is not file_item)
 136     wvpasseq(file_size, augmented.meta.size)
 137
 138     ## Ensure a meta mode is handled properly
 139     mode_item = file_item._replace(meta=vfs.default_file_mode)
 140     augmented = vfs.augment_item_meta(repo, mode_item)
 141     augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
 142     for item in (augmented, augmented_w_size):
 143         meta = item.meta
 144         wvpass(item is not file_item)
 145         wvpass(isinstance(meta, Metadata))
 146         wvpasseq(vfs.default_file_mode, meta.mode)
 147         wvpasseq((0, 0, 0, 0, 0),
 148                  (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
 149     wvpass(augmented.meta.size is None)
 150     wvpasseq(file_size, augmented_w_size.meta.size)
 151
 152     ## Ensure symlinks are handled properly
 153     mode_item = link_item._replace(meta=vfs.default_symlink_mode)
 154     augmented = vfs.augment_item_meta(repo, mode_item)
 155     wvpass(augmented is not mode_item)
 156     wvpass(isinstance(augmented.meta, Metadata))
 157     wvpasseq(link_target, augmented.meta.symlink_target)
 158     wvpasseq(len(link_target), augmented.meta.size)
 159     augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
 160     wvpass(augmented is not mode_item)
 161     wvpass(isinstance(augmented.meta, Metadata))
 162     wvpasseq(link_target, augmented.meta.symlink_target)
 163     wvpasseq(len(link_target), augmented.meta.size)
 164
 165
 166 @wvtest
 167 def test_item_mode():
 168     with no_lingering_errors():
 169         mode = S_IFDIR | 0o755
 170         meta = metadata.from_path('.')
 171         oid = '\0' * 20
 172         wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
 173         wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
 174
 175 @wvtest
 176 def test_reverse_suffix_duplicates():
 177     suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x))
 178     wvpasseq(('x',), suffix(('x',)))
 179     wvpasseq(('x', 'y'), suffix(('x', 'y')))
 180     wvpasseq(('x-1', 'x-0'), suffix(('x',) * 2))
 181     wvpasseq(['x-%02d' % n for n in reversed(range(11))],
 182              list(suffix(('x',) * 11)))
 183     wvpasseq(('x-1', 'x-0', 'y'), suffix(('x', 'x', 'y')))
 184     wvpasseq(('x', 'y-1', 'y-0'), suffix(('x', 'y', 'y')))
 185     wvpasseq(('x', 'y-1', 'y-0', 'z'), suffix(('x', 'y', 'y', 'z')))
 186
 187 @wvtest
 188 def test_misc():
 189     with no_lingering_errors():
 190         with test_tempdir('bup-tvfs-') as tmpdir:
 191             bup_dir = tmpdir + '/bup'
 192             environ['GIT_DIR'] = bup_dir
 193             environ['BUP_DIR'] = bup_dir
 194             git.repodir = bup_dir
 195             data_path = tmpdir + '/src'
 196             os.mkdir(data_path)
 197             with open(data_path + '/file', 'w+') as tmpfile:
 198                 tmpfile.write(b'canary\n')
 199             symlink('file', data_path + '/symlink')
 200             ex((bup_path, 'init'))
 201             ex((bup_path, 'index', '-v', data_path))
 202             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 203                 data_path))
 204             repo = LocalRepo()
 205
 206             wvstart('readlink')
 207             ls_tree = exo(('git', 'ls-tree', 'test', 'symlink'))
 208             mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
 209             assert name == 'symlink'
 210             link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
 211             wvpasseq('file', vfs.readlink(repo, link_item))
 212
 213             ls_tree = exo(('git', 'ls-tree', 'test', 'file'))
 214             mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
 215             assert name == 'file'
 216             file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
 217             wvexcept(Exception, vfs.readlink, repo, file_item)
 218
 219             wvstart('item_size')
 220             wvpasseq(4, vfs.item_size(repo, link_item))
 221             wvpasseq(7, vfs.item_size(repo, file_item))
 222             meta = metadata.from_path(__file__)
 223             meta.size = 42
 224             fake_item = file_item._replace(meta=meta)
 225             wvpasseq(42, vfs.item_size(repo, fake_item))
 226
 227             wvstart('augment_item_meta')
 228             run_augment_item_meta_tests(repo,
 229                                         '/test/latest/file', 7,
 230                                         '/test/latest/symlink', 'file')
 231
 232             wvstart('copy_item')
 233             # FIXME: this caused StopIteration
 234             #_, file_item = vfs.resolve(repo, '/file')[-1]
 235             _, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
 236             file_copy = vfs.copy_item(file_item)
 237             wvpass(file_copy is not file_item)
 238             wvpass(file_copy.meta is not file_item.meta)
 239             wvpass(isinstance(file_copy, tuple))
 240             wvpass(file_item.meta.user)
 241             wvpass(file_copy.meta.user)
 242             file_copy.meta.user = None
 243             wvpass(file_item.meta.user)
 244
 245 @wvtest
 246 def test_resolve():
 247     with no_lingering_errors():
 248         with test_tempdir('bup-tvfs-') as tmpdir:
 249             resolve = vfs.resolve
 250             lresolve = vfs.lresolve
 251             bup_dir = tmpdir + '/bup'
 252             environ['GIT_DIR'] = bup_dir
 253             environ['BUP_DIR'] = bup_dir
 254             git.repodir = bup_dir
 255             data_path = tmpdir + '/src'
 256             save_time = 100000
 257             save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time))
 258             os.mkdir(data_path)
 259             os.mkdir(data_path + '/dir')
 260             with open(data_path + '/file', 'w+') as tmpfile:
 261                 print('canary', file=tmpfile)
 262             symlink('file', data_path + '/file-symlink')
 263             symlink('dir', data_path + '/dir-symlink')
 264             symlink('not-there', data_path + '/bad-symlink')
 265             ex((bup_path, 'init'))
 266             ex((bup_path, 'index', '-v', data_path))
 267             ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test',
 268                 '--strip', data_path))
 269             ex((bup_path, 'tag', 'test-tag', 'test'))
 270             repo = LocalRepo()
 271
 272             tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0]
 273             tip_oidx = tip_hash.strip().split()[0]
 274             tip_oid = tip_oidx.decode('hex')
 275             tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
 276                                  tip_oidx))[0].strip()
 277             tip_tree_oid = tip_tree_oidx.decode('hex')
 278             tip_tree = tree_dict(repo, tip_tree_oid)
 279             test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta,
 280                                               oid=tip_oid)
 281             expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755,
 282                                               oid=tip_tree_oid,
 283                                               coid=tip_oid)
 284             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
 285                                                      oid=tip_tree_oid,
 286                                                      coid=tip_oid)
 287             expected_test_tag_item = expected_latest_item
 288
 289             wvstart('resolve: /')
 290             vfs.clear_cache()
 291             res = resolve(repo, '/')
 292             wvpasseq(1, len(res))
 293             wvpasseq((('', vfs._root),), res)
 294             ignore, root_item = res[0]
 295             root_content = frozenset(vfs.contents(repo, root_item))
 296             wvpasseq(frozenset([('.', root_item),
 297                                 ('.tag', vfs._tags),
 298                                 ('test', test_revlist_w_meta)]),
 299                      root_content)
 300             for path in ('//', '/.', '/./', '/..', '/../',
 301                          '/test/latest/dir/../../..',
 302                          '/test/latest/dir/../../../',
 303                          '/test/latest/dir/../../../.',
 304                          '/test/latest/dir/../../..//',
 305                          '/test//latest/dir/../../..',
 306                          '/test/./latest/dir/../../..',
 307                          '/test/././latest/dir/../../..',
 308                          '/test/.//./latest/dir/../../..',
 309                          '/test//.//.//latest/dir/../../..'
 310                          '/test//./latest/dir/../../..'):
 311                 wvstart('resolve: ' + path)
 312                 vfs.clear_cache()
 313                 res = resolve(repo, path)
 314                 wvpasseq((('', vfs._root),), res)
 315
 316             wvstart('resolve: /.tag')
 317             vfs.clear_cache()
 318             res = resolve(repo, '/.tag')
 319             wvpasseq(2, len(res))
 320             wvpasseq((('', vfs._root), ('.tag', vfs._tags)),
 321                      res)
 322             ignore, tag_item = res[1]
 323             tag_content = frozenset(vfs.contents(repo, tag_item))
 324             wvpasseq(frozenset([('.', tag_item),
 325                                 ('test-tag', expected_test_tag_item)]),
 326                      tag_content)
 327
 328             wvstart('resolve: /test')
 329             vfs.clear_cache()
 330             res = resolve(repo, '/test')
 331             wvpasseq(2, len(res))
 332             wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res)
 333             ignore, test_item = res[1]
 334             test_content = frozenset(vfs.contents(repo, test_item))
 335             # latest has metadata here due to caching
 336             wvpasseq(frozenset([('.', test_revlist_w_meta),
 337                                 (save_time_str, expected_latest_item_w_meta),
 338                                 ('latest', expected_latest_item_w_meta)]),
 339                      test_content)
 340
 341             wvstart('resolve: /test/latest')
 342             vfs.clear_cache()
 343             res = resolve(repo, '/test/latest')
 344             wvpasseq(3, len(res))
 345             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
 346                                                      oid=tip_tree_oid,
 347                                                      coid=tip_oid)
 348             expected = (('', vfs._root),
 349                         ('test', test_revlist_w_meta),
 350                         ('latest', expected_latest_item_w_meta))
 351             wvpasseq(expected, res)
 352             ignore, latest_item = res[2]
 353             latest_content = frozenset(vfs.contents(repo, latest_item))
 354             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
 355                                  for x in (tip_tree[name]
 356                                            for name in ('.',
 357                                                         'bad-symlink',
 358                                                         'dir',
 359                                                         'dir-symlink',
 360                                                         'file',
 361                                                         'file-symlink')))
 362             wvpasseq(expected, latest_content)
 363
 364             wvstart('resolve: /test/latest/file')
 365             vfs.clear_cache()
 366             res = resolve(repo, '/test/latest/file')
 367             wvpasseq(4, len(res))
 368             expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta,
 369                                                  oid=tip_tree['file'].oid)
 370             expected = (('', vfs._root),
 371                         ('test', test_revlist_w_meta),
 372                         ('latest', expected_latest_item_w_meta),
 373                         ('file', expected_file_item_w_meta))
 374             wvpasseq(expected, res)
 375
 376             wvstart('resolve: /test/latest/bad-symlink')
 377             vfs.clear_cache()
 378             res = resolve(repo, '/test/latest/bad-symlink')
 379             wvpasseq(4, len(res))
 380             expected = (('', vfs._root),
 381                         ('test', test_revlist_w_meta),
 382                         ('latest', expected_latest_item_w_meta),
 383                         ('not-there', None))
 384             wvpasseq(expected, res)
 385
 386             wvstart('lresolve: /test/latest/bad-symlink')
 387             vfs.clear_cache()
 388             res = lresolve(repo, '/test/latest/bad-symlink')
 389             wvpasseq(4, len(res))
 390             bad_symlink_value = tip_tree['bad-symlink']
 391             expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta,
 392                                                         oid=bad_symlink_value.oid)
 393             expected = (('', vfs._root),
 394                         ('test', test_revlist_w_meta),
 395                         ('latest', expected_latest_item_w_meta),
 396                         ('bad-symlink', expected_bad_symlink_item_w_meta))
 397             wvpasseq(expected, res)
 398
 399             wvstart('resolve: /test/latest/file-symlink')
 400             vfs.clear_cache()
 401             res = resolve(repo, '/test/latest/file-symlink')
 402             wvpasseq(4, len(res))
 403             expected = (('', vfs._root),
 404                         ('test', test_revlist_w_meta),
 405                         ('latest', expected_latest_item_w_meta),
 406                         ('file', expected_file_item_w_meta))
 407             wvpasseq(expected, res)
 408
 409             wvstart('lresolve: /test/latest/file-symlink')
 410             vfs.clear_cache()
 411             res = lresolve(repo, '/test/latest/file-symlink')
 412             wvpasseq(4, len(res))
 413             file_symlink_value = tip_tree['file-symlink']
 414             expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta,
 415                                                          oid=file_symlink_value.oid)
 416             expected = (('', vfs._root),
 417                         ('test', test_revlist_w_meta),
 418                         ('latest', expected_latest_item_w_meta),
 419                         ('file-symlink', expected_file_symlink_item_w_meta))
 420             wvpasseq(expected, res)
 421
 422             wvstart('resolve: /test/latest/missing')
 423             vfs.clear_cache()
 424             res = resolve(repo, '/test/latest/missing')
 425             wvpasseq(4, len(res))
 426             name, item = res[-1]
 427             wvpasseq('missing', name)
 428             wvpass(item is None)
 429
 430             for path in ('/test/latest/file/',
 431                          '/test/latest/file/.',
 432                          '/test/latest/file/..',
 433                          '/test/latest/file/../',
 434                          '/test/latest/file/../.',
 435                          '/test/latest/file/../..',
 436                          '/test/latest/file/foo'):
 437                 wvstart('resolve: ' + path)
 438                 vfs.clear_cache()
 439                 try:
 440                     resolve(repo, path)
 441                 except vfs.IOError as res_ex:
 442                     wvpasseq(ENOTDIR, res_ex.errno)
 443                     wvpasseq(['', 'test', 'latest', 'file'],
 444                              [name for name, item in res_ex.terminus])
 445
 446             for path in ('/test/latest/file-symlink/',
 447                          '/test/latest/file-symlink/.',
 448                          '/test/latest/file-symlink/..',
 449                          '/test/latest/file-symlink/../',
 450                          '/test/latest/file-symlink/../.',
 451                          '/test/latest/file-symlink/../..'):
 452                 wvstart('lresolve: ' + path)
 453                 vfs.clear_cache()
 454                 try:
 455                     lresolve(repo, path)
 456                 except vfs.IOError as res_ex:
 457                     wvpasseq(ENOTDIR, res_ex.errno)
 458                     wvpasseq(['', 'test', 'latest', 'file'],
 459                              [name for name, item in res_ex.terminus])
 460
 461             wvstart('resolve: non-directory parent')
 462             vfs.clear_cache()
 463             file_res = resolve(repo, '/test/latest/file')
 464             try:
 465                 resolve(repo, 'foo', parent=file_res)
 466             except vfs.IOError as res_ex:
 467                 wvpasseq(ENOTDIR, res_ex.errno)
 468                 wvpasseq(None, res_ex.terminus)
 469
 470             wvstart('lresolve: /test/latest/dir-symlink')
 471             vfs.clear_cache()
 472             res = lresolve(repo, '/test/latest/dir-symlink')
 473             wvpasseq(4, len(res))
 474             dir_symlink_value = tip_tree['dir-symlink']
 475             expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta,
 476                                                          oid=dir_symlink_value.oid)
 477             expected = (('', vfs._root),
 478                         ('test', test_revlist_w_meta),
 479                         ('latest', expected_latest_item_w_meta),
 480                         ('dir-symlink', expected_dir_symlink_item_w_meta))
 481             wvpasseq(expected, res)
 482
 483             dir_value = tip_tree['dir']
 484             expected_dir_item = vfs.Item(oid=dir_value.oid,
 485                                          meta=tree_dict(repo, dir_value.oid)['.'].meta)
 486             expected = (('', vfs._root),
 487                         ('test', test_revlist_w_meta),
 488                         ('latest', expected_latest_item_w_meta),
 489                         ('dir', expected_dir_item))
 490             for resname, resolver in (('resolve', resolve),
 491                                       ('lresolve', lresolve)):
 492                 for path in ('/test/latest/dir-symlink/',
 493                              '/test/latest/dir-symlink/.'):
 494                     wvstart(resname + ': ' + path)
 495                     vfs.clear_cache()
 496                     res = resolver(repo, path)
 497                     wvpasseq(4, len(res))
 498                     wvpasseq(expected, res)
 499             wvstart('resolve: /test/latest/dir-symlink')
 500             vfs.clear_cache()
 501             res = resolve(repo, path)
 502             wvpasseq(4, len(res))
 503             wvpasseq(expected, res)
 504
 505 def write_sized_random_content(parent_dir, size, seed):
 506     verbose = 0
 507     with open('%s/%d' % (parent_dir, size), 'wb') as f:
 508         write_random(f.fileno(), size, seed, verbose)
 509
 510 def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
 511     for read_size in read_sizes:
 512         with open(expected_path, 'rb') as expected:
 513             with vfs.fopen(repo, item) as actual:
 514                 ex_buf = expected.read(read_size)
 515                 act_buf = actual.read(read_size)
 516                 while ex_buf and act_buf:
 517                     wvpassge(read_size, len(ex_buf))
 518                     wvpassge(read_size, len(act_buf))
 519                     wvpasseq(len(ex_buf), len(act_buf))
 520                     wvpass(ex_buf == act_buf)
 521                     ex_buf = expected.read(read_size)
 522                     act_buf = actual.read(read_size)
 523                 wvpasseq('', ex_buf)
 524                 wvpasseq('', act_buf)
 525
 526 def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
 527     def read_act(act_pos):
 528         with vfs.fopen(repo, item) as actual:
 529             actual.seek(act_pos)
 530             wvpasseq(act_pos, actual.tell())
 531             act_buf = actual.read(read_size)
 532             act_pos += len(act_buf)
 533             wvpasseq(act_pos, actual.tell())
 534             return act_pos, act_buf
 535
 536     for read_size in read_sizes:
 537         with open(expected_path, 'rb') as expected:
 538                 ex_buf = expected.read(read_size)
 539                 act_buf = None
 540                 act_pos = 0
 541                 while ex_buf:
 542                     act_pos, act_buf = read_act(act_pos)
 543                     wvpassge(read_size, len(ex_buf))
 544                     wvpassge(read_size, len(act_buf))
 545                     wvpasseq(len(ex_buf), len(act_buf))
 546                     wvpass(ex_buf == act_buf)
 547                     if not act_buf:
 548                         break
 549                     ex_buf = expected.read(read_size)
 550                 else:  # hit expected eof first
 551                     act_pos, act_buf = read_act(act_pos)
 552                 wvpasseq('', ex_buf)
 553                 wvpasseq('', act_buf)
 554
 555 @wvtest
 556 def test_read_and_seek():
 557     # Write a set of randomly sized files containing random data whose
 558     # names are their sizes, and then verify that what we get back
 559     # from the vfs when seeking and reading with various block sizes
 560     # matches the original content.
 561     with no_lingering_errors():
 562         with test_tempdir('bup-tvfs-read-') as tmpdir:
 563             resolve = vfs.resolve
 564             bup_dir = tmpdir + '/bup'
 565             environ['GIT_DIR'] = bup_dir
 566             environ['BUP_DIR'] = bup_dir
 567             git.repodir = bup_dir
 568             repo = LocalRepo()
 569             data_path = tmpdir + '/src'
 570             os.mkdir(data_path)
 571             seed = randint(-(1 << 31), (1 << 31) - 1)
 572             rand = Random()
 573             rand.seed(seed)
 574             print('test_read seed:', seed, file=sys.stderr)
 575             max_size = 2 * 1024 * 1024
 576             sizes = set((rand.randint(1, max_size) for _ in xrange(5)))
 577             sizes.add(1)
 578             sizes.add(max_size)
 579             for size in sizes:
 580                 write_sized_random_content(data_path, size, seed)
 581             ex((bup_path, 'init'))
 582             ex((bup_path, 'index', '-v', data_path))
 583             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 584                 data_path))
 585             read_sizes = set((rand.randint(1, max_size) for _ in xrange(10)))
 586             sizes.add(1)
 587             sizes.add(max_size)
 588             print('test_read src sizes:', sizes, file=sys.stderr)
 589             print('test_read read sizes:', read_sizes, file=sys.stderr)
 590             for size in sizes:
 591                 res = resolve(repo, '/test/latest/' + str(size))
 592                 _, item = res[-1]
 593                 wvpasseq(size, vfs.item_size(repo, res[-1][1]))
 594                 validate_vfs_streaming_read(repo, item,
 595                                             '%s/%d' % (data_path, size),
 596                                             read_sizes)
 597                 validate_vfs_seeking_read(repo, item,
 598                                           '%s/%d' % (data_path, size),
 599                                           read_sizes)
 600
 601 @wvtest
 602 def test_resolve_loop():
 603     with no_lingering_errors():
 604         with test_tempdir('bup-tvfs-resloop-') as tmpdir:
 605             resolve = vfs.resolve
 606             lresolve = vfs.lresolve
 607             bup_dir = tmpdir + '/bup'
 608             environ['GIT_DIR'] = bup_dir
 609             environ['BUP_DIR'] = bup_dir
 610             git.repodir = bup_dir
 611             repo = LocalRepo()
 612             data_path = tmpdir + '/src'
 613             os.mkdir(data_path)
 614             symlink('loop', data_path + '/loop')
 615             ex((bup_path, 'init'))
 616             ex((bup_path, 'index', '-v', data_path))
 617             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 618                 data_path))
 619             try:
 620                 resolve(repo, '/test/latest/loop')
 621             except vfs.IOError as res_ex:
 622                 wvpasseq(ELOOP, res_ex.errno)
 623                 wvpasseq(['', 'test', 'latest', 'loop'],
 624                          [name for name, item in res_ex.terminus])
 625
 626 @wvtest
 627 def test_contents_with_mismatched_bupm_git_ordering():
 628     with no_lingering_errors():
 629         with test_tempdir('bup-tvfs-') as tmpdir:
 630             bup_dir = tmpdir + '/bup'
 631             environ['GIT_DIR'] = bup_dir
 632             environ['BUP_DIR'] = bup_dir
 633             git.repodir = bup_dir
 634             data_path = tmpdir + '/src'
 635             os.mkdir(data_path)
 636             os.mkdir(data_path + '/foo')
 637             with open(data_path + '/foo.', 'w+') as tmpfile:
 638                 tmpfile.write(b'canary\n')
 639             ex((bup_path, 'init'))
 640             ex((bup_path, 'index', '-v', data_path))
 641             ex((bup_path, 'save', '-tvvn', 'test', '--strip',
 642                 data_path))
 643             repo = LocalRepo()
 644             tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
 645             tip_oidx = tip_sref.strip().split()[0]
 646             tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
 647                                  tip_oidx))[0].strip()
 648             tip_tree_oid = tip_tree_oidx.decode('hex')
 649             tip_tree = tree_dict(repo, tip_tree_oid)
 650
 651             name, item = vfs.resolve(repo, '/test/latest')[2]
 652             wvpasseq('latest', name)
 653             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
 654                                  for x in (tip_tree[name]
 655                                            for name in ('.', 'foo', 'foo.')))
 656             contents = tuple(vfs.contents(repo, item))
 657             wvpasseq(expected, frozenset(contents))
 658             # Spot check, in case tree_dict shares too much code with the vfs
 659             name, item = next(((n, i) for n, i in contents if n == 'foo'))
 660             wvpass(S_ISDIR(item.meta))
 661             name, item = next(((n, i) for n, i in contents if n == 'foo.'))
 662             wvpass(S_ISREG(item.meta.mode))
 663
 664 @wvtest
 665 def test_duplicate_save_dates():
 666     with no_lingering_errors():
 667         with test_tempdir('bup-tvfs-') as tmpdir:
 668             bup_dir = tmpdir + '/bup'
 669             environ['GIT_DIR'] = bup_dir
 670             environ['BUP_DIR'] = bup_dir
 671             environ['TZ'] = 'UTC'
 672             git.repodir = bup_dir
 673             data_path = tmpdir + '/src'
 674             os.mkdir(data_path)
 675             with open(data_path + '/file', 'w+') as tmpfile:
 676                 tmpfile.write(b'canary\n')
 677             ex((bup_path, 'init'))
 678             ex((bup_path, 'index', '-v', data_path))
 679             for i in range(11):
 680                 ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
 681             repo = LocalRepo()
 682             res = vfs.resolve(repo, '/test')
 683             wvpasseq(2, len(res))
 684             name, revlist = res[-1]
 685             wvpasseq('test', name)
 686             wvpasseq(('.',
 687                       '1970-01-02-034640-00',
 688                       '1970-01-02-034640-01',
 689                       '1970-01-02-034640-02',
 690                       '1970-01-02-034640-03',
 691                       '1970-01-02-034640-04',
 692                       '1970-01-02-034640-05',
 693                       '1970-01-02-034640-06',
 694                       '1970-01-02-034640-07',
 695                       '1970-01-02-034640-08',
 696                       '1970-01-02-034640-09',
 697                       '1970-01-02-034640-10',
 698                       'latest'),
 699                      tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
 700
 701 # FIXME: add tests for the want_meta=False cases.