lib/bup/t/tvfs.py

   1
   2 from __future__ import absolute_import, print_function
   3 from collections import namedtuple
   4 from errno import ELOOP, ENOTDIR
   5 from io import BytesIO
   6 from os import environ, symlink
   7 from random import Random, randint
   8 from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG
   9 from sys import stderr
  10 from time import localtime, strftime
  11
  12 from wvtest import *
  13
  14 from bup._helpers import write_random
  15 from bup import git, metadata, vfs
  16 from bup.git import BUP_CHUNKED
  17 from bup.helpers import exc, exo, shstr
  18 from bup.metadata import Metadata
  19 from bup.repo import LocalRepo
  20 from buptest import no_lingering_errors, test_tempdir
  21
  22 top_dir = '../../..'
  23 bup_tmp = os.path.realpath('../../../t/tmp')
  24 bup_path = top_dir + '/bup'
  25 start_dir = os.getcwd()
  26
  27 def ex(cmd, **kwargs):
  28     print(shstr(cmd), file=stderr)
  29     return exc(cmd, **kwargs)
  30
  31 @wvtest
  32 def test_default_modes():
  33     wvpasseq(S_IFREG | 0o644, vfs.default_file_mode)
  34     wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode)
  35     wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode)
  36
  37 @wvtest
  38 def test_cache_behavior():
  39     orig_max = vfs._cache_max_items
  40     try:
  41         vfs._cache_max_items = 2
  42         vfs.clear_cache()
  43         wvpasseq({}, vfs._cache)
  44         wvpasseq([], vfs._cache_keys)
  45         wvfail(vfs._cache_keys)
  46         wvexcept(Exception, vfs.cache_notice, 'x', 1)
  47         key_0 = 'itm:' + b'\0' * 20
  48         key_1 = 'itm:' + b'\1' * 20
  49         key_2 = 'itm:' + b'\2' * 20
  50         vfs.cache_notice(key_0, 'something')
  51         wvpasseq({key_0 : 'something'}, vfs._cache)
  52         wvpasseq([key_0], vfs._cache_keys)
  53         vfs.cache_notice(key_1, 'something else')
  54         wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache)
  55         wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
  56         vfs.cache_notice(key_2, 'and also')
  57         wvpasseq(2, len(vfs._cache))
  58         wvpass(frozenset(vfs._cache.iteritems())
  59                < frozenset({key_0 : 'something',
  60                             key_1 : 'something else',
  61                             key_2 : 'and also'}.iteritems()))
  62         wvpasseq(2, len(vfs._cache_keys))
  63         wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
  64         vfs.clear_cache()
  65         wvpasseq({}, vfs._cache)
  66         wvpasseq([], vfs._cache_keys)
  67     finally:
  68         vfs._cache_max_items = orig_max
  69         vfs.clear_cache()
  70
  71 ## The clear_cache() calls below are to make sure that the test starts
  72 ## from a known state since at the moment the cache entry for a given
  73 ## item (like a commit) can change.  For example, its meta value might
  74 ## be promoted from a mode to a Metadata instance once the tree it
  75 ## refers to is traversed.
  76
  77 TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
  78
  79 def tree_items(repo, oid):
  80     """Yield (name, entry_oid, meta) for each entry in oid.  meta will be
  81     a Metadata object for any non-directories and for '.', otherwise
  82     None.
  83
  84     """
  85     # This is a simpler approach than the one in the vfs, used to
  86     # cross-check its behavior.
  87     tree_data, bupm_oid = vfs.tree_data_and_bupm(repo, oid)
  88     bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
  89     try:
  90         maybe_meta = lambda : Metadata.read(bupm) if bupm else None
  91         m = maybe_meta()
  92         if m and m.size is None:
  93             m.size = 0
  94         yield TreeDictValue(name='.', oid=oid, meta=m)
  95         tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
  96         for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
  97             if mangled_name == '.bupm':
  98                 continue
  99             assert name != '.'
 100             if S_ISDIR(gitmode):
 101                 if kind == BUP_CHUNKED:
 102                     yield TreeDictValue(name=name, oid=sub_oid,
 103                                         meta=maybe_meta())
 104                 else:
 105                     yield TreeDictValue(name=name, oid=sub_oid,
 106                                         meta=vfs.default_dir_mode)
 107             else:
 108                 yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
 109     finally:
 110         if bupm:
 111             bupm.close()
 112
 113 def tree_dict(repo, oid):
 114     return dict((x.name, x) for x in tree_items(repo, oid))
 115
 116 def run_augment_item_meta_tests(repo,
 117                                 file_path, file_size,
 118                                 link_path, link_target):
 119     _, file_item = vfs.resolve(repo, file_path)[-1]
 120     _, link_item = vfs.resolve(repo, link_path, follow=False)[-1]
 121     wvpass(isinstance(file_item.meta, Metadata))
 122     wvpass(isinstance(link_item.meta, Metadata))
 123     # Note: normally, modifying item.meta values is forbidden
 124     file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
 125     link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
 126
 127     ## Ensure a fully populated item is left alone
 128     augmented = vfs.augment_item_meta(repo, file_item)
 129     wvpass(augmented is file_item)
 130     wvpass(augmented.meta is file_item.meta)
 131     augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
 132     wvpass(augmented is file_item)
 133     wvpass(augmented.meta is file_item.meta)
 134
 135     ## Ensure a missing size is handled poperly
 136     file_item.meta.size = None
 137     augmented = vfs.augment_item_meta(repo, file_item)
 138     wvpass(augmented is file_item)
 139     wvpass(augmented.meta is file_item.meta)
 140     augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
 141     wvpass(augmented is not file_item)
 142     wvpasseq(file_size, augmented.meta.size)
 143
 144     ## Ensure a meta mode is handled properly
 145     mode_item = file_item._replace(meta=vfs.default_file_mode)
 146     augmented = vfs.augment_item_meta(repo, mode_item)
 147     augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
 148     for item in (augmented, augmented_w_size):
 149         meta = item.meta
 150         wvpass(item is not file_item)
 151         wvpass(isinstance(meta, Metadata))
 152         wvpasseq(vfs.default_file_mode, meta.mode)
 153         wvpasseq((0, 0, 0, 0, 0),
 154                  (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
 155     wvpass(augmented.meta.size is None)
 156     wvpasseq(file_size, augmented_w_size.meta.size)
 157
 158     ## Ensure symlinks are handled properly
 159     mode_item = link_item._replace(meta=vfs.default_symlink_mode)
 160     augmented = vfs.augment_item_meta(repo, mode_item)
 161     wvpass(augmented is not mode_item)
 162     wvpass(isinstance(augmented.meta, Metadata))
 163     wvpasseq(link_target, augmented.meta.symlink_target)
 164     wvpasseq(len(link_target), augmented.meta.size)
 165     augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
 166     wvpass(augmented is not mode_item)
 167     wvpass(isinstance(augmented.meta, Metadata))
 168     wvpasseq(link_target, augmented.meta.symlink_target)
 169     wvpasseq(len(link_target), augmented.meta.size)
 170
 171
 172 @wvtest
 173 def test_item_mode():
 174     with no_lingering_errors():
 175         mode = S_IFDIR | 0o755
 176         meta = metadata.from_path('.')
 177         oid = '\0' * 20
 178         wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
 179         wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
 180
 181 @wvtest
 182 def test_reverse_suffix_duplicates():
 183     suffix = lambda x: tuple(vfs._reverse_suffix_duplicates(x))
 184     wvpasseq(('x',), suffix(('x',)))
 185     wvpasseq(('x', 'y'), suffix(('x', 'y')))
 186     wvpasseq(('x-1', 'x-0'), suffix(('x',) * 2))
 187     wvpasseq(['x-%02d' % n for n in reversed(range(11))],
 188              list(suffix(('x',) * 11)))
 189     wvpasseq(('x-1', 'x-0', 'y'), suffix(('x', 'x', 'y')))
 190     wvpasseq(('x', 'y-1', 'y-0'), suffix(('x', 'y', 'y')))
 191     wvpasseq(('x', 'y-1', 'y-0', 'z'), suffix(('x', 'y', 'y', 'z')))
 192
 193 @wvtest
 194 def test_misc():
 195     with no_lingering_errors():
 196         with test_tempdir('bup-tvfs-') as tmpdir:
 197             bup_dir = tmpdir + '/bup'
 198             environ['GIT_DIR'] = bup_dir
 199             environ['BUP_DIR'] = bup_dir
 200             git.repodir = bup_dir
 201             data_path = tmpdir + '/src'
 202             os.mkdir(data_path)
 203             with open(data_path + '/file', 'w+') as tmpfile:
 204                 tmpfile.write(b'canary\n')
 205             symlink('file', data_path + '/symlink')
 206             ex((bup_path, 'init'))
 207             ex((bup_path, 'index', '-v', data_path))
 208             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 209                 data_path))
 210             repo = LocalRepo()
 211
 212             wvstart('readlink')
 213             ls_tree = exo(('git', 'ls-tree', 'test', 'symlink'))
 214             mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
 215             assert name == 'symlink'
 216             link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
 217             wvpasseq('file', vfs.readlink(repo, link_item))
 218
 219             ls_tree = exo(('git', 'ls-tree', 'test', 'file'))
 220             mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
 221             assert name == 'file'
 222             file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
 223             wvexcept(Exception, vfs.readlink, repo, file_item)
 224
 225             wvstart('item_size')
 226             wvpasseq(4, vfs.item_size(repo, link_item))
 227             wvpasseq(7, vfs.item_size(repo, file_item))
 228             meta = metadata.from_path(__file__)
 229             meta.size = 42
 230             fake_item = file_item._replace(meta=meta)
 231             wvpasseq(42, vfs.item_size(repo, fake_item))
 232
 233             wvstart('augment_item_meta')
 234             run_augment_item_meta_tests(repo,
 235                                         '/test/latest/file', 7,
 236                                         '/test/latest/symlink', 'file')
 237
 238             wvstart('copy_item')
 239             # FIXME: this caused StopIteration
 240             #_, file_item = vfs.resolve(repo, '/file')[-1]
 241             _, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
 242             file_copy = vfs.copy_item(file_item)
 243             wvpass(file_copy is not file_item)
 244             wvpass(file_copy.meta is not file_item.meta)
 245             wvpass(isinstance(file_copy, tuple))
 246             wvpass(file_item.meta.user)
 247             wvpass(file_copy.meta.user)
 248             file_copy.meta.user = None
 249             wvpass(file_item.meta.user)
 250
 251 @wvtest
 252 def test_resolve():
 253     with no_lingering_errors():
 254         with test_tempdir('bup-tvfs-') as tmpdir:
 255             resolve = vfs.resolve
 256             bup_dir = tmpdir + '/bup'
 257             environ['GIT_DIR'] = bup_dir
 258             environ['BUP_DIR'] = bup_dir
 259             git.repodir = bup_dir
 260             data_path = tmpdir + '/src'
 261             save_time = 100000
 262             save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time))
 263             os.mkdir(data_path)
 264             os.mkdir(data_path + '/dir')
 265             with open(data_path + '/file', 'w+') as tmpfile:
 266                 print('canary', file=tmpfile)
 267             symlink('file', data_path + '/file-symlink')
 268             symlink('dir', data_path + '/dir-symlink')
 269             symlink('not-there', data_path + '/bad-symlink')
 270             ex((bup_path, 'init'))
 271             ex((bup_path, 'index', '-v', data_path))
 272             ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test',
 273                 '--strip', data_path))
 274             ex((bup_path, 'tag', 'test-tag', 'test'))
 275             repo = LocalRepo()
 276
 277             tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0]
 278             tip_oidx = tip_hash.strip().split()[0]
 279             tip_oid = tip_oidx.decode('hex')
 280             tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
 281                                  tip_oidx))[0].strip()
 282             tip_tree_oid = tip_tree_oidx.decode('hex')
 283             tip_tree = tree_dict(repo, tip_tree_oid)
 284             test_revlist_w_meta = vfs.RevList(meta=tip_tree['.'].meta,
 285                                               oid=tip_oid)
 286             expected_latest_item = vfs.Commit(meta=S_IFDIR | 0o755,
 287                                               oid=tip_tree_oid,
 288                                               coid=tip_oid)
 289             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
 290                                                      oid=tip_tree_oid,
 291                                                      coid=tip_oid)
 292             expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode,
 293                                                 target=save_time_str)
 294             expected_test_tag_item = expected_latest_item
 295
 296             wvstart('resolve: /')
 297             vfs.clear_cache()
 298             res = resolve(repo, '/')
 299             wvpasseq(1, len(res))
 300             wvpasseq((('', vfs._root),), res)
 301             ignore, root_item = res[0]
 302             root_content = frozenset(vfs.contents(repo, root_item))
 303             wvpasseq(frozenset([('.', root_item),
 304                                 ('.tag', vfs._tags),
 305                                 ('test', test_revlist_w_meta)]),
 306                      root_content)
 307             for path in ('//', '/.', '/./', '/..', '/../',
 308                          '/test/latest/dir/../../..',
 309                          '/test/latest/dir/../../../',
 310                          '/test/latest/dir/../../../.',
 311                          '/test/latest/dir/../../..//',
 312                          '/test//latest/dir/../../..',
 313                          '/test/./latest/dir/../../..',
 314                          '/test/././latest/dir/../../..',
 315                          '/test/.//./latest/dir/../../..',
 316                          '/test//.//.//latest/dir/../../..'
 317                          '/test//./latest/dir/../../..'):
 318                 wvstart('resolve: ' + path)
 319                 vfs.clear_cache()
 320                 res = resolve(repo, path)
 321                 wvpasseq((('', vfs._root),), res)
 322
 323             wvstart('resolve: /.tag')
 324             vfs.clear_cache()
 325             res = resolve(repo, '/.tag')
 326             wvpasseq(2, len(res))
 327             wvpasseq((('', vfs._root), ('.tag', vfs._tags)),
 328                      res)
 329             ignore, tag_item = res[1]
 330             tag_content = frozenset(vfs.contents(repo, tag_item))
 331             wvpasseq(frozenset([('.', tag_item),
 332                                 ('test-tag', expected_test_tag_item)]),
 333                      tag_content)
 334
 335             wvstart('resolve: /test')
 336             vfs.clear_cache()
 337             res = resolve(repo, '/test')
 338             wvpasseq(2, len(res))
 339             wvpasseq((('', vfs._root), ('test', test_revlist_w_meta)), res)
 340             ignore, test_item = res[1]
 341             test_content = frozenset(vfs.contents(repo, test_item))
 342             # latest has metadata here due to caching
 343             wvpasseq(frozenset([('.', test_revlist_w_meta),
 344                                 (save_time_str, expected_latest_item_w_meta),
 345                                 ('latest', expected_latest_link)]),
 346                      test_content)
 347
 348             wvstart('resolve: /test/latest')
 349             vfs.clear_cache()
 350             res = resolve(repo, '/test/latest')
 351             wvpasseq(3, len(res))
 352             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
 353                                                      oid=tip_tree_oid,
 354                                                      coid=tip_oid)
 355             expected = (('', vfs._root),
 356                         ('test', test_revlist_w_meta),
 357                         (save_time_str, expected_latest_item_w_meta))
 358             wvpasseq(expected, res)
 359             ignore, latest_item = res[2]
 360             latest_content = frozenset(vfs.contents(repo, latest_item))
 361             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
 362                                  for x in (tip_tree[name]
 363                                            for name in ('.',
 364                                                         'bad-symlink',
 365                                                         'dir',
 366                                                         'dir-symlink',
 367                                                         'file',
 368                                                         'file-symlink')))
 369             wvpasseq(expected, latest_content)
 370
 371             wvstart('resolve: /test/latest/file')
 372             vfs.clear_cache()
 373             res = resolve(repo, '/test/latest/file')
 374             wvpasseq(4, len(res))
 375             expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta,
 376                                                  oid=tip_tree['file'].oid)
 377             expected = (('', vfs._root),
 378                         ('test', test_revlist_w_meta),
 379                         (save_time_str, expected_latest_item_w_meta),
 380                         ('file', expected_file_item_w_meta))
 381             wvpasseq(expected, res)
 382
 383             wvstart('resolve: /test/latest/bad-symlink')
 384             vfs.clear_cache()
 385             res = resolve(repo, '/test/latest/bad-symlink')
 386             wvpasseq(4, len(res))
 387             expected = (('', vfs._root),
 388                         ('test', test_revlist_w_meta),
 389                         (save_time_str, expected_latest_item_w_meta),
 390                         ('not-there', None))
 391             wvpasseq(expected, res)
 392
 393             wvstart('resolve nofollow: /test/latest/bad-symlink')
 394             vfs.clear_cache()
 395             res = resolve(repo, '/test/latest/bad-symlink', follow=False)
 396             wvpasseq(4, len(res))
 397             bad_symlink_value = tip_tree['bad-symlink']
 398             expected_bad_symlink_item_w_meta = vfs.Item(meta=bad_symlink_value.meta,
 399                                                         oid=bad_symlink_value.oid)
 400             expected = (('', vfs._root),
 401                         ('test', test_revlist_w_meta),
 402                         (save_time_str, expected_latest_item_w_meta),
 403                         ('bad-symlink', expected_bad_symlink_item_w_meta))
 404             wvpasseq(expected, res)
 405
 406             wvstart('resolve: /test/latest/file-symlink')
 407             vfs.clear_cache()
 408             res = resolve(repo, '/test/latest/file-symlink')
 409             wvpasseq(4, len(res))
 410             expected = (('', vfs._root),
 411                         ('test', test_revlist_w_meta),
 412                         (save_time_str, expected_latest_item_w_meta),
 413                         ('file', expected_file_item_w_meta))
 414             wvpasseq(expected, res)
 415
 416             wvstart('resolve nofollow: /test/latest/file-symlink')
 417             vfs.clear_cache()
 418             res = resolve(repo, '/test/latest/file-symlink', follow=False)
 419             wvpasseq(4, len(res))
 420             file_symlink_value = tip_tree['file-symlink']
 421             expected_file_symlink_item_w_meta = vfs.Item(meta=file_symlink_value.meta,
 422                                                          oid=file_symlink_value.oid)
 423             expected = (('', vfs._root),
 424                         ('test', test_revlist_w_meta),
 425                         (save_time_str, expected_latest_item_w_meta),
 426                         ('file-symlink', expected_file_symlink_item_w_meta))
 427             wvpasseq(expected, res)
 428
 429             wvstart('resolve: /test/latest/missing')
 430             vfs.clear_cache()
 431             res = resolve(repo, '/test/latest/missing')
 432             wvpasseq(4, len(res))
 433             name, item = res[-1]
 434             wvpasseq('missing', name)
 435             wvpass(item is None)
 436
 437             for path in ('/test/latest/file/',
 438                          '/test/latest/file/.',
 439                          '/test/latest/file/..',
 440                          '/test/latest/file/../',
 441                          '/test/latest/file/../.',
 442                          '/test/latest/file/../..',
 443                          '/test/latest/file/foo'):
 444                 wvstart('resolve: ' + path)
 445                 vfs.clear_cache()
 446                 try:
 447                     resolve(repo, path)
 448                 except vfs.IOError as res_ex:
 449                     wvpasseq(ENOTDIR, res_ex.errno)
 450                     wvpasseq(['', 'test', save_time_str, 'file'],
 451                              [name for name, item in res_ex.terminus])
 452
 453             for path in ('/test/latest/file-symlink/',
 454                          '/test/latest/file-symlink/.',
 455                          '/test/latest/file-symlink/..',
 456                          '/test/latest/file-symlink/../',
 457                          '/test/latest/file-symlink/../.',
 458                          '/test/latest/file-symlink/../..'):
 459                 wvstart('resolve nofollow: ' + path)
 460                 vfs.clear_cache()
 461                 try:
 462                     resolve(repo, path, follow=False)
 463                 except vfs.IOError as res_ex:
 464                     wvpasseq(ENOTDIR, res_ex.errno)
 465                     wvpasseq(['', 'test', save_time_str, 'file'],
 466                              [name for name, item in res_ex.terminus])
 467
 468             wvstart('resolve: non-directory parent')
 469             vfs.clear_cache()
 470             file_res = resolve(repo, '/test/latest/file')
 471             try:
 472                 resolve(repo, 'foo', parent=file_res)
 473             except vfs.IOError as res_ex:
 474                 wvpasseq(ENOTDIR, res_ex.errno)
 475                 wvpasseq(None, res_ex.terminus)
 476
 477             wvstart('resolve nofollow: /test/latest/dir-symlink')
 478             vfs.clear_cache()
 479             res = resolve(repo, '/test/latest/dir-symlink', follow=False)
 480             wvpasseq(4, len(res))
 481             dir_symlink_value = tip_tree['dir-symlink']
 482             expected_dir_symlink_item_w_meta = vfs.Item(meta=dir_symlink_value.meta,
 483                                                          oid=dir_symlink_value.oid)
 484             expected = (('', vfs._root),
 485                         ('test', test_revlist_w_meta),
 486                         (save_time_str, expected_latest_item_w_meta),
 487                         ('dir-symlink', expected_dir_symlink_item_w_meta))
 488             wvpasseq(expected, res)
 489
 490             dir_value = tip_tree['dir']
 491             expected_dir_item = vfs.Item(oid=dir_value.oid,
 492                                          meta=tree_dict(repo, dir_value.oid)['.'].meta)
 493             expected = (('', vfs._root),
 494                         ('test', test_revlist_w_meta),
 495                         (save_time_str, expected_latest_item_w_meta),
 496                         ('dir', expected_dir_item))
 497             def lresolve(*args, **keys):
 498                 return resolve(*args, **dict(keys, follow=False))
 499             for resname, resolver in (('resolve', resolve),
 500                                       ('resolve nofollow', lresolve)):
 501                 for path in ('/test/latest/dir-symlink/',
 502                              '/test/latest/dir-symlink/.'):
 503                     wvstart(resname + ': ' + path)
 504                     vfs.clear_cache()
 505                     res = resolver(repo, path)
 506                     wvpasseq(4, len(res))
 507                     wvpasseq(expected, res)
 508             wvstart('resolve: /test/latest/dir-symlink')
 509             vfs.clear_cache()
 510             res = resolve(repo, path)
 511             wvpasseq(4, len(res))
 512             wvpasseq(expected, res)
 513
 514 def write_sized_random_content(parent_dir, size, seed):
 515     verbose = 0
 516     with open('%s/%d' % (parent_dir, size), 'wb') as f:
 517         write_random(f.fileno(), size, seed, verbose)
 518
 519 def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
 520     for read_size in read_sizes:
 521         with open(expected_path, 'rb') as expected:
 522             with vfs.fopen(repo, item) as actual:
 523                 ex_buf = expected.read(read_size)
 524                 act_buf = actual.read(read_size)
 525                 while ex_buf and act_buf:
 526                     wvpassge(read_size, len(ex_buf))
 527                     wvpassge(read_size, len(act_buf))
 528                     wvpasseq(len(ex_buf), len(act_buf))
 529                     wvpass(ex_buf == act_buf)
 530                     ex_buf = expected.read(read_size)
 531                     act_buf = actual.read(read_size)
 532                 wvpasseq('', ex_buf)
 533                 wvpasseq('', act_buf)
 534
 535 def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
 536     def read_act(act_pos):
 537         with vfs.fopen(repo, item) as actual:
 538             actual.seek(act_pos)
 539             wvpasseq(act_pos, actual.tell())
 540             act_buf = actual.read(read_size)
 541             act_pos += len(act_buf)
 542             wvpasseq(act_pos, actual.tell())
 543             return act_pos, act_buf
 544
 545     for read_size in read_sizes:
 546         with open(expected_path, 'rb') as expected:
 547                 ex_buf = expected.read(read_size)
 548                 act_buf = None
 549                 act_pos = 0
 550                 while ex_buf:
 551                     act_pos, act_buf = read_act(act_pos)
 552                     wvpassge(read_size, len(ex_buf))
 553                     wvpassge(read_size, len(act_buf))
 554                     wvpasseq(len(ex_buf), len(act_buf))
 555                     wvpass(ex_buf == act_buf)
 556                     if not act_buf:
 557                         break
 558                     ex_buf = expected.read(read_size)
 559                 else:  # hit expected eof first
 560                     act_pos, act_buf = read_act(act_pos)
 561                 wvpasseq('', ex_buf)
 562                 wvpasseq('', act_buf)
 563
 564 @wvtest
 565 def test_read_and_seek():
 566     # Write a set of randomly sized files containing random data whose
 567     # names are their sizes, and then verify that what we get back
 568     # from the vfs when seeking and reading with various block sizes
 569     # matches the original content.
 570     with no_lingering_errors():
 571         with test_tempdir('bup-tvfs-read-') as tmpdir:
 572             resolve = vfs.resolve
 573             bup_dir = tmpdir + '/bup'
 574             environ['GIT_DIR'] = bup_dir
 575             environ['BUP_DIR'] = bup_dir
 576             git.repodir = bup_dir
 577             repo = LocalRepo()
 578             data_path = tmpdir + '/src'
 579             os.mkdir(data_path)
 580             seed = randint(-(1 << 31), (1 << 31) - 1)
 581             rand = Random()
 582             rand.seed(seed)
 583             print('test_read seed:', seed, file=sys.stderr)
 584             max_size = 2 * 1024 * 1024
 585             sizes = set((rand.randint(1, max_size) for _ in xrange(5)))
 586             sizes.add(1)
 587             sizes.add(max_size)
 588             for size in sizes:
 589                 write_sized_random_content(data_path, size, seed)
 590             ex((bup_path, 'init'))
 591             ex((bup_path, 'index', '-v', data_path))
 592             ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
 593                 data_path))
 594             read_sizes = set((rand.randint(1, max_size) for _ in xrange(10)))
 595             sizes.add(1)
 596             sizes.add(max_size)
 597             print('test_read src sizes:', sizes, file=sys.stderr)
 598             print('test_read read sizes:', read_sizes, file=sys.stderr)
 599             for size in sizes:
 600                 res = resolve(repo, '/test/latest/' + str(size))
 601                 _, item = res[-1]
 602                 wvpasseq(size, vfs.item_size(repo, res[-1][1]))
 603                 validate_vfs_streaming_read(repo, item,
 604                                             '%s/%d' % (data_path, size),
 605                                             read_sizes)
 606                 validate_vfs_seeking_read(repo, item,
 607                                           '%s/%d' % (data_path, size),
 608                                           read_sizes)
 609
 610 @wvtest
 611 def test_resolve_loop():
 612     with no_lingering_errors():
 613         with test_tempdir('bup-tvfs-resloop-') as tmpdir:
 614             resolve = vfs.resolve
 615             bup_dir = tmpdir + '/bup'
 616             environ['GIT_DIR'] = bup_dir
 617             environ['BUP_DIR'] = bup_dir
 618             git.repodir = bup_dir
 619             repo = LocalRepo()
 620             data_path = tmpdir + '/src'
 621             os.mkdir(data_path)
 622             symlink('loop', data_path + '/loop')
 623             ex((bup_path, 'init'))
 624             ex((bup_path, 'index', '-v', data_path))
 625             save_utc = 100000
 626             ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip',
 627                 data_path))
 628             save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
 629             try:
 630                 wvpasseq('this call should never return',
 631                          resolve(repo, '/test/%s/loop' % save_name))
 632             except vfs.IOError as res_ex:
 633                 wvpasseq(ELOOP, res_ex.errno)
 634                 wvpasseq(['', 'test', save_name, 'loop'],
 635                          [name for name, item in res_ex.terminus])
 636
 637 @wvtest
 638 def test_contents_with_mismatched_bupm_git_ordering():
 639     with no_lingering_errors():
 640         with test_tempdir('bup-tvfs-') as tmpdir:
 641             bup_dir = tmpdir + '/bup'
 642             environ['GIT_DIR'] = bup_dir
 643             environ['BUP_DIR'] = bup_dir
 644             git.repodir = bup_dir
 645             data_path = tmpdir + '/src'
 646             os.mkdir(data_path)
 647             os.mkdir(data_path + '/foo')
 648             with open(data_path + '/foo.', 'w+') as tmpfile:
 649                 tmpfile.write(b'canary\n')
 650             ex((bup_path, 'init'))
 651             ex((bup_path, 'index', '-v', data_path))
 652             save_utc = 100000
 653             save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
 654             ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc),
 655                 '--strip', data_path))
 656             repo = LocalRepo()
 657             tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
 658             tip_oidx = tip_sref.strip().split()[0]
 659             tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
 660                                  tip_oidx))[0].strip()
 661             tip_tree_oid = tip_tree_oidx.decode('hex')
 662             tip_tree = tree_dict(repo, tip_tree_oid)
 663
 664             name, item = vfs.resolve(repo, '/test/latest')[2]
 665             wvpasseq(save_name, name)
 666             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
 667                                  for x in (tip_tree[name]
 668                                            for name in ('.', 'foo', 'foo.')))
 669             contents = tuple(vfs.contents(repo, item))
 670             wvpasseq(expected, frozenset(contents))
 671             # Spot check, in case tree_dict shares too much code with the vfs
 672             name, item = next(((n, i) for n, i in contents if n == 'foo'))
 673             wvpass(S_ISDIR(item.meta))
 674             name, item = next(((n, i) for n, i in contents if n == 'foo.'))
 675             wvpass(S_ISREG(item.meta.mode))
 676
 677 @wvtest
 678 def test_duplicate_save_dates():
 679     with no_lingering_errors():
 680         with test_tempdir('bup-tvfs-') as tmpdir:
 681             bup_dir = tmpdir + '/bup'
 682             environ['GIT_DIR'] = bup_dir
 683             environ['BUP_DIR'] = bup_dir
 684             environ['TZ'] = 'UTC'
 685             git.repodir = bup_dir
 686             data_path = tmpdir + '/src'
 687             os.mkdir(data_path)
 688             with open(data_path + '/file', 'w+') as tmpfile:
 689                 tmpfile.write(b'canary\n')
 690             ex((bup_path, 'init'))
 691             ex((bup_path, 'index', '-v', data_path))
 692             for i in range(11):
 693                 ex((bup_path, 'save', '-d', '100000', '-n', 'test', data_path))
 694             repo = LocalRepo()
 695             res = vfs.resolve(repo, '/test')
 696             wvpasseq(2, len(res))
 697             name, revlist = res[-1]
 698             wvpasseq('test', name)
 699             wvpasseq(('.',
 700                       '1970-01-02-034640-00',
 701                       '1970-01-02-034640-01',
 702                       '1970-01-02-034640-02',
 703                       '1970-01-02-034640-03',
 704                       '1970-01-02-034640-04',
 705                       '1970-01-02-034640-05',
 706                       '1970-01-02-034640-06',
 707                       '1970-01-02-034640-07',
 708                       '1970-01-02-034640-08',
 709                       '1970-01-02-034640-09',
 710                       '1970-01-02-034640-10',
 711                       'latest'),
 712                      tuple(sorted(x[0] for x in vfs.contents(repo, revlist))))
 713
 714 # FIXME: add tests for the want_meta=False cases.