vfs: change /save/latest back to a symlink to the latest save

[bup.git] / lib / bup / t / tvfs.py

diff --git a/lib/bup/t/tvfs.py b/lib/bup/t/tvfs.py
index a8a9be447a77866e9b00d2fc205354eadd75e887..7cfe2f0330ae124ccaba3f01349b4c864cfbe8a7 100644 (file)
--- a/lib/bup/t/tvfs.py
+++ b/lib/bup/t/tvfs.py
@@ -1,15 +1,17 @@
 
 

-from __future__ import print_function
+from __future__ import absolute_import, print_function

 from collections import namedtuple
 from errno import ELOOP, ENOTDIR
 from io import BytesIO
 from os import environ, symlink
 from collections import namedtuple
 from errno import ELOOP, ENOTDIR
 from io import BytesIO
 from os import environ, symlink

-from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG
+from random import Random, randint
+from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISREG

 from sys import stderr
 from time import localtime, strftime
 
 from wvtest import *
 
 from sys import stderr
 from time import localtime, strftime
 
 from wvtest import *
 

+from bup._helpers import write_random

 from bup import git, metadata, vfs
 from bup.git import BUP_CHUNKED
 from bup.helpers import exc, exo, shstr
 from bup import git, metadata, vfs
 from bup.git import BUP_CHUNKED
 from bup.helpers import exc, exo, shstr


@@ -22,16 +24,56 @@ bup_tmp = os.path.realpath('../../../t/tmp')
 bup_path = top_dir + '/bup'
 start_dir = os.getcwd()
 
 bup_path = top_dir + '/bup'
 start_dir = os.getcwd()
 

+def ex(cmd, **kwargs):
+    print(shstr(cmd), file=stderr)
+    return exc(cmd, **kwargs)
+
+@wvtest
+def test_default_modes():
+    wvpasseq(S_IFREG | 0o644, vfs.default_file_mode)
+    wvpasseq(S_IFDIR | 0o755, vfs.default_dir_mode)
+    wvpasseq(S_IFLNK | 0o755, vfs.default_symlink_mode)
+
+@wvtest
+def test_cache_behavior():
+    orig_max = vfs._cache_max_items
+    try:
+        vfs._cache_max_items = 2
+        vfs.clear_cache()
+        wvpasseq({}, vfs._cache)
+        wvpasseq([], vfs._cache_keys)
+        wvfail(vfs._cache_keys)
+        wvexcept(Exception, vfs.cache_notice, 'x', 1)
+        key_0 = 'itm:' + b'\0' * 20
+        key_1 = 'itm:' + b'\1' * 20
+        key_2 = 'itm:' + b'\2' * 20
+        vfs.cache_notice(key_0, 'something')
+        wvpasseq({key_0 : 'something'}, vfs._cache)
+        wvpasseq([key_0], vfs._cache_keys)
+        vfs.cache_notice(key_1, 'something else')
+        wvpasseq({key_0 : 'something', key_1 : 'something else'}, vfs._cache)
+        wvpasseq(frozenset([key_0, key_1]), frozenset(vfs._cache_keys))
+        vfs.cache_notice(key_2, 'and also')
+        wvpasseq(2, len(vfs._cache))
+        wvpass(frozenset(vfs._cache.iteritems())
+               < frozenset({key_0 : 'something',
+                            key_1 : 'something else',
+                            key_2 : 'and also'}.iteritems()))
+        wvpasseq(2, len(vfs._cache_keys))
+        wvpass(frozenset(vfs._cache_keys) < frozenset([key_0, key_1, key_2]))
+        vfs.clear_cache()
+        wvpasseq({}, vfs._cache)
+        wvpasseq([], vfs._cache_keys)
+    finally:
+        vfs._cache_max_items = orig_max
+        vfs.clear_cache()
+

 ## The clear_cache() calls below are to make sure that the test starts
 ## from a known state since at the moment the cache entry for a given
 ## item (like a commit) can change.  For example, its meta value might
 ## be promoted from a mode to a Metadata instance once the tree it
 ## refers to is traversed.
 
 ## The clear_cache() calls below are to make sure that the test starts
 ## from a known state since at the moment the cache entry for a given
 ## item (like a commit) can change.  For example, its meta value might
 ## be promoted from a mode to a Metadata instance once the tree it
 ## refers to is traversed.
 

-def ex(cmd, **kwargs):
-    print(shstr(cmd), file=stderr)
-    return exc(cmd, **kwargs)
-

 TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
 
 def tree_items(repo, oid):
 TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
 
 def tree_items(repo, oid):


@@ -248,6 +290,8 @@ def test_resolve():
             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
                                                      oid=tip_tree_oid,
                                                      coid=tip_oid)
             expected_latest_item_w_meta = vfs.Commit(meta=tip_tree['.'].meta,
                                                      oid=tip_tree_oid,
                                                      coid=tip_oid)

+            expected_latest_link = vfs.FakeLink(meta=vfs.default_symlink_mode,
+                                                target=save_time_str)

             expected_test_tag_item = expected_latest_item
 
             wvstart('resolve: /')
             expected_test_tag_item = expected_latest_item
 
             wvstart('resolve: /')


@@ -299,7 +343,7 @@ def test_resolve():
             # latest has metadata here due to caching
             wvpasseq(frozenset([('.', test_revlist_w_meta),
                                 (save_time_str, expected_latest_item_w_meta),
             # latest has metadata here due to caching
             wvpasseq(frozenset([('.', test_revlist_w_meta),
                                 (save_time_str, expected_latest_item_w_meta),

-                                ('latest', expected_latest_item_w_meta)]),
+                                ('latest', expected_latest_link)]),

                      test_content)
 
             wvstart('resolve: /test/latest')
                      test_content)
 
             wvstart('resolve: /test/latest')


@@ -311,7 +355,7 @@ def test_resolve():
                                                      coid=tip_oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
                                                      coid=tip_oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta))
+                        (save_time_str, expected_latest_item_w_meta))

             wvpasseq(expected, res)
             ignore, latest_item = res[2]
             latest_content = frozenset(vfs.contents(repo, latest_item))
             wvpasseq(expected, res)
             ignore, latest_item = res[2]
             latest_content = frozenset(vfs.contents(repo, latest_item))


@@ -333,7 +377,7 @@ def test_resolve():
                                                  oid=tip_tree['file'].oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
                                                  oid=tip_tree['file'].oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('file', expected_file_item_w_meta))
             wvpasseq(expected, res)
 
                         ('file', expected_file_item_w_meta))
             wvpasseq(expected, res)
 


@@ -343,7 +387,7 @@ def test_resolve():
             wvpasseq(4, len(res))
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
             wvpasseq(4, len(res))
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('not-there', None))
             wvpasseq(expected, res)
 
                         ('not-there', None))
             wvpasseq(expected, res)
 


@@ -356,7 +400,7 @@ def test_resolve():
                                                         oid=bad_symlink_value.oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
                                                         oid=bad_symlink_value.oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('bad-symlink', expected_bad_symlink_item_w_meta))
             wvpasseq(expected, res)
 
                         ('bad-symlink', expected_bad_symlink_item_w_meta))
             wvpasseq(expected, res)
 


@@ -366,7 +410,7 @@ def test_resolve():
             wvpasseq(4, len(res))
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
             wvpasseq(4, len(res))
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('file', expected_file_item_w_meta))
             wvpasseq(expected, res)
 
                         ('file', expected_file_item_w_meta))
             wvpasseq(expected, res)
 


@@ -379,7 +423,7 @@ def test_resolve():
                                                          oid=file_symlink_value.oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
                                                          oid=file_symlink_value.oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('file-symlink', expected_file_symlink_item_w_meta))
             wvpasseq(expected, res)
 
                         ('file-symlink', expected_file_symlink_item_w_meta))
             wvpasseq(expected, res)
 


@@ -404,7 +448,7 @@ def test_resolve():
                     resolve(repo, path)
                 except vfs.IOError as res_ex:
                     wvpasseq(ENOTDIR, res_ex.errno)
                     resolve(repo, path)
                 except vfs.IOError as res_ex:
                     wvpasseq(ENOTDIR, res_ex.errno)

-                    wvpasseq(['', 'test', 'latest', 'file'],
+                    wvpasseq(['', 'test', save_time_str, 'file'],

                              [name for name, item in res_ex.terminus])
 
             for path in ('/test/latest/file-symlink/',
                              [name for name, item in res_ex.terminus])
 
             for path in ('/test/latest/file-symlink/',


@@ -419,7 +463,7 @@ def test_resolve():
                     lresolve(repo, path)
                 except vfs.IOError as res_ex:
                     wvpasseq(ENOTDIR, res_ex.errno)
                     lresolve(repo, path)
                 except vfs.IOError as res_ex:
                     wvpasseq(ENOTDIR, res_ex.errno)

-                    wvpasseq(['', 'test', 'latest', 'file'],
+                    wvpasseq(['', 'test', save_time_str, 'file'],

                              [name for name, item in res_ex.terminus])
 
             wvstart('resolve: non-directory parent')
                              [name for name, item in res_ex.terminus])
 
             wvstart('resolve: non-directory parent')


@@ -440,7 +484,7 @@ def test_resolve():
                                                          oid=dir_symlink_value.oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
                                                          oid=dir_symlink_value.oid)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('dir-symlink', expected_dir_symlink_item_w_meta))
             wvpasseq(expected, res)
 
                         ('dir-symlink', expected_dir_symlink_item_w_meta))
             wvpasseq(expected, res)
 


@@ -449,7 +493,7 @@ def test_resolve():
                                          meta=tree_dict(repo, dir_value.oid)['.'].meta)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),
                                          meta=tree_dict(repo, dir_value.oid)['.'].meta)
             expected = (('', vfs._root),
                         ('test', test_revlist_w_meta),

-                        ('latest', expected_latest_item_w_meta),
+                        (save_time_str, expected_latest_item_w_meta),

                         ('dir', expected_dir_item))
             for resname, resolver in (('resolve', resolve),
                                       ('lresolve', lresolve)):
                         ('dir', expected_dir_item))
             for resname, resolver in (('resolve', resolve),
                                       ('lresolve', lresolve)):


@@ -466,6 +510,102 @@ def test_resolve():
             wvpasseq(4, len(res))
             wvpasseq(expected, res)
 
             wvpasseq(4, len(res))
             wvpasseq(expected, res)
 

+def write_sized_random_content(parent_dir, size, seed):
+    verbose = 0
+    with open('%s/%d' % (parent_dir, size), 'wb') as f:
+        write_random(f.fileno(), size, seed, verbose)
+
+def validate_vfs_streaming_read(repo, item, expected_path, read_sizes):
+    for read_size in read_sizes:
+        with open(expected_path, 'rb') as expected:
+            with vfs.fopen(repo, item) as actual:
+                ex_buf = expected.read(read_size)
+                act_buf = actual.read(read_size)
+                while ex_buf and act_buf:
+                    wvpassge(read_size, len(ex_buf))
+                    wvpassge(read_size, len(act_buf))
+                    wvpasseq(len(ex_buf), len(act_buf))
+                    wvpass(ex_buf == act_buf)
+                    ex_buf = expected.read(read_size)
+                    act_buf = actual.read(read_size)
+                wvpasseq('', ex_buf)
+                wvpasseq('', act_buf)
+
+def validate_vfs_seeking_read(repo, item, expected_path, read_sizes):
+    def read_act(act_pos):
+        with vfs.fopen(repo, item) as actual:
+            actual.seek(act_pos)
+            wvpasseq(act_pos, actual.tell())
+            act_buf = actual.read(read_size)
+            act_pos += len(act_buf)
+            wvpasseq(act_pos, actual.tell())
+            return act_pos, act_buf
+
+    for read_size in read_sizes:
+        with open(expected_path, 'rb') as expected:
+                ex_buf = expected.read(read_size)
+                act_buf = None
+                act_pos = 0
+                while ex_buf:
+                    act_pos, act_buf = read_act(act_pos)
+                    wvpassge(read_size, len(ex_buf))
+                    wvpassge(read_size, len(act_buf))
+                    wvpasseq(len(ex_buf), len(act_buf))
+                    wvpass(ex_buf == act_buf)
+                    if not act_buf:
+                        break
+                    ex_buf = expected.read(read_size)
+                else:  # hit expected eof first
+                    act_pos, act_buf = read_act(act_pos)
+                wvpasseq('', ex_buf)
+                wvpasseq('', act_buf)
+
+@wvtest
+def test_read_and_seek():
+    # Write a set of randomly sized files containing random data whose
+    # names are their sizes, and then verify that what we get back
+    # from the vfs when seeking and reading with various block sizes
+    # matches the original content.
+    with no_lingering_errors():
+        with test_tempdir('bup-tvfs-read-') as tmpdir:
+            resolve = vfs.resolve
+            bup_dir = tmpdir + '/bup'
+            environ['GIT_DIR'] = bup_dir
+            environ['BUP_DIR'] = bup_dir
+            git.repodir = bup_dir
+            repo = LocalRepo()
+            data_path = tmpdir + '/src'
+            os.mkdir(data_path)
+            seed = randint(-(1 << 31), (1 << 31) - 1)
+            rand = Random()
+            rand.seed(seed)
+            print('test_read seed:', seed, file=sys.stderr)
+            max_size = 2 * 1024 * 1024
+            sizes = set((rand.randint(1, max_size) for _ in xrange(5)))
+            sizes.add(1)
+            sizes.add(max_size)
+            for size in sizes:
+                write_sized_random_content(data_path, size, seed)
+            ex((bup_path, 'init'))
+            ex((bup_path, 'index', '-v', data_path))
+            ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
+                data_path))
+            read_sizes = set((rand.randint(1, max_size) for _ in xrange(10)))
+            sizes.add(1)
+            sizes.add(max_size)
+            print('test_read src sizes:', sizes, file=sys.stderr)
+            print('test_read read sizes:', read_sizes, file=sys.stderr)
+            for size in sizes:
+                res = resolve(repo, '/test/latest/' + str(size))
+                _, item = res[-1]
+                wvpasseq(size, vfs.item_size(repo, res[-1][1]))
+                validate_vfs_streaming_read(repo, item,
+                                            '%s/%d' % (data_path, size),
+                                            read_sizes)
+                validate_vfs_seeking_read(repo, item,
+                                          '%s/%d' % (data_path, size),
+                                          read_sizes)
+

 @wvtest
 def test_resolve_loop():
     with no_lingering_errors():
 @wvtest
 def test_resolve_loop():
     with no_lingering_errors():


@@ -482,10 +622,12 @@ def test_resolve_loop():
             symlink('loop', data_path + '/loop')
             ex((bup_path, 'init'))
             ex((bup_path, 'index', '-v', data_path))
             symlink('loop', data_path + '/loop')
             ex((bup_path, 'init'))
             ex((bup_path, 'index', '-v', data_path))

-            ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
+            save_utc = 100000
+            ex((bup_path, 'save', '-d', str(save_utc), '-tvvn', 'test', '--strip',

                 data_path))
                 data_path))

+            save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))

             try:
             try:

-                resolve(repo, '/test/latest/loop')
+                resolve(repo, '/test/%s/loop' % save_utc)

             except vfs.IOError as res_ex:
                 wvpasseq(ELOOP, res_ex.errno)
                 wvpasseq(['', 'test', 'latest', 'loop'],
             except vfs.IOError as res_ex:
                 wvpasseq(ELOOP, res_ex.errno)
                 wvpasseq(['', 'test', 'latest', 'loop'],


@@ -506,8 +648,10 @@ def test_contents_with_mismatched_bupm_git_ordering():
                 tmpfile.write(b'canary\n')
             ex((bup_path, 'init'))
             ex((bup_path, 'index', '-v', data_path))
                 tmpfile.write(b'canary\n')
             ex((bup_path, 'init'))
             ex((bup_path, 'index', '-v', data_path))

-            ex((bup_path, 'save', '-tvvn', 'test', '--strip',
-                data_path))
+            save_utc = 100000
+            save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
+            ex((bup_path, 'save', '-tvvn', 'test', '-d', str(save_utc),
+                '--strip', data_path))

             repo = LocalRepo()
             tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
             tip_oidx = tip_sref.strip().split()[0]
             repo = LocalRepo()
             tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
             tip_oidx = tip_sref.strip().split()[0]


@@ -517,7 +661,7 @@ def test_contents_with_mismatched_bupm_git_ordering():
             tip_tree = tree_dict(repo, tip_tree_oid)
 
             name, item = vfs.resolve(repo, '/test/latest')[2]
             tip_tree = tree_dict(repo, tip_tree_oid)
 
             name, item = vfs.resolve(repo, '/test/latest')[2]

-            wvpasseq('latest', name)
+            wvpasseq(save_name, name)

             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
                                  for x in (tip_tree[name]
                                            for name in ('.', 'foo', 'foo.')))
             expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
                                  for x in (tip_tree[name]
                                            for name in ('.', 'foo', 'foo.')))