index: fix -H option

[bup.git] / t / test-prune-older

diff --git a/t/test-prune-older b/t/test-prune-older
index 717e4ce8579aa307d30de44e12d040cc17fdbe62..a2ea4df0f3216d84dcfb7cdd4d97f33aeecab8dc 100755 (executable)
--- a/t/test-prune-older
+++ b/t/test-prune-older
@@ -5,113 +5,84 @@ exec "$bup_python" "$0" ${1+"$@"}
 """
 # end of bup preamble
 
 """
 # end of bup preamble
 

-from __future__ import print_function
+from __future__ import absolute_import, print_function

 from collections import defaultdict
 from difflib import unified_diff
 from itertools import chain, dropwhile, groupby, takewhile
 from collections import defaultdict
 from difflib import unified_diff
 from itertools import chain, dropwhile, groupby, takewhile

-from os import environ, chdir
+from os import chdir

 from os.path import abspath, dirname
 from os.path import abspath, dirname

-from pipes import quote

 from random import choice, randint
 from shutil import copytree, rmtree
 from random import choice, randint
 from shutil import copytree, rmtree

-from subprocess import PIPE, Popen, check_call
+from subprocess import PIPE

 from sys import stderr
 from time import localtime, strftime, time
 import os, random, sys
 
 from sys import stderr
 from time import localtime, strftime, time
 import os, random, sys
 

-script_home = abspath(dirname(sys.argv[0] or '.'))
-sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
-top = os.getcwd()
-bup_cmd = top + '/bup'
+# For buptest, wvtest, ...
+sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),)

 
 

-from buptest import test_tempdir
+from buptest import ex, exo, test_tempdir

 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
 
 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
 

+from bup import compat
+from bup.compat import environ

 from bup.helpers import partition, period_as_secs, readpipe
 from bup.helpers import partition, period_as_secs, readpipe

+import bup.path

 
 
 
 

-def logcmd(cmd):
-    if isinstance(cmd, basestring):
-        print(cmd, file=stderr)
-    else:
-        print(' '.join(map(quote, cmd)), file=stderr)
-
-def exc(cmd, shell=False):
-    logcmd(cmd)
-    check_call(cmd, shell=shell)
-
-def exo(cmd, stdin=None, stdout=True, stderr=False, shell=False, check=True):
-    logcmd(cmd)
-    p = Popen(cmd,
-              stdin=None,
-              stdout=(PIPE if stdout else None),
-              stderr=PIPE,
-              shell=shell)
-    out, err = p.communicate()
-    if check and p.returncode != 0:
-        raise Exception('subprocess %r failed with status %d, stderr: %r'
-                        % (' '.join(argv), p.returncode, err))
-    return out, err, p
-
-def bup(*args):
-    return exo((bup_cmd,) + args)[0]
-
-def bupc(*args):
-    return exc((bup_cmd,) + args)
-

 def create_older_random_saves(n, start_utc, end_utc):
 def create_older_random_saves(n, start_utc, end_utc):

-    with open('foo', 'w') as f:
+    with open(b'foo', 'wb') as f:

         pass
         pass

-    exc(['git', 'add', 'foo'])
+    ex([b'git', b'add', b'foo'])

     utcs = set()
     while len(utcs) != n:
         utcs.add(randint(start_utc, end_utc))
     utcs = sorted(utcs)
     for utc in utcs:
     utcs = set()
     while len(utcs) != n:
         utcs.add(randint(start_utc, end_utc))
     utcs = sorted(utcs)
     for utc in utcs:

-        with open('foo', 'w') as f:
-            f.write(str(utc) + '\n')
-        exc(['git', 'commit', '--date', str(utc), '-qam', str(utc)])
-    exc(['git', 'gc', '--aggressive'])
+        with open(b'foo', 'wb') as f:
+            f.write(b'%d\n' % utc)
+        ex([b'git', b'commit', b'--date', b'%d' % utc, b'-qam', b'%d' % utc])
+    ex([b'git', b'gc', b'--aggressive'])

     return utcs
 
 # There is corresponding code in bup for some of this, but the
 # computation method is different here, in part so that the test can
 # provide a more effective cross-check.
 
     return utcs
 
 # There is corresponding code in bup for some of this, but the
 # computation method is different here, in part so that the test can
 # provide a more effective cross-check.
 

-period_kinds = ['all', 'dailies', 'monthlies', 'yearlies']
-period_scale = {'s': 1,
-                'min': 60,
-                'h': 60 * 60,
-                'd': 60 * 60 * 24,
-                'w': 60 * 60 * 24 * 7,
-                'm': 60 * 60 * 24 * 31,
-                'y': 60 * 60 * 24 * 366}
-period_scale_kinds = period_scale.keys()
+period_kinds = [b'all', b'dailies', b'monthlies', b'yearlies']
+period_scale = {b's': 1,
+                b'min': 60,
+                b'h': 60 * 60,
+                b'd': 60 * 60 * 24,
+                b'w': 60 * 60 * 24 * 7,
+                b'm': 60 * 60 * 24 * 31,
+                b'y': 60 * 60 * 24 * 366}
+period_scale_kinds = list(period_scale.keys())

 
 def expected_retentions(utcs, utc_start, spec):
     if not spec:
         return utcs
     utcs = sorted(utcs, reverse=True)
     period_start = dict(spec)
 
 def expected_retentions(utcs, utc_start, spec):
     if not spec:
         return utcs
     utcs = sorted(utcs, reverse=True)
     period_start = dict(spec)

-    for kind, duration in period_start.iteritems():
+    for kind, duration in compat.items(period_start):

         period_start[kind] = utc_start - period_as_secs(duration)
     period_start = defaultdict(lambda: float('inf'), period_start)
 
         period_start[kind] = utc_start - period_as_secs(duration)
     period_start = defaultdict(lambda: float('inf'), period_start)
 

-    all = list(takewhile(lambda x: x >= period_start['all'], utcs))
-    utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs))
+    all = list(takewhile(lambda x: x >= period_start[b'all'], utcs))
+    utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs))

 
 

-    matches = takewhile(lambda x: x >= period_start['dailies'], utcs)
-    dailies = [min(day_utcs) for yday, day_utcs
+    matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs)
+    dailies = [max(day_utcs) for yday, day_utcs

                in groupby(matches, lambda x: localtime(x).tm_yday)]
                in groupby(matches, lambda x: localtime(x).tm_yday)]

-    utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs))
+    utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs))

 
 

-    matches = takewhile(lambda x: x >= period_start['monthlies'], utcs)
-    monthlies = [min(month_utcs) for month, month_utcs
+    matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs)
+    monthlies = [max(month_utcs) for month, month_utcs

                  in groupby(matches, lambda x: localtime(x).tm_mon)]
                  in groupby(matches, lambda x: localtime(x).tm_mon)]

-    utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs)
+    utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs)

 
 

-    matches = takewhile(lambda x: x >= period_start['yearlies'], utcs)
-    yearlies = [min(year_utcs) for year, year_utcs
+    matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs)
+    yearlies = [max(year_utcs) for year, year_utcs

                 in groupby(matches, lambda x: localtime(x).tm_year)]
 
     return chain(all, dailies, monthlies, yearlies)
                 in groupby(matches, lambda x: localtime(x).tm_year)]
 
     return chain(all, dailies, monthlies, yearlies)


@@ -124,14 +95,14 @@ def period_spec(start_utc, end_utc):
     while len(result) < desired_specs:
         period = None
         if randint(1, 100) <= 5:
     while len(result) < desired_specs:
         period = None
         if randint(1, 100) <= 5:

-            period = 'forever'
+            period = b'forever'

         else:
             assert(end_utc > start_utc)
             period_secs = randint(1, end_utc - start_utc)
             scale = choice(period_scale_kinds)
             mag = int(float(period_secs) / period_scale[scale])
             if mag != 0:
         else:
             assert(end_utc > start_utc)
             period_secs = randint(1, end_utc - start_utc)
             scale = choice(period_scale_kinds)
             mag = int(float(period_secs) / period_scale[scale])
             if mag != 0:

-                period = str(mag) + scale
+                period = (b'%d' % mag) + scale

         if period:
             result += [(choice(period_kinds), period)]
     return tuple(result)
         if period:
             result += [(choice(period_kinds), period)]
     return tuple(result)


@@ -143,16 +114,17 @@ def unique_period_specs(n, start_utc, end_utc):
     return tuple(invocations)
 
 def period_spec_to_period_args(spec):
     return tuple(invocations)
 
 def period_spec_to_period_args(spec):

-    return tuple(chain(*(('--keep-' + kind + '-for', period)
+    return tuple(chain(*((b'--keep-' + kind + b'-for', period)

                          for kind, period in spec)))
 
 def result_diffline(x):
                          for kind, period in spec)))
 
 def result_diffline(x):

-    return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n'
+    return (b'%d %s\n'
+            % (x, strftime(' %Y-%m-%d-%H%M%S', localtime(x)).encode('ascii')))

 
 def check_prune_result(expected):
     actual = sorted([int(x)
 
 def check_prune_result(expected):
     actual = sorted([int(x)

-                     for x in exo(['git', 'log',
-                                   '--pretty=format:%at'])[0].splitlines()])
+                     for x in exo([b'git', b'log',
+                                   b'--pretty=format:%at']).out.splitlines()])

     if expected != actual:
         for x in expected:
             print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
     if expected != actual:
         for x in expected:
             print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),


@@ -164,44 +136,47 @@ def check_prune_result(expected):
     wvpass(expected == actual)
 
 
     wvpass(expected == actual)
 
 

-environ['GIT_AUTHOR_NAME'] = 'bup test'
-environ['GIT_COMMITTER_NAME'] = 'bup test'
-environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
-environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_AUTHOR_NAME'] = b'bup test'
+environ[b'GIT_COMMITTER_NAME'] = b'bup test'
+environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'

 
 

-seed = int(environ.get('BUP_TEST_SEED', time()))
+seed = int(environ.get(b'BUP_TEST_SEED', time()))

 random.seed(seed)
 print('random seed:', seed, file=stderr)
 
 random.seed(seed)
 print('random seed:', seed, file=stderr)
 

-save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000))
-prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20))
-prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
+save_population = int(environ.get(b'BUP_TEST_PRUNE_OLDER_SAVES', 2000))
+prune_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_CYCLES', 20))
+prune_gc_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
+
+bup_cmd = bup.path.exe()

 
 

-with test_tempdir('prune-older-') as tmpdir:
-    environ['BUP_DIR'] = tmpdir + '/work/.git'
-    environ['GIT_DIR'] = tmpdir + '/work/.git'
+with test_tempdir(b'prune-older-') as tmpdir:
+    environ[b'BUP_DIR'] = tmpdir + b'/work/.git'
+    environ[b'GIT_DIR'] = tmpdir + b'/work/.git'

     now = int(time())
     three_years_ago = now - (60 * 60 * 24 * 366 * 3)
     chdir(tmpdir)
     now = int(time())
     three_years_ago = now - (60 * 60 * 24 * 366 * 3)
     chdir(tmpdir)

-    exc(['git', 'init', 'work'])
+    ex([b'git', b'init', b'work'])
+    ex([b'git', b'config', b'gc.autoDetach', b'false'])

 
     wvstart('generating ' + str(save_population) + ' random saves')
 
     wvstart('generating ' + str(save_population) + ' random saves')

-    chdir(tmpdir + '/work')
+    chdir(tmpdir + b'/work')

     save_utcs = create_older_random_saves(save_population, three_years_ago, now)
     chdir(tmpdir)
     save_utcs = create_older_random_saves(save_population, three_years_ago, now)
     chdir(tmpdir)

-    test_set_hash = exo(['git', 'show-ref', '-s', 'master'])[0].rstrip()
-    ls_saves = bup('ls', 'master').splitlines()
+    test_set_hash = exo([b'git', b'show-ref', b'-s', b'master']).out.rstrip()
+    ls_saves = exo((bup_cmd, b'ls', b'master')).out.splitlines()

     wvpasseq(save_population + 1, len(ls_saves))
 
     wvstart('ensure everything kept, if no keep arguments')
     wvpasseq(save_population + 1, len(ls_saves))
 
     wvstart('ensure everything kept, if no keep arguments')

-    exc(['git', 'reset', '--hard', test_set_hash])
-    _, errmsg, proc = exo((bup_cmd,
-                           'prune-older', '-v', '--unsafe', '--no-gc',
-                           '--wrt', str(now)) \
-                          + ('master',),
-                          stdout=False, stderr=True, check=False)
-    wvpassne(proc.returncode, 0)
-    wvpass('at least one keep argument is required' in errmsg)
+    ex([b'git', b'reset', b'--hard', test_set_hash])
+    proc = ex((bup_cmd,
+               b'prune-older', b'-v', b'--unsafe', b'--no-gc',
+               b'--wrt', b'%d' % now) \
+              + (b'master',),
+              stdout=None, stderr=PIPE, check=False)
+    wvpassne(proc.rc, 0)
+    wvpass(b'at least one keep argument is required' in proc.err)

     check_prune_result(save_utcs)
 
 
     check_prune_result(save_utcs)
 
 


@@ -210,32 +185,33 @@ with test_tempdir('prune-older-') as tmpdir:
     for spec in unique_period_specs(prune_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.
     for spec in unique_period_specs(prune_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.

-                                    three_years_ago - period_scale['m'],
+                                    three_years_ago - period_scale[b'm'],

                                     now):
                                     now):

-        exc(['git', 'reset', '--hard', test_set_hash])
+        ex([b'git', b'reset', b'--hard', test_set_hash])

         expected = sorted(expected_retentions(save_utcs, now, spec))
         expected = sorted(expected_retentions(save_utcs, now, spec))

-        exc((bup_cmd,
-             'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \
-            + period_spec_to_period_args(spec) \
-            + ('master',))
+        ex((bup_cmd,
+            b'prune-older', b'-v', b'--unsafe', b'--no-gc', b'--wrt',
+            b'%d' % now) \
+           + period_spec_to_period_args(spec) \
+           + (b'master',))

         check_prune_result(expected)
 
 
     # More expensive because we have to recreate the repo each time
     wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
                                                             save_population))
         check_prune_result(expected)
 
 
     # More expensive because we have to recreate the repo each time
     wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
                                                             save_population))

-    exc(['git', 'reset', '--hard', test_set_hash])
-    copytree('work/.git', 'clean-test-repo', symlinks=True)
+    ex([b'git', b'reset', b'--hard', test_set_hash])
+    copytree(b'work/.git', b'clean-test-repo', symlinks=True)

     for spec in unique_period_specs(prune_gc_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.
     for spec in unique_period_specs(prune_gc_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.

-                                    three_years_ago - period_scale['m'],
+                                    three_years_ago - period_scale[b'm'],

                                     now):
                                     now):

-        rmtree('work/.git')
-        copytree('clean-test-repo', 'work/.git')
+        rmtree(b'work/.git')
+        copytree(b'clean-test-repo', b'work/.git')

         expected = sorted(expected_retentions(save_utcs, now, spec))
         expected = sorted(expected_retentions(save_utcs, now, spec))

-        exc((bup_cmd,
-             'prune-older', '-v', '--unsafe', '--wrt', str(now)) \
-            + period_spec_to_period_args(spec) \
-            + ('master',))
+        ex((bup_cmd,
+            b'prune-older', b'-v', b'--unsafe', b'--wrt', b'%d' % now) \
+           + period_spec_to_period_args(spec) \
+           + (b'master',))

         check_prune_result(expected)
         check_prune_result(expected)