prune-older: accommodate python 3 and test there

Signed-off-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/Makefile b/Makefile
index 99305dae22b79fc6e59484fa241d7bfa537596c4..67cc4b599e9eaac01e1355e559b4059cd6f0f944 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -186,6 +186,7 @@ cmdline_tests := \
   t/test-meta.sh \
   t/test-on.sh \
   t/test-packsizelimit \
+  t/test-prune-older \
   t/test-redundant-saves.sh \
   t/test-restore-single-file.sh \
   t/test-rm.sh \
@@ -202,7 +203,6 @@ cmdline_tests := \
 ifeq "2" "$(bup_python_majver)"
   cmdline_tests += \
     t/test-ftp \
-    t/test-prune-older \
     t/test-web.sh \
     t/test-fuse.sh \
     t/test-index-check-device.sh \

diff --git a/cmd/prune-older-cmd.py b/cmd/prune-older-cmd.py
index 96554233644b0bd789456627a31104834dc133a5..fcc0fbd2785f5378f2c9278c9a663a2befafe5d4 100755 (executable)
--- a/cmd/prune-older-cmd.py
+++ b/cmd/prune-older-cmd.py
@@ -6,6 +6,7 @@ exec "$bup_python" "$0" ${1+"$@"}
 # end of bup preamble
 
 from __future__ import absolute_import, print_function
+from binascii import hexlify, unhexlify
 from collections import defaultdict
 from itertools import groupby
 from sys import stderr
@@ -13,20 +14,22 @@ from time import localtime, strftime, time
 import re, sys
 
 from bup import git, options
-from bup.compat import int_types
+from bup.compat import argv_bytes, int_types
 from bup.gc import bup_gc
 from bup.helpers import die_if_errors, log, partition, period_as_secs
+from bup.io import byte_stream
 from bup.repo import LocalRepo
 from bup.rm import bup_rm
 
 
-def branches(refnames=()):
-    return ((name[11:], sha.encode('hex')) for (name,sha)
-            in git.list_refs(patterns=('refs/heads/' + n for n in refnames),
+def branches(refnames=tuple()):
+    return ((name[11:], hexlify(sha)) for (name,sha)
+            in git.list_refs(patterns=(b'refs/heads/' + n for n in refnames),
                              limit_to_heads=True))
 
 def save_name(branch, utc):
-    return branch + '/' + strftime('%Y-%m-%d-%H%M%S', localtime(utc))
+    return branch + b'/' \
+            + strftime('%Y-%m-%d-%H%M%S', localtime(utc)).encode('ascii')
 
 def classify_saves(saves, period_start):
     """For each (utc, id) in saves, yield (True, (utc, id)) if the save
@@ -82,6 +85,7 @@ unsafe        use the command even though it may be DANGEROUS
 
 o = options.Options(optspec)
 opt, flags, roots = o.parse(sys.argv[1:])
+roots = [argv_bytes(x) for x in roots]
 
 if not opt.unsafe:
     o.fatal('refusing to run dangerous, experimental command without --unsafe')
@@ -96,7 +100,7 @@ for period, extent in (('all', opt.keep_all_for),
                        ('monthlies', opt.keep_monthlies_for),
                        ('yearlies', opt.keep_yearlies_for)):
     if extent:
-        secs = period_as_secs(extent)
+        secs = period_as_secs(extent.encode('ascii'))
         if not secs:
             o.fatal('%r is not a valid period' % extent)
         period_start[period] = now - secs
@@ -136,16 +140,20 @@ def parse_info(f):
     author_secs = f.readline().strip()
     return int(author_secs)
 
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
 removals = []
 for branch, branch_id in branches(roots):
     die_if_errors()
-    saves = ((utc, oidx.decode('hex')) for (oidx, utc) in
-             git.rev_list(branch_id, format='%at', parse=parse_info))
+    saves = ((utc, unhexlify(oidx)) for (oidx, utc) in
+             git.rev_list(branch_id, format=b'%at', parse=parse_info))
     for keep_save, (utc, id) in classify_saves(saves, period_start):
         assert(keep_save in (False, True))
         # FIXME: base removals on hashes
         if opt.pretend:
-            print('+' if keep_save else '-', save_name(branch, utc))
+            out.write(b'+ ' if keep_save else b'- '
+                      + save_name(branch, utc) + b'\n')
         elif not keep_save:
             removals.append(save_name(branch, utc))
 

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index cfbdff49692bda777f82547f7dbcffa8030642fe..c77f630f1f0a19e5e628bb41910f355532fc6ab4 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -1166,20 +1166,20 @@ def valid_save_name(name):
     return True
 
 
-_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+_period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')
 
 def period_as_secs(s):
-    if s == 'forever':
+    if s == b'forever':
         return float('inf')
     match = _period_rx.match(s)
     if not match:
         return None
     mag = int(match.group(1))
     scale = match.group(2)
-    return mag * {'s': 1,
-                  'min': 60,
-                  'h': 60 * 60,
-                  'd': 60 * 60 * 24,
-                  'w': 60 * 60 * 24 * 7,
-                  'm': 60 * 60 * 24 * 31,
-                  'y': 60 * 60 * 24 * 366}[scale]
+    return mag * {b's': 1,
+                  b'min': 60,
+                  b'h': 60 * 60,
+                  b'd': 60 * 60 * 24,
+                  b'w': 60 * 60 * 24 * 7,
+                  b'm': 60 * 60 * 24 * 31,
+                  b'y': 60 * 60 * 24 * 366}[scale]

diff --git a/t/test-prune-older b/t/test-prune-older
index c460e821c677adbe7ac5c15ac5adb828a765f715..a2ea4df0f3216d84dcfb7cdd4d97f33aeecab8dc 100755 (executable)
--- a/t/test-prune-older
+++ b/t/test-prune-older
@@ -9,7 +9,7 @@ from __future__ import absolute_import, print_function
 from collections import defaultdict
 from difflib import unified_diff
 from itertools import chain, dropwhile, groupby, takewhile
-from os import environ, chdir
+from os import chdir
 from os.path import abspath, dirname
 from random import choice, randint
 from shutil import copytree, rmtree
@@ -18,46 +18,46 @@ from sys import stderr
 from time import localtime, strftime, time
 import os, random, sys
 
-script_home = abspath(dirname(sys.argv[0] or '.'))
-sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
-top = os.getcwd()
-bup_cmd = top + '/bup'
+# For buptest, wvtest, ...
+sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),)
 
 from buptest import ex, exo, test_tempdir
 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
 
 from bup import compat
+from bup.compat import environ
 from bup.helpers import partition, period_as_secs, readpipe
+import bup.path
 
 
 def create_older_random_saves(n, start_utc, end_utc):
-    with open('foo', 'w') as f:
+    with open(b'foo', 'wb') as f:
         pass
-    ex(['git', 'add', 'foo'])
+    ex([b'git', b'add', b'foo'])
     utcs = set()
     while len(utcs) != n:
         utcs.add(randint(start_utc, end_utc))
     utcs = sorted(utcs)
     for utc in utcs:
-        with open('foo', 'w') as f:
-            f.write(str(utc) + '\n')
-        ex(['git', 'commit', '--date', str(utc), '-qam', str(utc)])
-    ex(['git', 'gc', '--aggressive'])
+        with open(b'foo', 'wb') as f:
+            f.write(b'%d\n' % utc)
+        ex([b'git', b'commit', b'--date', b'%d' % utc, b'-qam', b'%d' % utc])
+    ex([b'git', b'gc', b'--aggressive'])
     return utcs
 
 # There is corresponding code in bup for some of this, but the
 # computation method is different here, in part so that the test can
 # provide a more effective cross-check.
 
-period_kinds = ['all', 'dailies', 'monthlies', 'yearlies']
-period_scale = {'s': 1,
-                'min': 60,
-                'h': 60 * 60,
-                'd': 60 * 60 * 24,
-                'w': 60 * 60 * 24 * 7,
-                'm': 60 * 60 * 24 * 31,
-                'y': 60 * 60 * 24 * 366}
-period_scale_kinds = period_scale.keys()
+period_kinds = [b'all', b'dailies', b'monthlies', b'yearlies']
+period_scale = {b's': 1,
+                b'min': 60,
+                b'h': 60 * 60,
+                b'd': 60 * 60 * 24,
+                b'w': 60 * 60 * 24 * 7,
+                b'm': 60 * 60 * 24 * 31,
+                b'y': 60 * 60 * 24 * 366}
+period_scale_kinds = list(period_scale.keys())
 
 def expected_retentions(utcs, utc_start, spec):
     if not spec:
@@ -68,20 +68,20 @@ def expected_retentions(utcs, utc_start, spec):
         period_start[kind] = utc_start - period_as_secs(duration)
     period_start = defaultdict(lambda: float('inf'), period_start)
 
-    all = list(takewhile(lambda x: x >= period_start['all'], utcs))
-    utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs))
+    all = list(takewhile(lambda x: x >= period_start[b'all'], utcs))
+    utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs))
 
-    matches = takewhile(lambda x: x >= period_start['dailies'], utcs)
+    matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs)
     dailies = [max(day_utcs) for yday, day_utcs
                in groupby(matches, lambda x: localtime(x).tm_yday)]
-    utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs))
+    utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs))
 
-    matches = takewhile(lambda x: x >= period_start['monthlies'], utcs)
+    matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs)
     monthlies = [max(month_utcs) for month, month_utcs
                  in groupby(matches, lambda x: localtime(x).tm_mon)]
-    utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs)
+    utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs)
 
-    matches = takewhile(lambda x: x >= period_start['yearlies'], utcs)
+    matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs)
     yearlies = [max(year_utcs) for year, year_utcs
                 in groupby(matches, lambda x: localtime(x).tm_year)]
 
@@ -95,14 +95,14 @@ def period_spec(start_utc, end_utc):
     while len(result) < desired_specs:
         period = None
         if randint(1, 100) <= 5:
-            period = 'forever'
+            period = b'forever'
         else:
             assert(end_utc > start_utc)
             period_secs = randint(1, end_utc - start_utc)
             scale = choice(period_scale_kinds)
             mag = int(float(period_secs) / period_scale[scale])
             if mag != 0:
-                period = str(mag) + scale
+                period = (b'%d' % mag) + scale
         if period:
             result += [(choice(period_kinds), period)]
     return tuple(result)
@@ -114,16 +114,17 @@ def unique_period_specs(n, start_utc, end_utc):
     return tuple(invocations)
 
 def period_spec_to_period_args(spec):
-    return tuple(chain(*(('--keep-' + kind + '-for', period)
+    return tuple(chain(*((b'--keep-' + kind + b'-for', period)
                          for kind, period in spec)))
 
 def result_diffline(x):
-    return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n'
+    return (b'%d %s\n'
+            % (x, strftime(' %Y-%m-%d-%H%M%S', localtime(x)).encode('ascii')))
 
 def check_prune_result(expected):
     actual = sorted([int(x)
-                     for x in exo(['git', 'log',
-                                   '--pretty=format:%at']).out.splitlines()])
+                     for x in exo([b'git', b'log',
+                                   b'--pretty=format:%at']).out.splitlines()])
     if expected != actual:
         for x in expected:
             print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
@@ -135,45 +136,47 @@ def check_prune_result(expected):
     wvpass(expected == actual)
 
 
-environ['GIT_AUTHOR_NAME'] = 'bup test'
-environ['GIT_COMMITTER_NAME'] = 'bup test'
-environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
-environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_AUTHOR_NAME'] = b'bup test'
+environ[b'GIT_COMMITTER_NAME'] = b'bup test'
+environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
 
-seed = int(environ.get('BUP_TEST_SEED', time()))
+seed = int(environ.get(b'BUP_TEST_SEED', time()))
 random.seed(seed)
 print('random seed:', seed, file=stderr)
 
-save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000))
-prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20))
-prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
+save_population = int(environ.get(b'BUP_TEST_PRUNE_OLDER_SAVES', 2000))
+prune_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_CYCLES', 20))
+prune_gc_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
 
-with test_tempdir('prune-older-') as tmpdir:
-    environ['BUP_DIR'] = tmpdir + '/work/.git'
-    environ['GIT_DIR'] = tmpdir + '/work/.git'
+bup_cmd = bup.path.exe()
+
+with test_tempdir(b'prune-older-') as tmpdir:
+    environ[b'BUP_DIR'] = tmpdir + b'/work/.git'
+    environ[b'GIT_DIR'] = tmpdir + b'/work/.git'
     now = int(time())
     three_years_ago = now - (60 * 60 * 24 * 366 * 3)
     chdir(tmpdir)
-    ex(['git', 'init', 'work'])
-    ex(['git', 'config', 'gc.autoDetach', 'false'])
+    ex([b'git', b'init', b'work'])
+    ex([b'git', b'config', b'gc.autoDetach', b'false'])
 
     wvstart('generating ' + str(save_population) + ' random saves')
-    chdir(tmpdir + '/work')
+    chdir(tmpdir + b'/work')
     save_utcs = create_older_random_saves(save_population, three_years_ago, now)
     chdir(tmpdir)
-    test_set_hash = exo(['git', 'show-ref', '-s', 'master']).out.rstrip()
-    ls_saves = exo((bup_cmd, 'ls', 'master')).out.splitlines()
+    test_set_hash = exo([b'git', b'show-ref', b'-s', b'master']).out.rstrip()
+    ls_saves = exo((bup_cmd, b'ls', b'master')).out.splitlines()
     wvpasseq(save_population + 1, len(ls_saves))
 
     wvstart('ensure everything kept, if no keep arguments')
-    ex(['git', 'reset', '--hard', test_set_hash])
+    ex([b'git', b'reset', b'--hard', test_set_hash])
     proc = ex((bup_cmd,
-               'prune-older', '-v', '--unsafe', '--no-gc',
-               '--wrt', str(now)) \
-              + ('master',),
+               b'prune-older', b'-v', b'--unsafe', b'--no-gc',
+               b'--wrt', b'%d' % now) \
+              + (b'master',),
               stdout=None, stderr=PIPE, check=False)
     wvpassne(proc.rc, 0)
-    wvpass('at least one keep argument is required' in proc.err)
+    wvpass(b'at least one keep argument is required' in proc.err)
     check_prune_result(save_utcs)
 
 
@@ -182,32 +185,33 @@ with test_tempdir('prune-older-') as tmpdir:
     for spec in unique_period_specs(prune_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.
-                                    three_years_ago - period_scale['m'],
+                                    three_years_ago - period_scale[b'm'],
                                     now):
-        ex(['git', 'reset', '--hard', test_set_hash])
+        ex([b'git', b'reset', b'--hard', test_set_hash])
         expected = sorted(expected_retentions(save_utcs, now, spec))
         ex((bup_cmd,
-            'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \
+            b'prune-older', b'-v', b'--unsafe', b'--no-gc', b'--wrt',
+            b'%d' % now) \
            + period_spec_to_period_args(spec) \
-           + ('master',))
+           + (b'master',))
         check_prune_result(expected)
 
 
     # More expensive because we have to recreate the repo each time
     wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
                                                             save_population))
-    ex(['git', 'reset', '--hard', test_set_hash])
-    copytree('work/.git', 'clean-test-repo', symlinks=True)
+    ex([b'git', b'reset', b'--hard', test_set_hash])
+    copytree(b'work/.git', b'clean-test-repo', symlinks=True)
     for spec in unique_period_specs(prune_gc_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.
-                                    three_years_ago - period_scale['m'],
+                                    three_years_ago - period_scale[b'm'],
                                     now):
-        rmtree('work/.git')
-        copytree('clean-test-repo', 'work/.git')
+        rmtree(b'work/.git')
+        copytree(b'clean-test-repo', b'work/.git')
         expected = sorted(expected_retentions(save_utcs, now, spec))
         ex((bup_cmd,
-            'prune-older', '-v', '--unsafe', '--wrt', str(now)) \
+            b'prune-older', b'-v', b'--unsafe', b'--wrt', b'%d' % now) \
            + period_spec_to_period_args(spec) \
-           + ('master',))
+           + (b'master',))
         check_prune_result(expected)