prune-older: accommodate python 3 and test there

Signed-off-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/Makefile b/Makefile
index 99305dae22b79fc6e59484fa241d7bfa537596c4..67cc4b599e9eaac01e1355e559b4059cd6f0f944 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -186,6 +186,7 @@ cmdline_tests := \
   t/test-meta.sh \
   t/test-on.sh \
   t/test-packsizelimit \
   t/test-meta.sh \
   t/test-on.sh \
   t/test-packsizelimit \

+  t/test-prune-older \

   t/test-redundant-saves.sh \
   t/test-restore-single-file.sh \
   t/test-rm.sh \
   t/test-redundant-saves.sh \
   t/test-restore-single-file.sh \
   t/test-rm.sh \


@@ -202,7 +203,6 @@ cmdline_tests := \
 ifeq "2" "$(bup_python_majver)"
   cmdline_tests += \
     t/test-ftp \
 ifeq "2" "$(bup_python_majver)"
   cmdline_tests += \
     t/test-ftp \

-    t/test-prune-older \

     t/test-web.sh \
     t/test-fuse.sh \
     t/test-index-check-device.sh \
     t/test-web.sh \
     t/test-fuse.sh \
     t/test-index-check-device.sh \

diff --git a/cmd/prune-older-cmd.py b/cmd/prune-older-cmd.py
index 96554233644b0bd789456627a31104834dc133a5..fcc0fbd2785f5378f2c9278c9a663a2befafe5d4 100755 (executable)
--- a/cmd/prune-older-cmd.py
+++ b/cmd/prune-older-cmd.py
@@ -6,6 +6,7 @@ exec "$bup_python" "$0" ${1+"$@"}
 # end of bup preamble
 
 from __future__ import absolute_import, print_function
 # end of bup preamble
 
 from __future__ import absolute_import, print_function

+from binascii import hexlify, unhexlify

 from collections import defaultdict
 from itertools import groupby
 from sys import stderr
 from collections import defaultdict
 from itertools import groupby
 from sys import stderr


@@ -13,20 +14,22 @@ from time import localtime, strftime, time
 import re, sys
 
 from bup import git, options
 import re, sys
 
 from bup import git, options

-from bup.compat import int_types
+from bup.compat import argv_bytes, int_types

 from bup.gc import bup_gc
 from bup.helpers import die_if_errors, log, partition, period_as_secs
 from bup.gc import bup_gc
 from bup.helpers import die_if_errors, log, partition, period_as_secs

+from bup.io import byte_stream

 from bup.repo import LocalRepo
 from bup.rm import bup_rm
 
 
 from bup.repo import LocalRepo
 from bup.rm import bup_rm
 
 

-def branches(refnames=()):
-    return ((name[11:], sha.encode('hex')) for (name,sha)
-            in git.list_refs(patterns=('refs/heads/' + n for n in refnames),
+def branches(refnames=tuple()):
+    return ((name[11:], hexlify(sha)) for (name,sha)
+            in git.list_refs(patterns=(b'refs/heads/' + n for n in refnames),

                              limit_to_heads=True))
 
 def save_name(branch, utc):
                              limit_to_heads=True))
 
 def save_name(branch, utc):

-    return branch + '/' + strftime('%Y-%m-%d-%H%M%S', localtime(utc))
+    return branch + b'/' \
+            + strftime('%Y-%m-%d-%H%M%S', localtime(utc)).encode('ascii')

 
 def classify_saves(saves, period_start):
     """For each (utc, id) in saves, yield (True, (utc, id)) if the save
 
 def classify_saves(saves, period_start):
     """For each (utc, id) in saves, yield (True, (utc, id)) if the save


@@ -82,6 +85,7 @@ unsafe        use the command even though it may be DANGEROUS
 
 o = options.Options(optspec)
 opt, flags, roots = o.parse(sys.argv[1:])
 
 o = options.Options(optspec)
 opt, flags, roots = o.parse(sys.argv[1:])

+roots = [argv_bytes(x) for x in roots]

 
 if not opt.unsafe:
     o.fatal('refusing to run dangerous, experimental command without --unsafe')
 
 if not opt.unsafe:
     o.fatal('refusing to run dangerous, experimental command without --unsafe')


@@ -96,7 +100,7 @@ for period, extent in (('all', opt.keep_all_for),
                        ('monthlies', opt.keep_monthlies_for),
                        ('yearlies', opt.keep_yearlies_for)):
     if extent:
                        ('monthlies', opt.keep_monthlies_for),
                        ('yearlies', opt.keep_yearlies_for)):
     if extent:

-        secs = period_as_secs(extent)
+        secs = period_as_secs(extent.encode('ascii'))

         if not secs:
             o.fatal('%r is not a valid period' % extent)
         period_start[period] = now - secs
         if not secs:
             o.fatal('%r is not a valid period' % extent)
         period_start[period] = now - secs


@@ -136,16 +140,20 @@ def parse_info(f):
     author_secs = f.readline().strip()
     return int(author_secs)
 
     author_secs = f.readline().strip()
     return int(author_secs)
 

+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+

 removals = []
 for branch, branch_id in branches(roots):
     die_if_errors()
 removals = []
 for branch, branch_id in branches(roots):
     die_if_errors()

-    saves = ((utc, oidx.decode('hex')) for (oidx, utc) in
-             git.rev_list(branch_id, format='%at', parse=parse_info))
+    saves = ((utc, unhexlify(oidx)) for (oidx, utc) in
+             git.rev_list(branch_id, format=b'%at', parse=parse_info))

     for keep_save, (utc, id) in classify_saves(saves, period_start):
         assert(keep_save in (False, True))
         # FIXME: base removals on hashes
         if opt.pretend:
     for keep_save, (utc, id) in classify_saves(saves, period_start):
         assert(keep_save in (False, True))
         # FIXME: base removals on hashes
         if opt.pretend:

-            print('+' if keep_save else '-', save_name(branch, utc))
+            out.write(b'+ ' if keep_save else b'- '
+                      + save_name(branch, utc) + b'\n')

         elif not keep_save:
             removals.append(save_name(branch, utc))
 
         elif not keep_save:
             removals.append(save_name(branch, utc))
 

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index cfbdff49692bda777f82547f7dbcffa8030642fe..c77f630f1f0a19e5e628bb41910f355532fc6ab4 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -1166,20 +1166,20 @@ def valid_save_name(name):
     return True
 
 
     return True
 
 

-_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+_period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')

 
 def period_as_secs(s):
 
 def period_as_secs(s):

-    if s == 'forever':
+    if s == b'forever':

         return float('inf')
     match = _period_rx.match(s)
     if not match:
         return None
     mag = int(match.group(1))
     scale = match.group(2)
         return float('inf')
     match = _period_rx.match(s)
     if not match:
         return None
     mag = int(match.group(1))
     scale = match.group(2)

-    return mag * {'s': 1,
-                  'min': 60,
-                  'h': 60 * 60,
-                  'd': 60 * 60 * 24,
-                  'w': 60 * 60 * 24 * 7,
-                  'm': 60 * 60 * 24 * 31,
-                  'y': 60 * 60 * 24 * 366}[scale]
+    return mag * {b's': 1,
+                  b'min': 60,
+                  b'h': 60 * 60,
+                  b'd': 60 * 60 * 24,
+                  b'w': 60 * 60 * 24 * 7,
+                  b'm': 60 * 60 * 24 * 31,
+                  b'y': 60 * 60 * 24 * 366}[scale]

diff --git a/t/test-prune-older b/t/test-prune-older
index c460e821c677adbe7ac5c15ac5adb828a765f715..a2ea4df0f3216d84dcfb7cdd4d97f33aeecab8dc 100755 (executable)
--- a/t/test-prune-older
+++ b/t/test-prune-older
@@ -9,7 +9,7 @@ from __future__ import absolute_import, print_function
 from collections import defaultdict
 from difflib import unified_diff
 from itertools import chain, dropwhile, groupby, takewhile
 from collections import defaultdict
 from difflib import unified_diff
 from itertools import chain, dropwhile, groupby, takewhile

-from os import environ, chdir
+from os import chdir

 from os.path import abspath, dirname
 from random import choice, randint
 from shutil import copytree, rmtree
 from os.path import abspath, dirname
 from random import choice, randint
 from shutil import copytree, rmtree


@@ -18,46 +18,46 @@ from sys import stderr
 from time import localtime, strftime, time
 import os, random, sys
 
 from time import localtime, strftime, time
 import os, random, sys
 

-script_home = abspath(dirname(sys.argv[0] or '.'))
-sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
-top = os.getcwd()
-bup_cmd = top + '/bup'
+# For buptest, wvtest, ...
+sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),)

 
 from buptest import ex, exo, test_tempdir
 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
 
 from bup import compat
 
 from buptest import ex, exo, test_tempdir
 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
 
 from bup import compat

+from bup.compat import environ

 from bup.helpers import partition, period_as_secs, readpipe
 from bup.helpers import partition, period_as_secs, readpipe

+import bup.path

 
 
 def create_older_random_saves(n, start_utc, end_utc):
 
 
 def create_older_random_saves(n, start_utc, end_utc):

-    with open('foo', 'w') as f:
+    with open(b'foo', 'wb') as f:

         pass
         pass

-    ex(['git', 'add', 'foo'])
+    ex([b'git', b'add', b'foo'])

     utcs = set()
     while len(utcs) != n:
         utcs.add(randint(start_utc, end_utc))
     utcs = sorted(utcs)
     for utc in utcs:
     utcs = set()
     while len(utcs) != n:
         utcs.add(randint(start_utc, end_utc))
     utcs = sorted(utcs)
     for utc in utcs:

-        with open('foo', 'w') as f:
-            f.write(str(utc) + '\n')
-        ex(['git', 'commit', '--date', str(utc), '-qam', str(utc)])
-    ex(['git', 'gc', '--aggressive'])
+        with open(b'foo', 'wb') as f:
+            f.write(b'%d\n' % utc)
+        ex([b'git', b'commit', b'--date', b'%d' % utc, b'-qam', b'%d' % utc])
+    ex([b'git', b'gc', b'--aggressive'])

     return utcs
 
 # There is corresponding code in bup for some of this, but the
 # computation method is different here, in part so that the test can
 # provide a more effective cross-check.
 
     return utcs
 
 # There is corresponding code in bup for some of this, but the
 # computation method is different here, in part so that the test can
 # provide a more effective cross-check.
 

-period_kinds = ['all', 'dailies', 'monthlies', 'yearlies']
-period_scale = {'s': 1,
-                'min': 60,
-                'h': 60 * 60,
-                'd': 60 * 60 * 24,
-                'w': 60 * 60 * 24 * 7,
-                'm': 60 * 60 * 24 * 31,
-                'y': 60 * 60 * 24 * 366}
-period_scale_kinds = period_scale.keys()
+period_kinds = [b'all', b'dailies', b'monthlies', b'yearlies']
+period_scale = {b's': 1,
+                b'min': 60,
+                b'h': 60 * 60,
+                b'd': 60 * 60 * 24,
+                b'w': 60 * 60 * 24 * 7,
+                b'm': 60 * 60 * 24 * 31,
+                b'y': 60 * 60 * 24 * 366}
+period_scale_kinds = list(period_scale.keys())

 
 def expected_retentions(utcs, utc_start, spec):
     if not spec:
 
 def expected_retentions(utcs, utc_start, spec):
     if not spec:


@@ -68,20 +68,20 @@ def expected_retentions(utcs, utc_start, spec):
         period_start[kind] = utc_start - period_as_secs(duration)
     period_start = defaultdict(lambda: float('inf'), period_start)
 
         period_start[kind] = utc_start - period_as_secs(duration)
     period_start = defaultdict(lambda: float('inf'), period_start)
 

-    all = list(takewhile(lambda x: x >= period_start['all'], utcs))
-    utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs))
+    all = list(takewhile(lambda x: x >= period_start[b'all'], utcs))
+    utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs))

 
 

-    matches = takewhile(lambda x: x >= period_start['dailies'], utcs)
+    matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs)

     dailies = [max(day_utcs) for yday, day_utcs
                in groupby(matches, lambda x: localtime(x).tm_yday)]
     dailies = [max(day_utcs) for yday, day_utcs
                in groupby(matches, lambda x: localtime(x).tm_yday)]

-    utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs))
+    utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs))

 
 

-    matches = takewhile(lambda x: x >= period_start['monthlies'], utcs)
+    matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs)

     monthlies = [max(month_utcs) for month, month_utcs
                  in groupby(matches, lambda x: localtime(x).tm_mon)]
     monthlies = [max(month_utcs) for month, month_utcs
                  in groupby(matches, lambda x: localtime(x).tm_mon)]

-    utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs)
+    utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs)

 
 

-    matches = takewhile(lambda x: x >= period_start['yearlies'], utcs)
+    matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs)

     yearlies = [max(year_utcs) for year, year_utcs
                 in groupby(matches, lambda x: localtime(x).tm_year)]
 
     yearlies = [max(year_utcs) for year, year_utcs
                 in groupby(matches, lambda x: localtime(x).tm_year)]
 


@@ -95,14 +95,14 @@ def period_spec(start_utc, end_utc):
     while len(result) < desired_specs:
         period = None
         if randint(1, 100) <= 5:
     while len(result) < desired_specs:
         period = None
         if randint(1, 100) <= 5:

-            period = 'forever'
+            period = b'forever'

         else:
             assert(end_utc > start_utc)
             period_secs = randint(1, end_utc - start_utc)
             scale = choice(period_scale_kinds)
             mag = int(float(period_secs) / period_scale[scale])
             if mag != 0:
         else:
             assert(end_utc > start_utc)
             period_secs = randint(1, end_utc - start_utc)
             scale = choice(period_scale_kinds)
             mag = int(float(period_secs) / period_scale[scale])
             if mag != 0:

-                period = str(mag) + scale
+                period = (b'%d' % mag) + scale

         if period:
             result += [(choice(period_kinds), period)]
     return tuple(result)
         if period:
             result += [(choice(period_kinds), period)]
     return tuple(result)


@@ -114,16 +114,17 @@ def unique_period_specs(n, start_utc, end_utc):
     return tuple(invocations)
 
 def period_spec_to_period_args(spec):
     return tuple(invocations)
 
 def period_spec_to_period_args(spec):

-    return tuple(chain(*(('--keep-' + kind + '-for', period)
+    return tuple(chain(*((b'--keep-' + kind + b'-for', period)

                          for kind, period in spec)))
 
 def result_diffline(x):
                          for kind, period in spec)))
 
 def result_diffline(x):

-    return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n'
+    return (b'%d %s\n'
+            % (x, strftime(' %Y-%m-%d-%H%M%S', localtime(x)).encode('ascii')))

 
 def check_prune_result(expected):
     actual = sorted([int(x)
 
 def check_prune_result(expected):
     actual = sorted([int(x)

-                     for x in exo(['git', 'log',
-                                   '--pretty=format:%at']).out.splitlines()])
+                     for x in exo([b'git', b'log',
+                                   b'--pretty=format:%at']).out.splitlines()])

     if expected != actual:
         for x in expected:
             print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
     if expected != actual:
         for x in expected:
             print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),


@@ -135,45 +136,47 @@ def check_prune_result(expected):
     wvpass(expected == actual)
 
 
     wvpass(expected == actual)
 
 

-environ['GIT_AUTHOR_NAME'] = 'bup test'
-environ['GIT_COMMITTER_NAME'] = 'bup test'
-environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
-environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_AUTHOR_NAME'] = b'bup test'
+environ[b'GIT_COMMITTER_NAME'] = b'bup test'
+environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'

 
 

-seed = int(environ.get('BUP_TEST_SEED', time()))
+seed = int(environ.get(b'BUP_TEST_SEED', time()))

 random.seed(seed)
 print('random seed:', seed, file=stderr)
 
 random.seed(seed)
 print('random seed:', seed, file=stderr)
 

-save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000))
-prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20))
-prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
+save_population = int(environ.get(b'BUP_TEST_PRUNE_OLDER_SAVES', 2000))
+prune_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_CYCLES', 20))
+prune_gc_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))

 
 

-with test_tempdir('prune-older-') as tmpdir:
-    environ['BUP_DIR'] = tmpdir + '/work/.git'
-    environ['GIT_DIR'] = tmpdir + '/work/.git'
+bup_cmd = bup.path.exe()
+
+with test_tempdir(b'prune-older-') as tmpdir:
+    environ[b'BUP_DIR'] = tmpdir + b'/work/.git'
+    environ[b'GIT_DIR'] = tmpdir + b'/work/.git'

     now = int(time())
     three_years_ago = now - (60 * 60 * 24 * 366 * 3)
     chdir(tmpdir)
     now = int(time())
     three_years_ago = now - (60 * 60 * 24 * 366 * 3)
     chdir(tmpdir)

-    ex(['git', 'init', 'work'])
-    ex(['git', 'config', 'gc.autoDetach', 'false'])
+    ex([b'git', b'init', b'work'])
+    ex([b'git', b'config', b'gc.autoDetach', b'false'])

 
     wvstart('generating ' + str(save_population) + ' random saves')
 
     wvstart('generating ' + str(save_population) + ' random saves')

-    chdir(tmpdir + '/work')
+    chdir(tmpdir + b'/work')

     save_utcs = create_older_random_saves(save_population, three_years_ago, now)
     chdir(tmpdir)
     save_utcs = create_older_random_saves(save_population, three_years_ago, now)
     chdir(tmpdir)

-    test_set_hash = exo(['git', 'show-ref', '-s', 'master']).out.rstrip()
-    ls_saves = exo((bup_cmd, 'ls', 'master')).out.splitlines()
+    test_set_hash = exo([b'git', b'show-ref', b'-s', b'master']).out.rstrip()
+    ls_saves = exo((bup_cmd, b'ls', b'master')).out.splitlines()

     wvpasseq(save_population + 1, len(ls_saves))
 
     wvstart('ensure everything kept, if no keep arguments')
     wvpasseq(save_population + 1, len(ls_saves))
 
     wvstart('ensure everything kept, if no keep arguments')

-    ex(['git', 'reset', '--hard', test_set_hash])
+    ex([b'git', b'reset', b'--hard', test_set_hash])

     proc = ex((bup_cmd,
     proc = ex((bup_cmd,

-               'prune-older', '-v', '--unsafe', '--no-gc',
-               '--wrt', str(now)) \
-              + ('master',),
+               b'prune-older', b'-v', b'--unsafe', b'--no-gc',
+               b'--wrt', b'%d' % now) \
+              + (b'master',),

               stdout=None, stderr=PIPE, check=False)
     wvpassne(proc.rc, 0)
               stdout=None, stderr=PIPE, check=False)
     wvpassne(proc.rc, 0)

-    wvpass('at least one keep argument is required' in proc.err)
+    wvpass(b'at least one keep argument is required' in proc.err)

     check_prune_result(save_utcs)
 
 
     check_prune_result(save_utcs)
 
 


@@ -182,32 +185,33 @@ with test_tempdir('prune-older-') as tmpdir:
     for spec in unique_period_specs(prune_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.
     for spec in unique_period_specs(prune_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.

-                                    three_years_ago - period_scale['m'],
+                                    three_years_ago - period_scale[b'm'],

                                     now):
                                     now):

-        ex(['git', 'reset', '--hard', test_set_hash])
+        ex([b'git', b'reset', b'--hard', test_set_hash])

         expected = sorted(expected_retentions(save_utcs, now, spec))
         ex((bup_cmd,
         expected = sorted(expected_retentions(save_utcs, now, spec))
         ex((bup_cmd,

-            'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \
+            b'prune-older', b'-v', b'--unsafe', b'--no-gc', b'--wrt',
+            b'%d' % now) \

            + period_spec_to_period_args(spec) \
            + period_spec_to_period_args(spec) \

-           + ('master',))
+           + (b'master',))

         check_prune_result(expected)
 
 
     # More expensive because we have to recreate the repo each time
     wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
                                                             save_population))
         check_prune_result(expected)
 
 
     # More expensive because we have to recreate the repo each time
     wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
                                                             save_population))

-    ex(['git', 'reset', '--hard', test_set_hash])
-    copytree('work/.git', 'clean-test-repo', symlinks=True)
+    ex([b'git', b'reset', b'--hard', test_set_hash])
+    copytree(b'work/.git', b'clean-test-repo', symlinks=True)

     for spec in unique_period_specs(prune_gc_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.
     for spec in unique_period_specs(prune_gc_cycles,
                                     # Make it more likely we'll have
                                     # some outside the save range.

-                                    three_years_ago - period_scale['m'],
+                                    three_years_ago - period_scale[b'm'],

                                     now):
                                     now):

-        rmtree('work/.git')
-        copytree('clean-test-repo', 'work/.git')
+        rmtree(b'work/.git')
+        copytree(b'clean-test-repo', b'work/.git')

         expected = sorted(expected_retentions(save_utcs, now, spec))
         ex((bup_cmd,
         expected = sorted(expected_retentions(save_utcs, now, spec))
         ex((bup_cmd,

-            'prune-older', '-v', '--unsafe', '--wrt', str(now)) \
+            b'prune-older', b'-v', b'--unsafe', b'--wrt', b'%d' % now) \

            + period_spec_to_period_args(spec) \
            + period_spec_to_period_args(spec) \

-           + ('master',))
+           + (b'master',))

         check_prune_result(expected)
         check_prune_result(expected)