From 0bbbc89f896e8b05b850b4e3198043060b30b0da Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Thu, 2 Jan 2020 11:30:39 -0600 Subject: [PATCH] prune-older: accommodate python 3 and test there Signed-off-by: Rob Browning --- Makefile | 2 +- cmd/prune-older-cmd.py | 26 +++++--- lib/bup/helpers.py | 18 +++--- t/test-prune-older | 134 +++++++++++++++++++++-------------------- 4 files changed, 96 insertions(+), 84 deletions(-) diff --git a/Makefile b/Makefile index 99305da..67cc4b5 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,7 @@ cmdline_tests := \ t/test-meta.sh \ t/test-on.sh \ t/test-packsizelimit \ + t/test-prune-older \ t/test-redundant-saves.sh \ t/test-restore-single-file.sh \ t/test-rm.sh \ @@ -202,7 +203,6 @@ cmdline_tests := \ ifeq "2" "$(bup_python_majver)" cmdline_tests += \ t/test-ftp \ - t/test-prune-older \ t/test-web.sh \ t/test-fuse.sh \ t/test-index-check-device.sh \ diff --git a/cmd/prune-older-cmd.py b/cmd/prune-older-cmd.py index 9655423..fcc0fbd 100755 --- a/cmd/prune-older-cmd.py +++ b/cmd/prune-older-cmd.py @@ -6,6 +6,7 @@ exec "$bup_python" "$0" ${1+"$@"} # end of bup preamble from __future__ import absolute_import, print_function +from binascii import hexlify, unhexlify from collections import defaultdict from itertools import groupby from sys import stderr @@ -13,20 +14,22 @@ from time import localtime, strftime, time import re, sys from bup import git, options -from bup.compat import int_types +from bup.compat import argv_bytes, int_types from bup.gc import bup_gc from bup.helpers import die_if_errors, log, partition, period_as_secs +from bup.io import byte_stream from bup.repo import LocalRepo from bup.rm import bup_rm -def branches(refnames=()): - return ((name[11:], sha.encode('hex')) for (name,sha) - in git.list_refs(patterns=('refs/heads/' + n for n in refnames), +def branches(refnames=tuple()): + return ((name[11:], hexlify(sha)) for (name,sha) + in git.list_refs(patterns=(b'refs/heads/' + n for n in refnames), limit_to_heads=True)) def save_name(branch, utc): - return branch + '/' + strftime('%Y-%m-%d-%H%M%S', localtime(utc)) + return branch + b'/' \ + + strftime('%Y-%m-%d-%H%M%S', localtime(utc)).encode('ascii') def classify_saves(saves, period_start): """For each (utc, id) in saves, yield (True, (utc, id)) if the save @@ -82,6 +85,7 @@ unsafe use the command even though it may be DANGEROUS o = options.Options(optspec) opt, flags, roots = o.parse(sys.argv[1:]) +roots = [argv_bytes(x) for x in roots] if not opt.unsafe: o.fatal('refusing to run dangerous, experimental command without --unsafe') @@ -96,7 +100,7 @@ for period, extent in (('all', opt.keep_all_for), ('monthlies', opt.keep_monthlies_for), ('yearlies', opt.keep_yearlies_for)): if extent: - secs = period_as_secs(extent) + secs = period_as_secs(extent.encode('ascii')) if not secs: o.fatal('%r is not a valid period' % extent) period_start[period] = now - secs @@ -136,16 +140,20 @@ def parse_info(f): author_secs = f.readline().strip() return int(author_secs) +sys.stdout.flush() +out = byte_stream(sys.stdout) + removals = [] for branch, branch_id in branches(roots): die_if_errors() - saves = ((utc, oidx.decode('hex')) for (oidx, utc) in - git.rev_list(branch_id, format='%at', parse=parse_info)) + saves = ((utc, unhexlify(oidx)) for (oidx, utc) in + git.rev_list(branch_id, format=b'%at', parse=parse_info)) for keep_save, (utc, id) in classify_saves(saves, period_start): assert(keep_save in (False, True)) # FIXME: base removals on hashes if opt.pretend: - print('+' if keep_save else '-', save_name(branch, utc)) + out.write(b'+ ' if keep_save else b'- ' + + save_name(branch, utc) + b'\n') elif not keep_save: removals.append(save_name(branch, utc)) diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py index cfbdff4..c77f630 100644 --- a/lib/bup/helpers.py +++ b/lib/bup/helpers.py @@ -1166,20 +1166,20 @@ def valid_save_name(name): return True -_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$') +_period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$') def period_as_secs(s): - if s == 'forever': + if s == b'forever': return float('inf') match = _period_rx.match(s) if not match: return None mag = int(match.group(1)) scale = match.group(2) - return mag * {'s': 1, - 'min': 60, - 'h': 60 * 60, - 'd': 60 * 60 * 24, - 'w': 60 * 60 * 24 * 7, - 'm': 60 * 60 * 24 * 31, - 'y': 60 * 60 * 24 * 366}[scale] + return mag * {b's': 1, + b'min': 60, + b'h': 60 * 60, + b'd': 60 * 60 * 24, + b'w': 60 * 60 * 24 * 7, + b'm': 60 * 60 * 24 * 31, + b'y': 60 * 60 * 24 * 366}[scale] diff --git a/t/test-prune-older b/t/test-prune-older index c460e82..a2ea4df 100755 --- a/t/test-prune-older +++ b/t/test-prune-older @@ -9,7 +9,7 @@ from __future__ import absolute_import, print_function from collections import defaultdict from difflib import unified_diff from itertools import chain, dropwhile, groupby, takewhile -from os import environ, chdir +from os import chdir from os.path import abspath, dirname from random import choice, randint from shutil import copytree, rmtree @@ -18,46 +18,46 @@ from sys import stderr from time import localtime, strftime, time import os, random, sys -script_home = abspath(dirname(sys.argv[0] or '.')) -sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')] -top = os.getcwd() -bup_cmd = top + '/bup' +# For buptest, wvtest, ... +sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),) from buptest import ex, exo, test_tempdir from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart from bup import compat +from bup.compat import environ from bup.helpers import partition, period_as_secs, readpipe +import bup.path def create_older_random_saves(n, start_utc, end_utc): - with open('foo', 'w') as f: + with open(b'foo', 'wb') as f: pass - ex(['git', 'add', 'foo']) + ex([b'git', b'add', b'foo']) utcs = set() while len(utcs) != n: utcs.add(randint(start_utc, end_utc)) utcs = sorted(utcs) for utc in utcs: - with open('foo', 'w') as f: - f.write(str(utc) + '\n') - ex(['git', 'commit', '--date', str(utc), '-qam', str(utc)]) - ex(['git', 'gc', '--aggressive']) + with open(b'foo', 'wb') as f: + f.write(b'%d\n' % utc) + ex([b'git', b'commit', b'--date', b'%d' % utc, b'-qam', b'%d' % utc]) + ex([b'git', b'gc', b'--aggressive']) return utcs # There is corresponding code in bup for some of this, but the # computation method is different here, in part so that the test can # provide a more effective cross-check. -period_kinds = ['all', 'dailies', 'monthlies', 'yearlies'] -period_scale = {'s': 1, - 'min': 60, - 'h': 60 * 60, - 'd': 60 * 60 * 24, - 'w': 60 * 60 * 24 * 7, - 'm': 60 * 60 * 24 * 31, - 'y': 60 * 60 * 24 * 366} -period_scale_kinds = period_scale.keys() +period_kinds = [b'all', b'dailies', b'monthlies', b'yearlies'] +period_scale = {b's': 1, + b'min': 60, + b'h': 60 * 60, + b'd': 60 * 60 * 24, + b'w': 60 * 60 * 24 * 7, + b'm': 60 * 60 * 24 * 31, + b'y': 60 * 60 * 24 * 366} +period_scale_kinds = list(period_scale.keys()) def expected_retentions(utcs, utc_start, spec): if not spec: @@ -68,20 +68,20 @@ def expected_retentions(utcs, utc_start, spec): period_start[kind] = utc_start - period_as_secs(duration) period_start = defaultdict(lambda: float('inf'), period_start) - all = list(takewhile(lambda x: x >= period_start['all'], utcs)) - utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs)) + all = list(takewhile(lambda x: x >= period_start[b'all'], utcs)) + utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs)) - matches = takewhile(lambda x: x >= period_start['dailies'], utcs) + matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs) dailies = [max(day_utcs) for yday, day_utcs in groupby(matches, lambda x: localtime(x).tm_yday)] - utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs)) + utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs)) - matches = takewhile(lambda x: x >= period_start['monthlies'], utcs) + matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs) monthlies = [max(month_utcs) for month, month_utcs in groupby(matches, lambda x: localtime(x).tm_mon)] - utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs) + utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs) - matches = takewhile(lambda x: x >= period_start['yearlies'], utcs) + matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs) yearlies = [max(year_utcs) for year, year_utcs in groupby(matches, lambda x: localtime(x).tm_year)] @@ -95,14 +95,14 @@ def period_spec(start_utc, end_utc): while len(result) < desired_specs: period = None if randint(1, 100) <= 5: - period = 'forever' + period = b'forever' else: assert(end_utc > start_utc) period_secs = randint(1, end_utc - start_utc) scale = choice(period_scale_kinds) mag = int(float(period_secs) / period_scale[scale]) if mag != 0: - period = str(mag) + scale + period = (b'%d' % mag) + scale if period: result += [(choice(period_kinds), period)] return tuple(result) @@ -114,16 +114,17 @@ def unique_period_specs(n, start_utc, end_utc): return tuple(invocations) def period_spec_to_period_args(spec): - return tuple(chain(*(('--keep-' + kind + '-for', period) + return tuple(chain(*((b'--keep-' + kind + b'-for', period) for kind, period in spec))) def result_diffline(x): - return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n' + return (b'%d %s\n' + % (x, strftime(' %Y-%m-%d-%H%M%S', localtime(x)).encode('ascii'))) def check_prune_result(expected): actual = sorted([int(x) - for x in exo(['git', 'log', - '--pretty=format:%at']).out.splitlines()]) + for x in exo([b'git', b'log', + b'--pretty=format:%at']).out.splitlines()]) if expected != actual: for x in expected: print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)), @@ -135,45 +136,47 @@ def check_prune_result(expected): wvpass(expected == actual) -environ['GIT_AUTHOR_NAME'] = 'bup test' -environ['GIT_COMMITTER_NAME'] = 'bup test' -environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f' -environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f' +environ[b'GIT_AUTHOR_NAME'] = b'bup test' +environ[b'GIT_COMMITTER_NAME'] = b'bup test' +environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f' +environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f' -seed = int(environ.get('BUP_TEST_SEED', time())) +seed = int(environ.get(b'BUP_TEST_SEED', time())) random.seed(seed) print('random seed:', seed, file=stderr) -save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000)) -prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20)) -prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10)) +save_population = int(environ.get(b'BUP_TEST_PRUNE_OLDER_SAVES', 2000)) +prune_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_CYCLES', 20)) +prune_gc_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10)) -with test_tempdir('prune-older-') as tmpdir: - environ['BUP_DIR'] = tmpdir + '/work/.git' - environ['GIT_DIR'] = tmpdir + '/work/.git' +bup_cmd = bup.path.exe() + +with test_tempdir(b'prune-older-') as tmpdir: + environ[b'BUP_DIR'] = tmpdir + b'/work/.git' + environ[b'GIT_DIR'] = tmpdir + b'/work/.git' now = int(time()) three_years_ago = now - (60 * 60 * 24 * 366 * 3) chdir(tmpdir) - ex(['git', 'init', 'work']) - ex(['git', 'config', 'gc.autoDetach', 'false']) + ex([b'git', b'init', b'work']) + ex([b'git', b'config', b'gc.autoDetach', b'false']) wvstart('generating ' + str(save_population) + ' random saves') - chdir(tmpdir + '/work') + chdir(tmpdir + b'/work') save_utcs = create_older_random_saves(save_population, three_years_ago, now) chdir(tmpdir) - test_set_hash = exo(['git', 'show-ref', '-s', 'master']).out.rstrip() - ls_saves = exo((bup_cmd, 'ls', 'master')).out.splitlines() + test_set_hash = exo([b'git', b'show-ref', b'-s', b'master']).out.rstrip() + ls_saves = exo((bup_cmd, b'ls', b'master')).out.splitlines() wvpasseq(save_population + 1, len(ls_saves)) wvstart('ensure everything kept, if no keep arguments') - ex(['git', 'reset', '--hard', test_set_hash]) + ex([b'git', b'reset', b'--hard', test_set_hash]) proc = ex((bup_cmd, - 'prune-older', '-v', '--unsafe', '--no-gc', - '--wrt', str(now)) \ - + ('master',), + b'prune-older', b'-v', b'--unsafe', b'--no-gc', + b'--wrt', b'%d' % now) \ + + (b'master',), stdout=None, stderr=PIPE, check=False) wvpassne(proc.rc, 0) - wvpass('at least one keep argument is required' in proc.err) + wvpass(b'at least one keep argument is required' in proc.err) check_prune_result(save_utcs) @@ -182,32 +185,33 @@ with test_tempdir('prune-older-') as tmpdir: for spec in unique_period_specs(prune_cycles, # Make it more likely we'll have # some outside the save range. - three_years_ago - period_scale['m'], + three_years_ago - period_scale[b'm'], now): - ex(['git', 'reset', '--hard', test_set_hash]) + ex([b'git', b'reset', b'--hard', test_set_hash]) expected = sorted(expected_retentions(save_utcs, now, spec)) ex((bup_cmd, - 'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \ + b'prune-older', b'-v', b'--unsafe', b'--no-gc', b'--wrt', + b'%d' % now) \ + period_spec_to_period_args(spec) \ - + ('master',)) + + (b'master',)) check_prune_result(expected) # More expensive because we have to recreate the repo each time wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles, save_population)) - ex(['git', 'reset', '--hard', test_set_hash]) - copytree('work/.git', 'clean-test-repo', symlinks=True) + ex([b'git', b'reset', b'--hard', test_set_hash]) + copytree(b'work/.git', b'clean-test-repo', symlinks=True) for spec in unique_period_specs(prune_gc_cycles, # Make it more likely we'll have # some outside the save range. - three_years_ago - period_scale['m'], + three_years_ago - period_scale[b'm'], now): - rmtree('work/.git') - copytree('clean-test-repo', 'work/.git') + rmtree(b'work/.git') + copytree(b'clean-test-repo', b'work/.git') expected = sorted(expected_retentions(save_utcs, now, spec)) ex((bup_cmd, - 'prune-older', '-v', '--unsafe', '--wrt', str(now)) \ + b'prune-older', b'-v', b'--unsafe', b'--wrt', b'%d' % now) \ + period_spec_to_period_args(spec) \ - + ('master',)) + + (b'master',)) check_prune_result(expected) -- 2.39.2