from collections import defaultdict
from difflib import unified_diff
from itertools import chain, dropwhile, groupby, takewhile
-from os import environ, chdir
+from os import chdir
from os.path import abspath, dirname
from random import choice, randint
from shutil import copytree, rmtree
from time import localtime, strftime, time
import os, random, sys
-script_home = abspath(dirname(sys.argv[0] or '.'))
-sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
-top = os.getcwd()
-bup_cmd = top + '/bup'
+# For buptest, wvtest, ...
+sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),)
from buptest import ex, exo, test_tempdir
from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
from bup import compat
+from bup.compat import environ
from bup.helpers import partition, period_as_secs, readpipe
+import bup.path
def create_older_random_saves(n, start_utc, end_utc):
- with open('foo', 'w') as f:
+ with open(b'foo', 'wb') as f:
pass
- ex(['git', 'add', 'foo'])
+ ex([b'git', b'add', b'foo'])
utcs = set()
while len(utcs) != n:
utcs.add(randint(start_utc, end_utc))
utcs = sorted(utcs)
for utc in utcs:
- with open('foo', 'w') as f:
- f.write(str(utc) + '\n')
- ex(['git', 'commit', '--date', str(utc), '-qam', str(utc)])
- ex(['git', 'gc', '--aggressive'])
+ with open(b'foo', 'wb') as f:
+ f.write(b'%d\n' % utc)
+ ex([b'git', b'commit', b'--date', b'%d' % utc, b'-qam', b'%d' % utc])
+ ex([b'git', b'gc', b'--aggressive'])
return utcs
# There is corresponding code in bup for some of this, but the
# computation method is different here, in part so that the test can
# provide a more effective cross-check.
-period_kinds = ['all', 'dailies', 'monthlies', 'yearlies']
-period_scale = {'s': 1,
- 'min': 60,
- 'h': 60 * 60,
- 'd': 60 * 60 * 24,
- 'w': 60 * 60 * 24 * 7,
- 'm': 60 * 60 * 24 * 31,
- 'y': 60 * 60 * 24 * 366}
-period_scale_kinds = period_scale.keys()
+period_kinds = [b'all', b'dailies', b'monthlies', b'yearlies']
+period_scale = {b's': 1,
+ b'min': 60,
+ b'h': 60 * 60,
+ b'd': 60 * 60 * 24,
+ b'w': 60 * 60 * 24 * 7,
+ b'm': 60 * 60 * 24 * 31,
+ b'y': 60 * 60 * 24 * 366}
+period_scale_kinds = list(period_scale.keys())
def expected_retentions(utcs, utc_start, spec):
if not spec:
period_start[kind] = utc_start - period_as_secs(duration)
period_start = defaultdict(lambda: float('inf'), period_start)
- all = list(takewhile(lambda x: x >= period_start['all'], utcs))
- utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs))
+ all = list(takewhile(lambda x: x >= period_start[b'all'], utcs))
+ utcs = list(dropwhile(lambda x: x >= period_start[b'all'], utcs))
- matches = takewhile(lambda x: x >= period_start['dailies'], utcs)
+ matches = takewhile(lambda x: x >= period_start[b'dailies'], utcs)
dailies = [max(day_utcs) for yday, day_utcs
in groupby(matches, lambda x: localtime(x).tm_yday)]
- utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs))
+ utcs = list(dropwhile(lambda x: x >= period_start[b'dailies'], utcs))
- matches = takewhile(lambda x: x >= period_start['monthlies'], utcs)
+ matches = takewhile(lambda x: x >= period_start[b'monthlies'], utcs)
monthlies = [max(month_utcs) for month, month_utcs
in groupby(matches, lambda x: localtime(x).tm_mon)]
- utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs)
+ utcs = dropwhile(lambda x: x >= period_start[b'monthlies'], utcs)
- matches = takewhile(lambda x: x >= period_start['yearlies'], utcs)
+ matches = takewhile(lambda x: x >= period_start[b'yearlies'], utcs)
yearlies = [max(year_utcs) for year, year_utcs
in groupby(matches, lambda x: localtime(x).tm_year)]
while len(result) < desired_specs:
period = None
if randint(1, 100) <= 5:
- period = 'forever'
+ period = b'forever'
else:
assert(end_utc > start_utc)
period_secs = randint(1, end_utc - start_utc)
scale = choice(period_scale_kinds)
mag = int(float(period_secs) / period_scale[scale])
if mag != 0:
- period = str(mag) + scale
+ period = (b'%d' % mag) + scale
if period:
result += [(choice(period_kinds), period)]
return tuple(result)
return tuple(invocations)
def period_spec_to_period_args(spec):
- return tuple(chain(*(('--keep-' + kind + '-for', period)
+ return tuple(chain(*((b'--keep-' + kind + b'-for', period)
for kind, period in spec)))
def result_diffline(x):
- return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n'
+ return (b'%d %s\n'
+ % (x, strftime(' %Y-%m-%d-%H%M%S', localtime(x)).encode('ascii')))
def check_prune_result(expected):
actual = sorted([int(x)
- for x in exo(['git', 'log',
- '--pretty=format:%at']).out.splitlines()])
+ for x in exo([b'git', b'log',
+ b'--pretty=format:%at']).out.splitlines()])
if expected != actual:
for x in expected:
print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
wvpass(expected == actual)
-environ['GIT_AUTHOR_NAME'] = 'bup test'
-environ['GIT_COMMITTER_NAME'] = 'bup test'
-environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
-environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_AUTHOR_NAME'] = b'bup test'
+environ[b'GIT_COMMITTER_NAME'] = b'bup test'
+environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
-seed = int(environ.get('BUP_TEST_SEED', time()))
+seed = int(environ.get(b'BUP_TEST_SEED', time()))
random.seed(seed)
print('random seed:', seed, file=stderr)
-save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000))
-prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20))
-prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
+save_population = int(environ.get(b'BUP_TEST_PRUNE_OLDER_SAVES', 2000))
+prune_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_CYCLES', 20))
+prune_gc_cycles = int(environ.get(b'BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
-with test_tempdir('prune-older-') as tmpdir:
- environ['BUP_DIR'] = tmpdir + '/work/.git'
- environ['GIT_DIR'] = tmpdir + '/work/.git'
+bup_cmd = bup.path.exe()
+
+with test_tempdir(b'prune-older-') as tmpdir:
+ environ[b'BUP_DIR'] = tmpdir + b'/work/.git'
+ environ[b'GIT_DIR'] = tmpdir + b'/work/.git'
now = int(time())
three_years_ago = now - (60 * 60 * 24 * 366 * 3)
chdir(tmpdir)
- ex(['git', 'init', 'work'])
+ ex([b'git', b'init', b'work'])
+ ex([b'git', b'config', b'gc.autoDetach', b'false'])
wvstart('generating ' + str(save_population) + ' random saves')
- chdir(tmpdir + '/work')
+ chdir(tmpdir + b'/work')
save_utcs = create_older_random_saves(save_population, three_years_ago, now)
chdir(tmpdir)
- test_set_hash = exo(['git', 'show-ref', '-s', 'master']).out.rstrip()
- ls_saves = exo((bup_cmd, 'ls', 'master')).out.splitlines()
+ test_set_hash = exo([b'git', b'show-ref', b'-s', b'master']).out.rstrip()
+ ls_saves = exo((bup_cmd, b'ls', b'master')).out.splitlines()
wvpasseq(save_population + 1, len(ls_saves))
wvstart('ensure everything kept, if no keep arguments')
- ex(['git', 'reset', '--hard', test_set_hash])
+ ex([b'git', b'reset', b'--hard', test_set_hash])
proc = ex((bup_cmd,
- 'prune-older', '-v', '--unsafe', '--no-gc',
- '--wrt', str(now)) \
- + ('master',),
+ b'prune-older', b'-v', b'--unsafe', b'--no-gc',
+ b'--wrt', b'%d' % now) \
+ + (b'master',),
stdout=None, stderr=PIPE, check=False)
wvpassne(proc.rc, 0)
- wvpass('at least one keep argument is required' in proc.err)
+ wvpass(b'at least one keep argument is required' in proc.err)
check_prune_result(save_utcs)
for spec in unique_period_specs(prune_cycles,
# Make it more likely we'll have
# some outside the save range.
- three_years_ago - period_scale['m'],
+ three_years_ago - period_scale[b'm'],
now):
- ex(['git', 'reset', '--hard', test_set_hash])
+ ex([b'git', b'reset', b'--hard', test_set_hash])
expected = sorted(expected_retentions(save_utcs, now, spec))
ex((bup_cmd,
- 'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \
+ b'prune-older', b'-v', b'--unsafe', b'--no-gc', b'--wrt',
+ b'%d' % now) \
+ period_spec_to_period_args(spec) \
- + ('master',))
+ + (b'master',))
check_prune_result(expected)
# More expensive because we have to recreate the repo each time
wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
save_population))
- ex(['git', 'reset', '--hard', test_set_hash])
- copytree('work/.git', 'clean-test-repo', symlinks=True)
+ ex([b'git', b'reset', b'--hard', test_set_hash])
+ copytree(b'work/.git', b'clean-test-repo', symlinks=True)
for spec in unique_period_specs(prune_gc_cycles,
# Make it more likely we'll have
# some outside the save range.
- three_years_ago - period_scale['m'],
+ three_years_ago - period_scale[b'm'],
now):
- rmtree('work/.git')
- copytree('clean-test-repo', 'work/.git')
+ rmtree(b'work/.git')
+ copytree(b'clean-test-repo', b'work/.git')
expected = sorted(expected_retentions(save_utcs, now, spec))
ex((bup_cmd,
- 'prune-older', '-v', '--unsafe', '--wrt', str(now)) \
+ b'prune-older', b'-v', b'--unsafe', b'--wrt', b'%d' % now) \
+ period_spec_to_period_args(spec) \
- + ('master',))
+ + (b'master',))
check_prune_result(expected)
projects / bup.git / blobdiff