3 bup_python="$(dirname "$0")/../cmd/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 from __future__ import absolute_import, print_function
9 from collections import defaultdict
10 from difflib import unified_diff
11 from itertools import chain, dropwhile, groupby, takewhile
12 from os import environ, chdir
13 from os.path import abspath, dirname
14 from random import choice, randint
15 from shutil import copytree, rmtree
16 from subprocess import PIPE
17 from sys import stderr
18 from time import localtime, strftime, time
19 import os, random, sys
21 script_home = abspath(dirname(sys.argv[0] or '.'))
22 sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
24 bup_cmd = top + '/bup'
26 from buptest import ex, exo, test_tempdir
27 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
29 from bup import compat
30 from bup.helpers import partition, period_as_secs, readpipe
33 def create_older_random_saves(n, start_utc, end_utc):
34 with open('foo', 'w') as f:
36 ex(['git', 'add', 'foo'])
39 utcs.add(randint(start_utc, end_utc))
42 with open('foo', 'w') as f:
43 f.write(str(utc) + '\n')
44 ex(['git', 'commit', '--date', str(utc), '-qam', str(utc)])
45 ex(['git', 'gc', '--aggressive'])
48 # There is corresponding code in bup for some of this, but the
49 # computation method is different here, in part so that the test can
50 # provide a more effective cross-check.
52 period_kinds = ['all', 'dailies', 'monthlies', 'yearlies']
53 period_scale = {'s': 1,
57 'w': 60 * 60 * 24 * 7,
58 'm': 60 * 60 * 24 * 31,
59 'y': 60 * 60 * 24 * 366}
60 period_scale_kinds = period_scale.keys()
62 def expected_retentions(utcs, utc_start, spec):
65 utcs = sorted(utcs, reverse=True)
66 period_start = dict(spec)
67 for kind, duration in compat.items(period_start):
68 period_start[kind] = utc_start - period_as_secs(duration)
69 period_start = defaultdict(lambda: float('inf'), period_start)
71 all = list(takewhile(lambda x: x >= period_start['all'], utcs))
72 utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs))
74 matches = takewhile(lambda x: x >= period_start['dailies'], utcs)
75 dailies = [max(day_utcs) for yday, day_utcs
76 in groupby(matches, lambda x: localtime(x).tm_yday)]
77 utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs))
79 matches = takewhile(lambda x: x >= period_start['monthlies'], utcs)
80 monthlies = [max(month_utcs) for month, month_utcs
81 in groupby(matches, lambda x: localtime(x).tm_mon)]
82 utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs)
84 matches = takewhile(lambda x: x >= period_start['yearlies'], utcs)
85 yearlies = [max(year_utcs) for year, year_utcs
86 in groupby(matches, lambda x: localtime(x).tm_year)]
88 return chain(all, dailies, monthlies, yearlies)
90 def period_spec(start_utc, end_utc):
91 global period_kinds, period_scale, period_scale_kinds
93 desired_specs = randint(1, 2 * len(period_kinds))
94 assert(desired_specs >= 1) # At least one --keep argument is required
95 while len(result) < desired_specs:
97 if randint(1, 100) <= 5:
100 assert(end_utc > start_utc)
101 period_secs = randint(1, end_utc - start_utc)
102 scale = choice(period_scale_kinds)
103 mag = int(float(period_secs) / period_scale[scale])
105 period = str(mag) + scale
107 result += [(choice(period_kinds), period)]
110 def unique_period_specs(n, start_utc, end_utc):
112 while len(invocations) < n:
113 invocations.add(period_spec(start_utc, end_utc))
114 return tuple(invocations)
116 def period_spec_to_period_args(spec):
117 return tuple(chain(*(('--keep-' + kind + '-for', period)
118 for kind, period in spec)))
120 def result_diffline(x):
121 return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n'
123 def check_prune_result(expected):
124 actual = sorted([int(x)
125 for x in exo(['git', 'log',
126 '--pretty=format:%at']).out.splitlines()])
127 if expected != actual:
129 print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
131 for line in unified_diff([result_diffline(x) for x in expected],
132 [result_diffline(x) for x in actual],
133 fromfile='expected', tofile='actual'):
134 sys.stderr.write(line)
135 wvpass(expected == actual)
138 environ['GIT_AUTHOR_NAME'] = 'bup test'
139 environ['GIT_COMMITTER_NAME'] = 'bup test'
140 environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
141 environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
143 seed = int(environ.get('BUP_TEST_SEED', time()))
145 print('random seed:', seed, file=stderr)
147 save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000))
148 prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20))
149 prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
151 with test_tempdir('prune-older-') as tmpdir:
152 environ['BUP_DIR'] = tmpdir + '/work/.git'
153 environ['GIT_DIR'] = tmpdir + '/work/.git'
155 three_years_ago = now - (60 * 60 * 24 * 366 * 3)
157 ex(['git', 'init', 'work'])
158 ex(['git', 'config', 'gc.autoDetach', 'false'])
160 wvstart('generating ' + str(save_population) + ' random saves')
161 chdir(tmpdir + '/work')
162 save_utcs = create_older_random_saves(save_population, three_years_ago, now)
164 test_set_hash = exo(['git', 'show-ref', '-s', 'master']).out.rstrip()
165 ls_saves = exo((bup_cmd, 'ls', 'master')).out.splitlines()
166 wvpasseq(save_population + 1, len(ls_saves))
168 wvstart('ensure everything kept, if no keep arguments')
169 ex(['git', 'reset', '--hard', test_set_hash])
171 'prune-older', '-v', '--unsafe', '--no-gc',
174 stdout=None, stderr=PIPE, check=False)
176 wvpass('at least one keep argument is required' in proc.err)
177 check_prune_result(save_utcs)
180 wvstart('running %d generative no-gc tests on %d saves' % (prune_cycles,
182 for spec in unique_period_specs(prune_cycles,
183 # Make it more likely we'll have
184 # some outside the save range.
185 three_years_ago - period_scale['m'],
187 ex(['git', 'reset', '--hard', test_set_hash])
188 expected = sorted(expected_retentions(save_utcs, now, spec))
190 'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \
191 + period_spec_to_period_args(spec) \
193 check_prune_result(expected)
196 # More expensive because we have to recreate the repo each time
197 wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
199 ex(['git', 'reset', '--hard', test_set_hash])
200 copytree('work/.git', 'clean-test-repo', symlinks=True)
201 for spec in unique_period_specs(prune_gc_cycles,
202 # Make it more likely we'll have
203 # some outside the save range.
204 three_years_ago - period_scale['m'],
207 copytree('clean-test-repo', 'work/.git')
208 expected = sorted(expected_retentions(save_utcs, now, spec))
210 'prune-older', '-v', '--unsafe', '--wrt', str(now)) \
211 + period_spec_to_period_args(spec) \
213 check_prune_result(expected)