--- /dev/null
+% bup-prune-older(1) bup %BUP_VERSION% | bup %BUP_VERSION%
+% Rob Browning <rlb@defaultvalue.org>
+% %BUP_DATE%
+
+# NAME
+
+bup-prune-older - remove older saves (CAUTION: EXPERIMENTAL)
+
+# SYNOPSIS
+
+bup prune-older [options...] <*branch*...>
+
+# DESCRIPTION
+
+`bup prune-older` removes (permanently deletes) all saves except those
+preserved by the various keep arguments detailed below. At least one
+keep argument must be specified. This command is equivalent to a
+suitable `bup rm` invocation followed by `bup gc`.
+
+WARNING: This is one of the few bup commands that modifies your
+archive in intentionally destructive ways. Though if an attempt to
+`join` or `restore` the data you still care about after a
+`prune-older` succeeds, that's a fairly encouraging sign that the
+commands worked correctly. (The `t/compare-trees` command in the
+source tree can be used to help test before/after results.)
+
+# KEEP PERIODS
+
+A `--keep` PERIOD (as required below) must be an integer followed by a
+scale, or "forever". For example, 12y specifies a PERIOD of twelve
+years. Here are the valid scales:
+
+ - s indicates seconds
+ - min indicates minutes (60s)
+ - h indicates hours (60m)
+ - d indicates days (24h)
+ - w indicates weeks (7d)
+ - m indicates months (31d)
+ - y indicates years (366d)
+ - forever is infinitely far in the past
+
+As indicated, the PERIODS are computed with respect to the current
+time, or the `--wrt` value if specified, and do not respect any
+calendar, so `--keep-dailies-for 5d` means a period starting exactly
+5 * 24 * 60 * 60 seconds before the starting point.
+
+# OPTIONS
+
+--keep-all-for PERIOD
+: when no smaller time scale --keep option applies, retain all saves
+ within the given period.
+
+--keep-dailies-for PERIOD
+: when no smaller time scale --keep option applies, retain the
+ oldest save for any day within the given period.
+
+--keep-monthlies-for PERIOD
+: when no smaller time scale --keep option applies, retain the
+ oldest save for any month within the given period.
+
+--keep-yearlies-for PERIOD
+: when no smaller time scale --keep option applies, retain the
+ oldest save for any year within the given period.
+
+--keep ALL,DAILY,MONTHLY,YEARLY
+: shorthand equivalent to specifying each of the related `--keep-*`
+ options for each period. If an underscore is specified for a
+ period, then that time-scale will be unaffected by this argument.
+
+--wrt UTC_SECONDS
+: when computing a keep period, place the most recent end of the
+ range at UTC\_SECONDS, and any saves newer than this will be kept.
+
+--pretend
+: don't do anything, just list the actions that would be taken to
+ standard output, one action per line like this:
+
+ - SAVE
+ + SAVE
+ ...
+
+--gc
+: garbage collect the repository after removing the relevant saves.
+ This is the default behavior, but it can be avoided with `--no-gc`.
+
+\--gc-threshold N
+: only rewrite a packfile if it's over N percent garbage; otherwise
+ leave it alone. The default threshold is 10%.
+
+-*#*, \--compress *#*
+: set the compression level when rewriting archive data to # (a
+ value from 0-9, where 9 is the highest and 0 is no compression).
+ The default is 1 (fast, loose compression).
+
+-v, \--verbose
+: increase verbosity (can be specified more than once).
+
+# NOTES
+
+When `--verbose` is specified, the save periods will be summarized to
+standard error with lines like this:
+
+ keeping monthlies since 1969-07-20-201800
+ keeping all yearlies
+ ...
+
+It's possible that the current implementation might not be able to
+format the date if, for example, it is far enough back in time. In
+that case, you will see something like this:
+
+ keeping yearlies since -30109891477 seconds before 1969-12-31-180000
+ ...
+
+# EXAMPLES
+
+ # Keep all saves for the past month, and any older monthlies for
+ # the past year. Delete everything else.
+ $ bup prune-older --keep-all-for 1m --keep-monthlies-for 1y
+
+ # Keep all saves for the past 6 months and delete everything else,
+ # but only on the semester branch.
+ $ bup prune-older --keep-all-for 6m semester
+
+# SEE ALSO
+
+`bup-rm`(1), `bup-gc`(1), and `bup-fsck`(1)
+
+# BUP
+
+Part of the `bup`(1) suite.
| tee -a t/tmp/test-log/$$$$.log
cmdline_tests := \
+ t/test-prune-older \
t/test-web.sh \
t/test-rm.sh \
t/test-gc.sh \
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import print_function
+from collections import defaultdict
+from itertools import groupby
+from sys import stderr
+from time import localtime, strftime, time
+import re, sys
+
+from bup import git, options
+from bup.gc import bup_gc
+from bup.helpers import die_if_errors, log, partition, period_as_secs
+from bup.rm import bup_rm
+
+
+def branches(refnames=()):
+ return ((name[11:], sha) for (name,sha)
+ in git.list_refs(refnames=('refs/heads/' + n for n in refnames),
+ limit_to_heads=True))
+
+def save_name(branch, utc):
+ return branch + '/' + strftime('%Y-%m-%d-%H%M%S', localtime(utc))
+
+def classify_saves(saves, period_start):
+ """For each (utc, id) in saves, yield (True, (utc, id)) if the save
+ should be kept and (False, (utc, id)) if the save should be removed.
+ The ids are binary hashes.
+ """
+
+ def retain_oldest_in_region(region):
+ prev = None
+ for save in region:
+ if prev:
+ yield False, prev
+ prev = save
+ if prev:
+ yield True, prev
+
+ matches, rest = partition(lambda s: s[0] >= period_start['all'], saves)
+ for save in matches:
+ yield True, save
+
+ tm_ranges = ((period_start['dailies'], lambda s: localtime(s[0]).tm_yday),
+ (period_start['monthlies'], lambda s: localtime(s[0]).tm_mon),
+ (period_start['yearlies'], lambda s: localtime(s[0]).tm_year))
+
+ for pstart, time_region_id in tm_ranges:
+ matches, rest = partition(lambda s: s[0] >= pstart, rest)
+ for region_id, region_saves in groupby(matches, time_region_id):
+ for action in retain_oldest_in_region(region_saves):
+ yield action
+
+ for save in rest:
+ yield False, save
+
+
+optspec = """
+bup prune-older [options...] [BRANCH...]
+--
+keep-all-for= retain all saves within the PERIOD
+keep-dailies-for= retain the oldest save per day within the PERIOD
+keep-monthlies-for= retain the oldest save per month within the PERIOD
+keep-yearlies-for= retain the oldest save per year within the PERIOD
+wrt= end all periods at this number of seconds since the epoch
+pretend don't prune, just report intended actions to standard output
+gc collect garbage after removals [1]
+gc-threshold= only rewrite a packfile if it's over this percent garbage [10]
+#,compress= set compression level to # (0-9, 9 is highest) [1]
+v,verbose increase log output (can be used more than once)
+unsafe use the command even though it may be DANGEROUS
+"""
+
+o = options.Options(optspec)
+opt, flags, roots = o.parse(sys.argv[1:])
+
+if not opt.unsafe:
+ o.fatal('refusing to run dangerous, experimental command without --unsafe')
+
+now = int(time()) if not opt.wrt else opt.wrt
+if not isinstance(now, (int, long)):
+ o.fatal('--wrt value ' + str(now) + ' is not an integer')
+
+period_start = {}
+for period, extent in (('all', opt.keep_all_for),
+ ('dailies', opt.keep_dailies_for),
+ ('monthlies', opt.keep_monthlies_for),
+ ('yearlies', opt.keep_yearlies_for)):
+ if extent:
+ secs = period_as_secs(extent)
+ if not secs:
+ o.fatal('%r is not a valid period' % extent)
+ period_start[period] = now - secs
+
+if not period_start:
+ o.fatal('at least one keep argument is required')
+
+period_start = defaultdict(lambda: float('inf'), period_start)
+
+if opt.verbose:
+ epoch_ymd = strftime('%Y-%m-%d-%H%M%S', localtime(0))
+ for kind in ['all', 'dailies', 'monthlies', 'yearlies']:
+ period_utc = period_start[kind]
+ if period_utc != float('inf'):
+ if not (period_utc > float('-inf')):
+ log('keeping all ' + kind)
+ else:
+ try:
+ when = strftime('%Y-%m-%d-%H%M%S', localtime(period_utc))
+ log('keeping ' + kind + ' since ' + when + '\n')
+ except ValueError as ex:
+ if period_utc < 0:
+ log('keeping %s since %d seconds before %s\n'
+ %(kind, abs(period_utc), epoch_ymd))
+ elif period_utc > 0:
+ log('keeping %s since %d seconds after %s\n'
+ %(kind, period_utc, epoch_ymd))
+ else:
+ log('keeping %s since %s\n' % (kind, epoch_ymd))
+
+git.check_repo_or_die()
+
+# This could be more efficient, but for now just build the whole list
+# in memory and let bup_rm() do some redundant work.
+
+removals = []
+for branch, branch_id in branches(roots):
+ die_if_errors()
+ saves = git.rev_list(branch_id.encode('hex'))
+ for keep_save, (utc, id) in classify_saves(saves, period_start):
+ assert(keep_save in (False, True))
+ # FIXME: base removals on hashes
+ if opt.pretend:
+ print('+' if keep_save else '-', save_name(branch, utc))
+ elif not keep_save:
+ removals.append(save_name(branch, utc))
+
+if not opt.pretend:
+ die_if_errors()
+ bup_rm(removals, compression=opt.compress, verbosity=opt.verbose)
+ if opt.gc:
+ die_if_errors()
+ bup_gc(threshold=opt.gc_threshold,
+ compression=opt.compress,
+ verbosity=opt.verbose)
+
+die_if_errors()
import glob, os, subprocess, sys, tempfile
from bup import bloom, git, midx
from bup.git import MissingObject, walk_object
-from bup.helpers import log, progress, qprogress
+from bup.helpers import Nonlocal, log, progress, qprogress
from os.path import basename
# This garbage collector uses a Bloom filter to track the live objects
# FIXME: add a bloom filter tuning parameter?
-class Nonlocal:
- pass
-
-
def count_objects(dir, verbosity):
# For now we'll just use open_idx(), but we could probably be much
# more efficient since all we need is a single integer (the last
from bup import _helpers
+
+class Nonlocal:
+ """Helper to deal with Python scoping issues"""
+ pass
+
+
sc_page_size = os.sysconf('SC_PAGE_SIZE')
assert(sc_page_size > 0)
fdatasync = _fdatasync
+def partition(predicate, stream):
+ """Returns (leading_matches_it, rest_it), where leading_matches_it
+ must be completely exhausted before traversing rest_it.
+
+ """
+ stream = iter(stream)
+ ns = Nonlocal()
+ ns.first_nonmatch = None
+ def leading_matches():
+ for x in stream:
+ if predicate(x):
+ yield x
+ else:
+ ns.first_nonmatch = (x,)
+ break
+ def rest():
+ if ns.first_nonmatch:
+ yield ns.first_nonmatch[0]
+ for x in stream:
+ yield x
+ return (leading_matches(), rest())
+
+
# Write (blockingly) to sockets that may or may not be in blocking mode.
# We need this because our stderr is sometimes eaten by subprocesses
# (probably ssh) that sometimes make it nonblocking, if only temporarily,
if part.startswith('.') or part.endswith('.lock'):
return False
return True
+
+
+_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+
+def period_as_secs(s):
+ if s == 'forever':
+ return float('inf')
+ match = _period_rx.match(s)
+ if not match:
+ return None
+ mag = int(match.group(1))
+ scale = match.group(2)
+ return mag * {'s': 1,
+ 'min': 60,
+ 'h': 60 * 60,
+ 'd': 60 * 60 * 24,
+ 'w': 60 * 60 * 24 * 7,
+ 'm': 60 * 60 * 24 * 31,
+ 'y': 60 * 60 * 24 * 366}[scale]
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/../cmd/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import print_function
+from collections import defaultdict
+from difflib import unified_diff
+from itertools import chain, dropwhile, groupby, takewhile
+from os import environ, chdir
+from os.path import abspath, dirname
+from pipes import quote
+from random import choice, randint
+from shutil import copytree, rmtree
+from subprocess import PIPE, Popen, check_call
+from sys import stderr
+from time import localtime, strftime, time
+import os, random, sys
+
+script_home = abspath(dirname(sys.argv[0] or '.'))
+sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
+top = os.getcwd()
+bup_cmd = top + '/bup'
+
+from buptest import test_tempdir
+from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
+
+from bup.helpers import partition, period_as_secs, readpipe
+
+
+def logcmd(cmd):
+ if isinstance(cmd, basestring):
+ print(cmd, file=stderr)
+ else:
+ print(' '.join(map(quote, cmd)), file=stderr)
+
+def exc(cmd, shell=False):
+ logcmd(cmd)
+ check_call(cmd, shell=shell)
+
+def exo(cmd, stdin=None, stdout=True, stderr=False, shell=False, check=True):
+ logcmd(cmd)
+ p = Popen(cmd,
+ stdin=None,
+ stdout=(PIPE if stdout else None),
+ stderr=PIPE,
+ shell=shell)
+ out, err = p.communicate()
+ if check and p.returncode != 0:
+ raise Exception('subprocess %r failed with status %d, stderr: %r'
+ % (' '.join(argv), p.returncode, err))
+ return out, err, p
+
+def bup(*args):
+ return exo((bup_cmd,) + args)[0]
+
+def bupc(*args):
+ return exc((bup_cmd,) + args)
+
+def create_older_random_saves(n, start_utc, end_utc):
+ with open('foo', 'w') as f:
+ pass
+ exc(['git', 'add', 'foo'])
+ utcs = sorted(randint(start_utc, end_utc) for x in xrange(n))
+ for utc in utcs:
+ with open('foo', 'w') as f:
+ f.write(str(utc) + '\n')
+ exc(['git', 'commit', '--date', str(utc), '-qam', str(utc)])
+ exc(['git', 'gc', '--aggressive'])
+ return utcs
+
+# There is corresponding code in bup for some of this, but the
+# computation method is different here, in part so that the test can
+# provide a more effective cross-check.
+
+period_kinds = ['all', 'dailies', 'monthlies', 'yearlies']
+period_scale = {'s': 1,
+ 'min': 60,
+ 'h': 60 * 60,
+ 'd': 60 * 60 * 24,
+ 'w': 60 * 60 * 24 * 7,
+ 'm': 60 * 60 * 24 * 31,
+ 'y': 60 * 60 * 24 * 366}
+period_scale_kinds = period_scale.keys()
+
+def expected_retentions(utcs, utc_start, spec):
+ if not spec:
+ return utcs
+ utcs = sorted(utcs, reverse=True)
+ period_start = dict(spec)
+ for kind, duration in period_start.iteritems():
+ period_start[kind] = utc_start - period_as_secs(duration)
+ period_start = defaultdict(lambda: float('inf'), period_start)
+
+ all = list(takewhile(lambda x: x >= period_start['all'], utcs))
+ utcs = list(dropwhile(lambda x: x >= period_start['all'], utcs))
+
+ matches = takewhile(lambda x: x >= period_start['dailies'], utcs)
+ dailies = [min(day_utcs) for yday, day_utcs
+ in groupby(matches, lambda x: localtime(x).tm_yday)]
+ utcs = list(dropwhile(lambda x: x >= period_start['dailies'], utcs))
+
+ matches = takewhile(lambda x: x >= period_start['monthlies'], utcs)
+ monthlies = [min(month_utcs) for month, month_utcs
+ in groupby(matches, lambda x: localtime(x).tm_mon)]
+ utcs = dropwhile(lambda x: x >= period_start['monthlies'], utcs)
+
+ matches = takewhile(lambda x: x >= period_start['yearlies'], utcs)
+ yearlies = [min(year_utcs) for year, year_utcs
+ in groupby(matches, lambda x: localtime(x).tm_year)]
+
+ return chain(all, dailies, monthlies, yearlies)
+
+def period_spec(start_utc, end_utc):
+ global period_kinds, period_scale, period_scale_kinds
+ result = []
+ desired_specs = randint(1, 2 * len(period_kinds))
+ assert(desired_specs >= 1) # At least one --keep argument is required
+ while len(result) < desired_specs:
+ period = None
+ if randint(1, 100) <= 5:
+ period = 'forever'
+ else:
+ assert(end_utc > start_utc)
+ period_secs = randint(1, end_utc - start_utc)
+ scale = choice(period_scale_kinds)
+ mag = int(float(period_secs) / period_scale[scale])
+ if mag != 0:
+ period = str(mag) + scale
+ if period:
+ result += [(choice(period_kinds), period)]
+ return tuple(result)
+
+def unique_period_specs(n, start_utc, end_utc):
+ invocations = set()
+ while len(invocations) < n:
+ invocations.add(period_spec(start_utc, end_utc))
+ return tuple(invocations)
+
+def period_spec_to_period_args(spec):
+ return tuple(chain(*(('--keep-' + kind + '-for', period)
+ for kind, period in spec)))
+
+def result_diffline(x):
+ return str(x) + strftime(' %Y-%m-%d-%H%M%S', localtime(x)) + '\n'
+
+def check_prune_result(expected):
+ actual = sorted([int(x)
+ for x in exo(['git', 'log',
+ '--pretty=format:%at'])[0].splitlines()])
+ if expected != actual:
+ for x in expected:
+ print('ex:', x, strftime('%Y-%m-%d-%H%M%S', localtime(x)),
+ file=stderr)
+ for line in unified_diff([result_diffline(x) for x in expected],
+ [result_diffline(x) for x in actual],
+ fromfile='expected', tofile='actual'):
+ sys.stderr.write(line)
+ wvpass(expected == actual)
+
+
+seed = int(environ.get('BUP_TEST_SEED', time()))
+random.seed(seed)
+print('random seed:', seed, file=stderr)
+
+save_population = int(environ.get('BUP_TEST_PRUNE_OLDER_SAVES', 2000))
+prune_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_CYCLES', 20))
+prune_gc_cycles = int(environ.get('BUP_TEST_PRUNE_OLDER_GC_CYCLES', 10))
+
+with test_tempdir('prune-older-') as tmpdir:
+ environ['BUP_DIR'] = tmpdir + '/work/.git'
+ environ['GIT_DIR'] = tmpdir + '/work/.git'
+ now = int(time())
+ three_years_ago = now - (60 * 60 * 24 * 366 * 3)
+ chdir(tmpdir)
+ exc(['git', 'init', 'work'])
+
+ wvstart('generating ' + str(save_population) + ' random saves')
+ chdir(tmpdir + '/work')
+ save_utcs = create_older_random_saves(save_population, three_years_ago, now)
+ chdir(tmpdir)
+ test_set_hash = exo(['git', 'show-ref', '-s', 'master'])[0].rstrip()
+ ls_saves = bup('ls', 'master').splitlines()
+ wvpasseq(save_population + 1, len(ls_saves))
+
+ wvstart('ensure everything kept, if no keep arguments')
+ exc(['git', 'reset', '--hard', test_set_hash])
+ _, errmsg, proc = exo((bup_cmd,
+ 'prune-older', '-v', '--unsafe', '--no-gc',
+ '--wrt', str(now)) \
+ + ('master',),
+ stdout=False, stderr=True, check=False)
+ wvpassne(proc.returncode, 0)
+ wvpass('at least one keep argument is required' in errmsg)
+ check_prune_result(save_utcs)
+
+
+ wvstart('running %d generative no-gc tests on %d saves' % (prune_cycles,
+ save_population))
+ for spec in unique_period_specs(prune_cycles,
+ # Make it more likely we'll have
+ # some outside the save range.
+ three_years_ago - period_scale['m'],
+ now):
+ exc(['git', 'reset', '--hard', test_set_hash])
+ expected = sorted(expected_retentions(save_utcs, now, spec))
+ exc((bup_cmd,
+ 'prune-older', '-v', '--unsafe', '--no-gc', '--wrt', str(now)) \
+ + period_spec_to_period_args(spec) \
+ + ('master',))
+ check_prune_result(expected)
+
+
+ # More expensive because we have to recreate the repo each time
+ wvstart('running %d generative gc tests on %d saves' % (prune_gc_cycles,
+ save_population))
+ exc(['git', 'reset', '--hard', test_set_hash])
+ copytree('work/.git', 'clean-test-repo', symlinks=True)
+ for spec in unique_period_specs(prune_gc_cycles,
+ # Make it more likely we'll have
+ # some outside the save range.
+ three_years_ago - period_scale['m'],
+ now):
+ rmtree('work/.git')
+ copytree('clean-test-repo', 'work/.git')
+ expected = sorted(expected_retentions(save_utcs, now, spec))
+ exc((bup_cmd,
+ 'prune-older', '-v', '--unsafe', '--wrt', str(now)) \
+ + period_spec_to_period_args(spec) \
+ + ('master',))
+ check_prune_result(expected)
projects / bup.git / commitdiff