3 # https://sourceware.org/bugzilla/show_bug.cgi?id=26034
4 export "BUP_ARGV_0"="$0"
7 export "BUP_ARGV_${arg_i}"="$arg"
11 # Here to end of preamble replaced during install
12 bup_python="$(dirname "$0")/../../config/bin/python" || exit $?
13 exec "$bup_python" "$0"
17 from __future__ import absolute_import
18 import glob, os, sys, tempfile
20 sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..']
22 from bup import compat, options, git, bloom
23 from bup.compat import argv_bytes, hexstr
24 from bup.helpers import (add_error, debug1, handle_ctrl_c, log, progress, qprogress,
26 from bup.io import path_msg
30 bup bloom [options...]
32 ruin ruin the specified bloom file (clearing the bitfield)
33 f,force ignore existing bloom file and regenerate it from scratch
34 o,output= output bloom filename (default: auto)
35 d,dir= input directory to look for idx files (default: auto)
36 k,hashes= number of hash functions to use (4 or 5) (default: auto)
37 c,check= check the given .idx file against the bloom filter
41 def ruin_bloom(bloomfilename):
42 rbloomfilename = git.repo_rel(bloomfilename)
43 if not os.path.exists(bloomfilename):
44 log(path_msg(bloomfilename) + '\n')
45 add_error('bloom: %s not found to ruin\n' % path_msg(rbloomfilename))
47 b = bloom.ShaBloom(bloomfilename, readwrite=True, expected=1)
48 b.map[16 : 16 + 2**b.bits] = b'\0' * 2**b.bits
51 def check_bloom(path, bloomfilename, idx):
52 rbloomfilename = git.repo_rel(bloomfilename)
53 ridx = git.repo_rel(idx)
54 if not os.path.exists(bloomfilename):
55 log('bloom: %s: does not exist.\n' % path_msg(rbloomfilename))
57 b = bloom.ShaBloom(bloomfilename)
59 add_error('bloom: %r is invalid.\n' % path_msg(rbloomfilename))
61 base = os.path.basename(idx)
62 if base not in b.idxnames:
63 log('bloom: %s does not contain the idx.\n' % path_msg(rbloomfilename))
66 idx = os.path.join(path, idx)
67 log('bloom: bloom file: %s\n' % path_msg(rbloomfilename))
68 log('bloom: checking %s\n' % path_msg(ridx))
69 for objsha in git.open_idx(idx):
70 if not b.exists(objsha):
71 add_error('bloom: ERROR: object %s missing' % hexstr(objsha))
75 def do_bloom(path, outfilename, k):
77 assert k in (None, 4, 5)
79 if os.path.exists(outfilename) and not opt.force:
80 b = bloom.ShaBloom(outfilename)
82 debug1("bloom: Existing invalid bloom found, regenerating.\n")
89 for i, name in enumerate(glob.glob(b'%s/*.idx' % path)):
90 progress('bloom: counting: %d\r' % i)
91 ix = git.open_idx(name)
92 ixbase = os.path.basename(name)
93 if b and (ixbase in b.idxnames):
101 debug1("bloom: nothing to do.\n")
105 if len(b) != rest_count:
106 debug1("bloom: size %d != idx total %d, regenerating\n"
107 % (len(b), rest_count))
109 elif k is not None and k != b.k:
110 debug1("bloom: new k %d != existing k %d, regenerating\n"
113 elif (b.bits < bloom.MAX_BLOOM_BITS[b.k] and
114 b.pfalse_positive(add_count) > bloom.MAX_PFALSE_POSITIVE):
115 debug1("bloom: regenerating: adding %d entries gives "
116 "%.2f%% false positives.\n"
117 % (add_count, b.pfalse_positive(add_count)))
120 b = bloom.ShaBloom(outfilename, readwrite=True, expected=add_count)
121 if not b: # Need all idxs to build from scratch
123 add_count += rest_count
127 msg = b is None and 'creating from' or 'adding'
128 if not _first: _first = path
129 dirprefix = (_first != path) and git.repo_rel(path) + b': ' or b''
130 progress('bloom: %s%s %d file%s (%d object%s).\r'
131 % (path_msg(dirprefix), msg,
132 len(add), len(add)!=1 and 's' or '',
133 add_count, add_count!=1 and 's' or ''))
137 tfname = os.path.join(path, b'bup.tmp.bloom')
138 b = bloom.create(tfname, expected=add_count, k=k)
142 ix = git.open_idx(name)
143 qprogress('bloom: writing %.2f%% (%d/%d objects)\r'
144 % (icount*100.0/add_count, icount, add_count))
149 # Currently, there's an open file object for tfname inside b.
150 # Make sure it's closed before rename.
154 os.rename(tfname, outfilename)
159 o = options.Options(optspec)
160 opt, flags, extra = o.parse(compat.argv[1:])
163 o.fatal('no positional parameters expected')
165 if not opt.check and opt.k and opt.k not in (4,5):
166 o.fatal('only k values of 4 and 5 are supported')
169 opt.check = argv_bytes(opt.check)
171 git.check_repo_or_die()
173 output = argv_bytes(opt.output) if opt.output else None
174 paths = opt.dir and [argv_bytes(opt.dir)] or git.all_packdirs()
176 debug1('bloom: scanning %s\n' % path_msg(path))
177 outfilename = output or os.path.join(path, b'bup.bloom')
179 check_bloom(path, outfilename, opt.check)
181 ruin_bloom(outfilename)
183 do_bloom(path, outfilename, opt.k)
186 log('WARNING: %d errors encountered during bloom.\n' % len(saved_errors))
189 log('All tests passed.\n')