Add a --check behavior to verify bloom

This new behavior is useful when diagnosing weird behavior, lets a bloom
filter claiming to contain a particular idx be verified against that idx
file.

Signed-off-by: Brandon Low <lostlogic@lostlogicx.com>

diff --git a/Documentation/bup-bloom.md b/Documentation/bup-bloom.md
index 01373bff13288a3102e3784825f52aa2a2cd3f2f..1501e2e0c517959afdc03ce39830157eb10f3f68 100644 (file)
--- a/Documentation/bup-bloom.md
+++ b/Documentation/bup-bloom.md
@@ -8,7 +8,7 @@ bup-bloom - generates, regenerates, updates bloom filters
 
 # SYNOPSIS
 
-bup daemon [-d dir] [-o outfile] [-k hashes]
+bup bloom [-d dir] [-o outfile] [-k hashes] [-c idxfile]
 
 # DESCRIPTION
 
@@ -31,6 +31,14 @@ it if needed.
     defaults to 5 for repositories < 2TiB and 4 otherwise.
     see comments in git.py for more on this value.
 
+-c, --check=*idxfile*
+:   checks the bloom file (counterintuitively outfile)
+    against the specified .idx file, first checks that the
+    bloom filter is claiming to contain the .idx, then
+    checks that it does actually contain all of the objects
+    in the .idx.  Does not write anything and ignores the
+    `-k` option.
+
 # BUP
 
 Part of the `bup`(1) suite.

diff --git a/cmd/bloom-cmd.py b/cmd/bloom-cmd.py
index 2fa7a68453cb5f304784356b4b193fe53499fb96..81ee392fb2d77a778e8cd8e4dc08dbf32f695728 100755 (executable)
--- a/cmd/bloom-cmd.py
+++ b/cmd/bloom-cmd.py
@@ -9,13 +9,31 @@ bup bloom [options...]
 o,output=  output bloom filename (default: auto-generated)
 d,dir=     input directory to look for idx files (default: auto-generated)
 k,hashes=  number of hash functions to use (4 or 5) (default: auto-generated)
+c,check=   an idx file to check against an existing bloom filter
 """
 
-def do_bloom(path, outfilename):
-    if not outfilename:
-        assert(path)
-        outfilename = os.path.join(path, 'bup.bloom')
+def check_bloom(path, bloomfilename, idx):
+    if not os.path.exists(bloomfilename):
+        log("bloom: %s not found to check\n" % bloomfilename)
+        return
+    b = git.ShaBloom(bloomfilename)
+    if not b.valid():
+        log("bloom: %s could not be opened to check\n" % bloomfilename)
+        return
+    base = os.path.basename(idx)
+    if base not in b.idxnames:
+        log("bloom: filter does not contain %s, nothing to check\n" % idx)
+        return
+    if base == idx:
+        idx = os.path.join(path, idx)
+    log("bloom: checking %s" % idx)
+    for objsha in git.open_idx(idx):
+        if not b.exists(objsha):
+            add_error("bloom: ERROR: %s missing from bloom" 
+                      % str(objsha).encode('hex'))
+
 
+def do_bloom(path, outfilename):
     b = None
     if os.path.exists(outfilename):
         b = git.ShaBloom(outfilename)
@@ -91,9 +109,22 @@ o = options.Options(optspec)
 if extra:
     o.fatal('no positional parameters expected')
 
-if opt.k and opt.k not in (4,5):
-    o.fatal('only k values of 4 and 5 are supported')
-
 git.check_repo_or_die()
+bloompath = opt.dir or git.repo('objects/pack')
+
+if not opt.output:
+    assert(bloompath)
+outfilename = opt.output or os.path.join(bloompath, 'bup.bloom')
+
+if opt.check:
+    check_bloom(bloompath, outfilename, opt.check)
+else:
+    if opt.k and opt.k not in (4,5):
+        o.fatal('only k values of 4 and 5 are supported')
+
+    do_bloom(bloompath, outfilename)
 
-do_bloom(opt.dir or git.repo('objects/pack'), opt.output)
+if saved_errors:
+    log('WARNING: %d errors encountered during bloom.\n' % len(saved_errors))
+    sys.exit(1)
+