bloom: avoid kernel disk flushes when we dirty a lot of pages.

[bup.git] / cmd / bloom-cmd.py

diff --git a/cmd/bloom-cmd.py b/cmd/bloom-cmd.py
index 768d5fe5fdd3243c0e21f6f85682a15c18596a86..9709239a612584f053def4528d8977107bde4ac7 100755 (executable)
--- a/cmd/bloom-cmd.py
+++ b/cmd/bloom-cmd.py
@@ -8,6 +8,7 @@ bup bloom [options...]
 --
 o,output=  output bloom filename (default: auto-generated)
 d,dir=     input directory to look for idx files (default: auto-generated)
+k,hashes=  number of hash functions to use (4 or 5) (default: auto-generated)
 """
 
 def do_bloom(path, outfilename):
@@ -17,8 +18,9 @@ def do_bloom(path, outfilename):
 
     b = None
     if os.path.exists(outfilename):
-        b = git.ShaBloom(outfilename, readwrite=True)
+        b = git.ShaBloom(outfilename)
         if not b.valid():
+            debug1("bloom: Existing invalid bloom found, regenerating.\n")
             b = None
 
     add = []
@@ -28,11 +30,11 @@ def do_bloom(path, outfilename):
     for name in glob.glob('%s/*.idx' % path):
         ix = git.open_idx(name)
         ixbase = os.path.basename(name)
-        if b is not None and ixbase in b.idxnames:
-            rest.append(ix)
+        if b and (ixbase in b.idxnames):
+            rest.append(name)
             rest_count += len(ix)
         else:
-            add.append(ix)
+            add.append(name)
             add_count += len(ix)
     total = add_count + rest_count
 
@@ -40,17 +42,19 @@ def do_bloom(path, outfilename):
         log("bloom: Nothing to do\n")
         return
 
-    if b is not None:
+    if b:
         if len(b) != rest_count:
             log("bloom: size %d != idx total %d, regenerating\n"
                     % (len(b), rest_count))
             b = None
-        elif b.bits < git.MAX_BLOOM_BITS and \
-             b.pfalse_positive(add_count) > git.MAX_PFALSE_POSITIVE:
+        elif (b.bits < git.MAX_BLOOM_BITS and
+              b.pfalse_positive(add_count) > git.MAX_PFALSE_POSITIVE):
             log("bloom: %d more entries => %.2f false positive, regenerating\n"
                     % (add_count, b.pfalse_positive(add_count)))
             b = None
-    if b is None: # Need all idxs to build from scratch
+        else:
+            b = git.ShaBloom(outfilename, readwrite=True, expected=add_count)
+    if not b: # Need all idxs to build from scratch
         add += rest
         add_count += rest_count
     del rest
@@ -59,22 +63,21 @@ def do_bloom(path, outfilename):
     msg = b is None and 'creating from' or 'adding'
     log('bloom: %s %d files (%d objects).\n' % (msg, len(add), add_count))
 
-    tempname = None
+    tfname = None
     if b is None:
-        tf = tempfile.NamedTemporaryFile(
-                dir=path, suffix='bup.bloom', delete=False)
-        tempname = tf.name
-        tf.close()
-        b = git.ShaBloom.create(tempname, readwrite=True, expected=add_count)
+        tfname = os.path.join(path, 'bup.tmp.bloom')
+        tf = open(tfname, 'w+')
+        b = git.ShaBloom.create(tfname, f=tf, expected=add_count, k=opt.k)
     count = 0
-    for ix in add:
+    for name in add:
+        ix = git.open_idx(name)
         progress('Writing bloom: %d/%d\r' % (count, len(add)))
         b.add_idx(ix)
         count += 1
     log('Writing bloom: %d/%d, done.\n' % (count, len(add)))
 
-    if tempname:
-        os.rename(tempname, outfilename)
+    if tfname:
+        os.rename(tfname, outfilename)
 
 
 handle_ctrl_c()
@@ -85,6 +88,9 @@ o = options.Options(optspec)
 if extra:
     o.fatal('no positional parameters expected')
 
+if opt.k and opt.k not in (4,5):
+    o.fatal('only k values of 4 and 5 are supported')
+
 git.check_repo_or_die()
 
 do_bloom(opt.dir or git.repo('objects/pack'), opt.output)