hashsplit: smallish files (less than BLOB_MAX) weren't getting split.

This buglet was introduced when doing my new fanout cleanups.  It's
relatively unimportant, but it would cause a bit of space wastage for
smallish files that changed by a bit, since we couldn't take advantage of
deduplication for their blocks.

This also explains why the --fanout argument test broke earlier.  I thought
I was going crazy (since the whole fanout implementation had changed and the
number now means something slightly different), so I just removed it.  But
now we can bring it back and it passes again.^

diff --git a/hashsplit.py b/hashsplit.py
index ca5682f60dc95ecdbb409a3ba49218b1527268fa..f9bc48d8f4c48be1dc149255034141d612b0ba34 100644 (file)
--- a/hashsplit.py
+++ b/hashsplit.py
@@ -54,26 +54,35 @@ def blobiter(files):
             yield b
 
 
-def hashsplit_iter(files):
-    assert(BLOB_HWM > BLOB_MAX)
-    buf = Buf()
-    fi = blobiter(files)
+def drainbuf(buf, finalize):
     while 1:
         (blob, bits) = splitbuf(buf)
         if blob:
             yield (blob, bits)
         else:
-            if buf.used() >= BLOB_MAX:
-                # limit max blob size
-                yield (buf.get(buf.used()), 0)
-            while buf.used() < BLOB_HWM:
-                bnew = next(fi)
-                if not bnew:
-                    # eof
-                    if buf.used():
-                        yield (buf.get(buf.used()), 0)
-                    return
-                buf.put(bnew)
+            break
+    if buf.used() > BLOB_MAX:
+        # limit max blob size
+        yield (buf.get(buf.used()), 0)
+    elif finalize and buf.used():
+        yield (buf.get(buf.used()), 0)
+
+
+def hashsplit_iter(files):
+    assert(BLOB_HWM > BLOB_MAX)
+    buf = Buf()
+    fi = blobiter(files)
+    while 1:
+        for i in drainbuf(buf, finalize=False):
+            yield i
+        while buf.used() < BLOB_HWM:
+            bnew = next(fi)
+            if not bnew:
+                # eof
+                for i in drainbuf(buf, finalize=True):
+                    yield i
+                return
+            buf.put(bnew)
 
 
 total_split = 0

diff --git a/t/test.sh b/t/test.sh
index f1ba8ca41f0240d7536ebbf9011ec6c083648d09..40230a550b0b56a692dda9ac5c3698ec63684b1e 100755 (executable)
--- a/t/test.sh
+++ b/t/test.sh
@@ -106,6 +106,7 @@ WVPASS bup margin
 WVPASS bup midx -f
 WVPASS bup margin
 WVPASS bup split -t t/testfile2 >tags2t.tmp
+WVPASS bup split -t t/testfile2 --fanout 3 >tags2tf.tmp
 WVPASS bup split -r "$BUP_DIR" -c t/testfile2 >tags2c.tmp
 WVPASS ls -lR \
    | WVPASS bup split -r "$BUP_DIR" -c --fanout 3 --max-pack-objects 3 -n lslr
@@ -115,6 +116,8 @@ WVPASS bup ls /lslr
 WVPASS bup ls /lslr/1971-01-01   # all dates always exist
 WVFAIL diff -u tags1.tmp tags2.tmp
 
+# fanout must be different from non-fanout
+WVFAIL diff -q tags2t.tmp tags2tf.tmp
 wc -c t/testfile1 t/testfile2
 wc -l tags1.tmp tags2.tmp