_helpers.c: midx4 didn't handle multiple index with the same object.

It *tried* to handle it, but would end up with a bunch of zero entries at
the end, which prevents .exists() from working correctly in some cases.

In midx2, it made sense to never include the same entry twice, because the
only informatin we had about a particular entry was that it existed.  In
midx4 this is no longer true; we might sometimes want to know *all* the idx
files that contain a particular object (for example, when we implement
expiry later).  So the easiest fix for this bug is to just include multiple
entries when we have them.

Signed-off-by: Avery Pennarun <apenwarr@gmail.com>

diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c
index f1b8d72252f0ac5e0f9786b476ab591b11a7ff28..4d12ddfe9a7ea8d7cbf6d1d34f2fd795fbd62799 100644 (file)
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@ -1,3 +1,4 @@
+#undef NDEBUG
 #include "bupsplit.h"
 #include <Python.h>
 #include <assert.h>
@@ -358,8 +359,8 @@ static PyObject *merge_into(PyObject *self, PyObject *args)
 {
     PyObject *ilist = NULL;
     unsigned char *fmap = NULL;
-    struct sha *sha_ptr, *last = NULL;
-    uint32_t *table_ptr, *name_ptr;
+    struct sha *sha_ptr, *sha_start, *last = NULL;
+    uint32_t *table_ptr, *name_ptr, *name_start;
     struct idx **idxs = NULL;
     int flen = 0, bits = 0, i;
     uint32_t total, count, prefix;
@@ -388,8 +389,8 @@ static PyObject *merge_into(PyObject *self, PyObject *args)
            idxs[i]->cur_name = NULL;
     }
     table_ptr = (uint32_t *)&fmap[MIDX4_HEADERLEN];
-    sha_ptr = (struct sha *)&table_ptr[1<<bits];
-    name_ptr = (uint32_t *)&sha_ptr[total];
+    sha_start = sha_ptr = (struct sha *)&table_ptr[1<<bits];
+    name_start = name_ptr = (uint32_t *)&sha_ptr[total];
 
     last_i = num_i-1;
     count = 0;
@@ -405,25 +406,27 @@ static PyObject *merge_into(PyObject *self, PyObject *args)
        new_prefix = _extract_bits((unsigned char *)idx->cur, bits);
        while (prefix < new_prefix)
            table_ptr[prefix++] = htonl(count);
-       if (last == NULL || _cmp_sha(last, idx->cur) != 0)
-       {
-           memcpy(sha_ptr++, idx->cur, sizeof(struct sha));
-           *name_ptr++ = htonl(_get_idx_i(idx));
-           last = idx->cur;
-       }
+       memcpy(sha_ptr++, idx->cur, sizeof(struct sha));
+       *name_ptr++ = htonl(_get_idx_i(idx));
+       last = idx->cur;
        ++idx->cur;
        if (idx->cur_name != NULL)
            ++idx->cur_name;
        _fix_idx_order(idxs, &last_i);
        ++count;
     }
-    table_ptr[prefix] = htonl(count);
+    while (prefix < (1<<bits))
+       table_ptr[prefix++] = htonl(count);
+    assert(count == total);
+    assert(prefix == (1<<bits));
+    assert(sha_ptr == sha_start+count);
+    assert(name_ptr == name_start+count);
 
     PyMem_Free(idxs);
     return PyLong_FromUnsignedLong(count);
 }
 
-// This function should techniclly be macro'd out if it's going to be used
+// This function should technically be macro'd out if it's going to be used
 // more than ocasionally.  As of this writing, it'll actually never be called
 // in real world bup scenarios (because our packs are < MAX_INT bytes).
 static uint64_t htonll(uint64_t value)

diff --git a/lib/bup/bupsplit.c b/lib/bup/bupsplit.c
index 532619af7bd486da9eaf4c7c0101c115ad358d2b..871e1205b147f7fbb9eee08c9f4ed1d077eff908 100644 (file)
--- a/lib/bup/bupsplit.c
+++ b/lib/bup/bupsplit.c
@@ -1,5 +1,4 @@
 #include "bupsplit.h"
-#include <assert.h>
 #include <stdint.h>
 #include <memory.h>
 #include <stdlib.h>