Move .idx file writing to C

[bup.git] / lib / bup / _helpers.c

diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c
index 4befda24a3917cf3a26bb7ba1d06fd981dfcab70..df104cfc66ba39022b29bbc683aab1d40b5003f5 100644 (file)
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@ -412,6 +412,75 @@ static PyObject *merge_into(PyObject *self, PyObject *args)
     return PyLong_FromUnsignedLong(count);
 }
 
+// This function should techniclly be macro'd out if it's going to be used
+// more than ocasionally.  As of this writing, it'll actually never be called
+// in real world bup scenarios (because our packs are < MAX_INT bytes).
+static uint64_t htonll(uint64_t value)
+{
+    static const int endian_test = 42;
+
+    if (*(char *)&endian_test == endian_test) // LSB-MSB
+       return ((uint64_t)htonl(value & 0xFFFFFFFF) << 32) | htonl(value >> 32);
+    return value; // already in network byte order MSB-LSB
+}
+
+static PyObject *write_idx(PyObject *self, PyObject *args)
+{
+    PyObject *pf = NULL, *idx = NULL;
+    PyObject *part;
+    FILE *f;
+    unsigned char *fmap = NULL;
+    int flen = 0;
+    uint32_t total = 0;
+    uint32_t count;
+    int i, j, ofs64_count;
+    uint32_t *fan_ptr, *crc_ptr, *ofs_ptr;
+    struct sha *sha_ptr;
+
+    if (!PyArg_ParseTuple(args, "Ow#OI", &pf, &fmap, &flen, &idx, &total))
+       return NULL;
+
+    fan_ptr = (uint32_t *)&fmap[8];
+    sha_ptr = (struct sha *)&fan_ptr[256];
+    crc_ptr = (uint32_t *)&sha_ptr[total];
+    ofs_ptr = (uint32_t *)&crc_ptr[total];
+    f = PyFile_AsFile(pf);
+
+    count = 0;
+    ofs64_count = 0;
+    for (i = 0; i < 256; ++i)
+    {
+       int plen;
+       part = PyList_GET_ITEM(idx, i);
+       PyList_Sort(part);
+       plen = PyList_GET_SIZE(part);
+       count += plen;
+       *fan_ptr++ = htonl(count);
+       for (j = 0; j < plen; ++j)
+       {
+           unsigned char *sha = NULL;
+           int sha_len = 0;
+           uint32_t crc = 0;
+           uint64_t ofs = 0;
+           if (!PyArg_ParseTuple(PyList_GET_ITEM(part, j), "t#IK",
+                                 &sha, &sha_len, &crc, &ofs))
+               return NULL;
+           if (sha_len != 20)
+               return NULL;
+           memcpy(sha_ptr++, sha, 20);
+           *crc_ptr++ = htonl(crc);
+           if (ofs > 0x7fffffff)
+           {
+               uint64_t nofs = htonll(ofs);
+               fwrite(&nofs, 8, 1, f);
+               ofs = 0x80000000 | ofs64_count++;
+           }
+           *ofs_ptr++ = htonl((uint32_t)ofs);
+       }
+    }
+    return PyLong_FromUnsignedLong(count);
+}
+
 
 // I would have made this a lower-level function that just fills in a buffer
 // with random values, and then written those values from python.  But that's
@@ -552,6 +621,8 @@ static PyMethodDef faster_methods[] = {
        "Take the first 'nbits' bits from 'buf' and return them as an int." },
     { "merge_into", merge_into, METH_VARARGS,
        "Merges a bunch of idx and midx files into a single midx." },
+    { "write_idx", write_idx, METH_VARARGS,
+       "Write a PackIdxV2 file from an idx list of lists of tuples" },
     { "write_random", write_random, METH_VARARGS,
        "Write random bytes to the given file descriptor" },
     { "random_sha", random_sha, METH_VARARGS,