Use t# instead of et# for hashsplit parameter type.

This lets us work with any kind of buffer object, which means there's no
unnecessary data copying when coming into our module.  Causes a bit of
speedup.

Also refactored the checksum code for easier experimentation.

diff --git a/Makefile b/Makefile
index bc956c895a82fd574202e87f2c9b36b69044c3a8..f57f6e957463ceb592c4f0b3d5a0a174090fae9a 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ hashjoin: hashjoin.sh
 hashsplit.so: hashsplitmodule.o
        $(CC) -shared -Wl,-Bsymbolic-functions -o $@ $<
 
-test: hashsplit hashjoin
+test: all
        ./hashsplit.py <testfile1 >tags1
        ./hashsplit.py <testfile2 >tags2
        diff -u tags1 tags2 || true

diff --git a/hashsplitmodule.c b/hashsplitmodule.c
index cf6305d040359428fa2d1716d9015c63f8cccfa7..98029a27426f806b105a0a86dc6c4555cd3d0464 100644 (file)
--- a/hashsplitmodule.c
+++ b/hashsplitmodule.c
@@ -16,34 +16,35 @@ static uint32_t stupidsum_add(uint32_t old, uint8_t drop, uint8_t add)
 }
 
 
-static PyObject *splitbuf(PyObject *self, PyObject *args)
+static int find_ofs(const unsigned char *buf, int len)
 {
-    char *buf = NULL;
-    int len = 0, count;
-
-    if (!PyArg_ParseTuple(args, "et#", "utf-8", &buf, &len))
-       return NULL;
+    unsigned char window[WINDOWSIZE];
+    uint32_t sum = 0;
+    int i = 0, count;
+    memset(window, 0, sizeof(window));
     
+    for (count = 0; count < len; count++)
     {
-       unsigned char window[WINDOWSIZE];
-       uint32_t sum = 0;
-       int i = 0;
-       memset(window, 0, sizeof(window));
-       
-       for (count = 0; count < len; count++)
-       {
-           sum = stupidsum_add(sum, window[i], buf[count]);
-           window[i] = buf[count];
-           i = (i + 1) % WINDOWSIZE;
-           if ((sum & (BLOBSIZE-1)) == ((~0) & (BLOBSIZE-1)))
-               goto done;
-       }
+       sum = stupidsum_add(sum, window[i], buf[count]);
+       window[i] = buf[count];
+       i = (i + 1) % WINDOWSIZE;
+       if ((sum & (BLOBSIZE-1)) == ((~0) & (BLOBSIZE-1)))
+           return count+1;
     }
-    
-    count = -1;
-done:
-    PyMem_Free(buf);
-    return Py_BuildValue("i", count+1);
+    return 0;
+}
+
+
+static PyObject *splitbuf(PyObject *self, PyObject *args)
+{
+    unsigned char *buf = NULL;
+    int len = 0, out = 0;
+
+    if (!PyArg_ParseTuple(args, "t#", &buf, &len))
+       return NULL;
+    out = find_ofs(buf, len);
+    //return Py_BuildValue("i", len);//len>BLOBSIZE ? BLOBSIZE : len);
+    return Py_BuildValue("i", out);
 }