vint: implement encoding in C

The vuint/vint encoding is quite slow, possibly due to the use
of BytesIO and byte-wise writing, for what's really almost always
a fixed-size buffer since we typically don't deal with values
that don't fit into a 64-bit signed integer.

Implement this in C to make it faster, at least for those that
fit into signed 64-bit integers (long long, because python makes
overflow checking for unsigned 64-bit integers hard), and if the
C version throws an OverflowError, fall back to the python code.

Also add a test for something that doesn't actually fit into a
64-bit integer.

Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
Reviewed-by: Rob Browning <rlb@defaultvalue.org>
[rlb@defaultvalue.org: adjusted bup_vuint_encode and bup_vint_encode
 to just directly return the PyBytes_FromStringAndSize result]
Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c
index 4ab4d4e92bd3ba5662e0928925a63fa35b99537d..95b679921f952068d0a7e275f72496b061cfd197 100644 (file)
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@ -1701,6 +1701,81 @@ static PyObject *bup_mincore(PyObject *self, PyObject *args)
 }
 #endif /* def BUP_MINCORE_BUF_TYPE */
 
+static unsigned int vuint_encode(long long val, char *buf)
+{
+    unsigned int len = 0;
+
+    if (val < 0) {
+        PyErr_SetString(PyExc_Exception, "vuints must not be negative");
+        return 0;
+    }
+
+    do {
+        buf[len] = val & 0x7f;
+
+        val >>= 7;
+        if (val)
+            buf[len] |= 0x80;
+
+        len++;
+    } while (val);
+
+    return len;
+}
+
+static unsigned int vint_encode(long long val, char *buf)
+{
+    unsigned int len = 1;
+    char sign = 0;
+
+    if (val < 0) {
+        sign = 0x40;
+        val = -val;
+    }
+
+    buf[0] = (val & 0x3f) | sign;
+    val >>= 6;
+    if (val)
+        buf[0] |= 0x80;
+
+    while (val) {
+        buf[len] = val & 0x7f;
+        val >>= 7;
+        if (val)
+            buf[len] |= 0x80;
+        len++;
+    }
+
+    return len;
+}
+
+static PyObject *bup_vuint_encode(PyObject *self, PyObject *args)
+{
+    long long val;
+    // size the buffer appropriately - need 8 bits to encode each 7
+    char buf[(sizeof(val) + 1) / 7 * 8];
+
+    if (!PyArg_ParseTuple(args, "L", &val))
+       return NULL;
+
+    unsigned int len = vuint_encode(val, buf);
+    if (!len)
+        return NULL;
+
+    return PyBytes_FromStringAndSize(buf, len);
+}
+
+static PyObject *bup_vint_encode(PyObject *self, PyObject *args)
+{
+    long long val;
+    // size the buffer appropriately - need 8 bits to encode each 7
+    char buf[(sizeof(val) + 1) / 7 * 8];
+
+    if (!PyArg_ParseTuple(args, "L", &val))
+       return NULL;
+
+    return PyBytes_FromStringAndSize(buf, vint_encode(val, buf));
+}
 
 static PyObject *tuple_from_cstrs(char **cstrs)
 {
@@ -2312,6 +2387,8 @@ static PyMethodDef helper_methods[] = {
       "apply_acl(name, acl, def=None)\n\n"
       "Given a file/dirname (bytes) and the ACLs to restore, do that." },
 #endif /* HAVE_ACLS */
+    { "vuint_encode", bup_vuint_encode, METH_VARARGS, "encode an int to vuint" },
+    { "vint_encode", bup_vint_encode, METH_VARARGS, "encode an int to vint" },
     { NULL, NULL, 0, NULL },  // sentinel
 };
 

diff --git a/lib/bup/vint.py b/lib/bup/vint.py
index 086cb302087d8edae343631c04c68a87440c35f8..8d6a39725bc8ccc093cdedec86678bcb0e9a53ed 100644 (file)
--- a/lib/bup/vint.py
+++ b/lib/bup/vint.py
@@ -12,25 +12,32 @@ from io import BytesIO
 import sys
 
 from bup import compat
+from bup import _helpers
 
 
 def write_vuint(port, x):
-    write = port.write
-    bytes_from_uint = compat.bytes_from_uint
-    if x < 0:
-        raise Exception("vuints must not be negative")
-    elif x == 0:
-        write(bytes_from_uint(0))
-    else:
+    port.write(encode_vuint(x))
+
+
+def encode_vuint(x):
+    try:
+        return _helpers.vuint_encode(x)
+    except OverflowError:
+        ret = b''
+        bytes_from_uint = compat.bytes_from_uint
+        if x < 0:
+            raise Exception("vuints must not be negative")
+        assert x, "the C version should have picked this up"
+
         while True:
             seven_bits = x & 0x7f
             x >>= 7
             if x:
-                write(bytes_from_uint(0x80 | seven_bits))
+                ret += bytes_from_uint(0x80 | seven_bits)
             else:
-                write(bytes_from_uint(seven_bits))
+                ret += bytes_from_uint(seven_bits)
                 break
-
+        return ret
 
 def read_vuint(port):
     c = port.read(1)
@@ -58,22 +65,23 @@ def read_vuint(port):
 def write_vint(port, x):
     # Sign is handled with the second bit of the first byte.  All else
     # matches vuint.
-    write = port.write
-    bytes_from_uint = compat.bytes_from_uint
-    if x == 0:
-        write(bytes_from_uint(0))
-    else:
+    port.write(encode_vint(x))
+
+
+def encode_vint(x):
+    try:
+        return _helpers.vint_encode(x)
+    except OverflowError:
+        bytes_from_uint = compat.bytes_from_uint
+        assert x != 0, "the C version should have picked this up"
         if x < 0:
             x = -x
             sign_and_six_bits = (x & 0x3f) | 0x40
         else:
             sign_and_six_bits = x & 0x3f
         x >>= 6
-        if x:
-            write(bytes_from_uint(0x80 | sign_and_six_bits))
-            write_vuint(port, x)
-        else:
-            write(bytes_from_uint(sign_and_six_bits))
+        assert x, "the C version should have picked this up"
+        return bytes_from_uint(0x80 | sign_and_six_bits) + encode_vuint(x)
 
 
 def read_vint(port):

diff --git a/test/int/test_vint.py b/test/int/test_vint.py
index c2b4818e967856a4cdc521824d3c95f6143d2a0c..7ef1d2437e15d1fc1032a57e8715d0e4b957b6d9 100644 (file)
--- a/test/int/test_vint.py
+++ b/test/int/test_vint.py
@@ -14,7 +14,7 @@ def encode_and_decode_vuint(x):
 
 
 def test_vuint():
-        for x in (0, 1, 42, 128, 10**16):
+        for x in (0, 1, 42, 128, 10**16, 10**100):
             WVPASSEQ(encode_and_decode_vuint(x), x)
         WVEXCEPT(Exception, vint.write_vuint, BytesIO(), -1)
         WVEXCEPT(EOFError, vint.read_vuint, BytesIO())
@@ -27,7 +27,7 @@ def encode_and_decode_vint(x):
 
 
 def test_vint():
-    values = (0, 1, 42, 64, 10**16)
+    values = (0, 1, 42, 64, 10**16, 10**100)
     for x in values:
         WVPASSEQ(encode_and_decode_vint(x), x)
     for x in [-x for x in values]: