all: bup Documentation/all
-bup: lib/bup/_version.py lib/bup/_faster$(SOEXT) cmds
+bup: lib/bup/_version.py lib/bup/_helpers$(SOEXT) cmds
Documentation/all: bup
%/clean:
$(MAKE) -C $* clean
-lib/bup/_faster$(SOEXT): \
- lib/bup/bupsplit.c lib/bup/_faster.c lib/bup/csetup.py
+lib/bup/_helpers$(SOEXT): \
+ lib/bup/bupsplit.c lib/bup/_helpers.c lib/bup/csetup.py
@rm -f $@
cd lib/bup && $(PYTHON) csetup.py build
- cp lib/bup/build/*/_faster$(SOEXT) lib/bup/
+ cp lib/bup/build/*/_helpers$(SOEXT) lib/bup/
.PHONY: lib/bup/_version.py
lib/bup/_version.py:
#!/usr/bin/env python
import sys
-from bup import options, git, _faster
+from bup import options, git, _helpers
from bup.helpers import *
if i == last:
continue
#assert(str(i) >= last)
- pm = _faster.bitmatch(last, i)
+ pm = _helpers.bitmatch(last, i)
longmatch = max(longmatch, pm)
last = i
print longmatch
#!/usr/bin/env python
import sys
-from bup import options, _faster
+from bup import options, _helpers
from bup.helpers import *
optspec = """
if opt.force or (not os.isatty(1) and
not atoi(os.environ.get('BUP_FORCE_TTY')) & 1):
- _faster.write_random(sys.stdout.fileno(), total, opt.seed)
+ _helpers.write_random(sys.stdout.fileno(), total, opt.seed)
else:
log('error: not writing binary data to a terminal. Use -f to force.\n')
sys.exit(1)
+++ /dev/null
-#include "bupsplit.h"
-#include <Python.h>
-#include <assert.h>
-#include <stdint.h>
-#include <fcntl.h>
-
-static PyObject *selftest(PyObject *self, PyObject *args)
-{
- if (!PyArg_ParseTuple(args, ""))
- return NULL;
-
- return Py_BuildValue("i", !bupsplit_selftest());
-}
-
-
-static PyObject *blobbits(PyObject *self, PyObject *args)
-{
- if (!PyArg_ParseTuple(args, ""))
- return NULL;
- return Py_BuildValue("i", BUP_BLOBBITS);
-}
-
-
-static PyObject *splitbuf(PyObject *self, PyObject *args)
-{
- unsigned char *buf = NULL;
- int len = 0, out = 0, bits = -1;
-
- if (!PyArg_ParseTuple(args, "t#", &buf, &len))
- return NULL;
- out = bupsplit_find_ofs(buf, len, &bits);
- return Py_BuildValue("ii", out, bits);
-}
-
-
-static PyObject *bitmatch(PyObject *self, PyObject *args)
-{
- unsigned char *buf1 = NULL, *buf2 = NULL;
- int len1 = 0, len2 = 0;
- int byte, bit;
-
- if (!PyArg_ParseTuple(args, "t#t#", &buf1, &len1, &buf2, &len2))
- return NULL;
-
- bit = 0;
- for (byte = 0; byte < len1 && byte < len2; byte++)
- {
- int b1 = buf1[byte], b2 = buf2[byte];
- if (b1 != b2)
- {
- for (bit = 0; bit < 8; bit++)
- if ( (b1 & (0x80 >> bit)) != (b2 & (0x80 >> bit)) )
- break;
- break;
- }
- }
-
- return Py_BuildValue("i", byte*8 + bit);
-}
-
-
-// I would have made this a lower-level function that just fills in a buffer
-// with random values, and then written those values from python. But that's
-// about 20% slower in my tests, and since we typically generate random
-// numbers for benchmarking other parts of bup, any slowness in generating
-// random bytes will make our benchmarks inaccurate. Plus nobody wants
-// pseudorandom bytes much except for this anyway.
-static PyObject *write_random(PyObject *self, PyObject *args)
-{
- uint32_t buf[1024/4];
- int fd = -1, seed = 0;
- ssize_t ret;
- long long len = 0, kbytes = 0, written = 0;
-
- if (!PyArg_ParseTuple(args, "iLi", &fd, &len, &seed))
- return NULL;
-
- srandom(seed);
-
- for (kbytes = 0; kbytes < len/1024; kbytes++)
- {
- unsigned i;
- for (i = 0; i < sizeof(buf)/sizeof(buf[0]); i++)
- buf[i] = random();
- ret = write(fd, buf, sizeof(buf));
- if (ret < 0)
- ret = 0;
- written += ret;
- if (ret < (int)sizeof(buf))
- break;
- if (kbytes/1024 > 0 && !(kbytes%1024))
- fprintf(stderr, "Random: %lld Mbytes\r", kbytes/1024);
- }
-
- // handle non-multiples of 1024
- if (len % 1024)
- {
- unsigned i;
- for (i = 0; i < sizeof(buf)/sizeof(buf[0]); i++)
- buf[i] = random();
- ret = write(fd, buf, len % 1024);
- if (ret < 0)
- ret = 0;
- written += ret;
- }
-
- if (kbytes/1024 > 0)
- fprintf(stderr, "Random: %lld Mbytes, done.\n", kbytes/1024);
- return Py_BuildValue("L", written);
-}
-
-
-static PyObject *open_noatime(PyObject *self, PyObject *args)
-{
- char *filename = NULL;
- int attrs, attrs_noatime, fd;
- if (!PyArg_ParseTuple(args, "s", &filename))
- return NULL;
- attrs = O_RDONLY;
-#ifdef O_NOFOLLOW
- attrs |= O_NOFOLLOW;
-#endif
-#ifdef O_LARGEFILE
- attrs |= O_LARGEFILE;
-#endif
- attrs_noatime = attrs;
-#ifdef O_NOATIME
- attrs_noatime |= O_NOATIME;
-#endif
- fd = open(filename, attrs_noatime);
- if (fd < 0 && errno == EPERM)
- {
- // older Linux kernels would return EPERM if you used O_NOATIME
- // and weren't the file's owner. This pointless restriction was
- // relaxed eventually, but we have to handle it anyway.
- // (VERY old kernels didn't recognized O_NOATIME, but they would
- // just harmlessly ignore it, so this branch won't trigger)
- fd = open(filename, attrs);
- }
- if (fd < 0)
- return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
- return Py_BuildValue("i", fd);
-}
-
-
-static PyObject *fadvise_done(PyObject *self, PyObject *args)
-{
- int fd = -1;
- long long ofs = 0;
- if (!PyArg_ParseTuple(args, "iL", &fd, &ofs))
- return NULL;
-#ifdef POSIX_FADV_DONTNEED
- posix_fadvise(fd, 0, ofs, POSIX_FADV_DONTNEED);
-#endif
- return Py_BuildValue("");
-}
-
-
-static PyMethodDef faster_methods[] = {
- { "selftest", selftest, METH_VARARGS,
- "Check that the rolling checksum rolls correctly (for unit tests)." },
- { "blobbits", blobbits, METH_VARARGS,
- "Return the number of bits in the rolling checksum." },
- { "splitbuf", splitbuf, METH_VARARGS,
- "Split a list of strings based on a rolling checksum." },
- { "bitmatch", bitmatch, METH_VARARGS,
- "Count the number of matching prefix bits between two strings." },
- { "write_random", write_random, METH_VARARGS,
- "Write random bytes to the given file descriptor" },
- { "open_noatime", open_noatime, METH_VARARGS,
- "open() the given filename for read with O_NOATIME if possible" },
- { "fadvise_done", fadvise_done, METH_VARARGS,
- "Inform the kernel that we're finished with earlier parts of a file" },
- { NULL, NULL, 0, NULL }, // sentinel
-};
-
-PyMODINIT_FUNC init_faster(void)
-{
- Py_InitModule("_faster", faster_methods);
-}
--- /dev/null
+#include "bupsplit.h"
+#include <Python.h>
+#include <assert.h>
+#include <stdint.h>
+#include <fcntl.h>
+
+static PyObject *selftest(PyObject *self, PyObject *args)
+{
+ if (!PyArg_ParseTuple(args, ""))
+ return NULL;
+
+ return Py_BuildValue("i", !bupsplit_selftest());
+}
+
+
+static PyObject *blobbits(PyObject *self, PyObject *args)
+{
+ if (!PyArg_ParseTuple(args, ""))
+ return NULL;
+ return Py_BuildValue("i", BUP_BLOBBITS);
+}
+
+
+static PyObject *splitbuf(PyObject *self, PyObject *args)
+{
+ unsigned char *buf = NULL;
+ int len = 0, out = 0, bits = -1;
+
+ if (!PyArg_ParseTuple(args, "t#", &buf, &len))
+ return NULL;
+ out = bupsplit_find_ofs(buf, len, &bits);
+ return Py_BuildValue("ii", out, bits);
+}
+
+
+static PyObject *bitmatch(PyObject *self, PyObject *args)
+{
+ unsigned char *buf1 = NULL, *buf2 = NULL;
+ int len1 = 0, len2 = 0;
+ int byte, bit;
+
+ if (!PyArg_ParseTuple(args, "t#t#", &buf1, &len1, &buf2, &len2))
+ return NULL;
+
+ bit = 0;
+ for (byte = 0; byte < len1 && byte < len2; byte++)
+ {
+ int b1 = buf1[byte], b2 = buf2[byte];
+ if (b1 != b2)
+ {
+ for (bit = 0; bit < 8; bit++)
+ if ( (b1 & (0x80 >> bit)) != (b2 & (0x80 >> bit)) )
+ break;
+ break;
+ }
+ }
+
+ return Py_BuildValue("i", byte*8 + bit);
+}
+
+
+// I would have made this a lower-level function that just fills in a buffer
+// with random values, and then written those values from python. But that's
+// about 20% slower in my tests, and since we typically generate random
+// numbers for benchmarking other parts of bup, any slowness in generating
+// random bytes will make our benchmarks inaccurate. Plus nobody wants
+// pseudorandom bytes much except for this anyway.
+static PyObject *write_random(PyObject *self, PyObject *args)
+{
+ uint32_t buf[1024/4];
+ int fd = -1, seed = 0;
+ ssize_t ret;
+ long long len = 0, kbytes = 0, written = 0;
+
+ if (!PyArg_ParseTuple(args, "iLi", &fd, &len, &seed))
+ return NULL;
+
+ srandom(seed);
+
+ for (kbytes = 0; kbytes < len/1024; kbytes++)
+ {
+ unsigned i;
+ for (i = 0; i < sizeof(buf)/sizeof(buf[0]); i++)
+ buf[i] = random();
+ ret = write(fd, buf, sizeof(buf));
+ if (ret < 0)
+ ret = 0;
+ written += ret;
+ if (ret < (int)sizeof(buf))
+ break;
+ if (kbytes/1024 > 0 && !(kbytes%1024))
+ fprintf(stderr, "Random: %lld Mbytes\r", kbytes/1024);
+ }
+
+ // handle non-multiples of 1024
+ if (len % 1024)
+ {
+ unsigned i;
+ for (i = 0; i < sizeof(buf)/sizeof(buf[0]); i++)
+ buf[i] = random();
+ ret = write(fd, buf, len % 1024);
+ if (ret < 0)
+ ret = 0;
+ written += ret;
+ }
+
+ if (kbytes/1024 > 0)
+ fprintf(stderr, "Random: %lld Mbytes, done.\n", kbytes/1024);
+ return Py_BuildValue("L", written);
+}
+
+
+static PyObject *open_noatime(PyObject *self, PyObject *args)
+{
+ char *filename = NULL;
+ int attrs, attrs_noatime, fd;
+ if (!PyArg_ParseTuple(args, "s", &filename))
+ return NULL;
+ attrs = O_RDONLY;
+#ifdef O_NOFOLLOW
+ attrs |= O_NOFOLLOW;
+#endif
+#ifdef O_LARGEFILE
+ attrs |= O_LARGEFILE;
+#endif
+ attrs_noatime = attrs;
+#ifdef O_NOATIME
+ attrs_noatime |= O_NOATIME;
+#endif
+ fd = open(filename, attrs_noatime);
+ if (fd < 0 && errno == EPERM)
+ {
+ // older Linux kernels would return EPERM if you used O_NOATIME
+ // and weren't the file's owner. This pointless restriction was
+ // relaxed eventually, but we have to handle it anyway.
+ // (VERY old kernels didn't recognized O_NOATIME, but they would
+ // just harmlessly ignore it, so this branch won't trigger)
+ fd = open(filename, attrs);
+ }
+ if (fd < 0)
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
+ return Py_BuildValue("i", fd);
+}
+
+
+static PyObject *fadvise_done(PyObject *self, PyObject *args)
+{
+ int fd = -1;
+ long long ofs = 0;
+ if (!PyArg_ParseTuple(args, "iL", &fd, &ofs))
+ return NULL;
+#ifdef POSIX_FADV_DONTNEED
+ posix_fadvise(fd, 0, ofs, POSIX_FADV_DONTNEED);
+#endif
+ return Py_BuildValue("");
+}
+
+
+static PyMethodDef faster_methods[] = {
+ { "selftest", selftest, METH_VARARGS,
+ "Check that the rolling checksum rolls correctly (for unit tests)." },
+ { "blobbits", blobbits, METH_VARARGS,
+ "Return the number of bits in the rolling checksum." },
+ { "splitbuf", splitbuf, METH_VARARGS,
+ "Split a list of strings based on a rolling checksum." },
+ { "bitmatch", bitmatch, METH_VARARGS,
+ "Count the number of matching prefix bits between two strings." },
+ { "write_random", write_random, METH_VARARGS,
+ "Write random bytes to the given file descriptor" },
+ { "open_noatime", open_noatime, METH_VARARGS,
+ "open() the given filename for read with O_NOATIME if possible" },
+ { "fadvise_done", fadvise_done, METH_VARARGS,
+ "Inform the kernel that we're finished with earlier parts of a file" },
+ { NULL, NULL, 0, NULL }, // sentinel
+};
+
+PyMODINIT_FUNC init_helpers(void)
+{
+ Py_InitModule("_helpers", faster_methods);
+}
from distutils.core import setup, Extension
-_faster_mod = Extension('_faster', sources=['_faster.c', 'bupsplit.c'])
+_helpers_mod = Extension('_helpers', sources=['_helpers.c', 'bupsplit.c'])
-setup(name='_faster',
+setup(name='_helpers',
version='0.1',
description='accelerator library for bup',
- ext_modules=[_faster_mod])
+ ext_modules=[_helpers_mod])
import math
-from bup import _faster
+from bup import _helpers
from bup.helpers import *
BLOB_LWM = 8192*2
def splitbuf(buf):
b = buf.peek(buf.used())
- (ofs, bits) = _faster.splitbuf(b)
+ (ofs, bits) = _helpers.splitbuf(b)
if ofs:
buf.eat(ofs)
return (buffer(b, 0, ofs), bits)
shal.append(('100644', sha, size))
return _make_shalist(shal)[0]
else:
- base_bits = _faster.blobbits()
+ base_bits = _helpers.blobbits()
fanout_bits = int(math.log(fanout, 2))
def bits_to_idx(n):
assert(n >= base_bits)
def open_noatime(name):
- fd = _faster.open_noatime(name)
+ fd = _helpers.open_noatime(name)
try:
return os.fdopen(fd, 'rb', 1024*1024)
except:
def fadvise_done(f, ofs):
assert(ofs >= 0)
if ofs > 0:
- _faster.fadvise_done(f.fileno(), ofs)
+ _helpers.fadvise_done(f.fileno(), ofs)
-from bup import hashsplit, _faster
+from bup import hashsplit, _helpers
from wvtest import *
@wvtest
def test_rolling_sums():
- WVPASS(_faster.selftest())
+ WVPASS(_helpers.selftest())
projects / bup.git / commitdiff