#endif
+#define INTEGRAL_ASSIGNMENT_FITS(dest, src) \
+ ({ \
+ *(dest) = (src); \
+ *(dest) == (src) && (*(dest) < 1) == ((src) < 1); \
+ })
+
+
// At the moment any code that calls INTGER_TO_PY() will have to
// disable -Wtautological-compare for clang. See below.
#endif // not __WIN32__ or __CYGWIN__
+static unsigned long long count_leading_zeros(const unsigned char * const buf,
+ unsigned long long len)
+{
+ const unsigned char *cur = buf;
+ while(len-- && *cur == 0)
+ cur++;
+ return cur - buf;
+}
+
+
+static int write_all(int fd, const void *buf, const size_t count)
+{
+ size_t written = 0;
+ while (written < count)
+ {
+ const ssize_t rc = write(fd, buf + written, count - written);
+ if (rc == -1)
+ return -1;
+ written += rc;
+ }
+ return 0;
+}
+
+
+static int uadd(unsigned long long *dest,
+ const unsigned long long x,
+ const unsigned long long y)
+{
+ const unsigned long long result = x + y;
+ if (result < x || result < y)
+ return 0;
+ *dest = result;
+ return 1;
+}
+
+static PyObject *append_sparse_region(const int fd, unsigned long long n)
+{
+ while(n)
+ {
+ off_t new_off;
+ if (!INTEGRAL_ASSIGNMENT_FITS(&new_off, n))
+ new_off = INT_MAX;
+ const off_t off = lseek(fd, new_off, SEEK_CUR);
+ if (off == (off_t) -1)
+ return PyErr_SetFromErrno(PyExc_IOError);
+ n -= new_off;
+ }
+ return NULL;
+}
+
+
+static PyObject *bup_write_sparsely(PyObject *self, PyObject *args)
+{
+ int fd;
+ unsigned char *buf = NULL;
+ Py_ssize_t sbuf_len;
+ PyObject *py_min_sparse_len, *py_prev_sparse_len;
+ if (!PyArg_ParseTuple(args, "it#OO",
+ &fd, &buf, &sbuf_len,
+ &py_min_sparse_len, &py_prev_sparse_len))
+ return NULL;
+ unsigned long long min_sparse_len, prev_sparse_len, buf_len;
+ if (!bup_ullong_from_py(&min_sparse_len, py_min_sparse_len, "min_sparse_len"))
+ return NULL;
+ if (!bup_ullong_from_py(&prev_sparse_len, py_prev_sparse_len, "prev_sparse_len"))
+ return NULL;
+ if (sbuf_len < 0)
+ return PyErr_Format(PyExc_ValueError, "negative bufer length");
+ if (!INTEGRAL_ASSIGNMENT_FITS(&buf_len, sbuf_len))
+ return PyErr_Format(PyExc_OverflowError, "buffer length too large");
+
+ // The value of zeros_read indicates the number of zeros read from
+ // buf that haven't been accounted for yet (with respect to cur),
+ // while zeros indicates the total number of pending zeros, which
+ // could be larger in the first iteration if prev_sparse_len
+ // wasn't zero.
+ int rc;
+ unsigned long long unexamined = buf_len;
+ unsigned char *block_start = buf, *cur = buf;
+ unsigned long long zeros, zeros_read = count_leading_zeros(cur, unexamined);
+ assert(zeros_read <= unexamined);
+ unexamined -= zeros_read;
+ if (!uadd(&zeros, prev_sparse_len, zeros_read))
+ {
+ PyObject *err = append_sparse_region(fd, prev_sparse_len);
+ if (err != NULL)
+ return err;
+ zeros = zeros_read;
+ }
+
+ while(unexamined)
+ {
+ if (zeros < min_sparse_len)
+ cur += zeros_read;
+ else
+ {
+ rc = write_all(fd, block_start, cur - block_start);
+ if (rc)
+ return PyErr_SetFromErrno(PyExc_IOError);
+ PyObject *err = append_sparse_region(fd, zeros);
+ if (err != NULL)
+ return err;
+ cur += zeros_read;
+ block_start = cur;
+ }
+ // Pending zeros have ether been made sparse, or are going to
+ // be rolled into the next non-sparse block since we know we
+ // now have at least one unexamined non-zero byte.
+ assert(unexamined && *cur != 0);
+ zeros = zeros_read = 0;
+ while (unexamined && *cur != 0)
+ {
+ cur++; unexamined--;
+ }
+ if (unexamined)
+ {
+ zeros_read = count_leading_zeros(cur, unexamined);
+ assert(zeros_read <= unexamined);
+ unexamined -= zeros_read;
+ zeros = zeros_read;
+ }
+ }
+ rc = write_all(fd, block_start, cur - block_start);
+ if (rc)
+ return PyErr_SetFromErrno(PyExc_IOError);
+ return PyLong_FromUnsignedLongLong(zeros);
+}
+
+
static PyObject *selftest(PyObject *self, PyObject *args)
{
if (!PyArg_ParseTuple(args, ""))
#endif
#endif
-
-#define INTEGRAL_ASSIGNMENT_FITS(dest, src) \
- ({ \
- *(dest) = (src); \
- *(dest) == (src) && (*(dest) < 1) == ((src) < 1); \
- })
-
-
#define ASSIGN_PYLONG_TO_INTEGRAL(dest, pylong, overflow) \
({ \
int result = 0; \
static PyMethodDef helper_methods[] = {
+ { "write_sparsely", bup_write_sparsely, METH_VARARGS,
+ "Write buf excepting zeros at the end. Return trailing zero count." },
{ "selftest", selftest, METH_VARARGS,
"Check that the rolling checksum rolls correctly (for unit tests)." },
{ "blobbits", blobbits, METH_VARARGS,
assert(sizeof(PY_LONG_LONG) <= sizeof(long long));
assert(sizeof(unsigned PY_LONG_LONG) <= sizeof(unsigned long long));
+ if (sizeof(off_t) < sizeof(int))
+ {
+ // Originally required by append_sparse_region().
+ fprintf(stderr, "sizeof(off_t) < sizeof(int); please report.\n");
+ exit(1);
+ }
+
char *e;
PyObject *m = Py_InitModule("_helpers", helper_methods);
if (m == NULL)