X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=lib%2Fbup%2F_helpers.c;h=808c728642ae2d0669ecebdb256ec85a294f7592;hb=9268378e79d20d66647240b926cfcc1d8e95901f;hp=2a5dc468b7b660a849c6e67df60c8fa91da26b1a;hpb=c7139a3fae51633f0f290ebf73f0cf5e4864e547;p=bup.git diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c index 2a5dc46..808c728 100644 --- a/lib/bup/_helpers.c +++ b/lib/bup/_helpers.c @@ -14,8 +14,10 @@ #include #include #include -#include +#ifdef HAVE_SYS_MMAN_H +#include +#endif #ifdef HAVE_SYS_TYPES_H #include #endif @@ -33,6 +35,10 @@ #include #endif +#ifdef HAVE_TM_TM_GMTOFF +#include +#endif + #include "bupsplit.h" #if defined(FS_IOC_GETFLAGS) && defined(FS_IOC_SETFLAGS) @@ -264,6 +270,21 @@ static int uadd(unsigned long long *dest, return 1; } +static PyObject *append_sparse_region(const int fd, unsigned long long n) +{ + while(n) + { + off_t new_off; + if (!INTEGRAL_ASSIGNMENT_FITS(&new_off, n)) + new_off = INT_MAX; + const off_t off = lseek(fd, new_off, SEEK_CUR); + if (off == (off_t) -1) + return PyErr_SetFromErrno(PyExc_IOError); + n -= new_off; + } + return NULL; +} + static PyObject *bup_write_sparsely(PyObject *self, PyObject *args) { @@ -285,69 +306,61 @@ static PyObject *bup_write_sparsely(PyObject *self, PyObject *args) if (!INTEGRAL_ASSIGNMENT_FITS(&buf_len, sbuf_len)) return PyErr_Format(PyExc_OverflowError, "buffer length too large"); - // For now, there are some cases where we just give up if the - // values are too large, but we could try to break up the relevant - // operations into chunks. - - // Deal with preceding zeros. Just make them sparse, along with - // any leading zeros in buf, even if the region's not >= min, - // since the alternative is a potentially extra small write. - if (prev_sparse_len) - { - const unsigned long long zeros = count_leading_zeros(buf, buf_len); - unsigned long long new_sparse_len = 0; - if (!uadd(&new_sparse_len, prev_sparse_len, zeros)) - return PyErr_Format (PyExc_OverflowError, "sparse region too large"); - if (zeros == buf_len) - return PyLong_FromUnsignedLongLong(new_sparse_len); - - off_t new_off; - if (!INTEGRAL_ASSIGNMENT_FITS(&new_off, new_sparse_len)) - return PyErr_Format(PyExc_OverflowError, - "sparse region too large for seek"); - const off_t off = lseek(fd, new_off, SEEK_CUR); - if (off == -1) - return PyErr_SetFromErrno(PyExc_IOError); - buf += zeros; - buf_len -= zeros; - } - + // The value of zeros_read indicates the number of zeros read from + // buf that haven't been accounted for yet (with respect to cur), + // while zeros indicates the total number of pending zeros, which + // could be larger in the first iteration if prev_sparse_len + // wasn't zero. int rc; unsigned long long unexamined = buf_len; unsigned char *block_start = buf, *cur = buf; + unsigned long long zeros, zeros_read = count_leading_zeros(cur, unexamined); + assert(zeros_read <= unexamined); + unexamined -= zeros_read; + if (!uadd(&zeros, prev_sparse_len, zeros_read)) + { + PyObject *err = append_sparse_region(fd, prev_sparse_len); + if (err != NULL) + return err; + zeros = zeros_read; + } + while(unexamined) { - const unsigned long long zeros = count_leading_zeros(cur, unexamined); - assert(zeros <= unexamined); - unexamined -= zeros; - if (unexamined == 0) // Runs off the end. + if (zeros < min_sparse_len) + cur += zeros_read; + else { rc = write_all(fd, block_start, cur - block_start); if (rc) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromUnsignedLongLong(zeros); - } - cur += zeros; - if (zeros >= min_sparse_len) - { - off_t new_off; - if (!INTEGRAL_ASSIGNMENT_FITS(&new_off, zeros)) - return PyErr_Format(PyExc_ValueError, - "zero count overflows off_t"); - off_t off = lseek(fd, new_off, SEEK_CUR); - if (off == -1) - return PyErr_SetFromErrno(PyExc_IOError); + PyObject *err = append_sparse_region(fd, zeros); + if (err != NULL) + return err; + cur += zeros_read; block_start = cur; } + // Pending zeros have ether been made sparse, or are going to + // be rolled into the next non-sparse block since we know we + // now have at least one unexamined non-zero byte. + assert(unexamined && *cur != 0); + zeros = zeros_read = 0; while (unexamined && *cur != 0) { cur++; unexamined--; } + if (unexamined) + { + zeros_read = count_leading_zeros(cur, unexamined); + assert(zeros_read <= unexamined); + unexamined -= zeros_read; + zeros = zeros_read; + } } rc = write_all(fd, block_start, cur - block_start); if (rc) return PyErr_SetFromErrno(PyExc_IOError); - return PyInt_FromLong(0); + return PyLong_FromUnsignedLongLong(zeros); } @@ -934,11 +947,18 @@ static PyObject *open_noatime(PyObject *self, PyObject *args) static PyObject *fadvise_done(PyObject *self, PyObject *args) { int fd = -1; - long long ofs = 0; - if (!PyArg_ParseTuple(args, "iL", &fd, &ofs)) + long long llofs, lllen = 0; + if (!PyArg_ParseTuple(args, "iLL", &fd, &llofs, &lllen)) return NULL; + off_t ofs, len; + if (!INTEGRAL_ASSIGNMENT_FITS(&ofs, llofs)) + return PyErr_Format(PyExc_OverflowError, + "fadvise offset overflows off_t"); + if (!INTEGRAL_ASSIGNMENT_FITS(&len, lllen)) + return PyErr_Format(PyExc_OverflowError, + "fadvise length overflows off_t"); #ifdef POSIX_FADV_DONTNEED - posix_fadvise(fd, 0, ofs, POSIX_FADV_DONTNEED); + posix_fadvise(fd, ofs, len, POSIX_FADV_DONTNEED); #endif return Py_BuildValue(""); } @@ -1290,6 +1310,75 @@ static PyObject *bup_fstat(PyObject *self, PyObject *args) } +#ifdef HAVE_TM_TM_GMTOFF +static PyObject *bup_localtime(PyObject *self, PyObject *args) +{ + long long lltime; + time_t ttime; + if (!PyArg_ParseTuple(args, "L", &lltime)) + return NULL; + if (!INTEGRAL_ASSIGNMENT_FITS(&ttime, lltime)) + return PyErr_Format(PyExc_OverflowError, "time value too large"); + + struct tm tm; + tzset(); + if(localtime_r(&ttime, &tm) == NULL) + return PyErr_SetFromErrno(PyExc_OSError); + + // Match the Python struct_time values. + return Py_BuildValue("[i,i,i,i,i,i,i,i,i,i,s]", + 1900 + tm.tm_year, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, + tm.tm_wday, tm.tm_yday + 1, + tm.tm_isdst, tm.tm_gmtoff, tm.tm_zone); +} +#endif /* def HAVE_TM_TM_GMTOFF */ + + +#ifdef BUP_MINCORE_BUF_TYPE +static PyObject *bup_mincore(PyObject *self, PyObject *args) +{ + const char *src; + Py_ssize_t src_ssize; + Py_buffer dest; + PyObject *py_src_n, *py_src_off, *py_dest_off; + if (!PyArg_ParseTuple(args, "s#OOw*O", + &src, &src_ssize, &py_src_n, &py_src_off, + &dest, &py_dest_off)) + return NULL; + + unsigned long long src_size, src_n, src_off, dest_size, dest_off; + if (!(bup_ullong_from_py(&src_n, py_src_n, "src_n") + && bup_ullong_from_py(&src_off, py_src_off, "src_off") + && bup_ullong_from_py(&dest_off, py_dest_off, "dest_off"))) + return NULL; + + if (!INTEGRAL_ASSIGNMENT_FITS(&src_size, src_ssize)) + return PyErr_Format(PyExc_OverflowError, "invalid src size"); + unsigned long long src_region_end; + + if (!uadd(&src_region_end, src_off, src_n)) + return PyErr_Format(PyExc_OverflowError, "(src_off + src_n) too large"); + if (src_region_end > src_size) + return PyErr_Format(PyExc_OverflowError, "region runs off end of src"); + + if (!INTEGRAL_ASSIGNMENT_FITS(&dest_size, dest.len)) + return PyErr_Format(PyExc_OverflowError, "invalid dest size"); + if (dest_off > dest_size) + return PyErr_Format(PyExc_OverflowError, "region runs off end of dest"); + + size_t length; + if (!INTEGRAL_ASSIGNMENT_FITS(&length, src_n)) + return PyErr_Format(PyExc_OverflowError, "src_n overflows size_t"); + int rc = mincore((void *)(src + src_off), src_n, + (BUP_MINCORE_BUF_TYPE *) (dest.buf + dest_off)); + if (rc != 0) + return PyErr_SetFromErrno(PyExc_OSError); + return Py_BuildValue("O", Py_None); +} +#endif /* def BUP_MINCORE_BUF_TYPE */ + + static PyMethodDef helper_methods[] = { { "write_sparsely", bup_write_sparsely, METH_VARARGS, "Write buf excepting zeros at the end. Return trailing zero count." }, @@ -1348,6 +1437,15 @@ static PyMethodDef helper_methods[] = { "Extended version of lstat." }, { "fstat", bup_fstat, METH_VARARGS, "Extended version of fstat." }, +#ifdef HAVE_TM_TM_GMTOFF + { "localtime", bup_localtime, METH_VARARGS, + "Return struct_time elements plus the timezone offset and name." }, +#endif +#ifdef BUP_MINCORE_BUF_TYPE + { "mincore", bup_mincore, METH_VARARGS, + "For mincore(src, src_n, src_off, dest, dest_off)" + " call the system mincore(src + src_off, src_n, &dest[dest_off])." }, +#endif { NULL, NULL, 0, NULL }, // sentinel }; @@ -1365,6 +1463,13 @@ PyMODINIT_FUNC init_helpers(void) assert(sizeof(PY_LONG_LONG) <= sizeof(long long)); assert(sizeof(unsigned PY_LONG_LONG) <= sizeof(unsigned long long)); + if (sizeof(off_t) < sizeof(int)) + { + // Originally required by append_sparse_region(). + fprintf(stderr, "sizeof(off_t) < sizeof(int); please report.\n"); + exit(1); + } + char *e; PyObject *m = Py_InitModule("_helpers", helper_methods); if (m == NULL) @@ -1372,6 +1477,15 @@ PyMODINIT_FUNC init_helpers(void) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wtautological-compare" // For INTEGER_TO_PY(). + { + PyObject *value; + value = INTEGER_TO_PY(INT_MAX); + PyObject_SetAttrString(m, "INT_MAX", value); + Py_DECREF(value); + value = INTEGER_TO_PY(UINT_MAX); + PyObject_SetAttrString(m, "UINT_MAX", value); + Py_DECREF(value); + } #ifdef HAVE_UTIMENSAT { PyObject *value; @@ -1386,18 +1500,14 @@ PyMODINIT_FUNC init_helpers(void) Py_DECREF(value); } #endif +#ifdef BUP_HAVE_MINCORE_INCORE { PyObject *value; - const long arg_max = sysconf(_SC_ARG_MAX); - if (arg_max == -1) - { - fprintf(stderr, "Cannot find SC_ARG_MAX, please report a bug.\n"); - exit(1); - } - value = INTEGER_TO_PY(arg_max); - PyObject_SetAttrString(m, "SC_ARG_MAX", value); + value = INTEGER_TO_PY(MINCORE_INCORE); + PyObject_SetAttrString(m, "MINCORE_INCORE", value); Py_DECREF(value); } +#endif #pragma clang diagnostic pop // ignored "-Wtautological-compare" e = getenv("BUP_FORCE_TTY");