1 #define _LARGEFILE64_SOURCE 1
13 #include <sys/ioctl.h>
19 static PyObject *selftest(PyObject *self, PyObject *args)
21 if (!PyArg_ParseTuple(args, ""))
24 return Py_BuildValue("i", !bupsplit_selftest());
28 static PyObject *blobbits(PyObject *self, PyObject *args)
30 if (!PyArg_ParseTuple(args, ""))
32 return Py_BuildValue("i", BUP_BLOBBITS);
36 static PyObject *splitbuf(PyObject *self, PyObject *args)
38 unsigned char *buf = NULL;
39 int len = 0, out = 0, bits = -1;
41 if (!PyArg_ParseTuple(args, "t#", &buf, &len))
43 out = bupsplit_find_ofs(buf, len, &bits);
44 return Py_BuildValue("ii", out, bits);
48 static PyObject *bitmatch(PyObject *self, PyObject *args)
50 unsigned char *buf1 = NULL, *buf2 = NULL;
51 int len1 = 0, len2 = 0;
54 if (!PyArg_ParseTuple(args, "t#t#", &buf1, &len1, &buf2, &len2))
58 for (byte = 0; byte < len1 && byte < len2; byte++)
60 int b1 = buf1[byte], b2 = buf2[byte];
63 for (bit = 0; bit < 8; bit++)
64 if ( (b1 & (0x80 >> bit)) != (b2 & (0x80 >> bit)) )
70 return Py_BuildValue("i", byte*8 + bit);
74 static PyObject *firstword(PyObject *self, PyObject *args)
76 unsigned char *buf = NULL;
80 if (!PyArg_ParseTuple(args, "t#", &buf, &len))
86 v = ntohl(*(uint32_t *)buf);
87 return Py_BuildValue("I", v);
91 static PyObject *extract_bits(PyObject *self, PyObject *args)
93 unsigned char *buf = NULL;
94 int len = 0, nbits = 0;
97 if (!PyArg_ParseTuple(args, "t#i", &buf, &len, &nbits))
103 mask = (1<<nbits) - 1;
104 v = ntohl(*(uint32_t *)buf);
105 v = (v >> (32-nbits)) & mask;
106 return Py_BuildValue("I", v);
110 // I would have made this a lower-level function that just fills in a buffer
111 // with random values, and then written those values from python. But that's
112 // about 20% slower in my tests, and since we typically generate random
113 // numbers for benchmarking other parts of bup, any slowness in generating
114 // random bytes will make our benchmarks inaccurate. Plus nobody wants
115 // pseudorandom bytes much except for this anyway.
116 static PyObject *write_random(PyObject *self, PyObject *args)
118 uint32_t buf[1024/4];
119 int fd = -1, seed = 0;
121 long long len = 0, kbytes = 0, written = 0;
123 if (!PyArg_ParseTuple(args, "iLi", &fd, &len, &seed))
128 for (kbytes = 0; kbytes < len/1024; kbytes++)
131 for (i = 0; i < sizeof(buf)/sizeof(buf[0]); i++)
133 ret = write(fd, buf, sizeof(buf));
137 if (ret < (int)sizeof(buf))
139 if (kbytes/1024 > 0 && !(kbytes%1024))
140 fprintf(stderr, "Random: %lld Mbytes\r", kbytes/1024);
143 // handle non-multiples of 1024
147 for (i = 0; i < sizeof(buf)/sizeof(buf[0]); i++)
149 ret = write(fd, buf, len % 1024);
156 fprintf(stderr, "Random: %lld Mbytes, done.\n", kbytes/1024);
157 return Py_BuildValue("L", written);
161 static PyObject *open_noatime(PyObject *self, PyObject *args)
163 char *filename = NULL;
164 int attrs, attrs_noatime, fd;
165 if (!PyArg_ParseTuple(args, "s", &filename))
172 attrs |= O_LARGEFILE;
174 attrs_noatime = attrs;
176 attrs_noatime |= O_NOATIME;
178 fd = open(filename, attrs_noatime);
179 if (fd < 0 && errno == EPERM)
181 // older Linux kernels would return EPERM if you used O_NOATIME
182 // and weren't the file's owner. This pointless restriction was
183 // relaxed eventually, but we have to handle it anyway.
184 // (VERY old kernels didn't recognized O_NOATIME, but they would
185 // just harmlessly ignore it, so this branch won't trigger)
186 fd = open(filename, attrs);
189 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
190 return Py_BuildValue("i", fd);
194 static PyObject *fadvise_done(PyObject *self, PyObject *args)
198 if (!PyArg_ParseTuple(args, "iL", &fd, &ofs))
200 #ifdef POSIX_FADV_DONTNEED
201 posix_fadvise(fd, 0, ofs, POSIX_FADV_DONTNEED);
203 return Py_BuildValue("");
208 static PyObject *bup_get_linux_file_attr(PyObject *self, PyObject *args)
215 if (!PyArg_ParseTuple(args, "s", &path))
218 fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
220 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
223 rc = ioctl(fd, FS_IOC_GETFLAGS, &attr);
227 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
231 return Py_BuildValue("k", attr);
235 static PyObject *bup_set_linux_file_attr(PyObject *self, PyObject *args)
242 if (!PyArg_ParseTuple(args, "sk", &path, &attr))
245 fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
247 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
249 rc = ioctl(fd, FS_IOC_SETFLAGS, &attr);
253 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
259 #endif /* def linux */
262 #if defined(_ATFILE_SOURCE) \
263 || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L
264 #define HAVE_BUP_UTIMENSAT 1
266 static PyObject *bup_utimensat(PyObject *self, PyObject *args)
268 int rc, dirfd, flags;
270 long access, access_ns, modification, modification_ns;
271 struct timespec ts[2];
273 if (!PyArg_ParseTuple(args, "is((ll)(ll))i",
277 &modification, &modification_ns,
283 PyErr_SetString(PyExc_ValueError, "access time is NaN");
286 else if (isinf(access))
288 PyErr_SetString(PyExc_ValueError, "access time is infinite");
291 else if (isnan(modification))
293 PyErr_SetString(PyExc_ValueError, "modification time is NaN");
296 else if (isinf(modification))
298 PyErr_SetString(PyExc_ValueError, "modification time is infinite");
302 if (isnan(access_ns))
304 PyErr_SetString(PyExc_ValueError, "access time ns is NaN");
307 else if (isinf(access_ns))
309 PyErr_SetString(PyExc_ValueError, "access time ns is infinite");
312 else if (isnan(modification_ns))
314 PyErr_SetString(PyExc_ValueError, "modification time ns is NaN");
317 else if (isinf(modification_ns))
319 PyErr_SetString(PyExc_ValueError, "modification time ns is infinite");
323 ts[0].tv_sec = access;
324 ts[0].tv_nsec = access_ns;
325 ts[1].tv_sec = modification;
326 ts[1].tv_nsec = modification_ns;
328 rc = utimensat(dirfd, path, ts, flags);
330 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
335 #endif /* defined(_ATFILE_SOURCE)
336 || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */
339 #ifdef linux /* and likely others */
341 #define HAVE_BUP_STAT 1
342 static PyObject *bup_stat(PyObject *self, PyObject *args)
347 if (!PyArg_ParseTuple(args, "s", &filename))
351 rc = stat(filename, &st);
353 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
355 return Py_BuildValue("kkkkkkkk"
359 (unsigned long) st.st_mode,
360 (unsigned long) st.st_ino,
361 (unsigned long) st.st_dev,
362 (unsigned long) st.st_nlink,
363 (unsigned long) st.st_uid,
364 (unsigned long) st.st_gid,
365 (unsigned long) st.st_rdev,
366 (unsigned long) st.st_size,
368 (long) st.st_atim.tv_nsec,
370 (long) st.st_mtim.tv_nsec,
372 (long) st.st_ctim.tv_nsec);
376 #define HAVE_BUP_LSTAT 1
377 static PyObject *bup_lstat(PyObject *self, PyObject *args)
382 if (!PyArg_ParseTuple(args, "s", &filename))
386 rc = lstat(filename, &st);
388 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
390 return Py_BuildValue("kkkkkkkk"
394 (unsigned long) st.st_mode,
395 (unsigned long) st.st_ino,
396 (unsigned long) st.st_dev,
397 (unsigned long) st.st_nlink,
398 (unsigned long) st.st_uid,
399 (unsigned long) st.st_gid,
400 (unsigned long) st.st_rdev,
401 (unsigned long) st.st_size,
403 (long) st.st_atim.tv_nsec,
405 (long) st.st_mtim.tv_nsec,
407 (long) st.st_ctim.tv_nsec);
411 #define HAVE_BUP_FSTAT 1
412 static PyObject *bup_fstat(PyObject *self, PyObject *args)
416 if (!PyArg_ParseTuple(args, "i", &fd))
422 return PyErr_SetFromErrno(PyExc_IOError);
424 return Py_BuildValue("kkkkkkkk"
428 (unsigned long) st.st_mode,
429 (unsigned long) st.st_ino,
430 (unsigned long) st.st_dev,
431 (unsigned long) st.st_nlink,
432 (unsigned long) st.st_uid,
433 (unsigned long) st.st_gid,
434 (unsigned long) st.st_rdev,
435 (unsigned long) st.st_size,
437 (long) st.st_atim.tv_nsec,
439 (long) st.st_mtim.tv_nsec,
441 (long) st.st_ctim.tv_nsec);
444 #endif /* def linux */
447 static PyMethodDef helper_methods[] = {
448 { "selftest", selftest, METH_VARARGS,
449 "Check that the rolling checksum rolls correctly (for unit tests)." },
450 { "blobbits", blobbits, METH_VARARGS,
451 "Return the number of bits in the rolling checksum." },
452 { "splitbuf", splitbuf, METH_VARARGS,
453 "Split a list of strings based on a rolling checksum." },
454 { "bitmatch", bitmatch, METH_VARARGS,
455 "Count the number of matching prefix bits between two strings." },
456 { "firstword", firstword, METH_VARARGS,
457 "Return an int corresponding to the first 32 bits of buf." },
458 { "extract_bits", extract_bits, METH_VARARGS,
459 "Take the first 'nbits' bits from 'buf' and return them as an int." },
460 { "write_random", write_random, METH_VARARGS,
461 "Write random bytes to the given file descriptor" },
462 { "open_noatime", open_noatime, METH_VARARGS,
463 "open() the given filename for read with O_NOATIME if possible" },
464 { "fadvise_done", fadvise_done, METH_VARARGS,
465 "Inform the kernel that we're finished with earlier parts of a file" },
467 { "get_linux_file_attr", bup_get_linux_file_attr, METH_VARARGS,
468 "Return the Linux attributes for the given file." },
469 { "set_linux_file_attr", bup_set_linux_file_attr, METH_VARARGS,
470 "Set the Linux attributes for the given file." },
472 #ifdef HAVE_BUP_UTIMENSAT
473 { "utimensat", bup_utimensat, METH_VARARGS,
474 "Change file timestamps with nanosecond precision." },
477 { "stat", bup_stat, METH_VARARGS,
478 "Extended version of stat." },
480 #ifdef HAVE_BUP_LSTAT
481 { "lstat", bup_lstat, METH_VARARGS,
482 "Extended version of lstat." },
484 #ifdef HAVE_BUP_FSTAT
485 { "fstat", bup_fstat, METH_VARARGS,
486 "Extended version of fstat." },
488 { NULL, NULL, 0, NULL }, // sentinel
492 PyMODINIT_FUNC init_helpers(void)
494 PyObject *m = Py_InitModule("_helpers", helper_methods);
497 #ifdef HAVE_BUP_UTIMENSAT
498 PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD));
499 PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW",
500 Py_BuildValue("i", AT_SYMLINK_NOFOLLOW));
504 PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_True);
507 PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_False);