save-cmd: open files with O_NOATIME on OSes that support it.

Backing up files normally changes their atime, which is bad for two reasons.

First, the files haven't really been "accessed" in a useful sense; the fact
that we backed them up isn't an indication that, say, they're any more
frequently used than they were before.

Secondly, when reading a file updates its atime, the kernel has to enqueue
an atime update (disk write) for every file we back up.  For programs that
read the same files repeatedly, this is no big deal, since the atime just
gets flushed out occasionally (after a lot of updates).  But since bup
accesses *every* file only once, you end up with a huge atime backlog, and
this can wastefully bog down your disks during a big backup.

Of course, mounting your filesystem with noatime would work too, but not
everybody does that.  So let's help them out.

diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py
index 1c4b6ddac833b33350a3af5123396bc7a215acb1..cdae7f683f91282320a3ae8487d58da1774f6d55 100755 (executable)
--- a/cmd/save-cmd.py
+++ b/cmd/save-cmd.py
@@ -199,7 +199,7 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     else:
         if stat.S_ISREG(ent.mode):
             try:
-                f = open(ent.name)
+                f = hashsplit.open_noatime(ent.name)
             except IOError, e:
                 add_error(e)
                 lastskip_name = ent.name

diff --git a/lib/bup/_hashsplit.c b/lib/bup/_hashsplit.c
index cde92017cc48d7896855bc1083293695423b3151..c8a3cde1b112100d36896b049e7cca02cce71080 100644 (file)
--- a/lib/bup/_hashsplit.c
+++ b/lib/bup/_hashsplit.c
@@ -1,6 +1,7 @@
 #include <Python.h>
 #include <assert.h>
 #include <stdint.h>
+#include <fcntl.h>
 
 #define BLOBBITS (13)
 #define BLOBSIZE (1<<BLOBBITS)
@@ -129,6 +130,39 @@ static PyObject *write_random(PyObject *self, PyObject *args)
 }
 
 
+static PyObject *open_noatime(PyObject *self, PyObject *args)
+{
+    char *filename = NULL;
+    int attrs, attrs_noatime, fd;
+    if (!PyArg_ParseTuple(args, "s", &filename))
+       return NULL;
+    attrs = O_RDONLY;
+#ifdef O_NOFOLLOW
+    attrs |= O_NOFOLLOW;
+#endif
+#ifdef O_LARGEFILE
+    attrs |= O_LARGEFILE;
+#endif
+    attrs_noatime = attrs;
+#ifdef O_NOATIME
+    attrs_noatime |= O_NOATIME;
+#endif
+    fd = open(filename, attrs_noatime);
+    if (fd < 0 && errno == EPERM)
+    {
+       // older Linux kernels would return EPERM if you used O_NOATIME
+       // and weren't the file's owner.  This pointless restriction was
+       // relaxed eventually, but we have to handle it anyway.
+       // (VERY old kernels didn't recognized O_NOATIME, but they would
+       // just harmlessly ignore it, so this branch won't trigger)
+       fd = open(filename, attrs);
+    }
+    if (fd < 0)
+       return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
+    return Py_BuildValue("i", fd);
+}
+
+
 static PyMethodDef hashsplit_methods[] = {
     { "blobbits", blobbits, METH_VARARGS,
        "Return the number of bits in the rolling checksum." },
@@ -138,6 +172,8 @@ static PyMethodDef hashsplit_methods[] = {
        "Count the number of matching prefix bits between two strings." },
     { "write_random", write_random, METH_VARARGS,
        "Write random bytes to the given file descriptor" },
+    { "open_noatime", open_noatime, METH_VARARGS,
+       "open() the given filename for read with O_NOATIME if possible" },
     { NULL, NULL, 0, NULL },  // sentinel
 };
 

diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py
index f85011d668a55a4956be894705bac7275eac8866..764ff55f0603c842ac83da2cef2d00b46496c28c 100644 (file)
--- a/lib/bup/hashsplit.py
+++ b/lib/bup/hashsplit.py
@@ -156,3 +156,15 @@ def split_to_blob_or_tree(w, files):
         return ('100644', w.new_blob(''))
     else:
         return ('40000', w.new_tree(shalist))
+
+
+def open_noatime(name):
+    fd = _hashsplit.open_noatime(name)
+    try:
+        return os.fdopen(fd, 'rb', 1024*1024)
+    except:
+        try:
+            os.close(fd)
+        except:
+            pass
+        raise