From: Avery Pennarun <apenwarr@gmail.com>
Date: Fri, 12 Mar 2010 21:49:32 +0000 (-0500)
Subject: save-cmd: open files with O_NOATIME on OSes that support it.
X-Git-Tag: bup-0.12a~4
X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?p=bup.git;a=commitdiff_plain;h=49611a965cc436095e257d572d51528477bcf52d

save-cmd: open files with O_NOATIME on OSes that support it.

Backing up files normally changes their atime, which is bad for two reasons.

First, the files haven't really been "accessed" in a useful sense; the fact
that we backed them up isn't an indication that, say, they're any more
frequently used than they were before.

Secondly, when reading a file updates its atime, the kernel has to enqueue
an atime update (disk write) for every file we back up.  For programs that
read the same files repeatedly, this is no big deal, since the atime just
gets flushed out occasionally (after a lot of updates).  But since bup
accesses *every* file only once, you end up with a huge atime backlog, and
this can wastefully bog down your disks during a big backup.

Of course, mounting your filesystem with noatime would work too, but not
everybody does that.  So let's help them out.
---

diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py
index 1c4b6dd..cdae7f6 100755
--- a/cmd/save-cmd.py
+++ b/cmd/save-cmd.py
@@ -199,7 +199,7 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     else:
         if stat.S_ISREG(ent.mode):
             try:
-                f = open(ent.name)
+                f = hashsplit.open_noatime(ent.name)
             except IOError, e:
                 add_error(e)
                 lastskip_name = ent.name
diff --git a/lib/bup/_hashsplit.c b/lib/bup/_hashsplit.c
index cde9201..c8a3cde 100644
--- a/lib/bup/_hashsplit.c
+++ b/lib/bup/_hashsplit.c
@@ -1,6 +1,7 @@
 #include <Python.h>
 #include <assert.h>
 #include <stdint.h>
+#include <fcntl.h>
 
 #define BLOBBITS (13)
 #define BLOBSIZE (1<<BLOBBITS)
@@ -129,6 +130,39 @@ static PyObject *write_random(PyObject *self, PyObject *args)
 }
 
 
+static PyObject *open_noatime(PyObject *self, PyObject *args)
+{
+    char *filename = NULL;
+    int attrs, attrs_noatime, fd;
+    if (!PyArg_ParseTuple(args, "s", &filename))
+	return NULL;
+    attrs = O_RDONLY;
+#ifdef O_NOFOLLOW
+    attrs |= O_NOFOLLOW;
+#endif
+#ifdef O_LARGEFILE
+    attrs |= O_LARGEFILE;
+#endif
+    attrs_noatime = attrs;
+#ifdef O_NOATIME
+    attrs_noatime |= O_NOATIME;
+#endif
+    fd = open(filename, attrs_noatime);
+    if (fd < 0 && errno == EPERM)
+    {
+	// older Linux kernels would return EPERM if you used O_NOATIME
+	// and weren't the file's owner.  This pointless restriction was
+	// relaxed eventually, but we have to handle it anyway.
+	// (VERY old kernels didn't recognized O_NOATIME, but they would
+	// just harmlessly ignore it, so this branch won't trigger)
+	fd = open(filename, attrs);
+    }
+    if (fd < 0)
+	return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
+    return Py_BuildValue("i", fd);
+}
+
+
 static PyMethodDef hashsplit_methods[] = {
     { "blobbits", blobbits, METH_VARARGS,
 	"Return the number of bits in the rolling checksum." },
@@ -138,6 +172,8 @@ static PyMethodDef hashsplit_methods[] = {
 	"Count the number of matching prefix bits between two strings." },
     { "write_random", write_random, METH_VARARGS,
 	"Write random bytes to the given file descriptor" },
+    { "open_noatime", open_noatime, METH_VARARGS,
+	"open() the given filename for read with O_NOATIME if possible" },
     { NULL, NULL, 0, NULL },  // sentinel
 };
 
diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py
index f85011d..764ff55 100644
--- a/lib/bup/hashsplit.py
+++ b/lib/bup/hashsplit.py
@@ -156,3 +156,15 @@ def split_to_blob_or_tree(w, files):
         return ('100644', w.new_blob(''))
     else:
         return ('40000', w.new_tree(shalist))
+
+
+def open_noatime(name):
+    fd = _hashsplit.open_noatime(name)
+    try:
+        return os.fdopen(fd, 'rb', 1024*1024)
+    except:
+        try:
+            os.close(fd)
+        except:
+            pass
+        raise