Revert "Avoid fadvise (since it doesn't work as expected)"

On some systems (the reported system had 2GB), not forcing out all of
the data traversed during save dramatically slows down save operations,
possibly due to competition with access to the indexes, etc.  So restore
the use of fadvise_done() for now.

This reverts commit db68c4fd4dc3087eb3f0b6e5846629cbf02d0b27.

Thanks to Tilo Schwarz for reporting the problem.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c
index 43526c221403e1e8f06e41fae17c442390ed7cd4..caa5c021c183d2a98f5d8f74e4fe2e44f71d4ec7 100644 (file)
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@ -942,6 +942,19 @@ static PyObject *open_noatime(PyObject *self, PyObject *args)
 }
 
 
+static PyObject *fadvise_done(PyObject *self, PyObject *args)
+{
+    int fd = -1;
+    long long ofs = 0;
+    if (!PyArg_ParseTuple(args, "iL", &fd, &ofs))
+       return NULL;
+#ifdef POSIX_FADV_DONTNEED
+    posix_fadvise(fd, 0, ofs, POSIX_FADV_DONTNEED);
+#endif    
+    return Py_BuildValue("");
+}
+
+
 // Currently the Linux kernel and FUSE disagree over the type for
 // FS_IOC_GETFLAGS and FS_IOC_SETFLAGS.  The kernel actually uses int,
 // but FUSE chose long (matching the declaration in linux/fs.h).  So
@@ -1341,6 +1354,8 @@ static PyMethodDef helper_methods[] = {
         "Return a random 20-byte string" },
     { "open_noatime", open_noatime, METH_VARARGS,
        "open() the given filename for read with O_NOATIME if possible" },
+    { "fadvise_done", fadvise_done, METH_VARARGS,
+       "Inform the kernel that we're finished with earlier parts of a file" },
 #ifdef BUP_HAVE_FILE_ATTRS
     { "get_linux_file_attr", bup_get_linux_file_attr, METH_VARARGS,
       "Return the Linux attributes for the given file." },

diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py
index 987cadb09142581c4a6e4941091c855e90d9a156..571ddf6434ac1723c1948ffc5c4ba6f59843a72c 100644 (file)
--- a/lib/bup/hashsplit.py
+++ b/lib/bup/hashsplit.py
@@ -50,6 +50,9 @@ def readfile_iter(files, progress=None):
                 progress(filenum, len(b))
             b = f.read(BLOB_READ_SIZE)
             ofs += len(b)
+            # Warning: ofs == 0 means 'done with the whole file'
+            # This will only happen here when the file is empty
+            fadvise_done(f, ofs)
             if not b:
                 break
             yield b
@@ -188,3 +191,9 @@ def open_noatime(name):
         except:
             pass
         raise
+
+
+def fadvise_done(f, ofs):
+    assert(ofs >= 0)
+    if ofs > 0 and hasattr(f, 'fileno'):
+        _helpers.fadvise_done(f.fileno(), ofs)