hashsplit: skip uncaching when mmap is unsupported

Skip our fmincore based uncaching for any file region where mmap throws
an mmap.error with errno set to ENODEV, i.e.:

  mmap.error: [Errno 19] Operation not supported by device

by having fmincore return None.  This happens on some platforms when the
input is a pipe, i.e.:

  ... | bup split ...

Thanks to Thomas Klausner and Greg Troxel for reporting the problem.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py
index 38681793ded155326863211872a12bf4d89b0fc2..33e91a64ba0830cb4d7e837312d54708b314efcb 100644 (file)
--- a/lib/bup/hashsplit.py
+++ b/lib/bup/hashsplit.py
@@ -96,10 +96,12 @@ def readfile_iter(files, progress=None):
         fd = rpr = rstart = rlen = None
         if _fmincore and hasattr(f, 'fileno'):
             fd = f.fileno()
-            max_chunk = max(1, (8 * 1024 * 1024) / sc_page_size)
-            rpr = _nonresident_page_regions(_fmincore(fd),
-                                            helpers.MINCORE_INCORE, max_chunk)
-            rstart, rlen = next(rpr, (None, None))
+            mcore = _fmincore(fd)
+            if mcore:
+                max_chunk = max(1, (8 * 1024 * 1024) / sc_page_size)
+                rpr = _nonresident_page_regions(mcore, helpers.MINCORE_INCORE,
+                                                max_chunk)
+                rstart, rlen = next(rpr, (None, None))
         while 1:
             if progress:
                 progress(filenum, len(b))

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index 3427e0397fe4e86cb31ed4949ad7aea951582bca..ff6fe62132749ec3b49fd1353d34f2249009d4e2 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -735,7 +735,8 @@ if _mincore:
 
     def fmincore(fd):
         """Return the mincore() data for fd as a bytearray whose values can be
-        tested via MINCORE_INCORE"""
+        tested via MINCORE_INCORE, or None if fd does not fully
+        support the operation."""
         st = os.fstat(fd)
         if (st.st_size == 0):
             return bytearray(0)
@@ -750,7 +751,13 @@ if _mincore:
         for ci in xrange(chunk_count):
             pos = _fmincore_chunk_size * ci;
             msize = min(_fmincore_chunk_size, st.st_size - pos)
-            m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
+            try:
+                m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
+            except mmap.error, ex:
+                if ex.errno == errno.ENODEV:
+                    # Perhaps the file was a pipe, i.e. "... | bup split ..."
+                    return None
+                raise ex
             _mincore(m, msize, 0, result, ci * pages_per_chunk);
         return result