From: Rob Browning <rlb@defaultvalue.org>
Date: Thu, 17 Dec 2015 23:32:36 +0000 (-0600)
Subject: hashsplit: skip uncaching when mmap is unsupported
X-Git-Tag: 0.28-rc1~62
X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?p=bup.git;a=commitdiff_plain;h=7391df7c374e492c212e504b44de976d77037523;ds=sidebyside

hashsplit: skip uncaching when mmap is unsupported

Skip our fmincore based uncaching for any file region where mmap throws
an mmap.error with errno set to ENODEV, i.e.:

  mmap.error: [Errno 19] Operation not supported by device

by having fmincore return None.  This happens on some platforms when the
input is a pipe, i.e.:

  ... | bup split ...

Thanks to Thomas Klausner and Greg Troxel for reporting the problem.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>
---

diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py
index 3868179..33e91a6 100644
--- a/lib/bup/hashsplit.py
+++ b/lib/bup/hashsplit.py
@@ -96,10 +96,12 @@ def readfile_iter(files, progress=None):
         fd = rpr = rstart = rlen = None
         if _fmincore and hasattr(f, 'fileno'):
             fd = f.fileno()
-            max_chunk = max(1, (8 * 1024 * 1024) / sc_page_size)
-            rpr = _nonresident_page_regions(_fmincore(fd),
-                                            helpers.MINCORE_INCORE, max_chunk)
-            rstart, rlen = next(rpr, (None, None))
+            mcore = _fmincore(fd)
+            if mcore:
+                max_chunk = max(1, (8 * 1024 * 1024) / sc_page_size)
+                rpr = _nonresident_page_regions(mcore, helpers.MINCORE_INCORE,
+                                                max_chunk)
+                rstart, rlen = next(rpr, (None, None))
         while 1:
             if progress:
                 progress(filenum, len(b))
diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index 3427e03..ff6fe62 100644
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -735,7 +735,8 @@ if _mincore:
 
     def fmincore(fd):
         """Return the mincore() data for fd as a bytearray whose values can be
-        tested via MINCORE_INCORE"""
+        tested via MINCORE_INCORE, or None if fd does not fully
+        support the operation."""
         st = os.fstat(fd)
         if (st.st_size == 0):
             return bytearray(0)
@@ -750,7 +751,13 @@ if _mincore:
         for ci in xrange(chunk_count):
             pos = _fmincore_chunk_size * ci;
             msize = min(_fmincore_chunk_size, st.st_size - pos)
-            m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
+            try:
+                m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
+            except mmap.error, ex:
+                if ex.errno == errno.ENODEV:
+                    # Perhaps the file was a pipe, i.e. "... | bup split ..."
+                    return None
+                raise ex
             _mincore(m, msize, 0, result, ci * pages_per_chunk);
         return result