bloom: avoid kernel disk flushes when we dirty a lot of pages.

[bup.git] / lib / bup / helpers.py

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index 20339a628852293564d124394aa6118bebd583f7..da27edb44b29330e6a9998e7862a9cb1747ba248 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -328,13 +328,12 @@ class DemuxConn(BaseConn):
         # multiplexed and can be assumed to be debug/log before mux init.
         tail = ''
         while tail != 'BUPMUX':
-            b = os.read(infd, 1024)
+            b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
             if not b:
                 raise IOError('demux: unexpected EOF during initialization')
             tail += b
-            buf = tail[:-6]
+            sys.stderr.write(tail[:-6])  # pre-mux log messages
             tail = tail[-6:]
-            sys.stderr.write(buf)
         self.infd = infd
         self.reader = None
         self.buf = None
@@ -351,7 +350,7 @@ class DemuxConn(BaseConn):
         assert(rl[0] == self.infd)
         ns = ''.join(checked_reader(self.infd, 5))
         n, fdw = struct.unpack('!IB', ns)
-        assert(n<=MAX_PACKET)
+        assert(n <= MAX_PACKET)
         if fdw == 1:
             self.reader = checked_reader(self.infd, n)
         elif fdw == 2:
@@ -451,7 +450,7 @@ def slashappend(s):
         return s
 
 
-def _mmap_do(f, sz, flags, prot):
+def _mmap_do(f, sz, flags, prot, close):
     if not sz:
         st = os.fstat(f.fileno())
         sz = st.st_size
@@ -461,24 +460,34 @@ def _mmap_do(f, sz, flags, prot):
         # no elements :)
         return ''
     map = mmap.mmap(f.fileno(), sz, flags, prot)
-    f.close()  # map will persist beyond file close
+    if close:
+        f.close()  # map will persist beyond file close
     return map
 
 
-def mmap_read(f, sz = 0):
+def mmap_read(f, sz = 0, close=True):
     """Create a read-only memory mapped region on file 'f'.
-
     If sz is 0, the region will cover the entire file.
     """
-    return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ)
+    return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
 
 
-def mmap_readwrite(f, sz = 0):
+def mmap_readwrite(f, sz = 0, close=True):
     """Create a read-write memory mapped region on file 'f'.
+    If sz is 0, the region will cover the entire file.
+    """
+    return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
+                    close)
 
+
+def mmap_readwrite_private(f, sz = 0, close=True):
+    """Create a read-write memory mapped region on file 'f'.
     If sz is 0, the region will cover the entire file.
+    The map is private, which means the changes are never flushed back to the
+    file.
     """
-    return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE)
+    return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
+                    close)
 
 
 def parse_num(s):