hashsplit: join buffers more efficiently

Drop buffer_concat in favor of join_bytes, since we can use b''.join()
to join the two put() chunks in python 3, which should be notably more
efficient than copying the memoryview bytes before joining them.

In python2, also use join when we just have bytes, otherwise
accommodate the fact that you can't join bytes and buffers there, and
you can't append bytes to a buffer.  The case we currently care about
should still end up being handled the same as befoere (in the first
clause), i.e. buf + bytes.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/compat.py b/lib/bup/compat.py
index 2458574f02dfc16ee274c81930e5da8ce847ef2a..b795b652f916c46022a0e1a034b2d7be4cd2af0f 100644 (file)
--- a/lib/bup/compat.py
+++ b/lib/bup/compat.py
@@ -40,12 +40,9 @@ if py3:
             return memoryview(object)[offset:]
         return memoryview(object)
 
-    def buffer_concat(b1, b2):
-        if isinstance(b1, memoryview):
-            b1 = b1.tobytes()
-        if isinstance(b1, memoryview):
-            b2 = b2.tobytes()
-        return b1 + b2
+    def join_bytes(*items):
+        """Return the concatenated bytes or memoryview arguments as bytes."""
+        return b''.join(*items)
 
 else:  # Python 2
 
@@ -95,9 +92,18 @@ else:  # Python 2
 
     byte_int = ord
 
-    def buffer_concat(b1, b2):
-        return b1 + b2
-
+    buffer = buffer
+
+    def join_bytes(x, y):
+        """Return the concatenated bytes or buffer arguments as bytes."""
+        if type(x) == buffer:
+            assert type(y) in (bytes, buffer)
+            return x + y
+        assert type(x) == bytes
+        if type(y) == bytes:
+            return b''.join((x, y))
+        assert type(y) in (bytes, buffer)
+        return buffer(x) + y
 
 def wrap_main(main):
     """Run main() and raise a SystemExit with the return value if it

diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py
index af0ad172be93a9e46188e0e3806b1cc6bdfa571c..5c481321dca0e9e2e855bba37b8943c0d136fc9d 100644 (file)
--- a/lib/bup/hashsplit.py
+++ b/lib/bup/hashsplit.py
@@ -3,12 +3,9 @@ from __future__ import absolute_import
 import io, math, os
 
 from bup import _helpers, compat, helpers
-from bup.compat import buffer_concat
+from bup.compat import buffer, join_bytes
 from bup.helpers import sc_page_size
 
-if compat.py_maj > 2:
-    from bup.compat import buffer, buffer_concat
-
 
 _fmincore = getattr(helpers, 'fmincore', None)
 
@@ -32,7 +29,7 @@ class Buf:
 
     def put(self, s):
         if s:
-            self.data = buffer_concat(buffer(self.data, self.start), s)
+            self.data = join_bytes(buffer(self.data, self.start), s)
             self.start = 0
             
     def peek(self, count):