git: split out idx file writing to a separate class

Split the idx file writing into a separate class, to make that
kind of action available separately. This will be useful for the
next patch where we use it to test some idx/midx code.

In the future, it'll also be useful for encrypted repositories
since the idx format there will be useful for local caching to
take advantage of midx and bloom code as is, but the packwriter
will of course not be useful.

Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
Reviewed-by: Rob Browning <rlb@defaultvalue.org>
(cherry picked from commit bbeea3a3f5408d8752fa38a0da9d0de6d6aa30d8)
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 41a49d01a4b0c93ae1a86139280ba00db8ffb945..40bfc0a46731b4c0b4d9a5c299ccca9ddae5477f 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -690,8 +690,8 @@ class PackWriter:
                 raise
             assert(name.endswith('.pack'))
             self.filename = name[:-5]
-            self.file.write('PACK\0\0\0\2\0\0\0\0')
-            self.idx = list(list() for i in xrange(256))
+            self.file.write(b'PACK\0\0\0\2\0\0\0\0')
+            self.idx = PackIdxV2Writer()
 
     def _raw_write(self, datalist, sha):
         self._open()
@@ -716,7 +716,7 @@ class PackWriter:
     def _update_idx(self, sha, crc, size):
         assert(sha)
         if self.idx:
-            self.idx[ord(sha[0])].append((sha, crc, self.file.tell() - size))
+            self.idx.add(sha, crc, self.file.tell() - size)
 
     def _write(self, sha, type, content):
         if verbose:
@@ -839,7 +839,7 @@ class PackWriter:
         finally:
             f.close()
 
-        obj_list_sha = self._write_pack_idx_v2(self.filename + '.idx', idx, packbin)
+        obj_list_sha = idx.write(self.filename + b'.idx', packbin)
         nameprefix = os.path.join(self.repo_dir,
                                   'objects/pack/pack-' +  obj_list_sha)
         if os.path.exists(self.filename + '.map'):
@@ -863,9 +863,20 @@ class PackWriter:
         """Close the pack file and move it to its definitive path."""
         return self._end(run_midx=run_midx)
 
-    def _write_pack_idx_v2(self, filename, idx, packbin):
+
+class PackIdxV2Writer:
+    def __init__(self):
+        self.idx = list(list() for i in range(256))
+        self.count = 0
+
+    def add(self, sha, crc, offs):
+        assert(sha)
+        self.count += 1
+        self.idx[ord(sha[0])].append((sha, crc, offs))
+
+    def write(self, filename, packbin):
         ofs64_count = 0
-        for section in idx:
+        for section in self.idx:
             for entry in section:
                 if entry[2] >= 2**31:
                     ofs64_count += 1
@@ -879,7 +890,8 @@ class PackWriter:
             fdatasync(idx_f.fileno())
             idx_map = mmap_readwrite(idx_f, close=False)
             try:
-                count = _helpers.write_idx(filename, idx_map, idx, self.count)
+                count = _helpers.write_idx(filename, idx_map, self.idx,
+                                           self.count)
                 assert(count == self.count)
                 idx_map.flush()
             finally:
@@ -896,7 +908,7 @@ class PackWriter:
             idx_sum.update(b)
 
             obj_list_sum = Sha1()
-            for b in chunkyreader(idx_f, 20*self.count):
+            for b in chunkyreader(idx_f, 20 * self.count):
                 idx_sum.update(b)
                 obj_list_sum.update(b)
             namebase = obj_list_sum.hexdigest()

diff --git a/lib/bup/t/tgit.py b/lib/bup/t/tgit.py
index f5cfc2a7548f0ea9bd3ccdfa536679a8e6eab10f..6697b0ed02d4651904fdd0ec9d2b38f069133d50 100644 (file)
--- a/lib/bup/t/tgit.py
+++ b/lib/bup/t/tgit.py
@@ -203,7 +203,7 @@ def test_long_index():
             os.environ['BUP_MAIN_EXE'] = bup_exe
             os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup"
             git.init_repo(bupdir)
-            w = git.PackWriter()
+            idx = git.PackIdxV2Writer()
             obj_bin = struct.pack('!IIIII',
                     0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
             obj2_bin = struct.pack('!IIIII',
@@ -212,13 +212,11 @@ def test_long_index():
                     0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
             pack_bin = struct.pack('!IIIII',
                     0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
-            idx = list(list() for i in range(256))
-            idx[0].append((obj_bin, 1, 0xfffffffff))
-            idx[0x11].append((obj2_bin, 2, 0xffffffffff))
-            idx[0x22].append((obj3_bin, 3, 0xff))
-            w.count = 3
-            name = tmpdir + '/tmp.idx'
-            r = w._write_pack_idx_v2(name, idx, pack_bin)
+            idx.add(obj_bin, 1, 0xfffffffff)
+            idx.add(obj2_bin, 2, 0xffffffffff)
+            idx.add(obj3_bin, 3, 0xff)
+            name = tmpdir + b'/tmp.idx'
+            r = idx.write(name, pack_bin)
             i = git.PackIdxV2(name, open(name, 'rb'))
             WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
             WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)