git: split out idx file writing to a separate class

Split the idx file writing into a separate class, to make that
kind of action available separately. This will be useful for the
next patch where we use it to test some idx/midx code.

In the future, it'll also be useful for encrypted repositories
since the idx format there will be useful for local caching to
take advantage of midx and bloom code as is, but the packwriter
will of course not be useful.

Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
Reviewed-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/git.py b/lib/bup/git.py
index f63daade595e94fa6a9b54ff56c679c4abfe5f1c..dd78aa459d9fec43378755b8659e6773aba029fd 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -722,7 +722,7 @@ class PackWriter:
             assert name.endswith(b'.pack')
             self.filename = name[:-5]
             self.file.write(b'PACK\0\0\0\2\0\0\0\0')
-            self.idx = list(list() for i in range(256))
+            self.idx = PackIdxV2Writer()
 
     def _raw_write(self, datalist, sha):
         self._open()
@@ -747,8 +747,7 @@ class PackWriter:
     def _update_idx(self, sha, crc, size):
         assert(sha)
         if self.idx:
-            self.idx[byte_int(sha[0])].append((sha, crc,
-                                               self.file.tell() - size))
+            self.idx.add(sha, crc, self.file.tell() - size)
 
     def _write(self, sha, type, content):
         if verbose:
@@ -871,8 +870,7 @@ class PackWriter:
         finally:
             f.close()
 
-        obj_list_sha = self._write_pack_idx_v2(self.filename + b'.idx', idx,
-                                               packbin)
+        obj_list_sha = idx.write(self.filename + b'.idx', packbin)
         nameprefix = os.path.join(self.repo_dir,
                                   b'objects/pack/pack-' +  obj_list_sha)
         if os.path.exists(self.filename + b'.map'):
@@ -896,9 +894,20 @@ class PackWriter:
         """Close the pack file and move it to its definitive path."""
         return self._end(run_midx=run_midx)
 
-    def _write_pack_idx_v2(self, filename, idx, packbin):
+
+class PackIdxV2Writer:
+    def __init__(self):
+        self.idx = list(list() for i in range(256))
+        self.count = 0
+
+    def add(self, sha, crc, offs):
+        assert(sha)
+        self.count += 1
+        self.idx[byte_int(sha[0])].append((sha, crc, offs))
+
+    def write(self, filename, packbin):
         ofs64_count = 0
-        for section in idx:
+        for section in self.idx:
             for entry in section:
                 if entry[2] >= 2**31:
                     ofs64_count += 1
@@ -912,7 +921,8 @@ class PackWriter:
             fdatasync(idx_f.fileno())
             idx_map = mmap_readwrite(idx_f, close=False)
             try:
-                count = _helpers.write_idx(filename, idx_map, idx, self.count)
+                count = _helpers.write_idx(filename, idx_map, self.idx,
+                                           self.count)
                 assert(count == self.count)
                 idx_map.flush()
             finally:
@@ -929,7 +939,7 @@ class PackWriter:
             idx_sum.update(b)
 
             obj_list_sum = Sha1()
-            for b in chunkyreader(idx_f, 20*self.count):
+            for b in chunkyreader(idx_f, 20 * self.count):
                 idx_sum.update(b)
                 obj_list_sum.update(b)
             namebase = hexlify(obj_list_sum.digest())

diff --git a/lib/bup/t/tgit.py b/lib/bup/t/tgit.py
index 370e0c1593d5560eb173f86147e8d0f191dcd238..821ec7354e879203ad3ca8f920b1f23f747c2959 100644 (file)
--- a/lib/bup/t/tgit.py
+++ b/lib/bup/t/tgit.py
@@ -199,7 +199,7 @@ def test_long_index():
         with test_tempdir(b'bup-tgit-') as tmpdir:
             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
             git.init_repo(bupdir)
-            w = git.PackWriter()
+            idx = git.PackIdxV2Writer()
             obj_bin = struct.pack('!IIIII',
                     0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
             obj2_bin = struct.pack('!IIIII',
@@ -208,13 +208,11 @@ def test_long_index():
                     0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
             pack_bin = struct.pack('!IIIII',
                     0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
-            idx = list(list() for i in range(256))
-            idx[0].append((obj_bin, 1, 0xfffffffff))
-            idx[0x11].append((obj2_bin, 2, 0xffffffffff))
-            idx[0x22].append((obj3_bin, 3, 0xff))
-            w.count = 3
+            idx.add(obj_bin, 1, 0xfffffffff)
+            idx.add(obj2_bin, 2, 0xffffffffff)
+            idx.add(obj3_bin, 3, 0xff)
             name = tmpdir + b'/tmp.idx'
-            r = w._write_pack_idx_v2(name, idx, pack_bin)
+            r = idx.write(name, pack_bin)
             i = git.PackIdxV2(name, open(name, 'rb'))
             WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
             WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)