From: Johannes Berg <johannes@sipsolutions.net>
Date: Fri, 3 Jan 2020 12:53:20 +0000 (+0100)
Subject: git: split out idx file writing to a separate class
X-Git-Tag: 0.31~84
X-Git-Url: https://arthur.barton.de/gitweb/?p=bup.git;a=commitdiff_plain;h=bbeea3a3f5408d8752fa38a0da9d0de6d6aa30d8

git: split out idx file writing to a separate class

Split the idx file writing into a separate class, to make that
kind of action available separately. This will be useful for the
next patch where we use it to test some idx/midx code.

In the future, it'll also be useful for encrypted repositories
since the idx format there will be useful for local caching to
take advantage of midx and bloom code as is, but the packwriter
will of course not be useful.

Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
Reviewed-by: Rob Browning <rlb@defaultvalue.org>
---

diff --git a/lib/bup/git.py b/lib/bup/git.py
index f63daad..dd78aa4 100644
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -722,7 +722,7 @@ class PackWriter:
             assert name.endswith(b'.pack')
             self.filename = name[:-5]
             self.file.write(b'PACK\0\0\0\2\0\0\0\0')
-            self.idx = list(list() for i in range(256))
+            self.idx = PackIdxV2Writer()
 
     def _raw_write(self, datalist, sha):
         self._open()
@@ -747,8 +747,7 @@ class PackWriter:
     def _update_idx(self, sha, crc, size):
         assert(sha)
         if self.idx:
-            self.idx[byte_int(sha[0])].append((sha, crc,
-                                               self.file.tell() - size))
+            self.idx.add(sha, crc, self.file.tell() - size)
 
     def _write(self, sha, type, content):
         if verbose:
@@ -871,8 +870,7 @@ class PackWriter:
         finally:
             f.close()
 
-        obj_list_sha = self._write_pack_idx_v2(self.filename + b'.idx', idx,
-                                               packbin)
+        obj_list_sha = idx.write(self.filename + b'.idx', packbin)
         nameprefix = os.path.join(self.repo_dir,
                                   b'objects/pack/pack-' +  obj_list_sha)
         if os.path.exists(self.filename + b'.map'):
@@ -896,9 +894,20 @@ class PackWriter:
         """Close the pack file and move it to its definitive path."""
         return self._end(run_midx=run_midx)
 
-    def _write_pack_idx_v2(self, filename, idx, packbin):
+
+class PackIdxV2Writer:
+    def __init__(self):
+        self.idx = list(list() for i in range(256))
+        self.count = 0
+
+    def add(self, sha, crc, offs):
+        assert(sha)
+        self.count += 1
+        self.idx[byte_int(sha[0])].append((sha, crc, offs))
+
+    def write(self, filename, packbin):
         ofs64_count = 0
-        for section in idx:
+        for section in self.idx:
             for entry in section:
                 if entry[2] >= 2**31:
                     ofs64_count += 1
@@ -912,7 +921,8 @@ class PackWriter:
             fdatasync(idx_f.fileno())
             idx_map = mmap_readwrite(idx_f, close=False)
             try:
-                count = _helpers.write_idx(filename, idx_map, idx, self.count)
+                count = _helpers.write_idx(filename, idx_map, self.idx,
+                                           self.count)
                 assert(count == self.count)
                 idx_map.flush()
             finally:
@@ -929,7 +939,7 @@ class PackWriter:
             idx_sum.update(b)
 
             obj_list_sum = Sha1()
-            for b in chunkyreader(idx_f, 20*self.count):
+            for b in chunkyreader(idx_f, 20 * self.count):
                 idx_sum.update(b)
                 obj_list_sum.update(b)
             namebase = hexlify(obj_list_sum.digest())
diff --git a/lib/bup/t/tgit.py b/lib/bup/t/tgit.py
index 370e0c1..821ec73 100644
--- a/lib/bup/t/tgit.py
+++ b/lib/bup/t/tgit.py
@@ -199,7 +199,7 @@ def test_long_index():
         with test_tempdir(b'bup-tgit-') as tmpdir:
             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
             git.init_repo(bupdir)
-            w = git.PackWriter()
+            idx = git.PackIdxV2Writer()
             obj_bin = struct.pack('!IIIII',
                     0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
             obj2_bin = struct.pack('!IIIII',
@@ -208,13 +208,11 @@ def test_long_index():
                     0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
             pack_bin = struct.pack('!IIIII',
                     0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
-            idx = list(list() for i in range(256))
-            idx[0].append((obj_bin, 1, 0xfffffffff))
-            idx[0x11].append((obj2_bin, 2, 0xffffffffff))
-            idx[0x22].append((obj3_bin, 3, 0xff))
-            w.count = 3
+            idx.add(obj_bin, 1, 0xfffffffff)
+            idx.add(obj2_bin, 2, 0xffffffffff)
+            idx.add(obj3_bin, 3, 0xff)
             name = tmpdir + b'/tmp.idx'
-            r = w._write_pack_idx_v2(name, idx, pack_bin)
+            r = idx.write(name, pack_bin)
             i = git.PackIdxV2(name, open(name, 'rb'))
             WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
             WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)