Use a PackBitmap file as a quicker way to check .idx files.

When we receive a new .idx file, we auto-generate a .map file from it.  It's
essentially an allocation bitmap: for each 20-bit prefix, we assign one bit
to tell us if that particular prefix is in that particular packfile.  If it
isn't, there's no point searching the .idx file at all, so we can avoid
mapping in a lot of pages.  If it is, though, we then have to search the
.idx *too*, so we suffer a bit.

On the whole this reduces memory thrashing quite a bit for me, though.
Probably the number of bits needs to be variable in order to work over a
wider range of packfile sizes/numbers.

diff --git a/git.py b/git.py
index 780634d0d1770ae6ddad5f12efaf427ae0db315e..4941e358fcc0d2bb90ff76d683d7ae1cf0384900 100644 (file)
--- a/git.py
+++ b/git.py
@@ -79,6 +79,52 @@ def _decode_packobj(buf):
     return (type, zlib.decompress(buf[i+1:]))
 
 
+MAP_BITS = 20
+
+
+class PackBitmap:
+    def __init__(self, idxname):
+        self.idxname = idxname
+        assert(idxname.endswith('.idx'))
+        self.mapname = idxname[:-4] + '.map'
+        if not os.path.exists(self.mapname):
+            self.gen_map()
+        self.num = 1 << (MAP_BITS-3)
+        f = open(self.mapname)
+        self.map = mmap.mmap(f.fileno(), self.num,
+                             mmap.MAP_PRIVATE, mmap.PROT_READ)
+        f.close()
+
+    def gen_map(self):
+        (dir,fn) = os.path.split(self.idxname)
+        log('Generating map for %s...\n' % fn)
+        count = 0
+        a = ['\0']*((1 << MAP_BITS)/8)
+        for bin in PackIndex(self.idxname):
+            idx = self.bin_to_idx(bin)
+            byte = idx / 8
+            bit = idx % 8
+            a[byte] = chr(ord(a[byte]) | (1 << (7-bit)))
+            count += 1
+        open(self.mapname, 'w+').write(''.join(a))
+
+    def bin_to_idx(self, bin):
+        v = 0
+        for i in range(MAP_BITS/8):
+            v = (v << 8) | ord(bin[i])
+        rest = MAP_BITS - MAP_BITS/8*8
+        x = ord(bin[MAP_BITS/8]) >> (8-rest)
+        v = (v << rest) | x
+        return v
+
+    def might_exist(self, bin):
+        idx = self.bin_to_idx(bin)
+        byte = idx / 8
+        bit = idx % 8
+        v = ord(self.map[byte])
+        return (v >> (7-bit)) & 1
+
+
 class PackIndex:
     def __init__(self, filename):
         self.name = filename
@@ -132,6 +178,10 @@ class PackIndex:
     def exists(self, hash):
         return hash and (self._idx_from_hash(hash) != None) and True or None
 
+    def __iter__(self):
+        for i in xrange(self.fanout[255]):
+            yield buffer(self.map, 8 + 256*4 + 20*i, 20)
+
 
 _mpi_count = 0
 class MultiPackIndex:
@@ -142,6 +192,7 @@ class MultiPackIndex:
         self.dir = dir
         self.also = {}
         self.packs = []
+        self.maps = []
         self.refresh()
 
     def __del__(self):
@@ -153,10 +204,15 @@ class MultiPackIndex:
         if hash in self.also:
             return True
         for i in range(len(self.packs)):
+            m = self.maps[i]
+            if not m.might_exist(hash):
+                # FIXME: this test should perhaps be inside PackIndex?
+                continue
             p = self.packs[i]
             if p.exists(hash):
                 # reorder so most recently used packs are searched first
                 self.packs = [p] + self.packs[:i] + self.packs[i+1:]
+                self.maps =  [m] + self.maps[:i]  + self.maps[i+1:]
                 return p.name
         return None
 
@@ -166,6 +222,7 @@ class MultiPackIndex:
             for f in os.listdir(self.dir):
                 full = os.path.join(self.dir, f)
                 if f.endswith('.idx') and not d.get(full):
+                    self.maps.append(PackBitmap(full))
                     self.packs.append(PackIndex(full))
         #log('MultiPackIndex: using %d packs.\n' % len(self.packs))
 
@@ -323,6 +380,8 @@ class PackWriter:
         if not out:
             raise GitError('git index-pack produced no output')
         nameprefix = repo('objects/pack/%s' % out)
+        if os.path.exists(self.filename + '.map'):
+            os.unlink(self.filename + '.map')
         os.rename(self.filename + '.pack', nameprefix + '.pack')
         os.rename(self.filename + '.idx', nameprefix + '.idx')
         return nameprefix

diff --git a/t/tgit.py b/t/tgit.py
index 43079b60276b006077c6fa05e255c7d3aabe22e0..321a3430f1962d10424ef16e2e391d6479255d5b 100644 (file)
--- a/t/tgit.py
+++ b/t/tgit.py
@@ -32,7 +32,8 @@ def testpacks():
     
     w = git.PackWriter()
     hashes = []
-    for i in range(1000):
+    nobj = 1000
+    for i in range(nobj):
         hashes.append(w.write('blob', str(i)))
     log('\n')
     nameprefix = w.close()
@@ -43,11 +44,15 @@ def testpacks():
     r = git.PackIndex(nameprefix + '.idx')
     print repr(r.fanout)
 
-    for i in range(1000):
+    for i in range(nobj):
         WVPASS(r.find_offset(hashes[i]) > 0)
     WVPASS(r.exists(hashes[99]))
     WVFAIL(r.exists('\0'*20))
 
+    pi = iter(r)
+    for h in sorted(hashes):
+        WVPASSEQ(str(pi.next()).encode('hex'), h.encode('hex'))
+
     WVFAIL(r.find_offset('\0'*20))
 
     r = git.MultiPackIndex('pybuptest.tmp/objects/pack')