cmd-midx: a command for merging multiple .idx files into one.

This introduces a new "multi-index" index format, as suggested by Lukasz
Kosewski.

.midx files have a variable-bit-width fanout table that's supposedly
optimized to be able to find any sha1 while dirtying only two pages (one for
the fanout table lookup, and one for the final binary search).  Each entry
in the fanout table should correspond to approximately one page's worth of
sha1sums.

Also adds a PackMidx class, which acts just like PackIndex, but for .midx
files.  Not using it for anything yet, though.  The idea is to greatly
reduce memory burn when searching through lots of pack files.

diff --git a/Makefile b/Makefile
index 1068de55bc27c1253ad2a2fc13c44cf7ff433e35..2723930586eaa4b6ddead0530101b9fcdb25b707 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -20,6 +20,7 @@ endif
 default: all
 
 all: bup-split bup-join bup-save bup-init bup-server bup-index bup-tick \
+       bup-midx \
        bup memtest randomgen$(EXT) _hashsplit$(SOEXT)
 
 randomgen$(EXT): randomgen.o

diff --git a/cmd-midx.py b/cmd-midx.py
new file mode 100755 (executable)
index 0000000..d3e87a7
--- /dev/null
+++ b/cmd-midx.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+import sys, math, struct
+import options, git
+from helpers import *
+
+PAGE_SIZE=4096
+SHA_PER_PAGE=PAGE_SIZE/200.
+
+
+def next(it):
+    try:
+        return it.next()
+    except StopIteration:
+        return None
+    
+    
+optspec = """
+bup midx -o outfile.midx <idxnames...>
+--
+o,output=  output midx file name
+"""
+o = options.Options('bup midx', optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if not extra:
+    log("bup midx: no input filenames given\n")
+    o.usage()
+if not opt.output:
+    log("bup midx: no output filename given\n")
+    o.usage()
+    
+inp = []
+total = 0
+for name in extra:
+    ix = git.PackIndex(name)
+    inp.append(ix)
+    total += len(ix)
+    
+log('total objects expected: %d\n' % total)
+pages = total/SHA_PER_PAGE
+log('pages: %d\n' % pages)
+bits = int(math.ceil(math.log(pages, 2)))
+log('table bits: %d\n' % bits)
+entries = 2**bits
+log('table entries: %d\n' % entries)
+log('table size: %d\n' % (entries*8))
+
+table = [0]*entries
+
+def merge(idxlist):
+    iters = [iter(i) for i in inp]
+    iters = [[next(it), it] for it in iters]
+    count = 0
+    while iters:
+        if (count % (total/100)) == 0:
+            log('\rMerging: %d%%' % (count*100/total))
+        e = min(iters)  # FIXME: very slow for long lists
+        assert(e[0])
+        yield e[0]
+        count += 1
+        prefix = git.extract_bits(e[0], bits)
+        table[prefix] = count
+        e[0] = next(e[1])
+        iters = filter(lambda x: x[0], iters)
+    log('\rMerging: done.\n')
+
+f = open(opt.output, 'w+')
+f.write('MIDX\0\0\0\1')
+f.write(struct.pack('!I', bits))
+assert(f.tell() == 12)
+f.write('\0'*8*entries)
+
+for e in merge(inp):
+    f.write(e)
+
+f.write('\0'.join([os.path.basename(p) for p in extra]))
+
+f.seek(12)
+f.write(struct.pack('!%dQ' % entries, *table))
+f.close()
+
+# this is just for testing
+if 1:
+    p = git.PackMidx(opt.output)
+    assert(len(p.idxnames) == len(extra))
+    print p.idxnames
+    assert(len(p) == total)
+    pi = iter(p)
+    for i in merge(inp):
+        assert(i == pi.next())
+        assert(p.exists(i))

diff --git a/git.py b/git.py
index 2839abb7404c85cf3408cae539159307e71bcaf8..efe70a9204bb5ffbd8c8cc370c4c3d46e9b7423e 100644 (file)
--- a/git.py
+++ b/git.py
@@ -176,6 +176,62 @@ class PackIndex:
         for i in xrange(self.fanout[255]):
             yield buffer(self.map, 8 + 256*4 + 20*i, 20)
 
+    def __len__(self):
+        return self.fanout[255]
+
+
+def extract_bits(buf, bits):
+    mask = (1<<bits) - 1
+    v = struct.unpack('!Q', buf[0:8])[0]
+    v = (v >> (64-bits)) & mask
+    return v
+
+
+class PackMidx:
+    def __init__(self, filename):
+        self.name = filename
+        assert(filename.endswith('.midx'))
+        self.map = mmap_read(open(filename))
+        assert(str(self.map[0:8]) == 'MIDX\0\0\0\1')
+        self.bits = struct.unpack('!I', self.map[8:12])[0]
+        self.entries = 2**self.bits
+        self.fanout = buffer(self.map, 12, self.entries*8)
+        shaofs = 12 + self.entries*8
+        nsha = self._fanget(self.entries-1)
+        self.shalist = buffer(self.map, shaofs, nsha*20)
+        self.idxnames = str(self.map[shaofs + 20*nsha:]).split('\0')
+
+    def _fanget(self, i):
+        start = i*8
+        s = self.fanout[start:start+8]
+        return struct.unpack('!Q', s)[0]
+    
+    def exists(self, hash):
+        want = str(hash)
+        el = extract_bits(want, self.bits)
+        if el:
+            start = self._fanget(el-1)
+        else:
+            start = 0
+        end = self._fanget(el)
+        while start < end:
+            mid = start + (end-start)/2
+            v = str(self.shalist[mid*20:(mid+1)*20])
+            if v < want:
+                start = mid+1
+            elif v > want:
+                end = mid
+            else: # got it!
+                return True
+        return None
+    
+    def __iter__(self):
+        for i in xrange(self._fanget(self.entries-1)):
+            yield buffer(self.shalist, i*20, 20)
+    
+    def __len__(self):
+        return self._fanget(self.entries-1)
+
 
 _mpi_count = 0
 class MultiPackIndex: