cmd/midx: --auto mode can combine existing midx files now.

[bup.git] / lib / bup / git.py

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 5ec94ebfa3f3c1f85705fc72e47ac24e723e3446..3612c836dbdec0bd296b63181657785767ae1ce7 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -7,6 +7,8 @@ import heapq
 from bup.helpers import *
 from bup import _helpers
 
+MIDX_VERSION = 2
+
 verbose = 0
 ignore_midx = 0
 home_repodir = os.path.expanduser('~/.bup')
@@ -131,6 +133,7 @@ class PackIdx:
     """Object representation of a Git pack index file."""
     def __init__(self, filename):
         self.name = filename
+        self.idxnames = [self.name]
         self.map = mmap_read(open(filename))
         assert(str(self.map[0:8]) == '\377tOc\0\0\0\2')
         self.fanout = list(struct.unpack('!256I',
@@ -192,12 +195,7 @@ class PackIdx:
         return int(self.fanout[255])
 
 
-def extract_bits(buf, nbits):
-    """Take the first 'nbits' bits from 'buf' and return them as an integer."""
-    mask = (1<<nbits) - 1
-    v = _helpers.firstword(buf)
-    v = (v >> (32-nbits)) & mask
-    return v
+extract_bits = _helpers.extract_bits
 
 
 class PackMidx:
@@ -208,24 +206,39 @@ class PackMidx:
     """
     def __init__(self, filename):
         self.name = filename
+        self.force_keep = False
         assert(filename.endswith('.midx'))
         self.map = mmap_read(open(filename))
-        if str(self.map[0:8]) == 'MIDX\0\0\0\1':
-            log('Warning: ignoring old-style midx %r\n' % filename)
-            self.bits = 0
-            self.entries = 1
-            self.fanout = buffer('\0\0\0\0')
-            self.shalist = buffer('\0'*20)
-            self.idxnames = []
-        else:
-            assert(str(self.map[0:8]) == 'MIDX\0\0\0\2')
-            self.bits = _helpers.firstword(self.map[8:12])
-            self.entries = 2**self.bits
-            self.fanout = buffer(self.map, 12, self.entries*4)
-            shaofs = 12 + self.entries*4
-            nsha = self._fanget(self.entries-1)
-            self.shalist = buffer(self.map, shaofs, nsha*20)
-            self.idxnames = str(self.map[shaofs + 20*nsha:]).split('\0')
+        if str(self.map[0:4]) != 'MIDX':
+            log('Warning: skipping: invalid MIDX header in %r\n' % filename)
+            self.force_keep = True
+            return self._init_failed()
+        ver = struct.unpack('!I', self.map[4:8])[0]
+        if ver < MIDX_VERSION:
+            log('Warning: ignoring old-style (v%d) midx %r\n' 
+                % (ver, filename))
+            self.force_keep = False  # old stuff is boring  
+            return self._init_failed()
+        if ver > MIDX_VERSION:
+            log('Warning: ignoring too-new (v%d) midx %r\n'
+                % (ver, filename))
+            self.force_keep = True  # new stuff is exciting
+            return self._init_failed()
+
+        self.bits = _helpers.firstword(self.map[8:12])
+        self.entries = 2**self.bits
+        self.fanout = buffer(self.map, 12, self.entries*4)
+        shaofs = 12 + self.entries*4
+        nsha = self._fanget(self.entries-1)
+        self.shalist = buffer(self.map, shaofs, nsha*20)
+        self.idxnames = str(self.map[shaofs + 20*nsha:]).split('\0')
+
+    def _init_failed(self):
+        self.bits = 0
+        self.entries = 1
+        self.fanout = buffer('\0\0\0\0')
+        self.shalist = buffer('\0'*20)
+        self.idxnames = []
 
     def _fanget(self, i):
         start = i*4
@@ -361,7 +374,7 @@ class PackIdxList:
                                 d[os.path.join(self.dir, name)] = ix
                             any += 1
                             break
-                    if not any:
+                    if not any and not ix.force_keep:
                         log('midx: removing redundant: %s\n'
                             % os.path.basename(ix.name))
                         unlink(ix.name)
@@ -399,6 +412,15 @@ def _shalist_sort_key(ent):
         return name
 
 
+def open_idx(filename):
+    if filename.endswith('.idx'):
+        return PackIdx(filename)
+    elif filename.endswith('.midx'):
+        return PackMidx(filename)
+    else:
+        raise GitError('idx filenames must end with .idx or .midx')
+
+
 def idxmerge(idxlist):
     """Generate a list of all the objects reachable in a PackIdxList."""
     total = sum(len(i) for i in idxlist)