Merge branch 'maint'

* maint:
  git.py: recover more elegantly if a MIDX file has the wrong version.
  cmd/midx: add a new --max-files parameter.

Conflicts:
	lib/bup/git.py

diff --git a/cmd/midx-cmd.py b/cmd/midx-cmd.py
index bbfd2acc0d2a562b26c755f64636f8debdd0467d..579f78a1504c2bbda55f9d7d5020872c9932b6c4 100755 (executable)
--- a/cmd/midx-cmd.py
+++ b/cmd/midx-cmd.py
@@ -7,6 +7,11 @@ PAGE_SIZE=4096
 SHA_PER_PAGE=PAGE_SIZE/20.
 
 
+def _group(l, count):
+    for i in xrange(0, len(l), count):
+        yield l[i:i+count]
+
+
 def merge(idxlist, bits, table):
     count = 0
     for e in git.idxmerge(idxlist):
@@ -81,6 +86,7 @@ bup midx [options...] <idxnames...>
 o,output=  output midx filename (default: auto-generated)
 a,auto     automatically create .midx from any unindexed .idx files
 f,force    automatically create .midx from *all* .idx files
+max-files= maximum number of idx files to open at once [500]
 """
 o = options.Options('bup midx', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
@@ -98,7 +104,9 @@ elif opt.auto or opt.force:
     for path in paths:
         log('midx: scanning %s\n' % path)
         if opt.force:
-            do_midx(path, opt.output, glob.glob('%s/*.idx' % path))
+            idxs = glob.glob('%s/*.idx' % path)
+            for sublist in _group(idxs, opt.max_files):
+                do_midx(path, opt.output, sublist)
         elif opt.auto:
             m = git.PackIdxList(path)
             needed = {}

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 8ff5ac05fd3211599eb77e87134871d1d24533db..3bbd25a59c65ea523612ccc2fb3e5b9a27b44a1e 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -7,6 +7,8 @@ import heapq
 from bup.helpers import *
 from bup import _helpers
 
+MIDX_VERSION = 2
+
 verbose = 0
 ignore_midx = 0
 home_repodir = os.path.expanduser('~/.bup')
@@ -203,24 +205,39 @@ class PackMidx:
     """
     def __init__(self, filename):
         self.name = filename
+        self.force_keep = False
         assert(filename.endswith('.midx'))
         self.map = mmap_read(open(filename))
-        if str(self.map[0:8]) == 'MIDX\0\0\0\1':
-            log('Warning: ignoring old-style midx %r\n' % filename)
-            self.bits = 0
-            self.entries = 1
-            self.fanout = buffer('\0\0\0\0')
-            self.shalist = buffer('\0'*20)
-            self.idxnames = []
-        else:
-            assert(str(self.map[0:8]) == 'MIDX\0\0\0\2')
-            self.bits = _helpers.firstword(self.map[8:12])
-            self.entries = 2**self.bits
-            self.fanout = buffer(self.map, 12, self.entries*4)
-            shaofs = 12 + self.entries*4
-            nsha = self._fanget(self.entries-1)
-            self.shalist = buffer(self.map, shaofs, nsha*20)
-            self.idxnames = str(self.map[shaofs + 20*nsha:]).split('\0')
+        if str(self.map[0:4]) != 'MIDX':
+            log('Warning: skipping: invalid MIDX header in %r\n' % filename)
+            self.force_keep = True
+            return self._init_failed()
+        ver = struct.unpack('!I', self.map[4:8])[0]
+        if ver < MIDX_VERSION:
+            log('Warning: ignoring old-style (v%d) midx %r\n' 
+                % (ver, filename))
+            self.force_keep = False  # old stuff is boring  
+            return self._init_failed()
+        if ver > MIDX_VERSION:
+            log('Warning: ignoring too-new (v%d) midx %r\n'
+                % (ver, filename))
+            self.force_keep = True  # new stuff is exciting
+            return self._init_failed()
+
+        self.bits = _helpers.firstword(self.map[8:12])
+        self.entries = 2**self.bits
+        self.fanout = buffer(self.map, 12, self.entries*4)
+        shaofs = 12 + self.entries*4
+        nsha = self._fanget(self.entries-1)
+        self.shalist = buffer(self.map, shaofs, nsha*20)
+        self.idxnames = str(self.map[shaofs + 20*nsha:]).split('\0')
+
+    def _init_failed(self):
+        self.bits = 0
+        self.entries = 1
+        self.fanout = buffer('\0\0\0\0')
+        self.shalist = buffer('\0'*20)
+        self.idxnames = []
 
     def _fanget(self, i):
         start = i*4
@@ -356,7 +373,7 @@ class PackIdxList:
                                 d[os.path.join(self.dir, name)] = ix
                             any += 1
                             break
-                    if not any:
+                    if not any and not ix.force_keep:
                         log('midx: removing redundant: %s\n'
                             % os.path.basename(ix.name))
                         unlink(ix.name)