midx: prune redundant midx files automatically.

After running 'bup midx -f', all previous midx files become redundant.
Throw them away if we end up opening a midx file that supercedes them.

Also cleans up some minor code bits in cmd-midx.py.

diff --git a/cmd-midx.py b/cmd-midx.py
index b63e20168011344fda34e07c5cdbbb3cf49e2abf..6f62e66d59cdc2da7c598dba8de9ccd63b66dcb1 100755 (executable)
--- a/cmd-midx.py
+++ b/cmd-midx.py
@@ -32,7 +32,7 @@ def do_midx(outdir, outfilename, infilenames):
     log('Merging %d indexes (%d objects).\n' % (len(infilenames), total))
     if (not opt.force and (total < 1024 and len(infilenames) < 3)) \
        or (opt.force and not total):
-        log('%s: not enough objects for a .midx to be useful.\n' % outdir)
+        log('midx: nothing to do.\n')
         return
 
     pages = int(total/SHA_PER_PAGE) or 1
@@ -96,11 +96,11 @@ if extra:
 elif opt.auto or opt.force:
     paths = [git.repo('objects/pack')]
     paths += glob.glob(git.repo('index-cache/*/.'))
-    if opt.force:
-        for path in paths:
+    for path in paths:
+        log('midx: scanning %s\n' % path)
+        if opt.force:
             do_midx(path, opt.output, glob.glob('%s/*.idx' % path))
-    elif opt.auto:
-        for path in paths:
+        elif opt.auto:
             m = git.MultiPackIndex(path)
             needed = {}
             for pack in m.packs:  # only .idx files without a .midx are open
@@ -108,6 +108,7 @@ elif opt.auto or opt.force:
                     needed[pack.name] = 1
             del m
             do_midx(path, opt.output, needed.keys())
+        log('\n')
 else:
     log("bup midx: you must use -f or -a or provide input filenames\n")
     o.usage()

diff --git a/git.py b/git.py
index 444752fd10d1b32b5b34041e00522b5985e87c10..3b62b19ab791de0406978ae60b7a9a4205a5e89e 100644 (file)
--- a/git.py
+++ b/git.py
@@ -251,13 +251,19 @@ class MultiPackIndex:
                             d[ix.name] = 1
                             for name in ix.idxnames:
                                 d[os.path.join(self.dir, name)] = 1
+                            any += 1
                             break
+                    if not any:
+                        log('midx: removing redundant: %s\n' 
+                            % os.path.basename(ix.name))
+                        unlink(ix.name)
             for f in os.listdir(self.dir):
                 full = os.path.join(self.dir, f)
                 if f.endswith('.idx') and not d.get(full):
                     self.packs.append(PackIndex(full))
                     d[full] = 1
-        #log('MultiPackIndex: using %d packs.\n' % len(self.packs))
+        log('MultiPackIndex: using %d index%s.\n' 
+            % (len(self.packs), len(self.packs)!=1 and 'es' or ''))
 
     def add(self, hash):
         self.also[hash] = 1
@@ -287,12 +293,15 @@ def idxmerge(idxlist):
     heap = [(next(it), it) for it in iters]
     heapq.heapify(heap)
     count = 0
+    last = None
     while heap:
         if (count % 10024) == 0:
             progress('Reading indexes: %.2f%% (%d/%d)\r'
                      % (count*100.0/total, count, total))
         (e, it) = heap[0]
-        yield e
+        if e != last:
+            yield e
+            last = e
         count += 1
         e = next(it)
         if e: