Combine and speed up idx->midx and bupindex merge

[bup.git] / lib / bup / git.py

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 518adc5631b4ce8ebe97d44862fdf49c40a46b60..cd1682afc009b2f3b09d46fb6a8cf0b00e0c3cf7 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -2,9 +2,9 @@
 bup repositories are in Git format. This library allows us to
 interact with the Git data structures.
 """
 bup repositories are in Git format. This library allows us to
 interact with the Git data structures.
 """

-import os, zlib, time, subprocess, struct, stat, re, tempfile, heapq
+import os, sys, zlib, time, subprocess, struct, stat, re, tempfile

 from bup.helpers import *
 from bup.helpers import *

-from bup import _helpers
+from bup import _helpers, path

 
 MIDX_VERSION = 2
 
 
 MIDX_VERSION = 2
 


@@ -39,9 +39,13 @@ def repo(sub = ''):
 
 
 def auto_midx(objdir):
 
 
 def auto_midx(objdir):

-    main_exe = os.environ.get('BUP_MAIN_EXE') or sys.argv[0]
-    args = [main_exe, 'midx', '--auto', '--dir', objdir]
-    rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
+    args = [path.exe(), 'midx', '--auto', '--dir', objdir]
+    try:
+        rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
+    except OSError, e:
+        # make sure 'args' gets printed to help with debugging
+        add_error('%r: exception: %s' % (args, e))
+        raise

     if rv:
         add_error('%r: returned %d' % (args, rv))
 
     if rv:
         add_error('%r: returned %d' % (args, rv))
 


@@ -496,33 +500,21 @@ def open_idx(filename):
 
 def idxmerge(idxlist, final_progress=True):
     """Generate a list of all the objects reachable in a PackIdxList."""
 
 def idxmerge(idxlist, final_progress=True):
     """Generate a list of all the objects reachable in a PackIdxList."""

-    total = sum(len(i) for i in idxlist)
-    iters = (iter(i) for i in idxlist)
-    heap = [(next(it), it) for it in iters]
-    heapq.heapify(heap)
-    count = 0
-    last = None
-    while heap:
-        if (count % 10024) == 0:
-            progress('Reading indexes: %.2f%% (%d/%d)\r'
-                     % (count*100.0/total, count, total))
-        (e, it) = heap[0]
-        if e != last:
-            yield e
-            last = e
-        count += 1
-        e = next(it)
-        if e:
-            heapq.heapreplace(heap, (e, it))
-        else:
-            heapq.heappop(heap)
-    if final_progress:
-        log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
+    def pfunc(count, total):
+        progress('Reading indexes: %.2f%% (%d/%d)\r'
+                 % (count*100.0/total, count, total))
+    def pfinal(count, total):
+        if final_progress:
+            log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
+    return merge_iter(idxlist, 10024, pfunc, pfinal)
+

 
 

+def _make_objcache():
+    return PackIdxList(repo('objects/pack'))

 
 class PackWriter:
     """Writes Git objects insid a pack file."""
 
 class PackWriter:
     """Writes Git objects insid a pack file."""

-    def __init__(self, objcache_maker=None):
+    def __init__(self, objcache_maker=_make_objcache):

         self.count = 0
         self.outbytes = 0
         self.filename = None
         self.count = 0
         self.outbytes = 0
         self.filename = None


@@ -534,16 +526,8 @@ class PackWriter:
     def __del__(self):
         self.close()
 
     def __del__(self):
         self.close()
 

-    def _make_objcache(self):
-        if self.objcache == None:
-            if self.objcache_maker:
-                self.objcache = self.objcache_maker()
-            else:
-                self.objcache = PackIdxList(repo('objects/pack'))
-

     def _open(self):
         if not self.file:
     def _open(self):
         if not self.file:

-            self._make_objcache()

             (fd,name) = tempfile.mkstemp(suffix='.pack', dir=repo('objects'))
             self.file = os.fdopen(fd, 'w+b')
             assert(name.endswith('.pack'))
             (fd,name) = tempfile.mkstemp(suffix='.pack', dir=repo('objects'))
             self.file = os.fdopen(fd, 'w+b')
             assert(name.endswith('.pack'))


@@ -562,7 +546,10 @@ class PackWriter:
         # to our hashsplit algorithm.)  f.write() does its own buffering,
         # but that's okay because we'll flush it in _end().
         oneblob = ''.join(datalist)
         # to our hashsplit algorithm.)  f.write() does its own buffering,
         # but that's okay because we'll flush it in _end().
         oneblob = ''.join(datalist)

-        f.write(oneblob)
+        try:
+            f.write(oneblob)
+        except IOError, e:
+            raise GitError, e, sys.exc_info()[2]

         nw = len(oneblob)
         crc = zlib.crc32(oneblob) & 0xffffffff
         self._update_idx(sha, crc, nw)
         nw = len(oneblob)
         crc = zlib.crc32(oneblob) & 0xffffffff
         self._update_idx(sha, crc, nw)


@@ -593,14 +580,21 @@ class PackWriter:
         """Write an object in this pack file."""
         return self._write(calc_hash(type, content), type, content)
 
         """Write an object in this pack file."""
         return self._write(calc_hash(type, content), type, content)
 

+    def _require_objcache(self):
+        if self.objcache is None and self.objcache_maker:
+            self.objcache = self.objcache_maker()
+        if self.objcache is None:
+            raise GitError(
+                    "PackWriter not opened or can't check exists w/o objcache")
+

     def exists(self, id):
         """Return non-empty if an object is found in the object cache."""
     def exists(self, id):
         """Return non-empty if an object is found in the object cache."""

-        if not self.objcache:
-            self._make_objcache()
+        self._require_objcache()

         return self.objcache.exists(id)
 
     def maybe_write(self, type, content):
         """Write an object to the pack file if not present and return its id."""
         return self.objcache.exists(id)
 
     def maybe_write(self, type, content):
         """Write an object to the pack file if not present and return its id."""

+        self._require_objcache()

         sha = calc_hash(type, content)
         if not self.exists(sha):
             self._write(sha, type, content)
         sha = calc_hash(type, content)
         if not self.exists(sha):
             self._write(sha, type, content)


@@ -860,7 +854,10 @@ def guess_repo(path=None):
 def init_repo(path=None):
     """Create the Git bare repository for bup in a given path."""
     guess_repo(path)
 def init_repo(path=None):
     """Create the Git bare repository for bup in a given path."""
     guess_repo(path)

-    d = repo()
+    d = repo()  # appends a / to the path
+    parent = os.path.dirname(os.path.dirname(d))
+    if parent and not os.path.exists(parent):
+        raise GitError('parent directory "%s" does not exist\n' % parent)

     if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
         raise GitError('"%d" exists but is not a directory\n' % d)
     p = subprocess.Popen(['git', '--bare', 'init'], stdout=sys.stderr,
     if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
         raise GitError('"%d" exists but is not a directory\n' % d)
     p = subprocess.Popen(['git', '--bare', 'init'], stdout=sys.stderr,