git/packwriter: open(..) prohibited in __del__

[bup.git] / lib / bup / git.py

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 826a2d1251d1f9d351004b22e3278e0e1d733501..4c1a95c52a985e5eaee1bb36ef4ca61651559c78 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -2,21 +2,42 @@
 bup repositories are in Git format. This library allows us to
 interact with the Git data structures.
 """
 bup repositories are in Git format. This library allows us to
 interact with the Git data structures.
 """

-import os, sys, zlib, time, subprocess, struct, stat, re, tempfile, glob
-from collections import namedtuple

 
 

-from bup.helpers import *
-from bup import _helpers, path, midx, bloom, xstat
+from __future__ import absolute_import, print_function
+import os, sys, zlib, subprocess, struct, stat, re, tempfile, glob
+from array import array
+from binascii import hexlify, unhexlify
+from collections import namedtuple
+from itertools import islice
+
+from bup import _helpers, hashsplit, path, midx, bloom, xstat
+from bup.compat import (buffer,
+                        byte_int, bytes_from_byte, bytes_from_uint,
+                        environ,
+                        items,
+                        pending_raise,
+                        range,
+                        reraise)
+from bup.io import path_msg
+from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2,
+                         exo,
+                         fdatasync,
+                         finalized,
+                         log,
+                         merge_dict,
+                         merge_iter,
+                         mmap_read, mmap_readwrite,
+                         progress, qprogress, stat_if_exists,
+                         unlink,
+                         utc_offset_str)

 
 

-max_pack_size = 1000*1000*1000  # larger packs will slow down pruning
-max_pack_objects = 200*1000  # cache memory usage is about 83 bytes per object

 
 verbose = 0
 
 verbose = 0

-ignore_midx = 0
-repodir = None
+repodir = None  # The default repository, once initialized
+
+_typemap =  {b'blob': 3, b'tree': 2, b'commit': 1, b'tag': 4}
+_typermap = {v: k for k, v in items(_typemap)}

 
 

-_typemap =  { 'blob':3, 'tree':2, 'commit':1, 'tag':4 }
-_typermap = { 3:'blob', 2:'tree', 1:'commit', 4:'tag' }

 
 _total_searches = 0
 _total_steps = 0
 
 _total_searches = 0
 _total_steps = 0


@@ -26,39 +47,111 @@ class GitError(Exception):
     pass
 
 
     pass
 
 

+def _gitenv(repo_dir=None):
+    if not repo_dir:
+        repo_dir = repo()
+    return merge_dict(environ, {b'GIT_DIR': os.path.abspath(repo_dir)})
+
+def _git_wait(cmd, p):
+    rv = p.wait()
+    if rv != 0:
+        raise GitError('%r returned %d' % (cmd, rv))
+
+def _git_exo(cmd, **kwargs):
+    kwargs['check'] = False
+    result = exo(cmd, **kwargs)
+    _, _, proc = result
+    if proc.returncode != 0:
+        raise GitError('%r returned %d' % (cmd, proc.returncode))
+    return result
+
+def git_config_get(option, repo_dir=None, opttype=None, cfg_file=None):
+    assert not (repo_dir and cfg_file), "repo_dir and cfg_file cannot both be used"
+    cmd = [b'git', b'config', b'--null']
+    if cfg_file:
+        cmd.extend([b'--file', cfg_file])
+    if opttype == 'int':
+        cmd.extend([b'--int'])
+    elif opttype == 'bool':
+        cmd.extend([b'--bool'])
+    else:
+        assert opttype is None
+    cmd.extend([b'--get', option])
+    env=None
+    if repo_dir:
+        env = _gitenv(repo_dir=repo_dir)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env,
+                         close_fds=True)
+    # with --null, git writes out a trailing \0 after the value
+    r = p.stdout.read()[:-1]
+    rc = p.wait()
+    if rc == 0:
+        if opttype == 'int':
+            return int(r)
+        elif opttype == 'bool':
+            # git converts to 'true' or 'false'
+            return r == b'true'
+        return r
+    if rc != 1:
+        raise GitError('%r returned %d' % (cmd, rc))
+    return None
+
+

 def parse_tz_offset(s):
     """UTC offset in seconds."""
     tz_off = (int(s[1:3]) * 60 * 60) + (int(s[3:5]) * 60)
 def parse_tz_offset(s):
     """UTC offset in seconds."""
     tz_off = (int(s[1:3]) * 60 * 60) + (int(s[3:5]) * 60)

-    if s[0] == '-':
+    if bytes_from_byte(s[0]) == b'-':

         return - tz_off
     return tz_off
 
         return - tz_off
     return tz_off
 

+def parse_commit_gpgsig(sig):
+    """Return the original signature bytes.
+
+    i.e. with the "gpgsig " header and the leading space character on
+    each continuation line removed.
+
+    """
+    if not sig:
+        return None
+    assert sig.startswith(b'gpgsig ')
+    sig = sig[7:]
+    return sig.replace(b'\n ', b'\n')

 
 # FIXME: derived from http://git.rsbx.net/Documents/Git_Data_Formats.txt
 # Make sure that's authoritative.
 
 # FIXME: derived from http://git.rsbx.net/Documents/Git_Data_Formats.txt
 # Make sure that's authoritative.

-_start_end_char = r'[^ .,:;<>"\'\0\n]'
-_content_char = r'[^\0\n<>]'
-_safe_str_rx = '(?:%s{1,2}|(?:%s%s*%s))' \
+
+# See also
+# https://github.com/git/git/blob/master/Documentation/technical/signature-format.txt
+# The continuation lines have only one leading space.
+
+_start_end_char = br'[^ .,:;<>"\'\0\n]'
+_content_char = br'[^\0\n<>]'
+_safe_str_rx = br'(?:%s{1,2}|(?:%s%s*%s))' \

     % (_start_end_char,
        _start_end_char, _content_char, _start_end_char)
     % (_start_end_char,
        _start_end_char, _content_char, _start_end_char)

-_tz_rx = r'[-+]\d\d[0-5]\d'
-_parent_rx = r'(?:parent [abcdefABCDEF0123456789]{40}\n)'
-_commit_rx = re.compile(r'''tree (?P<tree>[abcdefABCDEF0123456789]{40})
+_tz_rx = br'[-+]\d\d[0-5]\d'
+_parent_rx = br'(?:parent [abcdefABCDEF0123456789]{40}\n)'
+# Assumes every following line starting with a space is part of the
+# mergetag.  Is there a formal commit blob spec?
+_mergetag_rx = br'(?:\nmergetag object [abcdefABCDEF0123456789]{40}(?:\n [^\0\n]*)*)'
+_commit_rx = re.compile(br'''tree (?P<tree>[abcdefABCDEF0123456789]{40})

 (?P<parents>%s*)author (?P<author_name>%s) <(?P<author_mail>%s)> (?P<asec>\d+) (?P<atz>%s)
 (?P<parents>%s*)author (?P<author_name>%s) <(?P<author_mail>%s)> (?P<asec>\d+) (?P<atz>%s)

-committer (?P<committer_name>%s) <(?P<committer_mail>%s)> (?P<csec>\d+) (?P<ctz>%s)
-
+committer (?P<committer_name>%s) <(?P<committer_mail>%s)> (?P<csec>\d+) (?P<ctz>%s)(?P<mergetag>%s?)
+(?P<gpgsig>gpgsig .*\n(?: .*\n)*)?

 (?P<message>(?:.|\n)*)''' % (_parent_rx,
                              _safe_str_rx, _safe_str_rx, _tz_rx,
 (?P<message>(?:.|\n)*)''' % (_parent_rx,
                              _safe_str_rx, _safe_str_rx, _tz_rx,

-                             _safe_str_rx, _safe_str_rx, _tz_rx))
-_parent_hash_rx = re.compile(r'\s*parent ([abcdefABCDEF0123456789]{40})\s*')
-
+                             _safe_str_rx, _safe_str_rx, _tz_rx,
+                             _mergetag_rx))
+_parent_hash_rx = re.compile(br'\s*parent ([abcdefABCDEF0123456789]{40})\s*')

 
 

-# Note that the author_sec and committer_sec values are (UTC) epoch seconds.
+# Note that the author_sec and committer_sec values are (UTC) epoch
+# seconds, and for now the mergetag is not included.

 CommitInfo = namedtuple('CommitInfo', ['tree', 'parents',
                                        'author_name', 'author_mail',
                                        'author_sec', 'author_offset',
                                        'committer_name', 'committer_mail',
                                        'committer_sec', 'committer_offset',
 CommitInfo = namedtuple('CommitInfo', ['tree', 'parents',
                                        'author_name', 'author_mail',
                                        'author_sec', 'author_offset',
                                        'committer_name', 'committer_mail',
                                        'committer_sec', 'committer_offset',

+                                       'gpgsig',

                                        'message'])
 
 def parse_commit(content):
                                        'message'])
 
 def parse_commit(content):


@@ -76,68 +169,80 @@ def parse_commit(content):
                       committer_mail=matches['committer_mail'],
                       committer_sec=int(matches['csec']),
                       committer_offset=parse_tz_offset(matches['ctz']),
                       committer_mail=matches['committer_mail'],
                       committer_sec=int(matches['csec']),
                       committer_offset=parse_tz_offset(matches['ctz']),

+                      gpgsig=parse_commit_gpgsig(matches['gpgsig']),

                       message=matches['message'])
 
 
                       message=matches['message'])
 
 

+def get_cat_data(cat_iterator, expected_type):
+    _, kind, _ = next(cat_iterator)
+    if kind != expected_type:
+        raise Exception('expected %r, saw %r' % (expected_type, kind))
+    return b''.join(cat_iterator)
+

 def get_commit_items(id, cp):
 def get_commit_items(id, cp):

-    commit_it = cp.get(id)
-    assert(commit_it.next() == 'commit')
-    commit_content = ''.join(commit_it)
-    return parse_commit(commit_content)
+    return parse_commit(get_cat_data(cp.get(id), b'commit'))
+
+def _local_git_date_str(epoch_sec):
+    return b'%d %s' % (epoch_sec, utc_offset_str(epoch_sec))

 
 
 
 

-def repo(sub = ''):
+def _git_date_str(epoch_sec, tz_offset_sec):
+    offs =  tz_offset_sec // 60
+    return b'%d %s%02d%02d' \
+        % (epoch_sec,
+           b'+' if offs >= 0 else b'-',
+           abs(offs) // 60,
+           abs(offs) % 60)
+
+
+def repo(sub = b'', repo_dir=None):

     """Get the path to the git repository or one of its subdirectories."""
     """Get the path to the git repository or one of its subdirectories."""

-    global repodir
-    if not repodir:
+    repo_dir = repo_dir or repodir
+    if not repo_dir:

         raise GitError('You should call check_repo_or_die()')
 
     # If there's a .git subdirectory, then the actual repo is in there.
         raise GitError('You should call check_repo_or_die()')
 
     # If there's a .git subdirectory, then the actual repo is in there.

-    gd = os.path.join(repodir, '.git')
+    gd = os.path.join(repo_dir, b'.git')

     if os.path.exists(gd):
     if os.path.exists(gd):

-        repodir = gd
+        repo_dir = gd
+
+    return os.path.join(repo_dir, sub)

 
 

-    return os.path.join(repodir, sub)

 
 

+_shorten_hash_rx = \
+    re.compile(br'([^0-9a-z]|\b)([0-9a-z]{7})[0-9a-z]{33}([^0-9a-z]|\b)')

 
 def shorten_hash(s):
 
 def shorten_hash(s):

-    return re.sub(r'([^0-9a-z]|\b)([0-9a-z]{7})[0-9a-z]{33}([^0-9a-z]|\b)',
-                  r'\1\2*\3', s)
+    return _shorten_hash_rx.sub(br'\1\2*\3', s)

 
 
 def repo_rel(path):
     full = os.path.abspath(path)
 
 
 def repo_rel(path):
     full = os.path.abspath(path)

-    fullrepo = os.path.abspath(repo(''))
-    if not fullrepo.endswith('/'):
-        fullrepo += '/'
+    fullrepo = os.path.abspath(repo(b''))
+    if not fullrepo.endswith(b'/'):
+        fullrepo += b'/'

     if full.startswith(fullrepo):
         path = full[len(fullrepo):]
     if full.startswith(fullrepo):
         path = full[len(fullrepo):]

-    if path.startswith('index-cache/'):
-        path = path[len('index-cache/'):]
+    if path.startswith(b'index-cache/'):
+        path = path[len(b'index-cache/'):]

     return shorten_hash(path)
 
 
     return shorten_hash(path)
 
 

-def all_packdirs():
-    paths = [repo('objects/pack')]
-    paths += glob.glob(repo('index-cache/*/.'))
-    return paths
-
-

 def auto_midx(objdir):
 def auto_midx(objdir):

-    args = [path.exe(), 'midx', '--auto', '--dir', objdir]
+    args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]

     try:
     try:

-        rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
-    except OSError, e:
+        rv = subprocess.call(args, stdout=open(os.devnull, 'w'))
+    except OSError as e:

         # make sure 'args' gets printed to help with debugging
         add_error('%r: exception: %s' % (args, e))
         raise
     if rv:
         add_error('%r: returned %d' % (args, rv))
 
         # make sure 'args' gets printed to help with debugging
         add_error('%r: exception: %s' % (args, e))
         raise
     if rv:
         add_error('%r: returned %d' % (args, rv))
 

-    args = [path.exe(), 'bloom', '--dir', objdir]
+    args = [path.exe(), b'bloom', b'--dir', objdir]

     try:
     try:

-        rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
-    except OSError, e:
+        rv = subprocess.call(args, stdout=open(os.devnull, 'w'))
+    except OSError as e:

         # make sure 'args' gets printed to help with debugging
         add_error('%r: exception: %s' % (args, e))
         raise
         # make sure 'args' gets printed to help with debugging
         add_error('%r: exception: %s' % (args, e))
         raise


@@ -149,39 +254,42 @@ def mangle_name(name, mode, gitmode):
     """Mangle a file name to present an abstract name for segmented files.
     Mangled file names will have the ".bup" extension added to them. If a
     file's name already ends with ".bup", a ".bupl" extension is added to
     """Mangle a file name to present an abstract name for segmented files.
     Mangled file names will have the ".bup" extension added to them. If a
     file's name already ends with ".bup", a ".bupl" extension is added to

-    disambiguate normal files from semgmented ones.
+    disambiguate normal files from segmented ones.

     """
     if stat.S_ISREG(mode) and not stat.S_ISREG(gitmode):
     """
     if stat.S_ISREG(mode) and not stat.S_ISREG(gitmode):

-        return name + '.bup'
-    elif name.endswith('.bup') or name[:-1].endswith('.bup'):
-        return name + '.bupl'
+        assert(stat.S_ISDIR(gitmode))
+        return name + b'.bup'
+    elif name.endswith(b'.bup') or name[:-1].endswith(b'.bup'):
+        return name + b'.bupl'

     else:
         return name
 
 
 (BUP_NORMAL, BUP_CHUNKED) = (0,1)
     else:
         return name
 
 
 (BUP_NORMAL, BUP_CHUNKED) = (0,1)

-def demangle_name(name):
+def demangle_name(name, mode):

     """Remove name mangling from a file name, if necessary.
 
     The return value is a tuple (demangled_filename,mode), where mode is one of
     the following:
 
     * BUP_NORMAL  : files that should be read as-is from the repository
     """Remove name mangling from a file name, if necessary.
 
     The return value is a tuple (demangled_filename,mode), where mode is one of
     the following:
 
     * BUP_NORMAL  : files that should be read as-is from the repository

-    * BUP_CHUNKED : files that were chunked and need to be assembled
+    * BUP_CHUNKED : files that were chunked and need to be reassembled

 
 

-    For more information on the name mangling algorythm, see mangle_name()
+    For more information on the name mangling algorithm, see mangle_name()

     """
     """

-    if name.endswith('.bupl'):
+    if name.endswith(b'.bupl'):

         return (name[:-5], BUP_NORMAL)
         return (name[:-5], BUP_NORMAL)

-    elif name.endswith('.bup'):
+    elif name.endswith(b'.bup'):

         return (name[:-4], BUP_CHUNKED)
         return (name[:-4], BUP_CHUNKED)

-    else:
-        return (name, BUP_NORMAL)
+    elif name.endswith(b'.bupm'):
+        return (name[:-5],
+                BUP_CHUNKED if stat.S_ISDIR(mode) else BUP_NORMAL)
+    return (name, BUP_NORMAL)

 
 
 def calc_hash(type, content):
     """Calculate some content's hash in the Git fashion."""
 
 
 def calc_hash(type, content):
     """Calculate some content's hash in the Git fashion."""

-    header = '%s %d\0' % (type, len(content))
+    header = b'%s %d\0' % (type, len(content))

     sum = Sha1(header)
     sum.update(content)
     return sum.digest()
     sum = Sha1(header)
     sum.update(content)
     return sum.digest()


@@ -191,7 +299,7 @@ def shalist_item_sort_key(ent):
     (mode, name, id) = ent
     assert(mode+0 == mode)
     if stat.S_ISDIR(mode):
     (mode, name, id) = ent
     assert(mode+0 == mode)
     if stat.S_ISDIR(mode):

-        return name + '/'
+        return name + b'/'

     else:
         return name
 
     else:
         return name
 


@@ -205,19 +313,19 @@ def tree_encode(shalist):
         assert(mode+0 == mode)
         assert(name)
         assert(len(bin) == 20)
         assert(mode+0 == mode)
         assert(name)
         assert(len(bin) == 20)

-        s = '%o %s\0%s' % (mode,name,bin)
-        assert(s[0] != '0')  # 0-padded octal is not acceptable in a git tree
+        s = b'%o %s\0%s' % (mode,name,bin)
+        assert s[0] != b'0'  # 0-padded octal is not acceptable in a git tree

         l.append(s)
         l.append(s)

-    return ''.join(l)
+    return b''.join(l)

 
 
 def tree_decode(buf):
     """Generate a list of (mode,name,hash) from the git tree object in buf."""
     ofs = 0
     while ofs < len(buf):
 
 
 def tree_decode(buf):
     """Generate a list of (mode,name,hash) from the git tree object in buf."""
     ofs = 0
     while ofs < len(buf):

-        z = buf.find('\0', ofs)
+        z = buf.find(b'\0', ofs)

         assert(z > ofs)
         assert(z > ofs)

-        spl = buf[ofs:z].split(' ', 1)
+        spl = buf[ofs:z].split(b' ', 1)

         assert(len(spl) == 2)
         mode,name = spl
         sha = buf[z+1:z+1+20]
         assert(len(spl) == 2)
         mode,name = spl
         sha = buf[z+1:z+1+20]


@@ -226,58 +334,35 @@ def tree_decode(buf):
 
 
 def _encode_packobj(type, content, compression_level=1):
 
 
 def _encode_packobj(type, content, compression_level=1):

-    szout = ''
+    if compression_level not in (0, 1, 2, 3, 4, 5, 6, 7, 8, 9):
+        raise ValueError('invalid compression level %s' % compression_level)
+    szout = b''

     sz = len(content)
     szbits = (sz & 0x0f) | (_typemap[type]<<4)
     sz >>= 4
     while 1:
         if sz: szbits |= 0x80
     sz = len(content)
     szbits = (sz & 0x0f) | (_typemap[type]<<4)
     sz >>= 4
     while 1:
         if sz: szbits |= 0x80

-        szout += chr(szbits)
+        szout += bytes_from_uint(szbits)

         if not sz:
             break
         szbits = sz & 0x7f
         sz >>= 7
         if not sz:
             break
         szbits = sz & 0x7f
         sz >>= 7

-    if compression_level > 9:
-        compression_level = 9
-    elif compression_level < 0:
-        compression_level = 0

     z = zlib.compressobj(compression_level)
     yield szout
     yield z.compress(content)
     yield z.flush()
 
 
     z = zlib.compressobj(compression_level)
     yield szout
     yield z.compress(content)
     yield z.flush()
 
 

-def _encode_looseobj(type, content, compression_level=1):
-    z = zlib.compressobj(compression_level)
-    yield z.compress('%s %d\0' % (type, len(content)))
-    yield z.compress(content)
-    yield z.flush()
-
-
-def _decode_looseobj(buf):
-    assert(buf);
-    s = zlib.decompress(buf)
-    i = s.find('\0')
-    assert(i > 0)
-    l = s[:i].split(' ')
-    type = l[0]
-    sz = int(l[1])
-    content = s[i+1:]
-    assert(type in _typemap)
-    assert(sz == len(content))
-    return (type, content)
-
-

 def _decode_packobj(buf):
     assert(buf)
 def _decode_packobj(buf):
     assert(buf)

-    c = ord(buf[0])
+    c = byte_int(buf[0])

     type = _typermap[(c & 0x70) >> 4]
     sz = c & 0x0f
     shift = 4
     i = 0
     while c & 0x80:
         i += 1
     type = _typermap[(c & 0x70) >> 4]
     sz = c & 0x0f
     shift = 4
     i = 0
     while c & 0x80:
         i += 1

-        c = ord(buf[i])
+        c = byte_int(buf[i])

         sz |= (c & 0x7f) << shift
         shift += 7
         if not (c & 0x80):
         sz |= (c & 0x7f) << shift
         shift += 7
         if not (c & 0x80):


@@ -302,21 +387,18 @@ class PackIdx:
             return want_source and os.path.basename(self.name) or True
         return None
 
             return want_source and os.path.basename(self.name) or True
         return None
 

-    def __len__(self):
-        return int(self.fanout[255])
-

     def _idx_from_hash(self, hash):
         global _total_searches, _total_steps
         _total_searches += 1
         assert(len(hash) == 20)
     def _idx_from_hash(self, hash):
         global _total_searches, _total_steps
         _total_searches += 1
         assert(len(hash) == 20)

-        b1 = ord(hash[0])
+        b1 = byte_int(hash[0])

         start = self.fanout[b1-1] # range -1..254
         end = self.fanout[b1] # range 0..255
         start = self.fanout[b1-1] # range -1..254
         end = self.fanout[b1] # range 0..255

-        want = str(hash)
+        want = hash

         _total_steps += 1  # lookup table is a step
         while start < end:
             _total_steps += 1
         _total_steps += 1  # lookup table is a step
         while start < end:
             _total_steps += 1

-            mid = start + (end-start)/2
+            mid = start + (end - start) // 2

             v = self._idx_to_hash(mid)
             if v < want:
                 start = mid+1
             v = self._idx_to_hash(mid)
             if v < want:
                 start = mid+1


@@ -333,22 +415,46 @@ class PackIdxV1(PackIdx):
         self.name = filename
         self.idxnames = [self.name]
         self.map = mmap_read(f)
         self.name = filename
         self.idxnames = [self.name]
         self.map = mmap_read(f)

-        self.fanout = list(struct.unpack('!256I',
-                                         str(buffer(self.map, 0, 256*4))))
+        # Min size for 'L' is 4, which is sufficient for struct's '!I'
+        self.fanout = array('L', struct.unpack('!256I', self.map))

         self.fanout.append(0)  # entry "-1"
         self.fanout.append(0)  # entry "-1"

-        nsha = self.fanout[255]
-        self.sha_ofs = 256*4
-        self.shatable = buffer(self.map, self.sha_ofs, nsha*24)
+        self.nsha = self.fanout[255]
+        self.sha_ofs = 256 * 4
+        # Avoid slicing shatable for individual hashes (very high overhead)
+        self.shatable = buffer(self.map, self.sha_ofs, self.nsha * 24)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        with pending_raise(value, rethrow=False):
+            self.close()
+
+    def __len__(self):
+        return int(self.nsha)  # int() from long for python 2

 
     def _ofs_from_idx(self, idx):
 
     def _ofs_from_idx(self, idx):

-        return struct.unpack('!I', str(self.shatable[idx*24 : idx*24+4]))[0]
+        if idx >= self.nsha or idx < 0:
+            raise IndexError('invalid pack index index %d' % idx)
+        ofs = self.sha_ofs + idx * 24
+        return struct.unpack_from('!I', self.map, offset=ofs)[0]

 
     def _idx_to_hash(self, idx):
 
     def _idx_to_hash(self, idx):

-        return str(self.shatable[idx*24+4 : idx*24+24])
+        if idx >= self.nsha or idx < 0:
+            raise IndexError('invalid pack index index %d' % idx)
+        ofs = self.sha_ofs + idx * 24 + 4
+        return self.map[ofs : ofs + 20]

 
     def __iter__(self):
 
     def __iter__(self):

-        for i in xrange(self.fanout[255]):
-            yield buffer(self.map, 256*4 + 24*i + 4, 20)
+        start = self.sha_ofs + 4
+        for ofs in range(start, start + 24 * self.nsha, 24):
+            yield self.map[ofs : ofs + 20]
+
+    def close(self):
+        if self.map is not None:
+            self.shatable = None
+            self.map.close()
+            self.map = None

 
 
 class PackIdxV2(PackIdx):
 
 
 class PackIdxV2(PackIdx):


@@ -357,38 +463,59 @@ class PackIdxV2(PackIdx):
         self.name = filename
         self.idxnames = [self.name]
         self.map = mmap_read(f)
         self.name = filename
         self.idxnames = [self.name]
         self.map = mmap_read(f)

-        assert(str(self.map[0:8]) == '\377tOc\0\0\0\2')
-        self.fanout = list(struct.unpack('!256I',
-                                         str(buffer(self.map, 8, 256*4))))
-        self.fanout.append(0)  # entry "-1"
-        nsha = self.fanout[255]
+        assert self.map[0:8] == b'\377tOc\0\0\0\2'
+        # Min size for 'L' is 4, which is sufficient for struct's '!I'
+        self.fanout = array('L', struct.unpack_from('!256I', self.map, offset=8))
+        self.fanout.append(0)
+        self.nsha = self.fanout[255]

         self.sha_ofs = 8 + 256*4
         self.sha_ofs = 8 + 256*4

-        self.shatable = buffer(self.map, self.sha_ofs, nsha*20)
-        self.ofstable = buffer(self.map,
-                               self.sha_ofs + nsha*20 + nsha*4,
-                               nsha*4)
-        self.ofs64table = buffer(self.map,
-                                 8 + 256*4 + nsha*20 + nsha*4 + nsha*4)
+        self.ofstable_ofs = self.sha_ofs + self.nsha * 20 + self.nsha * 4
+        self.ofs64table_ofs = self.ofstable_ofs + self.nsha * 4
+        # Avoid slicing this for individual hashes (very high overhead)
+        self.shatable = buffer(self.map, self.sha_ofs, self.nsha*20)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        with pending_raise(value, rethrow=False):
+            self.close()
+
+    def __len__(self):
+        return int(self.nsha)  # int() from long for python 2

 
     def _ofs_from_idx(self, idx):
 
     def _ofs_from_idx(self, idx):

-        ofs = struct.unpack('!I', str(buffer(self.ofstable, idx*4, 4)))[0]
+        if idx >= self.nsha or idx < 0:
+            raise IndexError('invalid pack index index %d' % idx)
+        ofs_ofs = self.ofstable_ofs + idx * 4
+        ofs = struct.unpack_from('!I', self.map, offset=ofs_ofs)[0]

         if ofs & 0x80000000:
             idx64 = ofs & 0x7fffffff
         if ofs & 0x80000000:
             idx64 = ofs & 0x7fffffff

-            ofs = struct.unpack('!Q',
-                                str(buffer(self.ofs64table, idx64*8, 8)))[0]
+            ofs64_ofs = self.ofs64table_ofs + idx64 * 8
+            ofs = struct.unpack_from('!Q', self.map, offset=ofs64_ofs)[0]

         return ofs
 
     def _idx_to_hash(self, idx):
         return ofs
 
     def _idx_to_hash(self, idx):

-        return str(self.shatable[idx*20:(idx+1)*20])
+        if idx >= self.nsha or idx < 0:
+            raise IndexError('invalid pack index index %d' % idx)
+        ofs = self.sha_ofs + idx * 20
+        return self.map[ofs : ofs + 20]

 
     def __iter__(self):
 
     def __iter__(self):

-        for i in xrange(self.fanout[255]):
-            yield buffer(self.map, 8 + 256*4 + 20*i, 20)
+        start = self.sha_ofs
+        for ofs in range(start, start + 20 * self.nsha, 20):
+            yield self.map[ofs : ofs + 20]
+
+    def close(self):
+        if self.map is not None:
+            self.shatable = None
+            self.map.close()
+            self.map = None

 
 
 _mpi_count = 0
 class PackIdxList:
 
 
 _mpi_count = 0
 class PackIdxList:

-    def __init__(self, dir):
+    def __init__(self, dir, ignore_midx=False):

         global _mpi_count
         assert(_mpi_count == 0) # these things suck tons of VM; don't waste it
         _mpi_count += 1
         global _mpi_count
         assert(_mpi_count == 0) # these things suck tons of VM; don't waste it
         _mpi_count += 1


@@ -397,6 +524,7 @@ class PackIdxList:
         self.packs = []
         self.do_bloom = False
         self.bloom = None
         self.packs = []
         self.do_bloom = False
         self.bloom = None

+        self.ignore_midx = ignore_midx

         self.refresh()
 
     def __del__(self):
         self.refresh()
 
     def __del__(self):


@@ -422,7 +550,7 @@ class PackIdxList:
             else:
                 _total_searches -= 1  # was counted by bloom
                 return None
             else:
                 _total_searches -= 1  # was counted by bloom
                 return None

-        for i in xrange(len(self.packs)):
+        for i in range(len(self.packs)):

             p = self.packs[i]
             _total_searches -= 1  # will be incremented by sub-pack
             ix = p.exists(hash, want_source=want_source)
             p = self.packs[i]
             _total_searches -= 1  # will be incremented by sub-pack
             ix = p.exists(hash, want_source=want_source)


@@ -442,30 +570,44 @@ class PackIdxList:
         If skip_midx is True, all work on .midx files will be skipped and .midx
         files will be removed from the list.
 
         If skip_midx is True, all work on .midx files will be skipped and .midx
         files will be removed from the list.
 

-        The module-global variable 'ignore_midx' can force this function to
+        The instance variable 'ignore_midx' can force this function to

         always act as if skip_midx was True.
         """
         always act as if skip_midx was True.
         """

+        if self.bloom is not None:
+            self.bloom.close()

         self.bloom = None # Always reopen the bloom as it may have been relaced
         self.do_bloom = False
         self.bloom = None # Always reopen the bloom as it may have been relaced
         self.do_bloom = False

-        skip_midx = skip_midx or ignore_midx
+        skip_midx = skip_midx or self.ignore_midx

         d = dict((p.name, p) for p in self.packs
                  if not skip_midx or not isinstance(p, midx.PackMidx))
         if os.path.exists(self.dir):
             if not skip_midx:
                 midxl = []
         d = dict((p.name, p) for p in self.packs
                  if not skip_midx or not isinstance(p, midx.PackMidx))
         if os.path.exists(self.dir):
             if not skip_midx:
                 midxl = []

+                midxes = set(glob.glob(os.path.join(self.dir, b'*.midx')))
+                # remove any *.midx files from our list that no longer exist
+                for ix in list(d.values()):
+                    if not isinstance(ix, midx.PackMidx):
+                        continue
+                    if ix.name in midxes:
+                        continue
+                    # remove the midx
+                    del d[ix.name]
+                    ix.close()
+                    self.packs.remove(ix)

                 for ix in self.packs:
                     if isinstance(ix, midx.PackMidx):
                         for name in ix.idxnames:
                             d[os.path.join(self.dir, name)] = ix
                 for ix in self.packs:
                     if isinstance(ix, midx.PackMidx):
                         for name in ix.idxnames:
                             d[os.path.join(self.dir, name)] = ix

-                for full in glob.glob(os.path.join(self.dir,'*.midx')):
+                for full in midxes:

                     if not d.get(full):
                         mx = midx.PackMidx(full)
                         (mxd, mxf) = os.path.split(mx.name)
                         broken = False
                         for n in mx.idxnames:
                             if not os.path.exists(os.path.join(mxd, n)):
                     if not d.get(full):
                         mx = midx.PackMidx(full)
                         (mxd, mxf) = os.path.split(mx.name)
                         broken = False
                         for n in mx.idxnames:
                             if not os.path.exists(os.path.join(mxd, n)):

-                                log(('warning: index %s missing\n' +
-                                    '  used by %s\n') % (n, mxf))
+                                log(('warning: index %s missing\n'
+                                     '  used by %s\n')
+                                    % (path_msg(n), path_msg(mxf)))

                                 broken = True
                         if broken:
                             mx.close()
                                 broken = True
                         if broken:
                             mx.close()


@@ -489,22 +631,22 @@ class PackIdxList:
                             d[os.path.join(self.dir, name)] = ix
                     elif not ix.force_keep:
                         debug1('midx: removing redundant: %s\n'
                             d[os.path.join(self.dir, name)] = ix
                     elif not ix.force_keep:
                         debug1('midx: removing redundant: %s\n'

-                               % os.path.basename(ix.name))
+                               % path_msg(os.path.basename(ix.name)))

                         ix.close()
                         unlink(ix.name)
                         ix.close()
                         unlink(ix.name)

-            for full in glob.glob(os.path.join(self.dir,'*.idx')):
+            for full in glob.glob(os.path.join(self.dir, b'*.idx')):

                 if not d.get(full):
                     try:
                         ix = open_idx(full)
                 if not d.get(full):
                     try:
                         ix = open_idx(full)

-                    except GitError, e:
+                    except GitError as e:

                         add_error(e)
                         continue
                     d[full] = ix
                         add_error(e)
                         continue
                     d[full] = ix

-            bfull = os.path.join(self.dir, 'bup.bloom')
+            bfull = os.path.join(self.dir, b'bup.bloom')

             if self.bloom is None and os.path.exists(bfull):
                 self.bloom = bloom.ShaBloom(bfull)
             self.packs = list(set(d.values()))
             if self.bloom is None and os.path.exists(bfull):
                 self.bloom = bloom.ShaBloom(bfull)
             self.packs = list(set(d.values()))

-            self.packs.sort(lambda x,y: -cmp(len(x),len(y)))
+            self.packs.sort(reverse=True, key=lambda x: len(x))

             if self.bloom and self.bloom.valid() and len(self.bloom) >= len(self):
                 self.do_bloom = True
             else:
             if self.bloom and self.bloom.valid() and len(self.bloom) >= len(self):
                 self.do_bloom = True
             else:


@@ -518,21 +660,22 @@ class PackIdxList:
 
 
 def open_idx(filename):
 
 
 def open_idx(filename):

-    if filename.endswith('.idx'):
+    if filename.endswith(b'.idx'):

         f = open(filename, 'rb')
         header = f.read(8)
         f = open(filename, 'rb')
         header = f.read(8)

-        if header[0:4] == '\377tOc':
+        if header[0:4] == b'\377tOc':

             version = struct.unpack('!I', header[4:8])[0]
             if version == 2:
                 return PackIdxV2(filename, f)
             else:
                 raise GitError('%s: expected idx file version 2, got %d'
             version = struct.unpack('!I', header[4:8])[0]
             if version == 2:
                 return PackIdxV2(filename, f)
             else:
                 raise GitError('%s: expected idx file version 2, got %d'

-                               % (filename, version))
-        elif len(header) == 8 and header[0:4] < '\377tOc':
+                               % (path_msg(filename), version))
+        elif len(header) == 8 and header[0:4] < b'\377tOc':

             return PackIdxV1(filename, f)
         else:
             return PackIdxV1(filename, f)
         else:

-            raise GitError('%s: unrecognized idx file header' % filename)
-    elif filename.endswith('.midx'):
+            raise GitError('%s: unrecognized idx file header'
+                           % path_msg(filename))
+    elif filename.endswith(b'.midx'):

         return midx.PackMidx(filename)
     else:
         raise GitError('idx filenames must end with .idx or .midx')
         return midx.PackMidx(filename)
     else:
         raise GitError('idx filenames must end with .idx or .midx')


@@ -550,32 +693,90 @@ def idxmerge(idxlist, final_progress=True):
     return merge_iter(idxlist, 10024, pfunc, pfinal)
 
 
     return merge_iter(idxlist, 10024, pfunc, pfinal)
 
 

+def create_commit_blob(tree, parent,
+                       author, adate_sec, adate_tz,
+                       committer, cdate_sec, cdate_tz,
+                       msg):
+    if adate_tz is not None:
+        adate_str = _git_date_str(adate_sec, adate_tz)
+    else:
+        adate_str = _local_git_date_str(adate_sec)
+    if cdate_tz is not None:
+        cdate_str = _git_date_str(cdate_sec, cdate_tz)
+    else:
+        cdate_str = _local_git_date_str(cdate_sec)
+    l = []
+    if tree: l.append(b'tree %s' % hexlify(tree))
+    if parent: l.append(b'parent %s' % hexlify(parent))
+    if author: l.append(b'author %s %s' % (author, adate_str))
+    if committer: l.append(b'committer %s %s' % (committer, cdate_str))
+    l.append(b'')
+    l.append(msg)
+    return b'\n'.join(l)
+
+

 def _make_objcache():
 def _make_objcache():

-    return PackIdxList(repo('objects/pack'))
+    return PackIdxList(repo(b'objects/pack'))
+
+# bup-gc assumes that it can disable all PackWriter activities
+# (bloom/midx/cache) via the constructor and close() arguments.

 
 class PackWriter:
     """Writes Git objects inside a pack file."""
 
 class PackWriter:
     """Writes Git objects inside a pack file."""

-    def __init__(self, objcache_maker=_make_objcache, compression_level=1):
+    def __init__(self, objcache_maker=_make_objcache, compression_level=1,
+                 run_midx=True, on_pack_finish=None,
+                 max_pack_size=None, max_pack_objects=None, repo_dir=None):
+        self.repo_dir = repo_dir or repo()
+        self.file = None
+        self.parentfd = None

         self.count = 0
         self.outbytes = 0
         self.filename = None
         self.count = 0
         self.outbytes = 0
         self.filename = None

-        self.file = None

         self.idx = None
         self.objcache_maker = objcache_maker
         self.objcache = None
         self.compression_level = compression_level
         self.idx = None
         self.objcache_maker = objcache_maker
         self.objcache = None
         self.compression_level = compression_level

+        self.run_midx=run_midx
+        self.on_pack_finish = on_pack_finish
+        if not max_pack_size:
+            max_pack_size = git_config_get(b'pack.packSizeLimit',
+                                           repo_dir=self.repo_dir,
+                                           opttype='int')
+            if not max_pack_size:
+                # larger packs slow down pruning
+                max_pack_size = 1000 * 1000 * 1000
+        self.max_pack_size = max_pack_size
+        # cache memory usage is about 83 bytes per object
+        self.max_pack_objects = max_pack_objects if max_pack_objects \
+                                else max(1, self.max_pack_size // 5000)
+
+    def __enter__(self):
+        return self

 
 

-    def __del__(self):
-        self.close()
+    def __exit__(self, type, value, traceback):
+        with pending_raise(value, rethrow=False):
+            self.close()

 
     def _open(self):
         if not self.file:
 
     def _open(self):
         if not self.file:

-            (fd,name) = tempfile.mkstemp(suffix='.pack', dir=repo('objects'))
-            self.file = os.fdopen(fd, 'w+b')
-            assert(name.endswith('.pack'))
+            objdir = dir = os.path.join(self.repo_dir, b'objects')
+            fd, name = tempfile.mkstemp(suffix=b'.pack', dir=objdir)
+            try:
+                self.file = os.fdopen(fd, 'w+b')
+            except:
+                os.close(fd)
+                raise
+            try:
+                self.parentfd = os.open(objdir, os.O_RDONLY)
+            except:
+                f = self.file
+                self.file = None
+                f.close()
+                raise
+            assert name.endswith(b'.pack')

             self.filename = name[:-5]
             self.filename = name[:-5]

-            self.file.write('PACK\0\0\0\2\0\0\0\0')
-            self.idx = list(list() for i in xrange(256))
+            self.file.write(b'PACK\0\0\0\2\0\0\0\0')
+            self.idx = PackIdxV2Writer()

 
     def _raw_write(self, datalist, sha):
         self._open()
 
     def _raw_write(self, datalist, sha):
         self._open()


@@ -585,11 +786,11 @@ class PackWriter:
         # all-or-nothing.  (The blob shouldn't be very big anyway, thanks
         # to our hashsplit algorithm.)  f.write() does its own buffering,
         # but that's okay because we'll flush it in _end().
         # all-or-nothing.  (The blob shouldn't be very big anyway, thanks
         # to our hashsplit algorithm.)  f.write() does its own buffering,
         # but that's okay because we'll flush it in _end().

-        oneblob = ''.join(datalist)
+        oneblob = b''.join(datalist)

         try:
             f.write(oneblob)
         try:
             f.write(oneblob)

-        except IOError, e:
-            raise GitError, e, sys.exc_info()[2]
+        except IOError as e:
+            reraise(GitError(e))

         nw = len(oneblob)
         crc = zlib.crc32(oneblob) & 0xffffffff
         self._update_idx(sha, crc, nw)
         nw = len(oneblob)
         crc = zlib.crc32(oneblob) & 0xffffffff
         self._update_idx(sha, crc, nw)


@@ -600,7 +801,7 @@ class PackWriter:
     def _update_idx(self, sha, crc, size):
         assert(sha)
         if self.idx:
     def _update_idx(self, sha, crc, size):
         assert(sha)
         if self.idx:

-            self.idx[ord(sha[0])].append((sha, crc, self.file.tell() - size))
+            self.idx.add(sha, crc, self.file.tell() - size)

 
     def _write(self, sha, type, content):
         if verbose:
 
     def _write(self, sha, type, content):
         if verbose:


@@ -610,16 +811,11 @@ class PackWriter:
         size, crc = self._raw_write(_encode_packobj(type, content,
                                                     self.compression_level),
                                     sha=sha)
         size, crc = self._raw_write(_encode_packobj(type, content,
                                                     self.compression_level),
                                     sha=sha)

-        if self.outbytes >= max_pack_size or self.count >= max_pack_objects:
+        if self.outbytes >= self.max_pack_size \
+           or self.count >= self.max_pack_objects:

             self.breakpoint()
         return sha
 
             self.breakpoint()
         return sha
 

-    def breakpoint(self):
-        """Clear byte and object counts and return the last processed id."""
-        id = self._end()
-        self.outbytes = self.count = 0
-        return id
-

     def _require_objcache(self):
         if self.objcache is None and self.objcache_maker:
             self.objcache = self.objcache_maker()
     def _require_objcache(self):
         if self.objcache is None and self.objcache_maker:
             self.objcache = self.objcache_maker()


@@ -632,93 +828,117 @@ class PackWriter:
         self._require_objcache()
         return self.objcache.exists(id, want_source=want_source)
 
         self._require_objcache()
         return self.objcache.exists(id, want_source=want_source)
 

+    def just_write(self, sha, type, content):
+        """Write an object to the pack file without checking for duplication."""
+        self._write(sha, type, content)
+        # If nothing else, gc doesn't have/want an objcache
+        if self.objcache is not None:
+            self.objcache.add(sha)
+

     def maybe_write(self, type, content):
         """Write an object to the pack file if not present and return its id."""
         sha = calc_hash(type, content)
         if not self.exists(sha):
     def maybe_write(self, type, content):
         """Write an object to the pack file if not present and return its id."""
         sha = calc_hash(type, content)
         if not self.exists(sha):

-            self._write(sha, type, content)

             self._require_objcache()
             self._require_objcache()

-            self.objcache.add(sha)
+            self.just_write(sha, type, content)

         return sha
 
     def new_blob(self, blob):
         """Create a blob object in the pack with the supplied content."""
         return sha
 
     def new_blob(self, blob):
         """Create a blob object in the pack with the supplied content."""

-        return self.maybe_write('blob', blob)
+        return self.maybe_write(b'blob', blob)

 
     def new_tree(self, shalist):
         """Create a tree object in the pack."""
         content = tree_encode(shalist)
 
     def new_tree(self, shalist):
         """Create a tree object in the pack."""
         content = tree_encode(shalist)

-        return self.maybe_write('tree', content)
-
-    def _new_commit(self, tree, parent, author, adate, committer, cdate, msg):
-        l = []
-        if tree: l.append('tree %s' % tree.encode('hex'))
-        if parent: l.append('parent %s' % parent.encode('hex'))
-        if author: l.append('author %s %s' % (author, _git_date(adate)))
-        if committer: l.append('committer %s %s' % (committer, _git_date(cdate)))
-        l.append('')
-        l.append(msg)
-        return self.maybe_write('commit', '\n'.join(l))
-
-    def new_commit(self, parent, tree, date, msg):
-        """Create a commit object in the pack."""
-        userline = '%s <%s@%s>' % (userfullname(), username(), hostname())
-        commit = self._new_commit(tree, parent,
-                                  userline, date, userline, date,
-                                  msg)
-        return commit
+        return self.maybe_write(b'tree', content)
+
+    def new_commit(self, tree, parent,
+                   author, adate_sec, adate_tz,
+                   committer, cdate_sec, cdate_tz,
+                   msg):
+        """Create a commit object in the pack.  The date_sec values must be
+        epoch-seconds, and if a tz is None, the local timezone is assumed."""
+        content = create_commit_blob(tree, parent,
+                                     author, adate_sec, adate_tz,
+                                     committer, cdate_sec, cdate_tz,
+                                     msg)
+        return self.maybe_write(b'commit', content)
+
+    def _end(self, run_midx=True, abort=False):
+        # Ignores run_midx during abort
+        if not self.file:
+            return None
+        self.file, f = None, self.file
+        self.idx, idx = None, self.idx
+        self.parentfd, pfd, = None, self.parentfd
+        self.objcache = None

 
 

-    def abort(self):
-        """Remove the pack file from disk."""
-        f = self.file
-        if f:
-            self.idx = None
-            self.file = None
+        with finalized(pfd, lambda x: x is not None and os.close(x)), \
+             f:
+
+            if abort:
+                os.unlink(self.filename + b'.pack')
+                return None
+
+            # update object count
+            f.seek(8)
+            cp = struct.pack('!i', self.count)
+            assert len(cp) == 4
+            f.write(cp)
+
+            # calculate the pack sha1sum
+            f.seek(0)
+            sum = Sha1()
+            for b in chunkyreader(f):
+                sum.update(b)
+            packbin = sum.digest()
+            f.write(packbin)
+            f.flush()
+            fdatasync(f.fileno())

             f.close()
             f.close()

-            os.unlink(self.filename + '.pack')

 
 

-    def _end(self, run_midx=True):
-        f = self.file
-        if not f: return None
-        self.file = None
-        self.objcache = None
-        idx = self.idx
-        self.idx = None
+            idx.write(self.filename + b'.idx', packbin)
+            nameprefix = os.path.join(self.repo_dir,
+                                      b'objects/pack/pack-' +  hexlify(packbin))
+            if os.path.exists(self.filename + b'.map'):
+                os.unlink(self.filename + b'.map')
+            os.rename(self.filename + b'.pack', nameprefix + b'.pack')
+            os.rename(self.filename + b'.idx', nameprefix + b'.idx')
+            os.fsync(pfd)
+            if run_midx:
+                auto_midx(os.path.join(self.repo_dir, b'objects/pack'))
+            if self.on_pack_finish:
+                self.on_pack_finish(nameprefix)
+            return nameprefix
+
+    def abort(self):
+        """Remove the pack file from disk."""
+        self._end(abort=True)

 
 

-        # update object count
-        f.seek(8)
-        cp = struct.pack('!i', self.count)
-        assert(len(cp) == 4)
-        f.write(cp)
-
-        # calculate the pack sha1sum
-        f.seek(0)
-        sum = Sha1()
-        for b in chunkyreader(f):
-            sum.update(b)
-        packbin = sum.digest()
-        f.write(packbin)
-        f.close()
-
-        obj_list_sha = self._write_pack_idx_v2(self.filename + '.idx', idx, packbin)
-
-        nameprefix = repo('objects/pack/pack-%s' % obj_list_sha)
-        if os.path.exists(self.filename + '.map'):
-            os.unlink(self.filename + '.map')
-        os.rename(self.filename + '.pack', nameprefix + '.pack')
-        os.rename(self.filename + '.idx', nameprefix + '.idx')
-
-        if run_midx:
-            auto_midx(repo('objects/pack'))
-        return nameprefix
+    def breakpoint(self):
+        """Clear byte and object counts and return the last processed id."""
+        id = self._end(self.run_midx)
+        self.outbytes = self.count = 0
+        return id

 
     def close(self, run_midx=True):
         """Close the pack file and move it to its definitive path."""
         return self._end(run_midx=run_midx)
 
 
     def close(self, run_midx=True):
         """Close the pack file and move it to its definitive path."""
         return self._end(run_midx=run_midx)
 

-    def _write_pack_idx_v2(self, filename, idx, packbin):
+
+class PackIdxV2Writer:
+    def __init__(self):
+        self.idx = list(list() for i in range(256))
+        self.count = 0
+
+    def add(self, sha, crc, offs):
+        assert(sha)
+        self.count += 1
+        self.idx[byte_int(sha[0])].append((sha, crc, offs))
+
+    def write(self, filename, packbin):

         ofs64_count = 0
         ofs64_count = 0

-        for section in idx:
+        for section in self.idx:

             for entry in section:
                 if entry[2] >= 2**31:
                     ofs64_count += 1
             for entry in section:
                 if entry[2] >= 2**31:
                     ofs64_count += 1


@@ -729,11 +949,16 @@ class PackWriter:
         idx_f = open(filename, 'w+b')
         try:
             idx_f.truncate(index_len)
         idx_f = open(filename, 'w+b')
         try:
             idx_f.truncate(index_len)

+            fdatasync(idx_f.fileno())

             idx_map = mmap_readwrite(idx_f, close=False)
             idx_map = mmap_readwrite(idx_f, close=False)

-            count = _helpers.write_idx(filename, idx_map, idx, self.count)
-            assert(count == self.count)
+            try:
+                count = _helpers.write_idx(filename, idx_map, self.idx,
+                                           self.count)
+                assert(count == self.count)
+                idx_map.flush()
+            finally:
+                idx_map.close()

         finally:
         finally:

-            if idx_map: idx_map.close()

             idx_f.close()
 
         idx_f = open(filename, 'a+b')
             idx_f.close()
 
         idx_f = open(filename, 'a+b')


@@ -744,49 +969,50 @@ class PackWriter:
             b = idx_f.read(8 + 4*256)
             idx_sum.update(b)
 
             b = idx_f.read(8 + 4*256)
             idx_sum.update(b)
 

-            obj_list_sum = Sha1()
-            for b in chunkyreader(idx_f, 20*self.count):
+            for b in chunkyreader(idx_f, 20 * self.count):

                 idx_sum.update(b)
                 idx_sum.update(b)

-                obj_list_sum.update(b)
-            namebase = obj_list_sum.hexdigest()

 
             for b in chunkyreader(idx_f):
                 idx_sum.update(b)
             idx_f.write(idx_sum.digest())
 
             for b in chunkyreader(idx_f):
                 idx_sum.update(b)
             idx_f.write(idx_sum.digest())

-            return namebase
+            fdatasync(idx_f.fileno())

         finally:
             idx_f.close()
 
 
         finally:
             idx_f.close()
 
 

-def _git_date(date):
-    return '%d %s' % (date, time.strftime('%z', time.localtime(date)))
-
-
-def _gitenv():
-    os.environ['GIT_DIR'] = os.path.abspath(repo())
-
+def list_refs(patterns=None, repo_dir=None,
+              limit_to_heads=False, limit_to_tags=False):
+    """Yield (refname, hash) tuples for all repository refs unless
+    patterns are specified.  In that case, only include tuples for
+    refs matching those patterns (cf. git-show-ref(1)).  The limits
+    restrict the result items to refs/heads or refs/tags.  If both
+    limits are specified, items from both sources will be included.

 
 

-def list_refs(refname = None):
-    """Generate a list of tuples in the form (refname,hash).
-    If a ref name is specified, list only this particular ref.

     """
     """

-    argv = ['git', 'show-ref', '--']
-    if refname:
-        argv += [refname]
-    p = subprocess.Popen(argv, preexec_fn = _gitenv, stdout = subprocess.PIPE)
+    argv = [b'git', b'show-ref']
+    if limit_to_heads:
+        argv.append(b'--heads')
+    if limit_to_tags:
+        argv.append(b'--tags')
+    argv.append(b'--')
+    if patterns:
+        argv.extend(patterns)
+    p = subprocess.Popen(argv, env=_gitenv(repo_dir), stdout=subprocess.PIPE,
+                         close_fds=True)

     out = p.stdout.read().strip()
     rv = p.wait()  # not fatal
     if rv:
         assert(not out)
     if out:
     out = p.stdout.read().strip()
     rv = p.wait()  # not fatal
     if rv:
         assert(not out)
     if out:

-        for d in out.split('\n'):
-            (sha, name) = d.split(' ', 1)
-            yield (name, sha.decode('hex'))
+        for d in out.split(b'\n'):
+            sha, name = d.split(b' ', 1)
+            yield name, unhexlify(sha)

 
 
 
 

-def read_ref(refname):
+def read_ref(refname, repo_dir = None):

     """Get the commit id of the most recent commit made on a given ref."""
     """Get the commit id of the most recent commit made on a given ref."""

-    l = list(list_refs(refname))
+    refs = list_refs(patterns=[refname], repo_dir=repo_dir, limit_to_heads=True)
+    l = tuple(islice(refs, 2))

     if l:
         assert(len(l) == 1)
         return l[0][1]
     if l:
         assert(len(l) == 1)
         return l[0][1]


@@ -794,47 +1020,57 @@ def read_ref(refname):
         return None
 
 
         return None
 
 

-def rev_list(ref, count=None):
-    """Generate a list of reachable commits in reverse chronological order.
+def rev_list_invocation(ref_or_refs, format=None):
+    if isinstance(ref_or_refs, bytes):
+        refs = (ref_or_refs,)
+    else:
+        refs = ref_or_refs
+    argv = [b'git', b'rev-list']
+
+    if format:
+        argv.append(b'--pretty=format:' + format)
+    for ref in refs:
+        assert not ref.startswith(b'-')
+        argv.append(ref)
+    argv.append(b'--')
+    return argv

 
 

-    This generator walks through commits, from child to parent, that are
-    reachable via the specified ref and yields a series of tuples of the form
-    (date,hash).

 
 

-    If count is a non-zero integer, limit the number of commits to "count"
-    objects.
+def rev_list(ref_or_refs, parse=None, format=None, repo_dir=None):
+    """Yield information about commits as per "git rev-list".  If a format
+    is not provided, yield one hex hash at a time.  If a format is
+    provided, pass it to rev-list and call parse(git_stdout) for each
+    commit with the stream positioned just after the rev-list "commit
+    HASH" header line.  When a format is provided yield (oidx,
+    parse(git_stdout)) for each commit.
+

     """
     """

-    assert(not ref.startswith('-'))
-    opts = []
-    if count:
-        opts += ['-n', str(atoi(count))]
-    argv = ['git', 'rev-list', '--pretty=format:%at'] + opts + [ref, '--']
-    p = subprocess.Popen(argv, preexec_fn = _gitenv, stdout = subprocess.PIPE)
-    commit = None
-    for row in p.stdout:
-        s = row.strip()
-        if s.startswith('commit '):
-            commit = s[7:].decode('hex')
-        else:
-            date = int(s)
-            yield (date, commit)
+    assert bool(parse) == bool(format)
+    p = subprocess.Popen(rev_list_invocation(ref_or_refs,
+                                             format=format),
+                         env=_gitenv(repo_dir),
+                         stdout = subprocess.PIPE,
+                         close_fds=True)
+    if not format:
+        for line in p.stdout:
+            yield line.strip()
+    else:
+        line = p.stdout.readline()
+        while line:
+            s = line.strip()
+            if not s.startswith(b'commit '):
+                raise Exception('unexpected line ' + repr(s))
+            s = s[7:]
+            assert len(s) == 40
+            yield s, parse(p.stdout)
+            line = p.stdout.readline()
+

     rv = p.wait()  # not fatal
     if rv:
     rv = p.wait()  # not fatal
     if rv:

-        raise GitError, 'git rev-list returned error %d' % rv
+        raise GitError('git rev-list returned error %d' % rv)

 
 
 
 

-def get_commit_dates(refs):
-    """Get the dates for the specified commit refs.  For now, every unique
-       string in refs must resolve to a different commit or this
-       function will fail."""
-    result = []
-    for ref in refs:
-        commit = get_commit_items(ref, cp())
-        result.append(commit.author_sec)
-    return result
-
-
-def rev_parse(committish):
+def rev_parse(committish, repo_dir=None):

     """Resolve the full hash for 'committish', if it exists.
 
     Should be roughly equivalent to 'git rev-parse'.
     """Resolve the full hash for 'committish', if it exists.
 
     Should be roughly equivalent to 'git rev-parse'.


@@ -842,16 +1078,16 @@ def rev_parse(committish):
     Returns the hex value of the hash if it is found, None if 'committish' does
     not correspond to anything.
     """
     Returns the hex value of the hash if it is found, None if 'committish' does
     not correspond to anything.
     """

-    head = read_ref(committish)
+    head = read_ref(committish, repo_dir=repo_dir)

     if head:
     if head:

-        debug2("resolved from ref: commit = %s\n" % head.encode('hex'))
+        debug2("resolved from ref: commit = %s\n" % hexlify(head))

         return head
 
         return head
 

-    pL = PackIdxList(repo('objects/pack'))
+    pL = PackIdxList(repo(b'objects/pack', repo_dir=repo_dir))

 
     if len(committish) == 40:
         try:
 
     if len(committish) == 40:
         try:

-            hash = committish.decode('hex')
+            hash = unhexlify(committish)

         except TypeError:
             return None
 
         except TypeError:
             return None
 


@@ -861,14 +1097,26 @@ def rev_parse(committish):
     return None
 
 
     return None
 
 

-def update_ref(refname, newval, oldval):
-    """Change the commit pointed to by a branch."""
+def update_ref(refname, newval, oldval, repo_dir=None):
+    """Update a repository reference."""

     if not oldval:
     if not oldval:

-        oldval = ''
-    assert(refname.startswith('refs/heads/'))
-    p = subprocess.Popen(['git', 'update-ref', refname,
-                          newval.encode('hex'), oldval.encode('hex')],
-                         preexec_fn = _gitenv)
+        oldval = b''
+    assert refname.startswith(b'refs/heads/') \
+        or refname.startswith(b'refs/tags/')
+    p = subprocess.Popen([b'git', b'update-ref', refname,
+                          hexlify(newval), hexlify(oldval)],
+                         env=_gitenv(repo_dir),
+                         close_fds=True)
+    _git_wait(b'git update-ref', p)
+
+
+def delete_ref(refname, oldvalue=None):
+    """Delete a repository reference (see git update-ref(1))."""
+    assert refname.startswith(b'refs/')
+    oldvalue = [] if not oldvalue else [oldvalue]
+    p = subprocess.Popen([b'git', b'update-ref', b'-d', refname] + oldvalue,
+                         env=_gitenv(),
+                         close_fds=True)

     _git_wait('git update-ref', p)
 
 
     _git_wait('git update-ref', p)
 
 


@@ -883,9 +1131,9 @@ def guess_repo(path=None):
     if path:
         repodir = path
     if not repodir:
     if path:
         repodir = path
     if not repodir:

-        repodir = os.environ.get('BUP_DIR')
+        repodir = environ.get(b'BUP_DIR')

         if not repodir:
         if not repodir:

-            repodir = os.path.expanduser('~/.bup')
+            repodir = os.path.expanduser(b'~/.bup')

 
 
 def init_repo(path=None):
 
 
 def init_repo(path=None):


@@ -894,79 +1142,90 @@ def init_repo(path=None):
     d = repo()  # appends a / to the path
     parent = os.path.dirname(os.path.dirname(d))
     if parent and not os.path.exists(parent):
     d = repo()  # appends a / to the path
     parent = os.path.dirname(os.path.dirname(d))
     if parent and not os.path.exists(parent):

-        raise GitError('parent directory "%s" does not exist\n' % parent)
-    if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
-        raise GitError('"%s" exists but is not a directory\n' % d)
-    p = subprocess.Popen(['git', '--bare', 'init'], stdout=sys.stderr,
-                         preexec_fn = _gitenv)
+        raise GitError('parent directory "%s" does not exist\n'
+                       % path_msg(parent))
+    if os.path.exists(d) and not os.path.isdir(os.path.join(d, b'.')):
+        raise GitError('"%s" exists but is not a directory\n' % path_msg(d))
+    p = subprocess.Popen([b'git', b'--bare', b'init'], stdout=sys.stderr,
+                         env=_gitenv(),
+                         close_fds=True)

     _git_wait('git init', p)
     # Force the index version configuration in order to ensure bup works
     # regardless of the version of the installed Git binary.
     _git_wait('git init', p)
     # Force the index version configuration in order to ensure bup works
     # regardless of the version of the installed Git binary.

-    p = subprocess.Popen(['git', 'config', 'pack.indexVersion', '2'],
-                         stdout=sys.stderr, preexec_fn = _gitenv)
+    p = subprocess.Popen([b'git', b'config', b'pack.indexVersion', '2'],
+                         stdout=sys.stderr, env=_gitenv(), close_fds=True)

     _git_wait('git config', p)
     # Enable the reflog
     _git_wait('git config', p)
     # Enable the reflog

-    p = subprocess.Popen(['git', 'config', 'core.logAllRefUpdates', 'true'],
-                         stdout=sys.stderr, preexec_fn = _gitenv)
+    p = subprocess.Popen([b'git', b'config', b'core.logAllRefUpdates', b'true'],
+                         stdout=sys.stderr, env=_gitenv(), close_fds=True)

     _git_wait('git config', p)
 
 
 def check_repo_or_die(path=None):
     _git_wait('git config', p)
 
 
 def check_repo_or_die(path=None):

-    """Make sure a bup repository exists, and abort if not.
-    If the path to a particular repository was not specified, this function
-    initializes the default repository automatically.
-    """
+    """Check to see if a bup repository probably exists, and abort if not."""

     guess_repo(path)
     guess_repo(path)

-    try:
-        os.stat(repo('objects/pack/.'))
-    except OSError, e:
-        if e.errno == errno.ENOENT:
-            log('error: %r is not a bup repository; run "bup init"\n'
-                % repo())
+    top = repo()
+    pst = stat_if_exists(top + b'/objects/pack')
+    if pst and stat.S_ISDIR(pst.st_mode):
+        return
+    if not pst:
+        top_st = stat_if_exists(top)
+        if not top_st:
+            log('error: repository %r does not exist (see "bup help init")\n'
+                % top)

             sys.exit(15)
             sys.exit(15)

-        else:
-            log('error: %s\n' % e)
-            sys.exit(14)
+    log('error: %s is not a repository\n' % path_msg(top))
+    sys.exit(14)
+
+
+def is_suitable_git(ver_str):
+    if not ver_str.startswith(b'git version '):
+        return 'unrecognized'
+    ver_str = ver_str[len(b'git version '):]
+    if ver_str.startswith(b'0.'):
+        return 'insufficient'
+    if ver_str.startswith(b'1.'):
+        if re.match(br'1\.[012345]rc', ver_str):
+            return 'insufficient'
+        if re.match(br'1\.[01234]\.', ver_str):
+            return 'insufficient'
+        if re.match(br'1\.5\.[012345]($|\.)', ver_str):
+            return 'insufficient'
+        if re.match(br'1\.5\.6-rc', ver_str):
+            return 'insufficient'
+        return 'suitable'
+    if re.match(br'[0-9]+(\.|$)?', ver_str):
+        return 'suitable'
+    sys.exit(13)
+
+_git_great = None
+
+def require_suitable_git(ver_str=None):
+    """Raise GitError if the version of git isn't suitable.
+
+    Rely on ver_str when provided, rather than invoking the git in the
+    path.

 
 

-
-_ver = None
-def ver():
-    """Get Git's version and ensure a usable version is installed.
-
-    The returned version is formatted as an ordered tuple with each position
-    representing a digit in the version tag. For example, the following tuple
-    would represent version 1.6.6.9:
-
-        ('1', '6', '6', '9')

     """
     """

-    global _ver
-    if not _ver:
-        p = subprocess.Popen(['git', '--version'],
-                             stdout=subprocess.PIPE)
-        gvs = p.stdout.read()
-        _git_wait('git --version', p)
-        m = re.match(r'git version (\S+.\S+)', gvs)
-        if not m:
-            raise GitError('git --version weird output: %r' % gvs)
-        _ver = tuple(m.group(1).split('.'))
-    needed = ('1','5', '3', '1')
-    if _ver < needed:
-        raise GitError('git version %s or higher is required; you have %s'
-                       % ('.'.join(needed), '.'.join(_ver)))
-    return _ver
-
-
-def _git_wait(cmd, p):
-    rv = p.wait()
-    if rv != 0:
-        raise GitError('%s returned %d' % (cmd, rv))
-
-
-def _git_capture(argv):
-    p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn = _gitenv)
-    r = p.stdout.read()
-    _git_wait(repr(argv), p)
-    return r
+    global _git_great
+    if _git_great is not None:
+        return
+    if environ.get(b'BUP_GIT_VERSION_IS_FINE', b'').lower() \
+       in (b'yes', b'true', b'1'):
+        _git_great = True
+        return
+    if not ver_str:
+        ver_str, _, _ = _git_exo([b'git', b'--version'])
+    status = is_suitable_git(ver_str)
+    if status == 'unrecognized':
+        raise GitError('Unexpected git --version output: %r' % ver_str)
+    if status == 'insufficient':
+        log('error: git version must be at least 1.5.6\n')
+        sys.exit(1)
+    if status == 'suitable':
+        _git_great = True
+        return
+    assert False

 
 
 class _AbortableIter:
 
 
 class _AbortableIter:


@@ -978,16 +1237,18 @@ class _AbortableIter:
     def __iter__(self):
         return self
 
     def __iter__(self):
         return self
 

-    def next(self):
+    def __next__(self):

         try:
         try:

-            return self.it.next()
-        except StopIteration, e:
+            return next(self.it)
+        except StopIteration as e:

             self.done = True
             raise
         except:
             self.abort()
             raise
 
             self.done = True
             raise
         except:
             self.abort()
             raise
 

+    next = __next__
+

     def abort(self):
         """Abort iteration and call the abortion callback, if needed."""
         if not self.done:
     def abort(self):
         """Abort iteration and call the abortion callback, if needed."""
         if not self.done:


@@ -999,108 +1260,97 @@ class _AbortableIter:
         self.abort()
 
 
         self.abort()
 
 

-_ver_warned = 0

 class CatPipe:
     """Link to 'git cat-file' that is used to retrieve blob data."""
 class CatPipe:
     """Link to 'git cat-file' that is used to retrieve blob data."""

-    def __init__(self):
-        global _ver_warned
-        wanted = ('1','5','6')
-        if ver() < wanted:
-            if not _ver_warned:
-                log('warning: git version < %s; bup will be slow.\n'
-                    % '.'.join(wanted))
-                _ver_warned = 1
-            self.get = self._slow_get
-        else:
-            self.p = self.inprogress = None
-            self.get = self._fast_get
-
-    def _abort(self):
-        if self.p:
-            self.p.stdout.close()
-            self.p.stdin.close()
+    def __init__(self, repo_dir = None):
+        require_suitable_git()
+        self.repo_dir = repo_dir
+        self.p = self.inprogress = None
+
+    def close(self, wait=False):
+        p = self.p
+        if p:
+            p.stdout.close()
+            p.stdin.close()

         self.p = None
         self.inprogress = None
         self.p = None
         self.inprogress = None

+        if wait:
+            p.wait()
+            return p.returncode
+        return None

 
 

-    def _restart(self):
-        self._abort()
-        self.p = subprocess.Popen(['git', 'cat-file', '--batch'],
+    def restart(self):
+        self.close()
+        self.p = subprocess.Popen([b'git', b'cat-file', b'--batch'],

                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   close_fds = True,
                                   bufsize = 4096,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   close_fds = True,
                                   bufsize = 4096,

-                                  preexec_fn = _gitenv)
+                                  env=_gitenv(self.repo_dir))
+
+    def get(self, ref):
+        """Yield (oidx, type, size), followed by the data referred to by ref.
+        If ref does not exist, only yield (None, None, None).

 
 

-    def _fast_get(self, id):
+        """

         if not self.p or self.p.poll() != None:
         if not self.p or self.p.poll() != None:

-            self._restart()
+            self.restart()

         assert(self.p)
         poll_result = self.p.poll()
         assert(poll_result == None)
         if self.inprogress:
         assert(self.p)
         poll_result = self.p.poll()
         assert(poll_result == None)
         if self.inprogress:

-            log('_fast_get: opening %r while %r is open\n'
-                % (id, self.inprogress))
+            log('get: opening %r while %r is open\n' % (ref, self.inprogress))

         assert(not self.inprogress)
         assert(not self.inprogress)

-        assert(id.find('\n') < 0)
-        assert(id.find('\r') < 0)
-        assert(not id.startswith('-'))
-        self.inprogress = id
-        self.p.stdin.write('%s\n' % id)
+        assert ref.find(b'\n') < 0
+        assert ref.find(b'\r') < 0
+        assert not ref.startswith(b'-')
+        self.inprogress = ref
+        self.p.stdin.write(ref + b'\n')

         self.p.stdin.flush()
         hdr = self.p.stdout.readline()
         self.p.stdin.flush()
         hdr = self.p.stdout.readline()

-        if hdr.endswith(' missing\n'):
+        if not hdr:
+            raise GitError('unexpected cat-file EOF (last request: %r, exit: %s)'
+                           % (ref, self.p.poll() or 'none'))
+        if hdr.endswith(b' missing\n'):

             self.inprogress = None
             self.inprogress = None

-            raise KeyError('blob %r is missing' % id)
-        spl = hdr.split(' ')
-        if len(spl) != 3 or len(spl[0]) != 40:
-            raise GitError('expected blob, got %r' % spl)
-        (hex, type, size) = spl
-
-        it = _AbortableIter(chunkyreader(self.p.stdout, int(spl[2])),
-                           onabort = self._abort)
+            yield None, None, None
+            return
+        info = hdr.split(b' ')
+        if len(info) != 3 or len(info[0]) != 40:
+            raise GitError('expected object (id, type, size), got %r' % info)
+        oidx, typ, size = info
+        size = int(size)
+        it = _AbortableIter(chunkyreader(self.p.stdout, size),
+                            onabort=self.close)

         try:
         try:

-            yield type
+            yield oidx, typ, size

             for blob in it:
                 yield blob
             readline_result = self.p.stdout.readline()
             for blob in it:
                 yield blob
             readline_result = self.p.stdout.readline()

-            assert(readline_result == '\n')
+            assert readline_result == b'\n'

             self.inprogress = None
             self.inprogress = None

-        except Exception, e:
+        except Exception as e:

             it.abort()
             raise
 
             it.abort()
             raise
 

-    def _slow_get(self, id):
-        assert(id.find('\n') < 0)
-        assert(id.find('\r') < 0)
-        assert(id[0] != '-')
-        type = _git_capture(['git', 'cat-file', '-t', id]).strip()
-        yield type
-
-        p = subprocess.Popen(['git', 'cat-file', type, id],
-                             stdout=subprocess.PIPE,
-                             preexec_fn = _gitenv)
-        for blob in chunkyreader(p.stdout):
-            yield blob
-        _git_wait('git cat-file', p)
-

     def _join(self, it):
     def _join(self, it):

-        type = it.next()
-        if type == 'blob':
+        _, typ, _ = next(it)
+        if typ == b'blob':

             for blob in it:
                 yield blob
             for blob in it:
                 yield blob

-        elif type == 'tree':
-            treefile = ''.join(it)
+        elif typ == b'tree':
+            treefile = b''.join(it)

             for (mode, name, sha) in tree_decode(treefile):
             for (mode, name, sha) in tree_decode(treefile):

-                for blob in self.join(sha.encode('hex')):
+                for blob in self.join(hexlify(sha)):

                     yield blob
                     yield blob

-        elif type == 'commit':
-            treeline = ''.join(it).split('\n')[0]
-            assert(treeline.startswith('tree '))
+        elif typ == b'commit':
+            treeline = b''.join(it).split(b'\n')[0]
+            assert treeline.startswith(b'tree ')

             for blob in self.join(treeline[5:]):
                 yield blob
         else:
             raise GitError('invalid object type %r: expected blob/tree/commit'
             for blob in self.join(treeline[5:]):
                 yield blob
         else:
             raise GitError('invalid object type %r: expected blob/tree/commit'

-                           % type)
+                           % typ)

 
     def join(self, id):
         """Generate a list of the content of all blobs that can be reached
 
     def join(self, id):
         """Generate a list of the content of all blobs that can be reached


@@ -1108,35 +1358,128 @@ class CatPipe:
         or a commit. The content of all blobs that can be seen from trees or
         commits will be added to the list.
         """
         or a commit. The content of all blobs that can be seen from trees or
         commits will be added to the list.
         """

-        try:
-            for d in self._join(self.get(id)):
-                yield d
-        except StopIteration:
-            log('booger!\n')
-
-
-_cp = (None, None)
-
-def cp():
-    """Create a CatPipe object or reuse an already existing one."""
-    global _cp
-    cp_dir, cp = _cp
-    cur_dir = os.path.realpath(repo())
-    if cur_dir != cp_dir:
-        cp = CatPipe()
-        _cp = (cur_dir, cp)
+        for d in self._join(self.get(id)):
+            yield d
+
+
+_cp = {}
+
+def cp(repo_dir=None):
+    """Create a CatPipe object or reuse the already existing one."""
+    global _cp, repodir
+    if not repo_dir:
+        repo_dir = repodir or repo()
+    repo_dir = os.path.abspath(repo_dir)
+    cp = _cp.get(repo_dir)
+    if not cp:
+        cp = CatPipe(repo_dir)
+        _cp[repo_dir] = cp

     return cp
 
 
     return cp
 
 

-def tags():
+def close_catpipes():
+    # FIXME: chain exceptions
+    while _cp:
+        _, cp = _cp.popitem()
+        cp.close(wait=True)
+
+
+def tags(repo_dir = None):

     """Return a dictionary of all tags in the form {hash: [tag_names, ...]}."""
     tags = {}
     """Return a dictionary of all tags in the form {hash: [tag_names, ...]}."""
     tags = {}

-    for (n,c) in list_refs():
-        if n.startswith('refs/tags/'):
-            name = n[10:]
-            if not c in tags:
-                tags[c] = []
+    for n, c in list_refs(repo_dir = repo_dir, limit_to_tags=True):
+        assert n.startswith(b'refs/tags/')
+        name = n[10:]
+        if not c in tags:
+            tags[c] = []
+        tags[c].append(name)  # more than one tag can point at 'c'
+    return tags

 
 

-            tags[c].append(name)  # more than one tag can point at 'c'

 
 

-    return tags
+class MissingObject(KeyError):
+    def __init__(self, oid):
+        self.oid = oid
+        KeyError.__init__(self, 'object %r is missing' % hexlify(oid))
+
+
+WalkItem = namedtuple('WalkItem', ['oid', 'type', 'mode',
+                                   'path', 'chunk_path', 'data'])
+# The path is the mangled path, and if an item represents a fragment
+# of a chunked file, the chunk_path will be the chunked subtree path
+# for the chunk, i.e. ['', '2d3115e', ...].  The top-level path for a
+# chunked file will have a chunk_path of [''].  So some chunk subtree
+# of the file '/foo/bar/baz' might look like this:
+#
+#   item.path = ['foo', 'bar', 'baz.bup']
+#   item.chunk_path = ['', '2d3115e', '016b097']
+#   item.type = 'tree'
+#   ...
+
+
+def walk_object(get_ref, oidx, stop_at=None, include_data=None):
+    """Yield everything reachable from oidx via get_ref (which must behave
+    like CatPipe get) as a WalkItem, stopping whenever stop_at(oidx)
+    returns true.  Throw MissingObject if a hash encountered is
+    missing from the repository, and don't read or return blob content
+    in the data field unless include_data is set.
+
+    """
+    # Maintain the pending stack on the heap to avoid stack overflow
+    pending = [(oidx, [], [], None)]
+    while len(pending):
+        oidx, parent_path, chunk_path, mode = pending.pop()
+        oid = unhexlify(oidx)
+        if stop_at and stop_at(oidx):
+            continue
+
+        if (not include_data) and mode and stat.S_ISREG(mode):
+            # If the object is a "regular file", then it's a leaf in
+            # the graph, so we can skip reading the data if the caller
+            # hasn't requested it.
+            yield WalkItem(oid=oid, type=b'blob',
+                           chunk_path=chunk_path, path=parent_path,
+                           mode=mode,
+                           data=None)
+            continue
+
+        item_it = get_ref(oidx)
+        get_oidx, typ, _ = next(item_it)
+        if not get_oidx:
+            raise MissingObject(unhexlify(oidx))
+        if typ not in (b'blob', b'commit', b'tree'):
+            raise Exception('unexpected repository object type %r' % typ)
+
+        # FIXME: set the mode based on the type when the mode is None
+        if typ == b'blob' and not include_data:
+            # Dump data until we can ask cat_pipe not to fetch it
+            for ignored in item_it:
+                pass
+            data = None
+        else:
+            data = b''.join(item_it)
+
+        yield WalkItem(oid=oid, type=typ,
+                       chunk_path=chunk_path, path=parent_path,
+                       mode=mode,
+                       data=(data if include_data else None))
+
+        if typ == b'commit':
+            commit_items = parse_commit(data)
+            for pid in commit_items.parents:
+                pending.append((pid, parent_path, chunk_path, mode))
+            pending.append((commit_items.tree, parent_path, chunk_path,
+                            hashsplit.GIT_MODE_TREE))
+        elif typ == b'tree':
+            for mode, name, ent_id in tree_decode(data):
+                demangled, bup_type = demangle_name(name, mode)
+                if chunk_path:
+                    sub_path = parent_path
+                    sub_chunk_path = chunk_path + [name]
+                else:
+                    sub_path = parent_path + [name]
+                    if bup_type == BUP_CHUNKED:
+                        sub_chunk_path = [b'']
+                    else:
+                        sub_chunk_path = chunk_path
+                pending.append((hexlify(ent_id), sub_path, sub_chunk_path,
+                                mode))