rev_list: support custom formats

Signed-off-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/cmd/prune-older-cmd.py b/cmd/prune-older-cmd.py
index 208ddf73bb8d6f69ded65bfbd99fa538a91929de..12a440a5cbf23964d7ce79e929a8602b684e243a 100755 (executable)
--- a/cmd/prune-older-cmd.py
+++ b/cmd/prune-older-cmd.py
@@ -19,7 +19,7 @@ from bup.rm import bup_rm
 
 
 def branches(refnames=()):
-    return ((name[11:], sha) for (name,sha)
+    return ((name[11:], sha.encode('hex')) for (name,sha)
             in git.list_refs(patterns=('refs/heads/' + n for n in refnames),
                              limit_to_heads=True))
 
@@ -130,10 +130,15 @@ git.check_repo_or_die()
 # This could be more efficient, but for now just build the whole list
 # in memory and let bup_rm() do some redundant work.
 
+def parse_info(f):
+    author_secs = f.readline().strip()
+    return int(author_secs)
+
 removals = []
 for branch, branch_id in branches(roots):
     die_if_errors()
-    saves = git.rev_list(branch_id.encode('hex'))
+    saves = ((utc, oidx.decode('hex')) for (oidx, utc) in
+             git.rev_list(branch_id, format='%at', parse=parse_info))
     for keep_save, (utc, id) in classify_saves(saves, period_start):
         assert(keep_save in (False, True))
         # FIXME: base removals on hashes

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 5e52ab6f1a6ae627dfee5bd375682405eaf80a25..8458b5cadee6f29c9d1299e863b79ab0795524d5 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -923,34 +923,42 @@ def read_ref(refname, repo_dir = None):
         return None
 
 
-def rev_list(ref, count=None, repo_dir=None):
-    """Generate a list of reachable commits in reverse chronological order.
+def rev_list(ref, count=None, parse=None, format=None, repo_dir=None):
+    """Yield information about commits as per "git rev-list".  If a format
+    is not provided, yield one hex hash at a time.  If a format is
+    provided, pass it to rev-list and call parse(git_stdout) for each
+    commit with the stream positioned just after the rev-list "commit
+    HASH" header line.  When a format is provided yield (oidx,
+    parse(git_stdout)) for each commit.
 
-    This generator walks through commits, from child to parent, that are
-    reachable via the specified ref and yields a series of tuples of the form
-    (date,hash).
-
-    If count is a non-zero integer, limit the number of commits to "count"
-    objects.
     """
-    assert(not ref.startswith('-'))
-    opts = []
+    assert bool(parse) == bool(format)
+    assert not ref.startswith('-')
+    argv = ['git', 'rev-list']
     if isinstance(count, Integral):
-        opts += ['-n', str(count)]
+        argv.extend(['-n', str(count)])
     else:
         assert not count
-    argv = ['git', 'rev-list', '--pretty=format:%at'] + opts + [ref, '--']
+    if format:
+        argv.append('--pretty=format:' + format)
+    if ref:
+        argv.append(ref)
+    argv.append('--')
     p = subprocess.Popen(argv,
                          preexec_fn = _gitenv(repo_dir),
                          stdout = subprocess.PIPE)
-    commit = None
-    for row in p.stdout:
-        s = row.strip()
-        if s.startswith('commit '):
-            commit = s[7:].decode('hex')
-        else:
-            date = int(s)
-            yield (date, commit)
+    if not format:
+        for line in p.stdout:
+            yield line.strip()
+    else:
+        line = p.stdout.readline()
+        while line:
+            s = line.strip()
+            if not s.startswith('commit '):
+                raise Exception('unexpected line ' + s)
+            yield s[7:], parse(p.stdout)
+            line = p.stdout.readline()
+
     rv = p.wait()  # not fatal
     if rv:
         raise GitError, 'git rev-list returned error %d' % rv

diff --git a/lib/bup/rm.py b/lib/bup/rm.py
index 31155fc9718ea1be391ccd3b5ddf1265abb5caf6..4f5742d2ef922f8010c0e8450490f0ebc99923bf 100644 (file)
--- a/lib/bup/rm.py
+++ b/lib/bup/rm.py
@@ -21,7 +21,7 @@ def append_commit(hash, parent, cp, writer):
 
 def filter_branch(tip_commit_hex, exclude, writer):
     # May return None if everything is excluded.
-    commits = [c for _, c in git.rev_list(tip_commit_hex)]
+    commits = [x.decode('hex') for x in git.rev_list(tip_commit_hex)]
     commits.reverse()
     last_c, tree = None, None
     # Rather than assert that we always find an exclusion here, we'll

diff --git a/lib/bup/vfs.py b/lib/bup/vfs.py
index b9e90f3415c18569a2c72fc2e4f3121066652063..554d40232373dc12f9f7e31a11f976fb6a748fae 100644 (file)
--- a/lib/bup/vfs.py
+++ b/lib/bup/vfs.py
@@ -483,10 +483,13 @@ class CommitDir(Node):
         refs = git.list_refs(repo_dir = self._repo_dir)
         for ref in refs:
             #debug2('ref name: %s\n' % ref[0])
-            revs = git.rev_list(ref[1].encode('hex'), repo_dir = self._repo_dir)
-            for (date, commit) in revs:
+            revs = git.rev_list(ref[1].encode('hex'),
+                                format='%at',
+                                parse=lambda f: int(f.readline().strip()),
+                                repo_dir=self._repo_dir)
+            for commithex, date in revs:
                 #debug2('commit: %s  date: %s\n' % (commit.encode('hex'), date))
-                commithex = commit.encode('hex')
+                commit = commithex.decode('hex')
                 containername = commithex[:2]
                 dirname = commithex[2:]
                 n1 = self._subs.get(containername)
@@ -544,22 +547,23 @@ class BranchList(Node):
         Node.__init__(self, parent, name, GIT_MODE_TREE, hash, repo_dir)
 
     def _mksubs(self):
+
         self._subs = {}
 
         revs = list(git.rev_list(self.hash.encode('hex'),
+                                 format='%at',
+                                 parse=lambda f: int(f.readline().strip()),
                                  repo_dir=self._repo_dir))
         latest = revs[0]
-        for (date, commit) in revs:
+        for commithex, date in revs:
             l = time.localtime(date)
             ls = time.strftime('%Y-%m-%d-%H%M%S', l)
-            commithex = commit.encode('hex')
             target = '../.commit/%s/%s' % (commithex[:2], commithex[2:])
             n1 = FakeSymlink(self, ls, target, self._repo_dir)
             n1.ctime = n1.mtime = date
             self._subs[ls] = n1
 
-        (date, commit) = latest
-        commithex = commit.encode('hex')
+        commithex, date = latest
         target = '../.commit/%s/%s' % (commithex[:2], commithex[2:])
         n1 = FakeSymlink(self, 'latest', target, self._repo_dir)
         n1.ctime = n1.mtime = date
@@ -588,8 +592,8 @@ class RefList(Node):
         self._subs['.tag'] = tag_dir
 
         refs_info = [(name[11:], sha) for (name,sha)
-                     in git.list_refs(repo_dir=self._repo_dir)
-                     if name.startswith('refs/heads/')]
+                     in git.list_refs(limit_to_heads=True,
+                                      repo_dir=self._repo_dir)]
         dates = git.get_commit_dates([sha.encode('hex')
                                       for (name, sha) in refs_info],
                                      repo_dir=self._repo_dir)