CatPipe.get: always return (oidx, type, size) for obj info

Return the size unconditionally, and add the oidx, which can be useful
if the get id argument was a ref rather than a hash.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py
index 5f52e7086ce5c7172ffc06761354115ef17f9f33..44d63230bf21a05601c755aae71d4509181cb8f6 100755 (executable)
--- a/cmd/split-cmd.py
+++ b/cmd/split-cmd.py
@@ -136,7 +136,7 @@ if opt.git_ids:
                 line = line.strip()
             try:
                 it = cp.get(line.strip())
-                next(it, None)  # skip the file type
+                next(it, None)  # skip the file info
             except KeyError as e:
                 add_error('error: %s' % e)
                 continue

diff --git a/lib/bup/gc.py b/lib/bup/gc.py
index 6ee631fba8ace74f913b9687ec45b0f1e16445a1..1926522c612f7ecb8123543591a692d531c205f6 100644 (file)
--- a/lib/bup/gc.py
+++ b/lib/bup/gc.py
@@ -186,8 +186,8 @@ def sweep(live_objects, existing_count, cat_pipe, threshold, compression,
             sha = idx.shatable[i * 20 : (i + 1) * 20]
             if live_objects.exists(sha):
                 item_it = cat_pipe.get(sha.encode('hex'))
-                type = next(item_it)
-                writer.just_write(sha, type, ''.join(item_it))
+                _, typ, _ = next(item_it)
+                writer.just_write(sha, typ, ''.join(item_it))
 
         ns.stale_files.append(idx_name)
         ns.stale_files.append(idx_name[:-3] + 'pack')

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 3ab36857b65b68c105926b22be59fd73cad4ff09..6787ac0a1cffa52e9dc9baf382c9f142e28bead1 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -112,7 +112,8 @@ def parse_commit(content):
 
 def get_commit_items(id, cp):
     commit_it = cp.get(id)
-    assert(next(commit_it) == 'commit')
+    _, typ, _ = next(commit_it)
+    assert(typ == 'commit')
     commit_content = ''.join(commit_it)
     return parse_commit(commit_content)
 
@@ -1155,12 +1156,9 @@ class CatPipe:
                                   bufsize = 4096,
                                   preexec_fn = _gitenv(self.repo_dir))
 
-    def get(self, id, size=False):
-        """Yield info about object id, and then if the object exists, all of
-        the data referred to by the object.  If size is false the info
-        will just be the object type name.  If size is true, the info
-        will be (type, size).  When the object does not exist, in both
-        cases the type will be None.
+    def get(self, ref):
+        """Yield (oidx, type, size), followed by the data referred to by ref.
+        If ref does not exist, only yield (None, None, None).
 
         """
         if not self.p or self.p.poll() != None:
@@ -1169,34 +1167,28 @@ class CatPipe:
         poll_result = self.p.poll()
         assert(poll_result == None)
         if self.inprogress:
-            log('get: opening %r while %r is open\n' % (id, self.inprogress))
+            log('get: opening %r while %r is open\n' % (ref, self.inprogress))
         assert(not self.inprogress)
-        assert(id.find('\n') < 0)
-        assert(id.find('\r') < 0)
-        assert(not id.startswith('-'))
-        self.inprogress = id
-        self.p.stdin.write('%s\n' % id)
+        assert(ref.find('\n') < 0)
+        assert(ref.find('\r') < 0)
+        assert(not ref.startswith('-'))
+        self.inprogress = ref
+        self.p.stdin.write('%s\n' % ref)
         self.p.stdin.flush()
         hdr = self.p.stdout.readline()
         if hdr.endswith(' missing\n'):
             self.inprogress = None
-            if size:
-                yield None, None
-            else:
-                yield None
+            yield None, None, None
             return
-        spl = hdr.split(' ')
-        if len(spl) != 3 or len(spl[0]) != 40:
-            raise GitError('expected blob, got %r' % spl)
-        hex, typ, sz = spl
-        sz = int(sz)
-        it = _AbortableIter(chunkyreader(self.p.stdout, sz),
+        info = hdr.split(' ')
+        if len(info) != 3 or len(info[0]) != 40:
+            raise GitError('expected object (id, type, size), got %r' % spl)
+        oidx, typ, size = info
+        size = int(size)
+        it = _AbortableIter(chunkyreader(self.p.stdout, size),
                             onabort=self._abort)
         try:
-            if size:
-                yield typ, sz
-            else:
-                yield typ
+            yield oidx, typ, size
             for blob in it:
                 yield blob
             readline_result = self.p.stdout.readline()
@@ -1207,23 +1199,23 @@ class CatPipe:
             raise
 
     def _join(self, it):
-        type = next(it)
-        if type == 'blob':
+        _, typ, _ = next(it)
+        if typ == 'blob':
             for blob in it:
                 yield blob
-        elif type == 'tree':
+        elif typ == 'tree':
             treefile = ''.join(it)
             for (mode, name, sha) in tree_decode(treefile):
                 for blob in self.join(sha.encode('hex')):
                     yield blob
-        elif type == 'commit':
+        elif typ == 'commit':
             treeline = ''.join(it).split('\n')[0]
             assert(treeline.startswith('tree '))
             for blob in self.join(treeline[5:]):
                 yield blob
         else:
             raise GitError('invalid object type %r: expected blob/tree/commit'
-                           % type)
+                           % typ)
 
     def join(self, id):
         """Generate a list of the content of all blobs that can be reached
@@ -1312,14 +1304,14 @@ def walk_object(cat_pipe, id,
             continue
 
         item_it = cat_pipe.get(id)
-        type = next(item_it)
-        if not type:
+        get_oidx, typ, _ = next(item_it)
+        if not get_oidx:
             raise MissingObject(id.decode('hex'))
-        if type not in ('blob', 'commit', 'tree'):
-            raise Exception('unexpected repository object type %r' % type)
+        if typ not in ('blob', 'commit', 'tree'):
+            raise Exception('unexpected repository object type %r' % typ)
 
         # FIXME: set the mode based on the type when the mode is None
-        if type == 'blob' and not include_data:
+        if typ == 'blob' and not include_data:
             # Dump data until we can ask cat_pipe not to fetch it
             for ignored in item_it:
                 pass
@@ -1327,18 +1319,18 @@ def walk_object(cat_pipe, id,
         else:
             data = ''.join(item_it)
 
-        yield WalkItem(id=id, type=type,
+        yield WalkItem(id=id, type=typ,
                        chunk_path=chunk_path, path=parent_path,
                        mode=mode,
                        data=(data if include_data else None))
 
-        if type == 'commit':
+        if typ == 'commit':
             commit_items = parse_commit(data)
             for pid in commit_items.parents:
                 pending.append((pid, parent_path, chunk_path, mode))
             pending.append((commit_items.tree, parent_path, chunk_path,
                             hashsplit.GIT_MODE_TREE))
-        elif type == 'tree':
+        elif typ == 'tree':
             for mode, name, ent_id in tree_decode(data):
                 demangled, bup_type = demangle_name(name, mode)
                 if chunk_path:

diff --git a/lib/bup/t/tgit.py b/lib/bup/t/tgit.py
index c330f66453c7313e21705435b6d4346677d92d82..3d2e903f9337159136e5d7c633a03ab3ac5afe3e 100644 (file)
--- a/lib/bup/t/tgit.py
+++ b/lib/bup/t/tgit.py
@@ -431,14 +431,13 @@ def test_cat_pipe():
                 print f, 'something else'
             git.init_repo(bupdir)
             exc(bup_exe, 'index', src)
-            exc(bup_exe, 'save', '-n', 'src', '--strip', src)
-            git_type = exo('git', '--git-dir', bupdir,
-                           'cat-file', '-t', 'src').strip()
-            git_size = int(exo('git', '--git-dir', bupdir,
+            oidx = exo(bup_exe, 'save', '-cn', 'src', '--strip', src).strip()
+            typ = exo('git', '--git-dir', bupdir,
+                      'cat-file', '-t', 'src').strip()
+            size = int(exo('git', '--git-dir', bupdir,
                                'cat-file', '-s', 'src'))
-            it = git.cp().get('src', size=True)
-            get_type, get_size = next(it)
-            for buf in next(it):
+            it = git.cp().get('src')
+            get_info = it.next()
+            for buf in it.next():
                 pass
-            WVPASSEQ(get_type, git_type)
-            WVPASSEQ(get_size, git_size)
+            WVPASSEQ((oidx, typ, size), get_info)

diff --git a/lib/bup/vfs.py b/lib/bup/vfs.py
index 6110a094b43d7e9cf1e63bbf8fb7996ac6de88c1..b9e90f3415c18569a2c72fc2e4f3121066652063 100644 (file)
--- a/lib/bup/vfs.py
+++ b/lib/bup/vfs.py
@@ -37,7 +37,7 @@ class TooManySymlinks(NodeError):
 
 def _treeget(hash, repo_dir=None):
     it = cp(repo_dir).get(hash.encode('hex'))
-    type = next(it)
+    _, type, _ = next(it)
     assert(type == 'tree')
     return git.tree_decode(''.join(it))
 
@@ -424,11 +424,11 @@ class Dir(Node):
     def _mksubs(self):
         self._subs = {}
         it = cp(self._repo_dir).get(self.hash.encode('hex'))
-        type = next(it)
+        _, type, _ = next(it)
         if type == 'commit':
             del it
             it = cp(self._repo_dir).get(self.hash.encode('hex') + ':')
-            type = next(it)
+            _, type, _ = next(it)
         assert(type == 'tree')
         for (mode,mangled_name,sha) in git.tree_decode(''.join(it)):
             if mangled_name == '.bupm':