Rewrite bup-join in python and add remote server support.

diff --git a/client.py b/client.py
index 3380956aa7d6b5d02baaf76f53c60adc67b95c8f..f01d7e71b9f2617e93c59405a54c108dfa0464da 100644 (file)
--- a/client.py
+++ b/client.py
@@ -69,9 +69,8 @@ class Client:
                 break
             all[line] = 1
             assert(line.find('/') < 0)
-            if (not os.path.exists(os.path.join(self.cachedir, line)) and
-                not os.path.exists(os.path.join(packdir, line))):
-                    needed[line] = 1
+            if not os.path.exists(os.path.join(self.cachedir, line)):
+                needed[line] = 1
         conn.check_ok()
 
         for f in os.listdir(self.cachedir):
@@ -123,3 +122,14 @@ class Client:
                         % (refname, newval.encode('hex'),
                            (oldval or '').encode('hex')))
         self.conn.check_ok()
+
+    def cat(self, id):
+        self.check_busy()
+        self._busy = 'cat'
+        self.conn.write('cat %s\n' % re.sub(r'[\n\r]', '_', id))
+        while 1:
+            sz = struct.unpack('!I', self.conn.read(4))[0]
+            if not sz: break
+            yield self.conn.read(sz)
+        self.conn.check_ok()
+        self._not_busy()

diff --git a/cmd-join.py b/cmd-join.py
new file mode 100755 (executable)
index 0000000..b87319a
--- /dev/null
+++ b/cmd-join.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python2.5
+import sys, time, struct
+import hashsplit, git, options, client
+from helpers import *
+from subprocess import PIPE
+
+
+optspec = """
+bup join [-r host:path] [refs or hashes...]
+--
+r,remote=  remote repository path
+"""
+o = options.Options('bup join', optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+git.check_repo_or_die()
+
+if not extra:
+    extra = linereader(sys.stdin)
+
+if opt.remote:
+    cli = client.Client(opt.remote)
+    for id in extra:
+        for blob in cli.cat(id):
+            sys.stdout.write(blob)
+    cli.close()
+else:
+    for id in extra:
+        for blob in git.cat(id):
+            sys.stdout.write(blob)

diff --git a/cmd-join.sh b/cmd-join.sh
deleted file mode 100755 (executable)
index 4fe1b49..0000000
--- a/cmd-join.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/sh
-set -e
-
-if [ -z "$BUP_DIR" ]; then
-    BUP_DIR="$HOME/.bup"
-fi
-
-export GIT_DIR="$BUP_DIR"
-
-get_one()
-{
-    local typ="$1"
-    local sha="$2"
-    if [ "$typ" = "tree" -o "$typ" = "commit" ]; then
-        git cat-file -p "$x:" | while read nmode ntyp nsha njunk; do
-           get_one $ntyp $nsha
-       done
-    else
-        git cat-file blob "$sha"
-    fi
-}
-
-
-get_from_stdin()
-{
-    while read x junk; do
-        [ -z "$x" ] && continue
-        typ="$(git cat-file -t "$x")"
-        get_one "$typ" "$x"
-    done
-}
-
-
-if [ -z "$*" ]; then
-    get_from_stdin
-else
-    for d in "$@"; do
-        echo "$d"
-    done | get_from_stdin
-fi

diff --git a/cmd-server.py b/cmd-server.py
index 6baea700bb7e90114da39ffc0fd64f8fd3fe70b9..2ad240a55825d2619094f5be5e580d42c1ea5c4c 100755 (executable)
--- a/cmd-server.py
+++ b/cmd-server.py
@@ -75,6 +75,15 @@ def update_ref(conn, refname):
     conn.ok()
 
 
+def cat(conn, id):
+    git.check_repo_or_die()
+    for blob in git.cat(id):
+        conn.write(struct.pack('!I', len(blob)))
+        conn.write(blob)
+    conn.write('\0\0\0\0')
+    conn.ok()
+
+
 optspec = """
 bup server
 """
@@ -95,6 +104,7 @@ commands = {
     'receive-objects': receive_objects,
     'read-ref': read_ref,
     'update-ref': update_ref,
+    'cat': cat,
 }
 
 # FIXME: this protocol is totally lame and not at all future-proof.

diff --git a/git.py b/git.py
index 3b758eaee6f4097602d095a068c3023f845cca66..dc641ffb70cab4a7f65ebac1d52d83d9205ccc71 100644 (file)
--- a/git.py
+++ b/git.py
@@ -332,3 +332,65 @@ def check_repo_or_die(path=None):
         else:
             log('error: %r is not a bup/git repository\n' % repo())
             exit(15)
+
+
+def _treeparse(buf):
+    ofs = 0
+    while ofs < len(buf):
+        z = buf[ofs:].find('\0')
+        assert(z > 0)
+        spl = buf[ofs:ofs+z].split(' ', 1)
+        assert(len(spl) == 2)
+        sha = buf[ofs+z+1:ofs+z+1+20]
+        ofs += z+1+20
+        yield (spl[0], spl[1], sha)
+
+
+class CatPipe:
+    def __init__(self):
+        self.p = subprocess.Popen(['git', 'cat-file', '--batch'],
+                                  stdin=subprocess.PIPE, 
+                                  stdout=subprocess.PIPE,
+                                  preexec_fn = _gitenv)
+
+    def get(self, id):
+        assert(id.find('\n') < 0)
+        assert(id.find('\r') < 0)
+        self.p.stdin.write('%s\n' % id)
+        hdr = self.p.stdout.readline()
+        spl = hdr.split(' ')
+        assert(len(spl) == 3)
+        assert(len(spl[0]) == 40)
+        (hex, type, size) = spl
+        yield type
+        for blob in chunkyreader(self.p.stdout, int(spl[2])):
+            yield blob
+        assert(self.p.stdout.readline() == '\n')
+
+    def _join(self, it):
+        type = it.next()
+        if type == 'blob':
+            for blob in it:
+                yield blob
+        elif type == 'tree':
+            treefile = ''.join(it)
+            for (mode, name, sha) in _treeparse(treefile):
+                for blob in self.join(sha.encode('hex')):
+                    yield blob
+        elif type == 'commit':
+            treeline = ''.join(it).split('\n')[0]
+            assert(treeline.startswith('tree '))
+            for blob in self.join(treeline[5:]):
+                yield blob
+        else:
+            raise GitError('unknown object type %r' % type)
+
+    def join(self, id):
+        for d in self._join(self.get(id)):
+            yield d
+        
+
+def cat(id):
+    c = CatPipe()
+    for d in c.join(id):
+        yield d

diff --git a/helpers.py b/helpers.py
index 0cb4332968ac8659bb81ffb67b531c603d403613..e7e70bda9f5dabe525640847e967f842e5396733 100644 (file)
--- a/helpers.py
+++ b/helpers.py
@@ -97,10 +97,16 @@ def linereader(f):
         yield line[:-1]
 
 
-def chunkyreader(f, count):
-    while count > 0:
-        b = f.read(min(count, 65536))
-        if not b:
-            raise IOError('EOF with %d bytes remaining' % count)
-        yield b
-        count -= len(b)
+def chunkyreader(f, count = None):
+    if count != None:
+        while count > 0:
+            b = f.read(min(count, 65536))
+            if not b:
+                raise IOError('EOF with %d bytes remaining' % count)
+            yield b
+            count -= len(b)
+    else:
+        while 1:
+            b = f.read(65536)
+            if not b: break
+            yield b