Still not updating refs, however, so it remains a bit stupid.
--- /dev/null
+import re, struct
+import git
+from helpers import *
+from subprocess import Popen, PIPE
+
+class ClientError(Exception):
+ pass
+
+class Client:
+ def __init__(self, remote, create=False):
+ self._busy = None
+ self._indexes_synced = 0
+ rs = remote.split(':', 1)
+ if len(rs) == 1:
+ (host, dir) = ('NONE', remote)
+ argv = ['bup', 'server']
+ else:
+ (host, dir) = rs
+ argv = ['ssh', host, '--', 'bup', 'server']
+ (self.host, self.dir) = (host, dir)
+ self.cachedir = git.repo('index-cache/%s'
+ % re.sub(r'[^@:\w]', '_',
+ "%s:%s" % (host, dir)))
+ self.p = p = Popen(argv, stdin=PIPE, stdout=PIPE)
+ self.conn = conn = Conn(p.stdout, p.stdin)
+ if dir:
+ dir = re.sub(r'[\r\n]', ' ', dir)
+ if create:
+ conn.write('init-dir %s\n' % dir)
+ else:
+ conn.write('set-dir %s\n' % dir)
+ conn.check_ok()
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ if self.conn and not self._busy:
+ self.conn.write('quit\n')
+ if self.p:
+ self.p.stdin.close()
+ while self.p.stdout.read(65536):
+ pass
+ self.p.stdout.close()
+ self.p.wait()
+ rv = self.p.wait()
+ if rv:
+ raise ClientError('server tunnel returned exit code %d' % rv)
+ self.conn = None
+ self.p = None
+
+ def check_busy(self):
+ if self._busy:
+ raise ClientError('already busy with command %r' % self._busy)
+
+ def _not_busy(self):
+ self._busy = None
+
+ def sync_indexes(self):
+ self.check_busy()
+ conn = self.conn
+ conn.write('list-indexes\n')
+ packdir = git.repo('objects/pack')
+ mkdirp(self.cachedir)
+ all = {}
+ needed = {}
+ for line in linereader(conn):
+ if not line:
+ break
+ all[line] = 1
+ assert(line.find('/') < 0)
+ if (not os.path.exists(os.path.join(self.cachedir, line)) and
+ not os.path.exists(os.path.join(packdir, line))):
+ needed[line] = 1
+ conn.check_ok()
+
+ for f in os.listdir(self.cachedir):
+ if f.endswith('.idx') and not f in all:
+ log('pruning old index: %r\n' % f)
+ os.unlink(os.path.join(self.cachedir, f))
+
+ # FIXME this should be pipelined: request multiple indexes at a time, or
+ # we waste lots of network turnarounds.
+ for name in needed.keys():
+ log('requesting %r\n' % name)
+ conn.write('send-index %s\n' % name)
+ n = struct.unpack('!I', conn.read(4))[0]
+ assert(n)
+ log(' expect %d bytes\n' % n)
+ fn = os.path.join(self.cachedir, name)
+ f = open(fn + '.tmp', 'w')
+ for b in chunkyreader(conn, n):
+ f.write(b)
+ conn.check_ok()
+ f.close()
+ os.rename(fn + '.tmp', fn)
+
+ self._indexes_synced = 1
+
+ def new_packwriter(self):
+ assert(self._indexes_synced)
+ self.check_busy()
+ self._busy = 'receive-objects'
+ self.conn.write('receive-objects\n')
+ objcache = git.MultiPackIndex(self.cachedir)
+ return git.PackWriter_Remote(self.conn, objcache = objcache,
+ onclose = self._not_busy)
#!/usr/bin/env python2.5
-import git, options
+import git, options, client
from helpers import *
optspec = """
-[BUP_DIR=...] bup init
+[BUP_DIR=...] bup init [-r host:path]
+--
+r,remote= remote repository path
"""
o = options.Options('bup init', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
log("bup init: no arguments expected\n")
o.usage()
-exit(git.init_repo())
+if opt.remote:
+ git.init_repo() # local repo
+ git.check_repo_or_die()
+ cli = client.Client(opt.remote, create=True)
+ cli.close()
+ exit(0) # if close() didn't throw an exception
+else:
+ exit(git.init_repo())
#!/usr/bin/env python2.5
-import sys, re, errno, stat
+import sys, re, errno, stat, client
import hashsplit, git, options
from helpers import *
optspec = """
bup save [-tc] [-n name] <filenames...>
--
+r,remote= remote repository path
t,tree output a tree id
c,commit output a commit id
n,name= name of backup set to update (if any)
git.verbose = opt.verbose - 1
hashsplit.split_verbosely = opt.verbose - 1
-w = git.PackWriter()
+if opt.remote:
+ cli = client.Client(opt.remote)
+ cli.sync_indexes()
+ w = cli.new_packwriter()
+else:
+ cli = None
+ w = git.PackWriter()
+
root = Tree(None, '')
for (fn,st) in direxpand(extra):
if opt.verbose:
print commit.encode('hex')
w.close()
+if cli:
+ cli.close()
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
from helpers import *
-def list_indexes(conn):
+def init_dir(conn, arg):
+ git.init_repo(arg)
+ log('bup server: bupdir initialized: %r\n' % git.repodir)
+ conn.ok()
+
+
+def set_dir(conn, arg):
+ git.check_repo_or_die(arg)
+ log('bup server: bupdir is %r\n' % git.repodir)
+ conn.ok()
+
+
+def list_indexes(conn, junk):
+ git.check_repo_or_die()
for f in os.listdir(git.repo('objects/pack')):
if f.endswith('.idx'):
conn.write('%s\n' % f)
def send_index(conn, name):
+ git.check_repo_or_die()
assert(name.find('/') < 0)
assert(name.endswith('.idx'))
idx = git.PackIndex(git.repo('objects/pack/%s' % name))
conn.ok()
-def receive_objects(conn):
+def receive_objects(conn, junk):
+ git.check_repo_or_die()
w = git.PackWriter()
while 1:
ns = conn.read(4)
% (n, len(buf)))
w._raw_write(buf)
w.close()
+ conn.ok()
optspec = """
log('bup server: reading from stdin.\n')
-# FIXME: this protocol is totally lame and not at all future-proof
+commands = {
+ 'init-dir': init_dir,
+ 'set-dir': set_dir,
+ 'list-indexes': list_indexes,
+ 'send-index': send_index,
+ 'receive-objects': receive_objects,
+}
+
+# FIXME: this protocol is totally lame and not at all future-proof.
+# (Especially since we abort completely as soon as *anything* bad happens)
conn = Conn(sys.stdin, sys.stdout)
lr = linereader(conn)
for _line in lr:
if not line:
continue
log('bup server: command: %r\n' % line)
- if line == 'quit':
+ words = line.split(' ', 1)
+ cmd = words[0]
+ rest = len(words)>1 and words[1] or ''
+ if cmd == 'quit':
break
- elif line.startswith('set-dir '):
- git.repodir = line[8:]
- git.check_repo_or_die()
- log('bup server: bupdir is %r\n' % git.repodir)
- conn.ok()
- elif line == 'list-indexes':
- list_indexes(conn)
- elif line.startswith('send-index '):
- send_index(conn, line[11:])
- elif line == 'receive-objects':
- git.check_repo_or_die()
- receive_objects(conn)
- conn.ok()
else:
- raise Exception('unknown server command: %r\n' % line)
+ cmd = commands.get(cmd)
+ if cmd:
+ cmd(conn, rest)
+ else:
+ raise Exception('unknown server command: %r\n' % line)
log('bup server: done\n')
#!/usr/bin/env python2.5
-import sys, time, re, struct
-import hashsplit, git, options
+import sys, time, struct
+import hashsplit, git, options, client
from helpers import *
+from subprocess import PIPE
+
optspec = """
bup split [-tcb] [-n name] [--bench] [filenames...]
start_time = time.time()
-def server_connect(remote):
- rs = remote.split(':', 1)
- if len(rs) == 1:
- (host, dir) = ('NONE', remote)
- p = subprocess.Popen(['bup', 'server'],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- else:
- (host, dir) = rs
- p = subprocess.Popen(['ssh', host, '--', 'bup', 'server'],
- stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- conn = Conn(p.stdout, p.stdin)
- dir = re.sub(r'[\r\n]', ' ', dir)
- conn.write('set-dir %s\n' % dir)
- conn.check_ok()
-
- conn.write('list-indexes\n')
- cachedir = git.repo('index-cache/%s' % re.sub(r'[^@:\w]', '_',
- "%s:%s" % (host, dir)))
- packdir = git.repo('objects/pack')
- mkdirp(cachedir)
- all = {}
- needed = {}
- for line in linereader(conn):
- if not line:
- break
- all[line] = 1
- assert(line.find('/') < 0)
- if (not os.path.exists(os.path.join(cachedir, line)) and
- not os.path.exists(os.path.join(packdir, line))):
- needed[line] = 1
- conn.check_ok()
-
- for f in os.listdir(cachedir):
- if f.endswith('.idx') and not f in all:
- log('pruning old index: %r\n' % f)
- os.unlink(os.path.join(cachedir, f))
-
- # FIXME this should be pipelined: request multiple indexes at a time, or
- # we waste lots of network turnarounds.
- for name in needed.keys():
- log('requesting %r\n' % name)
- conn.write('send-index %s\n' % name)
- n = struct.unpack('!I', conn.read(4))[0]
- assert(n)
- log(' expect %d bytes\n' % n)
- fn = os.path.join(cachedir, name)
- f = open(fn + '.tmp', 'w')
- for b in chunkyreader(conn, n):
- f.write(b)
- conn.check_ok()
- f.close()
- os.rename(fn + '.tmp', fn)
- return (p, conn, cachedir)
-
if opt.remote:
- (p, conn, cachedir) = server_connect(opt.remote)
- conn.write('receive-objects\n')
- w = git.PackWriter_Remote(conn, objcache = git.MultiPackIndex(cachedir))
+ cli = client.Client(opt.remote)
+ cli.sync_indexes()
+ w = cli.new_packwriter()
else:
+ cli = None
w = git.PackWriter()
(shalist,tree) = hashsplit.split_to_tree(w, hashsplit.autofiles(extra))
if opt.commit:
print commit.encode('hex')
-if opt.remote:
- w.close()
- p.stdin.write('quit\n')
- p.wait()
+w.close()
+if cli:
+ cli.close()
secs = time.time() - start_time
size = hashsplit.total_split
from helpers import *
verbose = 0
-repodir = os.environ.get('BUP_DIR', '.git')
+home_repodir = os.path.expanduser('~/.bup')
+repodir = None
+
+
+class GitError(Exception):
+ pass
+
def repo(sub = ''):
global repodir
+ if not repodir:
+ raise GitError('You should call check_repo_or_die()')
gd = os.path.join(repodir, '.git')
if os.path.exists(gd):
repodir = gd
stdout = subprocess.PIPE)
out = p.stdout.read().strip()
if p.wait() or not out:
- raise Exception('git index-pack returned an error')
+ raise GitError('git index-pack returned an error')
nameprefix = repo('objects/pack/%s' % out)
os.rename(self.filename + '.pack', nameprefix + '.pack')
os.rename(self.filename + '.idx', nameprefix + '.idx')
class PackWriter_Remote(PackWriter):
- def __init__(self, conn, objcache=None):
+ def __init__(self, conn, objcache=None, onclose=None):
PackWriter.__init__(self, objcache)
self.file = conn
self.filename = 'remote socket'
+ self.onclose = onclose
def _open(self):
assert(not "can't reopen a PackWriter_Remote")
def close(self):
if self.file:
self.file.write('\0\0\0\0')
+ if self.onclose:
+ self.onclose()
self.file = None
+ def abort(self):
+ raise GitError("don't know how to abort remote pack writing")
+
def _raw_write(self, datalist):
assert(self.file)
data = ''.join(datalist)
return newval
-def init_repo():
+def guess_repo(path=None):
+ global repodir
+ if path:
+ repodir = path
+ if not repodir:
+ repodir = os.environ.get('BUP_DIR')
+ if not repodir:
+ repodir = os.path.expanduser('~/.bup')
+
+
+def init_repo(path=None):
+ guess_repo(path)
d = repo()
if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
- raise Exception('"%d" exists but is not a directory\n' % d)
- p = subprocess.Popen(['git', 'init', '--bare'],
+ raise GitError('"%d" exists but is not a directory\n' % d)
+ p = subprocess.Popen(['git', 'init', '--bare'], stdout=sys.stderr,
preexec_fn = _gitenv)
return p.wait()
-def check_repo_or_die():
+def check_repo_or_die(path=None):
+ guess_repo(path)
if not os.path.isdir(repo('objects/pack/.')):
- log('error: %r is not a bup/git repository\n' % repo())
- exit(15)
+ if repodir == home_repodir:
+ init_repo()
+ else:
+ log('error: %r is not a bup/git repository\n' % repo())
+ exit(15)
@wvtest
def testpacks():
+ git.init_repo('pybuptest.tmp')
git.verbose = 1
now = str(time.time()) # hopefully not in any packs yet
projects / bup.git / commitdiff