X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=cmd%2Fserver-cmd.py;h=a60bf2305eefaa1b9e5693a52affdb34628e4d91;hb=c40b3dd5fd74e72024fbaad3daf5a958aefa1c54;hp=3fea0a9346f8c3b992c51aec3ccb749456fa11ff;hpb=2116dbc8c010d07b518088197f26726f7c1ebab5;p=bup.git diff --git a/cmd/server-cmd.py b/cmd/server-cmd.py index 3fea0a9..a60bf23 100755 --- a/cmd/server-cmd.py +++ b/cmd/server-cmd.py @@ -1,128 +1,165 @@ -#!/usr/bin/env python -import sys, struct, mmap +#!/bin/sh +"""": # -*-python-*- +bup_python="$(dirname "$0")/bup-python" || exit $? +exec "$bup_python" "$0" ${1+"$@"} +""" +# end of bup preamble + +from __future__ import absolute_import +import os, sys, struct, subprocess + from bup import options, git -from bup.helpers import * +from bup.git import MissingObject +from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel, + log) + suspended_w = None +dumb_server_mode = False + + +def do_help(conn, junk): + conn.write('Commands:\n %s\n' % '\n '.join(sorted(commands))) + conn.ok() + + +def _set_mode(): + global dumb_server_mode + dumb_server_mode = os.path.exists(git.repo('bup-dumb-server')) + debug1('bup server: serving in %s mode\n' + % (dumb_server_mode and 'dumb' or 'smart')) + + +def _init_session(reinit_with_new_repopath=None): + if reinit_with_new_repopath is None and git.repodir: + return + git.check_repo_or_die(reinit_with_new_repopath) + # OK. we now know the path is a proper repository. Record this path in the + # environment so that subprocesses inherit it and know where to operate. + os.environ['BUP_DIR'] = git.repodir + debug1('bup server: bupdir is %r\n' % git.repodir) + _set_mode() def init_dir(conn, arg): git.init_repo(arg) - log('bup server: bupdir initialized: %r\n' % git.repodir) + debug1('bup server: bupdir initialized: %r\n' % git.repodir) + _init_session(arg) conn.ok() def set_dir(conn, arg): - git.check_repo_or_die(arg) - log('bup server: bupdir is %r\n' % git.repodir) + _init_session(arg) conn.ok() def list_indexes(conn, junk): - git.check_repo_or_die() + _init_session() + suffix = '' + if dumb_server_mode: + suffix = ' load' for f in os.listdir(git.repo('objects/pack')): if f.endswith('.idx'): - conn.write('%s\n' % f) + conn.write('%s%s\n' % (f, suffix)) conn.ok() def send_index(conn, name): - git.check_repo_or_die() + _init_session() assert(name.find('/') < 0) assert(name.endswith('.idx')) - idx = git.PackIdx(git.repo('objects/pack/%s' % name)) + idx = git.open_idx(git.repo('objects/pack/%s' % name)) conn.write(struct.pack('!I', len(idx.map))) conn.write(idx.map) conn.ok() -def receive_objects(conn, junk): +def receive_objects_v2(conn, junk): global suspended_w - git.check_repo_or_die() - suggested = {} + _init_session() + suggested = set() if suspended_w: w = suspended_w suspended_w = None else: - w = git.PackWriter() + if dumb_server_mode: + w = git.PackWriter(objcache_maker=None) + else: + w = git.PackWriter() while 1: ns = conn.read(4) if not ns: w.abort() raise Exception('object read: expected length header, got EOF\n') n = struct.unpack('!I', ns)[0] - #log('expecting %d bytes\n' % n) + #debug2('expecting %d bytes\n' % n) if not n: - log('bup server: received %d object%s.\n' + debug1('bup server: received %d object%s.\n' % (w.count, w.count!=1 and "s" or '')) - fullpath = w.close() - (dir, name) = os.path.split(fullpath) - conn.write('%s.idx\n' % name) + fullpath = w.close(run_midx=not dumb_server_mode) + if fullpath: + (dir, name) = os.path.split(fullpath) + conn.write('%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: - log('bup server: receive-objects suspended.\n') + debug2('bup server: receive-objects suspended.\n') suspended_w = w conn.ok() return + shar = conn.read(20) + crcr = struct.unpack('!I', conn.read(4))[0] + n -= 20 + 4 buf = conn.read(n) # object sizes in bup are reasonably small - #log('read %d bytes\n' % n) - if len(buf) < n: - w.abort() - raise Exception('object read: expected %d bytes, got %d\n' - % (n, len(buf))) - (type, content) = git._decode_packobj(buf) - sha = git.calc_hash(type, content) - oldpack = w.exists(sha) - if oldpack and (oldpack == True or oldpack.endswith('.midx')): - # FIXME: we shouldn't really have to know about midx files - # at this layer. But exists() on a midx doesn't return the - # packname (since it doesn't know)... probably we should just - # fix that deficiency of midx files eventually, although it'll - # make the files bigger. This method is certainly not very - # efficient. - w.objcache.refresh(skip_midx = True, forget_packs = True) - oldpack = w.objcache.exists(sha) - log('new suggestion: %r\n' % oldpack) - assert(oldpack) - assert(oldpack != True) - assert(not oldpack.endswith('.midx')) - w.objcache.refresh(skip_midx = False) - if oldpack: - assert(oldpack.endswith('.idx')) - (dir,name) = os.path.split(oldpack) - if not (name in suggested): - log("bup server: suggesting index %s\n" % name) - conn.write('index %s\n' % name) - suggested[name] = 1 - else: - w._raw_write([buf]) + #debug2('read %d bytes\n' % n) + _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n') + if not dumb_server_mode: + oldpack = w.exists(shar, want_source=True) + if oldpack: + assert(not oldpack == True) + assert(oldpack.endswith('.idx')) + (dir,name) = os.path.split(oldpack) + if not (name in suggested): + debug1("bup server: suggesting index %s\n" + % git.shorten_hash(name)) + debug1("bup server: because of object %s\n" + % shar.encode('hex')) + conn.write('index %s\n' % name) + suggested.add(name) + continue + nw, crc = w._raw_write((buf,), sha=shar) + _check(w, crcr, crc, 'object read: expected crc %d, got %d\n') # NOTREACHED + + +def _check(w, expected, actual, msg): + if expected != actual: + w.abort() + raise Exception(msg % (expected, actual)) def read_ref(conn, refname): - git.check_repo_or_die() + _init_session() r = git.read_ref(refname) conn.write('%s\n' % (r or '').encode('hex')) conn.ok() def update_ref(conn, refname): - git.check_repo_or_die() + _init_session() newval = conn.readline().strip() oldval = conn.readline().strip() git.update_ref(refname, newval.decode('hex'), oldval.decode('hex')) conn.ok() - -def cat(conn, id): - git.check_repo_or_die() +def join(conn, id): + _init_session() try: - for blob in git.cat(id): + for blob in git.cp().join(id): conn.write(struct.pack('!I', len(blob))) conn.write(blob) - except KeyError, e: + except KeyError as e: log('server: error: %s\n' % e) conn.write('\0\0\0\0') conn.error(e) @@ -130,27 +167,92 @@ def cat(conn, id): conn.write('\0\0\0\0') conn.ok() +def cat_batch(conn, dummy): + _init_session() + cat_pipe = git.cp() + # For now, avoid potential deadlock by just reading them all + for ref in tuple(lines_until_sentinel(conn, '\n', Exception)): + ref = ref[:-1] + it = cat_pipe.get(ref) + info = next(it) + if not info[0]: + conn.write('missing\n') + continue + conn.write('%s %s %d\n' % info) + for buf in it: + conn.write(buf) + conn.ok() + +def refs(conn, args): + limit_to_heads, limit_to_tags = args.split() + assert limit_to_heads in ('0', '1') + assert limit_to_tags in ('0', '1') + limit_to_heads = int(limit_to_heads) + limit_to_tags = int(limit_to_tags) + _init_session() + patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception)) + for name, oid in git.list_refs(patterns=patterns, + limit_to_heads=limit_to_heads, + limit_to_tags=limit_to_tags): + assert '\n' not in name + conn.write('%s %s\n' % (oid.encode('hex'), name)) + conn.write('\n') + conn.ok() + +def rev_list(conn, _): + _init_session() + count = conn.readline() + if not count: + raise Exception('Unexpected EOF while reading rev-list count') + count = None if count == '\n' else int(count) + fmt = conn.readline() + if not fmt: + raise Exception('Unexpected EOF while reading rev-list format') + fmt = None if fmt == '\n' else fmt[:-1] + refs = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception)) + args = git.rev_list_invocation(refs, count=count, format=fmt) + p = subprocess.Popen(git.rev_list_invocation(refs, count=count, format=fmt), + preexec_fn=git._gitenv(git.repodir), + stdout=subprocess.PIPE) + while True: + out = p.stdout.read(64 * 1024) + if not out: + break + conn.write(out) + rv = p.wait() # not fatal + if rv: + msg = 'git rev-list returned error %d' % rv + conn.error(msg) + raise GitError(msg) + conn.ok() + optspec = """ bup server """ -o = options.Options('bup server', optspec) +o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) if extra: o.fatal('no arguments expected') -log('bup server: reading from stdin.\n') +debug2('bup server: reading from stdin.\n') commands = { + 'quit': None, + 'help': do_help, 'init-dir': init_dir, 'set-dir': set_dir, 'list-indexes': list_indexes, 'send-index': send_index, - 'receive-objects': receive_objects, + 'receive-objects-v2': receive_objects_v2, 'read-ref': read_ref, 'update-ref': update_ref, - 'cat': cat, + 'join': join, + 'cat': join, # apocryphal alias + 'cat-batch' : cat_batch, + 'refs': refs, + 'rev-list': rev_list } # FIXME: this protocol is totally lame and not at all future-proof. @@ -161,7 +263,7 @@ for _line in lr: line = _line.strip() if not line: continue - log('bup server: command: %r\n' % line) + debug1('bup server: command: %r\n' % line) words = line.split(' ', 1) cmd = words[0] rest = len(words)>1 and words[1] or '' @@ -174,4 +276,4 @@ for _line in lr: else: raise Exception('unknown server command: %r\n' % line) -log('bup server: done\n') +debug1('bup server: done\n')