X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=cmd%2Fserver-cmd.py;h=acd70077d199699635b90da66ef52319a392c197;hb=3eb23e4243379195f0e5cfff932ad3ec949b4f98;hp=42ec1d4ef4a59242928ce9a5d55f5b249b288369;hpb=ead021518687ff81296756366b31f4edd9610d7a;p=bup.git diff --git a/cmd/server-cmd.py b/cmd/server-cmd.py index 42ec1d4..acd7007 100755 --- a/cmd/server-cmd.py +++ b/cmd/server-cmd.py @@ -1,48 +1,84 @@ -#!/usr/bin/env python -import os, sys, struct -from bup import options, git -from bup.helpers import * +#!/bin/sh +"""": # -*-python-*- +bup_python="$(dirname "$0")/bup-python" || exit $? +exec "$bup_python" "$0" ${1+"$@"} +""" +# end of bup preamble + +from __future__ import absolute_import +from binascii import hexlify, unhexlify +import os, sys, struct, subprocess + +from bup import options, git, vfs, vint +from bup.compat import environ, hexstr +from bup.git import MissingObject +from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel, + log) +from bup.io import byte_stream, path_msg +from bup.repo import LocalRepo + suspended_w = None dumb_server_mode = False +repo = None + +def do_help(conn, junk): + conn.write(b'Commands:\n %s\n' % b'\n '.join(sorted(commands))) + conn.ok() + def _set_mode(): global dumb_server_mode - dumb_server_mode = os.path.exists(git.repo('bup-dumb-server')) + dumb_server_mode = os.path.exists(git.repo(b'bup-dumb-server')) debug1('bup server: serving in %s mode\n' % (dumb_server_mode and 'dumb' or 'smart')) +def _init_session(reinit_with_new_repopath=None): + global repo + if reinit_with_new_repopath is None and git.repodir: + if not repo: + repo = LocalRepo() + return + git.check_repo_or_die(reinit_with_new_repopath) + if repo: + repo.close() + repo = LocalRepo() + # OK. we now know the path is a proper repository. Record this path in the + # environment so that subprocesses inherit it and know where to operate. + environ[b'BUP_DIR'] = git.repodir + debug1('bup server: bupdir is %s\n' % path_msg(git.repodir)) + _set_mode() + + def init_dir(conn, arg): git.init_repo(arg) - debug1('bup server: bupdir initialized: %r\n' % git.repodir) - _set_mode() + debug1('bup server: bupdir initialized: %s\n' % path_msg(git.repodir)) + _init_session(arg) conn.ok() def set_dir(conn, arg): - git.check_repo_or_die(arg) - debug1('bup server: bupdir is %r\n' % git.repodir) - _set_mode() + _init_session(arg) conn.ok() def list_indexes(conn, junk): - git.check_repo_or_die() - suffix = '' + _init_session() + suffix = b'' if dumb_server_mode: - suffix = ' load' - for f in os.listdir(git.repo('objects/pack')): - if f.endswith('.idx'): - conn.write('%s%s\n' % (f, suffix)) + suffix = b' load' + for f in os.listdir(git.repo(b'objects/pack')): + if f.endswith(b'.idx'): + conn.write(b'%s%s\n' % (f, suffix)) conn.ok() def send_index(conn, name): - git.check_repo_or_die() - assert(name.find('/') < 0) - assert(name.endswith('.idx')) - idx = git.open_idx(git.repo('objects/pack/%s' % name)) + _init_session() + assert name.find(b'/') < 0 + assert name.endswith(b'.idx') + idx = git.open_idx(git.repo(b'objects/pack/%s' % name)) conn.write(struct.pack('!I', len(idx.map))) conn.write(idx.map) conn.ok() @@ -50,13 +86,16 @@ def send_index(conn, name): def receive_objects_v2(conn, junk): global suspended_w - git.check_repo_or_die() - suggested = {} + _init_session() + suggested = set() if suspended_w: w = suspended_w suspended_w = None else: - w = git.PackWriter() + if dumb_server_mode: + w = git.PackWriter(objcache_maker=None) + else: + w = git.PackWriter() while 1: ns = conn.read(4) if not ns: @@ -70,7 +109,7 @@ def receive_objects_v2(conn, junk): fullpath = w.close(run_midx=not dumb_server_mode) if fullpath: (dir, name) = os.path.split(fullpath) - conn.write('%s.idx\n' % name) + conn.write(b'%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: @@ -84,43 +123,23 @@ def receive_objects_v2(conn, junk): n -= 20 + 4 buf = conn.read(n) # object sizes in bup are reasonably small #debug2('read %d bytes\n' % n) - if len(buf) < n: - w.abort() - raise Exception('object read: expected %d bytes, got %d\n' - % (n, len(buf))) - if dumb_server_mode: - oldpack = None - else: - oldpack = w.exists(shar) - # FIXME: we only suggest a single index per cycle, because the client - # is currently too dumb to download more than one per cycle anyway. - # Actually we should fix the client, but this is a minor optimization - # on the server side. - if not suggested and \ - oldpack and (oldpack == True or oldpack.endswith('.midx')): - # FIXME: we shouldn't really have to know about midx files - # at this layer. But exists() on a midx doesn't return the - # packname (since it doesn't know)... probably we should just - # fix that deficiency of midx files eventually, although it'll - # make the files bigger. This method is certainly not very - # efficient. - oldpack = w.objcache.packname_containing(shar) - debug2('new suggestion: %r\n' % oldpack) - assert(oldpack) - assert(oldpack != True) - assert(not oldpack.endswith('.midx')) - w.objcache.refresh() - if not suggested and oldpack: - assert(oldpack.endswith('.idx')) - (dir,name) = os.path.split(oldpack) - if not (name in suggested): - debug1("bup server: suggesting index %s\n" % name) - conn.write('index %s\n' % name) - suggested[name] = 1 - else: - nw, crc = w._raw_write([buf], sha=shar) - _check(w, crcr, crc, 'object read: expected crc %d, got %d\n') - _check(w, n, nw, 'object read: expected %d bytes, got %d\n') + _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n') + if not dumb_server_mode: + oldpack = w.exists(shar, want_source=True) + if oldpack: + assert(not oldpack == True) + assert(oldpack.endswith(b'.idx')) + (dir,name) = os.path.split(oldpack) + if not (name in suggested): + debug1("bup server: suggesting index %s\n" + % git.shorten_hash(name).decode('ascii')) + debug1("bup server: because of object %s\n" + % hexstr(shar)) + conn.write(b'index %s\n' % name) + suggested.add(name) + continue + nw, crc = w._raw_write((buf,), sha=shar) + _check(w, crcr, crc, 'object read: expected crc %d, got %d\n') # NOTREACHED @@ -131,43 +150,121 @@ def _check(w, expected, actual, msg): def read_ref(conn, refname): - git.check_repo_or_die() + _init_session() r = git.read_ref(refname) - conn.write('%s\n' % (r or '').encode('hex')) + conn.write(b'%s\n' % hexlify(r) if r else b'') conn.ok() def update_ref(conn, refname): - git.check_repo_or_die() + _init_session() newval = conn.readline().strip() oldval = conn.readline().strip() - git.update_ref(refname, newval.decode('hex'), oldval.decode('hex')) + git.update_ref(refname, unhexlify(newval), unhexlify(oldval)) conn.ok() - -cat_pipe = None -def cat(conn, id): - global cat_pipe - git.check_repo_or_die() - if not cat_pipe: - cat_pipe = git.CatPipe() +def join(conn, id): + _init_session() try: - for blob in cat_pipe.join(id): + for blob in git.cp().join(id): conn.write(struct.pack('!I', len(blob))) conn.write(blob) - except KeyError, e: + except KeyError as e: log('server: error: %s\n' % e) - conn.write('\0\0\0\0') + conn.write(b'\0\0\0\0') conn.error(e) else: - conn.write('\0\0\0\0') + conn.write(b'\0\0\0\0') conn.ok() +def cat_batch(conn, dummy): + _init_session() + cat_pipe = git.cp() + # For now, avoid potential deadlock by just reading them all + for ref in tuple(lines_until_sentinel(conn, b'\n', Exception)): + ref = ref[:-1] + it = cat_pipe.get(ref) + info = next(it) + if not info[0]: + conn.write(b'missing\n') + continue + conn.write(b'%s %s %d\n' % info) + for buf in it: + conn.write(buf) + conn.ok() + +def refs(conn, args): + limit_to_heads, limit_to_tags = args.split() + assert limit_to_heads in (b'0', b'1') + assert limit_to_tags in (b'0', b'1') + limit_to_heads = int(limit_to_heads) + limit_to_tags = int(limit_to_tags) + _init_session() + patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception)) + for name, oid in git.list_refs(patterns=patterns, + limit_to_heads=limit_to_heads, + limit_to_tags=limit_to_tags): + assert b'\n' not in name + conn.write(b'%s %s\n' % (hexlify(oid), name)) + conn.write(b'\n') + conn.ok() + +def rev_list(conn, _): + _init_session() + count = conn.readline() + if not count: + raise Exception('Unexpected EOF while reading rev-list count') + count = None if count == b'\n' else int(count) + fmt = conn.readline() + if not fmt: + raise Exception('Unexpected EOF while reading rev-list format') + fmt = None if fmt == b'\n' else fmt[:-1] + refs = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception)) + args = git.rev_list_invocation(refs, count=count, format=fmt) + p = subprocess.Popen(git.rev_list_invocation(refs, count=count, format=fmt), + env=git._gitenv(git.repodir), + stdout=subprocess.PIPE) + while True: + out = p.stdout.read(64 * 1024) + if not out: + break + conn.write(out) + conn.write(b'\n') + rv = p.wait() # not fatal + if rv: + msg = 'git rev-list returned error %d' % rv + conn.error(msg) + raise GitError(msg) + conn.ok() + +def resolve(conn, args): + _init_session() + (flags,) = args.split() + flags = int(flags) + want_meta = bool(flags & 1) + follow = bool(flags & 2) + have_parent = bool(flags & 4) + parent = vfs.read_resolution(conn) if have_parent else None + path = vint.read_bvec(conn) + if not len(path): + raise Exception('Empty resolve path') + try: + res = list(vfs.resolve(repo, path, parent=parent, want_meta=want_meta, + follow=follow)) + except vfs.IOError as ex: + res = ex + if isinstance(res, vfs.IOError): + conn.write(b'\x00') # error + vfs.write_ioerror(conn, res) + else: + conn.write(b'\x01') # success + vfs.write_resolution(conn, res) + conn.ok() optspec = """ bup server """ -o = options.Options('bup server', optspec) +o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) if extra: @@ -176,29 +273,37 @@ if extra: debug2('bup server: reading from stdin.\n') commands = { - 'init-dir': init_dir, - 'set-dir': set_dir, - 'list-indexes': list_indexes, - 'send-index': send_index, - 'receive-objects-v2': receive_objects_v2, - 'read-ref': read_ref, - 'update-ref': update_ref, - 'cat': cat, + b'quit': None, + b'help': do_help, + b'init-dir': init_dir, + b'set-dir': set_dir, + b'list-indexes': list_indexes, + b'send-index': send_index, + b'receive-objects-v2': receive_objects_v2, + b'read-ref': read_ref, + b'update-ref': update_ref, + b'join': join, + b'cat': join, # apocryphal alias + b'cat-batch' : cat_batch, + b'refs': refs, + b'rev-list': rev_list, + b'resolve': resolve } # FIXME: this protocol is totally lame and not at all future-proof. # (Especially since we abort completely as soon as *anything* bad happens) -conn = Conn(sys.stdin, sys.stdout) +sys.stdout.flush() +conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout)) lr = linereader(conn) for _line in lr: line = _line.strip() if not line: continue debug1('bup server: command: %r\n' % line) - words = line.split(' ', 1) + words = line.split(b' ', 1) cmd = words[0] - rest = len(words)>1 and words[1] or '' - if cmd == 'quit': + rest = len(words)>1 and words[1] or b'' + if cmd == b'quit': break else: cmd = commands.get(cmd)