python_tests := \
lib/bup/t/tbloom.py \
+ lib/bup/t/tclient.py \
lib/bup/t/tgit.py \
lib/bup/t/thashsplit.py \
lib/bup/t/thelpers.py \
ifeq "2" "$(bup_python_majver)"
python_tests += \
- lib/bup/t/tclient.py \
lib/bup/t/tresolve.py
endif
# end of bup preamble
from __future__ import absolute_import
+from binascii import hexlify, unhexlify
import os, sys, struct, subprocess
from bup import options, git, vfs, vint
-from bup.compat import hexstr
+from bup.compat import environ, hexstr
from bup.git import MissingObject
from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel,
log)
+from bup.io import byte_stream, path_msg
from bup.repo import LocalRepo
repo = None
def do_help(conn, junk):
- conn.write('Commands:\n %s\n' % '\n '.join(sorted(commands)))
+ conn.write(b'Commands:\n %s\n' % b'\n '.join(sorted(commands)))
conn.ok()
def _set_mode():
global dumb_server_mode
- dumb_server_mode = os.path.exists(git.repo('bup-dumb-server'))
+ dumb_server_mode = os.path.exists(git.repo(b'bup-dumb-server'))
debug1('bup server: serving in %s mode\n'
% (dumb_server_mode and 'dumb' or 'smart'))
repo = LocalRepo()
# OK. we now know the path is a proper repository. Record this path in the
# environment so that subprocesses inherit it and know where to operate.
- os.environ['BUP_DIR'] = git.repodir
- debug1('bup server: bupdir is %r\n' % git.repodir)
+ environ[b'BUP_DIR'] = git.repodir
+ debug1('bup server: bupdir is %s\n' % path_msg(git.repodir))
_set_mode()
def init_dir(conn, arg):
git.init_repo(arg)
- debug1('bup server: bupdir initialized: %r\n' % git.repodir)
+ debug1('bup server: bupdir initialized: %s\n' % path_msg(git.repodir))
_init_session(arg)
conn.ok()
def list_indexes(conn, junk):
_init_session()
- suffix = ''
+ suffix = b''
if dumb_server_mode:
- suffix = ' load'
- for f in os.listdir(git.repo('objects/pack')):
- if f.endswith('.idx'):
- conn.write('%s%s\n' % (f, suffix))
+ suffix = b' load'
+ for f in os.listdir(git.repo(b'objects/pack')):
+ if f.endswith(b'.idx'):
+ conn.write(b'%s%s\n' % (f, suffix))
conn.ok()
def send_index(conn, name):
_init_session()
- assert(name.find('/') < 0)
- assert(name.endswith('.idx'))
- idx = git.open_idx(git.repo('objects/pack/%s' % name))
+ assert name.find(b'/') < 0
+ assert name.endswith(b'.idx')
+ idx = git.open_idx(git.repo(b'objects/pack/%s' % name))
conn.write(struct.pack('!I', len(idx.map)))
conn.write(idx.map)
conn.ok()
fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
- conn.write('%s.idx\n' % name)
+ conn.write(b'%s.idx\n' % name)
conn.ok()
return
elif n == 0xffffffff:
oldpack = w.exists(shar, want_source=True)
if oldpack:
assert(not oldpack == True)
- assert(oldpack.endswith('.idx'))
+ assert(oldpack.endswith(b'.idx'))
(dir,name) = os.path.split(oldpack)
if not (name in suggested):
debug1("bup server: suggesting index %s\n"
- % git.shorten_hash(name))
+ % git.shorten_hash(name).decode('ascii'))
debug1("bup server: because of object %s\n"
% hexstr(shar))
- conn.write('index %s\n' % name)
+ conn.write(b'index %s\n' % name)
suggested.add(name)
continue
nw, crc = w._raw_write((buf,), sha=shar)
def read_ref(conn, refname):
_init_session()
r = git.read_ref(refname)
- conn.write('%s\n' % (r or '').encode('hex'))
+ conn.write(b'%s\n' % hexlify(r) if r else b'')
conn.ok()
_init_session()
newval = conn.readline().strip()
oldval = conn.readline().strip()
- git.update_ref(refname, newval.decode('hex'), oldval.decode('hex'))
+ git.update_ref(refname, unhexlify(newval), unhexlify(oldval))
conn.ok()
def join(conn, id):
conn.write(blob)
except KeyError as e:
log('server: error: %s\n' % e)
- conn.write('\0\0\0\0')
+ conn.write(b'\0\0\0\0')
conn.error(e)
else:
- conn.write('\0\0\0\0')
+ conn.write(b'\0\0\0\0')
conn.ok()
def cat_batch(conn, dummy):
_init_session()
cat_pipe = git.cp()
# For now, avoid potential deadlock by just reading them all
- for ref in tuple(lines_until_sentinel(conn, '\n', Exception)):
+ for ref in tuple(lines_until_sentinel(conn, b'\n', Exception)):
ref = ref[:-1]
it = cat_pipe.get(ref)
info = next(it)
if not info[0]:
- conn.write('missing\n')
+ conn.write(b'missing\n')
continue
- conn.write('%s %s %d\n' % info)
+ conn.write(b'%s %s %d\n' % info)
for buf in it:
conn.write(buf)
conn.ok()
def refs(conn, args):
limit_to_heads, limit_to_tags = args.split()
- assert limit_to_heads in ('0', '1')
- assert limit_to_tags in ('0', '1')
+ assert limit_to_heads in (b'0', b'1')
+ assert limit_to_tags in (b'0', b'1')
limit_to_heads = int(limit_to_heads)
limit_to_tags = int(limit_to_tags)
_init_session()
- patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception))
+ patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception))
for name, oid in git.list_refs(patterns=patterns,
limit_to_heads=limit_to_heads,
limit_to_tags=limit_to_tags):
- assert '\n' not in name
- conn.write('%s %s\n' % (oid.encode('hex'), name))
- conn.write('\n')
+ assert b'\n' not in name
+ conn.write(b'%s %s\n' % (hexlify(oid), name))
+ conn.write(b'\n')
conn.ok()
def rev_list(conn, _):
count = conn.readline()
if not count:
raise Exception('Unexpected EOF while reading rev-list count')
- count = None if count == '\n' else int(count)
+ count = None if count == b'\n' else int(count)
fmt = conn.readline()
if not fmt:
raise Exception('Unexpected EOF while reading rev-list format')
- fmt = None if fmt == '\n' else fmt[:-1]
- refs = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception))
+ fmt = None if fmt == b'\n' else fmt[:-1]
+ refs = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception))
args = git.rev_list_invocation(refs, count=count, format=fmt)
p = subprocess.Popen(git.rev_list_invocation(refs, count=count, format=fmt),
env=git._gitenv(git.repodir),
if not out:
break
conn.write(out)
- conn.write('\n')
+ conn.write(b'\n')
rv = p.wait() # not fatal
if rv:
msg = 'git rev-list returned error %d' % rv
except vfs.IOError as ex:
res = ex
if isinstance(res, vfs.IOError):
- conn.write(b'\0') # error
+ conn.write(b'\x00') # error
vfs.write_ioerror(conn, res)
else:
- conn.write(b'\1') # success
+ conn.write(b'\x01') # success
vfs.write_resolution(conn, res)
conn.ok()
debug2('bup server: reading from stdin.\n')
commands = {
- 'quit': None,
- 'help': do_help,
- 'init-dir': init_dir,
- 'set-dir': set_dir,
- 'list-indexes': list_indexes,
- 'send-index': send_index,
- 'receive-objects-v2': receive_objects_v2,
- 'read-ref': read_ref,
- 'update-ref': update_ref,
- 'join': join,
- 'cat': join, # apocryphal alias
- 'cat-batch' : cat_batch,
- 'refs': refs,
- 'rev-list': rev_list,
- 'resolve': resolve
+ b'quit': None,
+ b'help': do_help,
+ b'init-dir': init_dir,
+ b'set-dir': set_dir,
+ b'list-indexes': list_indexes,
+ b'send-index': send_index,
+ b'receive-objects-v2': receive_objects_v2,
+ b'read-ref': read_ref,
+ b'update-ref': update_ref,
+ b'join': join,
+ b'cat': join, # apocryphal alias
+ b'cat-batch' : cat_batch,
+ b'refs': refs,
+ b'rev-list': rev_list,
+ b'resolve': resolve
}
# FIXME: this protocol is totally lame and not at all future-proof.
# (Especially since we abort completely as soon as *anything* bad happens)
-conn = Conn(sys.stdin, sys.stdout)
+sys.stdout.flush()
+conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout))
lr = linereader(conn)
for _line in lr:
line = _line.strip()
if not line:
continue
debug1('bup server: command: %r\n' % line)
- words = line.split(' ', 1)
+ words = line.split(b' ', 1)
cmd = words[0]
- rest = len(words)>1 and words[1] or ''
- if cmd == 'quit':
+ rest = len(words)>1 and words[1] or b''
+ if cmd == b'quit':
break
else:
cmd = commands.get(cmd)
# sentinel must end with \n and must contain only one \n
while True:
line = f.readline()
- if not (line and line.endswith('\n')):
+ if not (line and line.endswith(b'\n')):
raise ex_type('Hit EOF while reading line')
if line == sentinel:
return
def ok(self):
"""Indicate end of output from last sent command."""
- self.write('\nok\n')
+ self.write(b'\nok\n')
def error(self, s):
"""Indicate server error to the client."""
- s = re.sub(r'\s+', ' ', str(s))
- self.write('\nerror %s\n' % s)
+ s = re.sub(br'\s+', b' ', s)
+ self.write(b'\nerror %s\n' % s)
def _check_ok(self, onempty):
self.outp.flush()
- rl = ''
+ rl = b''
for rl in linereader(self):
#log('%d got line: %r\n' % (os.getpid(), rl))
if not rl: # empty line
continue
- elif rl == 'ok':
+ elif rl == b'ok':
return None
- elif rl.startswith('error '):
+ elif rl.startswith(b'error '):
#log('client: error: %s\n' % rl[6:])
return NotOk(rl[6:])
else:
from __future__ import absolute_import, print_function
import sys, os, re, subprocess
-from bup import helpers, path
+from bup import helpers, path
+from bup.compat import environ
def connect(rhost, port, subcmd, stderr=None):
"""Connect to 'rhost' and execute the bup subcommand 'subcmd' on it."""
- assert(not re.search(r'[^\w-]', subcmd))
- nicedir = re.sub(r':', "_", path.exedir())
- if rhost == '-':
+ assert not re.search(br'[^\w-]', subcmd)
+ nicedir = re.sub(b':', b'_', path.exedir())
+ if rhost == b'-':
rhost = None
if not rhost:
- argv = ['bup', subcmd]
+ argv = [b'bup', subcmd]
else:
# WARNING: shell quoting security holes are possible here, so we
# have to be super careful. We have to use 'sh -c' because
# can't exec *safely* using argv, because *both* ssh and 'sh -c'
# allow shellquoting. So we end up having to double-shellquote
# stuff here.
- escapedir = re.sub(r'([^\w/])', r'\\\\\\\1', nicedir)
- buglvl = helpers.atoi(os.environ.get('BUP_DEBUG'))
- force_tty = helpers.atoi(os.environ.get('BUP_FORCE_TTY'))
- cmd = r"""
+ escapedir = re.sub(br'([^\w/])', br'\\\\\\\1', nicedir)
+ buglvl = helpers.atoi(environ.get(b'BUP_DEBUG'))
+ force_tty = helpers.atoi(environ.get(b'BUP_FORCE_TTY'))
+ cmd = b"""
sh -c PATH=%s:'$PATH BUP_DEBUG=%s BUP_FORCE_TTY=%s bup %s'
""" % (escapedir, buglvl, force_tty, subcmd)
- argv = ['ssh']
+ argv = [b'ssh']
if port:
- argv.extend(('-p', port))
- argv.extend((rhost, '--', cmd.strip()))
+ argv.extend((b'-p', port))
+ argv.extend((rhost, b'--', cmd.strip()))
#helpers.log('argv is: %r\n' % argv)
if rhost:
- env = os.environ
+ env = environ
else:
- envpath = os.environ.get('PATH')
- env = os.environ.copy()
- env['PATH'] = nicedir if not envpath else nicedir + ':' + envpath
+ envpath = environ.get(b'PATH')
+ env = environ.copy()
+ env[b'PATH'] = nicedir if not envpath else nicedir + b':' + envpath
if sys.version_info[0] < 3:
return subprocess.Popen(argv,
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
from wvtest import *
from bup import client, git, path
+from bup.compat import bytes_from_uint, environ, range
from bup.helpers import mkdirp
from buptest import no_lingering_errors, test_tempdir
def randbytes(sz):
- s = ''
- for i in xrange(sz):
- s += chr(random.randrange(0,256))
+ s = b''
+ for i in range(sz):
+ s += bytes_from_uint(random.randrange(0,256))
return s
s2 = randbytes(10000)
s3 = randbytes(10000)
-IDX_PAT = '/*.idx'
+IDX_PAT = b'/*.idx'
@wvtest
def test_server_split_with_indexes():
with no_lingering_errors():
- with test_tempdir('bup-tclient-') as tmpdir:
- os.environ['BUP_DIR'] = bupdir = tmpdir
+ with test_tempdir(b'bup-tclient-') as tmpdir:
+ environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
c = client.Client(bupdir, create=True)
@wvtest
def test_multiple_suggestions():
with no_lingering_errors():
- with test_tempdir('bup-tclient-') as tmpdir:
- os.environ['BUP_DIR'] = bupdir = tmpdir
+ with test_tempdir(b'bup-tclient-') as tmpdir:
+ environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
lw = git.PackWriter()
lw = git.PackWriter()
lw.new_blob(s2)
lw.close()
- WVPASSEQ(len(glob.glob(git.repo('objects/pack'+IDX_PAT))), 2)
+ WVPASSEQ(len(glob.glob(git.repo(b'objects/pack'+IDX_PAT))), 2)
c = client.Client(bupdir, create=True)
WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 0)
@wvtest
def test_dumb_client_server():
with no_lingering_errors():
- with test_tempdir('bup-tclient-') as tmpdir:
- os.environ['BUP_DIR'] = bupdir = tmpdir
+ with test_tempdir(b'bup-tclient-') as tmpdir:
+ environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
- open(git.repo('bup-dumb-server'), 'w').close()
+ open(git.repo(b'bup-dumb-server'), 'w').close()
lw = git.PackWriter()
lw.new_blob(s1)
@wvtest
def test_midx_refreshing():
with no_lingering_errors():
- with test_tempdir('bup-tclient-') as tmpdir:
- os.environ['BUP_DIR'] = bupdir = tmpdir
+ with test_tempdir(b'bup-tclient-') as tmpdir:
+ environ[b'BUP_DIR'] = bupdir = tmpdir
git.init_repo(bupdir)
c = client.Client(bupdir, create=True)
rw = c.new_packwriter()
p2name = os.path.join(c.cachedir, p2base)
del rw
- pi = git.PackIdxList(bupdir + '/objects/pack')
+ pi = git.PackIdxList(bupdir + b'/objects/pack')
WVPASSEQ(len(pi.packs), 2)
pi.refresh()
WVPASSEQ(len(pi.packs), 2)
WVFAIL(p2.exists(s1sha))
WVPASS(p2.exists(s2sha))
- subprocess.call([path.exe(), 'midx', '-f'])
+ subprocess.call([path.exe(), b'midx', b'-f'])
pi.refresh()
WVPASSEQ(len(pi.packs), 1)
pi.refresh(skip_midx=True)
def test_remote_parsing():
with no_lingering_errors():
tests = (
- (':/bup', ('file', None, None, '/bup')),
- ('file:///bup', ('file', None, None, '/bup')),
- ('192.168.1.1:/bup', ('ssh', '192.168.1.1', None, '/bup')),
- ('ssh://192.168.1.1:2222/bup', ('ssh', '192.168.1.1', '2222', '/bup')),
- ('ssh://[ff:fe::1]:2222/bup', ('ssh', 'ff:fe::1', '2222', '/bup')),
- ('bup://foo.com:1950', ('bup', 'foo.com', '1950', None)),
- ('bup://foo.com:1950/bup', ('bup', 'foo.com', '1950', '/bup')),
- ('bup://[ff:fe::1]/bup', ('bup', 'ff:fe::1', None, '/bup')),)
+ (b':/bup', (b'file', None, None, b'/bup')),
+ (b'file:///bup', (b'file', None, None, b'/bup')),
+ (b'192.168.1.1:/bup', (b'ssh', b'192.168.1.1', None, b'/bup')),
+ (b'ssh://192.168.1.1:2222/bup', (b'ssh', b'192.168.1.1', b'2222', b'/bup')),
+ (b'ssh://[ff:fe::1]:2222/bup', (b'ssh', b'ff:fe::1', b'2222', b'/bup')),
+ (b'bup://foo.com:1950', (b'bup', b'foo.com', b'1950', None)),
+ (b'bup://foo.com:1950/bup', (b'bup', b'foo.com', b'1950', b'/bup')),
+ (b'bup://[ff:fe::1]/bup', (b'bup', b'ff:fe::1', None, b'/bup')),)
for remote, values in tests:
WVPASSEQ(client.parse_remote(remote), values)
try:
- client.parse_remote('http://asdf.com/bup')
+ client.parse_remote(b'http://asdf.com/bup')
WVFAIL()
except client.ClientError:
WVPASS()
projects / bup.git / commitdiff