From 3eb23e4243379195f0e5cfff932ad3ec949b4f98 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Tue, 31 Dec 2019 15:37:02 -0600 Subject: [PATCH] Adjust server and client to accommodate python 3 Make the typical bytes-related adjustments to bup-server, bup.client, and bup.ssh and enable the tclient tests with python 3. Signed-off-by: Rob Browning Tested-by: Rob Browning --- Makefile | 2 +- cmd/server-cmd.py | 115 ++++++++++++++++++++++--------------------- lib/bup/helpers.py | 14 +++--- lib/bup/ssh.py | 33 +++++++------ lib/bup/t/tclient.py | 51 +++++++++---------- 5 files changed, 110 insertions(+), 105 deletions(-) diff --git a/Makefile b/Makefile index 24d75fa..1423e13 100644 --- a/Makefile +++ b/Makefile @@ -145,6 +145,7 @@ runtests: runtests-python runtests-cmdline python_tests := \ lib/bup/t/tbloom.py \ + lib/bup/t/tclient.py \ lib/bup/t/tgit.py \ lib/bup/t/thashsplit.py \ lib/bup/t/thelpers.py \ @@ -158,7 +159,6 @@ python_tests := \ ifeq "2" "$(bup_python_majver)" python_tests += \ - lib/bup/t/tclient.py \ lib/bup/t/tresolve.py endif diff --git a/cmd/server-cmd.py b/cmd/server-cmd.py index b1b7729..acd7007 100755 --- a/cmd/server-cmd.py +++ b/cmd/server-cmd.py @@ -6,13 +6,15 @@ exec "$bup_python" "$0" ${1+"$@"} # end of bup preamble from __future__ import absolute_import +from binascii import hexlify, unhexlify import os, sys, struct, subprocess from bup import options, git, vfs, vint -from bup.compat import hexstr +from bup.compat import environ, hexstr from bup.git import MissingObject from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel, log) +from bup.io import byte_stream, path_msg from bup.repo import LocalRepo @@ -21,13 +23,13 @@ dumb_server_mode = False repo = None def do_help(conn, junk): - conn.write('Commands:\n %s\n' % '\n '.join(sorted(commands))) + conn.write(b'Commands:\n %s\n' % b'\n '.join(sorted(commands))) conn.ok() def _set_mode(): global dumb_server_mode - dumb_server_mode = os.path.exists(git.repo('bup-dumb-server')) + dumb_server_mode = os.path.exists(git.repo(b'bup-dumb-server')) debug1('bup server: serving in %s mode\n' % (dumb_server_mode and 'dumb' or 'smart')) @@ -44,14 +46,14 @@ def _init_session(reinit_with_new_repopath=None): repo = LocalRepo() # OK. we now know the path is a proper repository. Record this path in the # environment so that subprocesses inherit it and know where to operate. - os.environ['BUP_DIR'] = git.repodir - debug1('bup server: bupdir is %r\n' % git.repodir) + environ[b'BUP_DIR'] = git.repodir + debug1('bup server: bupdir is %s\n' % path_msg(git.repodir)) _set_mode() def init_dir(conn, arg): git.init_repo(arg) - debug1('bup server: bupdir initialized: %r\n' % git.repodir) + debug1('bup server: bupdir initialized: %s\n' % path_msg(git.repodir)) _init_session(arg) conn.ok() @@ -63,20 +65,20 @@ def set_dir(conn, arg): def list_indexes(conn, junk): _init_session() - suffix = '' + suffix = b'' if dumb_server_mode: - suffix = ' load' - for f in os.listdir(git.repo('objects/pack')): - if f.endswith('.idx'): - conn.write('%s%s\n' % (f, suffix)) + suffix = b' load' + for f in os.listdir(git.repo(b'objects/pack')): + if f.endswith(b'.idx'): + conn.write(b'%s%s\n' % (f, suffix)) conn.ok() def send_index(conn, name): _init_session() - assert(name.find('/') < 0) - assert(name.endswith('.idx')) - idx = git.open_idx(git.repo('objects/pack/%s' % name)) + assert name.find(b'/') < 0 + assert name.endswith(b'.idx') + idx = git.open_idx(git.repo(b'objects/pack/%s' % name)) conn.write(struct.pack('!I', len(idx.map))) conn.write(idx.map) conn.ok() @@ -107,7 +109,7 @@ def receive_objects_v2(conn, junk): fullpath = w.close(run_midx=not dumb_server_mode) if fullpath: (dir, name) = os.path.split(fullpath) - conn.write('%s.idx\n' % name) + conn.write(b'%s.idx\n' % name) conn.ok() return elif n == 0xffffffff: @@ -126,14 +128,14 @@ def receive_objects_v2(conn, junk): oldpack = w.exists(shar, want_source=True) if oldpack: assert(not oldpack == True) - assert(oldpack.endswith('.idx')) + assert(oldpack.endswith(b'.idx')) (dir,name) = os.path.split(oldpack) if not (name in suggested): debug1("bup server: suggesting index %s\n" - % git.shorten_hash(name)) + % git.shorten_hash(name).decode('ascii')) debug1("bup server: because of object %s\n" % hexstr(shar)) - conn.write('index %s\n' % name) + conn.write(b'index %s\n' % name) suggested.add(name) continue nw, crc = w._raw_write((buf,), sha=shar) @@ -150,7 +152,7 @@ def _check(w, expected, actual, msg): def read_ref(conn, refname): _init_session() r = git.read_ref(refname) - conn.write('%s\n' % (r or '').encode('hex')) + conn.write(b'%s\n' % hexlify(r) if r else b'') conn.ok() @@ -158,7 +160,7 @@ def update_ref(conn, refname): _init_session() newval = conn.readline().strip() oldval = conn.readline().strip() - git.update_ref(refname, newval.decode('hex'), oldval.decode('hex')) + git.update_ref(refname, unhexlify(newval), unhexlify(oldval)) conn.ok() def join(conn, id): @@ -169,42 +171,42 @@ def join(conn, id): conn.write(blob) except KeyError as e: log('server: error: %s\n' % e) - conn.write('\0\0\0\0') + conn.write(b'\0\0\0\0') conn.error(e) else: - conn.write('\0\0\0\0') + conn.write(b'\0\0\0\0') conn.ok() def cat_batch(conn, dummy): _init_session() cat_pipe = git.cp() # For now, avoid potential deadlock by just reading them all - for ref in tuple(lines_until_sentinel(conn, '\n', Exception)): + for ref in tuple(lines_until_sentinel(conn, b'\n', Exception)): ref = ref[:-1] it = cat_pipe.get(ref) info = next(it) if not info[0]: - conn.write('missing\n') + conn.write(b'missing\n') continue - conn.write('%s %s %d\n' % info) + conn.write(b'%s %s %d\n' % info) for buf in it: conn.write(buf) conn.ok() def refs(conn, args): limit_to_heads, limit_to_tags = args.split() - assert limit_to_heads in ('0', '1') - assert limit_to_tags in ('0', '1') + assert limit_to_heads in (b'0', b'1') + assert limit_to_tags in (b'0', b'1') limit_to_heads = int(limit_to_heads) limit_to_tags = int(limit_to_tags) _init_session() - patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception)) + patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception)) for name, oid in git.list_refs(patterns=patterns, limit_to_heads=limit_to_heads, limit_to_tags=limit_to_tags): - assert '\n' not in name - conn.write('%s %s\n' % (oid.encode('hex'), name)) - conn.write('\n') + assert b'\n' not in name + conn.write(b'%s %s\n' % (hexlify(oid), name)) + conn.write(b'\n') conn.ok() def rev_list(conn, _): @@ -212,12 +214,12 @@ def rev_list(conn, _): count = conn.readline() if not count: raise Exception('Unexpected EOF while reading rev-list count') - count = None if count == '\n' else int(count) + count = None if count == b'\n' else int(count) fmt = conn.readline() if not fmt: raise Exception('Unexpected EOF while reading rev-list format') - fmt = None if fmt == '\n' else fmt[:-1] - refs = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception)) + fmt = None if fmt == b'\n' else fmt[:-1] + refs = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception)) args = git.rev_list_invocation(refs, count=count, format=fmt) p = subprocess.Popen(git.rev_list_invocation(refs, count=count, format=fmt), env=git._gitenv(git.repodir), @@ -227,7 +229,7 @@ def rev_list(conn, _): if not out: break conn.write(out) - conn.write('\n') + conn.write(b'\n') rv = p.wait() # not fatal if rv: msg = 'git rev-list returned error %d' % rv @@ -252,10 +254,10 @@ def resolve(conn, args): except vfs.IOError as ex: res = ex if isinstance(res, vfs.IOError): - conn.write(b'\0') # error + conn.write(b'\x00') # error vfs.write_ioerror(conn, res) else: - conn.write(b'\1') # success + conn.write(b'\x01') # success vfs.write_resolution(conn, res) conn.ok() @@ -271,36 +273,37 @@ if extra: debug2('bup server: reading from stdin.\n') commands = { - 'quit': None, - 'help': do_help, - 'init-dir': init_dir, - 'set-dir': set_dir, - 'list-indexes': list_indexes, - 'send-index': send_index, - 'receive-objects-v2': receive_objects_v2, - 'read-ref': read_ref, - 'update-ref': update_ref, - 'join': join, - 'cat': join, # apocryphal alias - 'cat-batch' : cat_batch, - 'refs': refs, - 'rev-list': rev_list, - 'resolve': resolve + b'quit': None, + b'help': do_help, + b'init-dir': init_dir, + b'set-dir': set_dir, + b'list-indexes': list_indexes, + b'send-index': send_index, + b'receive-objects-v2': receive_objects_v2, + b'read-ref': read_ref, + b'update-ref': update_ref, + b'join': join, + b'cat': join, # apocryphal alias + b'cat-batch' : cat_batch, + b'refs': refs, + b'rev-list': rev_list, + b'resolve': resolve } # FIXME: this protocol is totally lame and not at all future-proof. # (Especially since we abort completely as soon as *anything* bad happens) -conn = Conn(sys.stdin, sys.stdout) +sys.stdout.flush() +conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout)) lr = linereader(conn) for _line in lr: line = _line.strip() if not line: continue debug1('bup server: command: %r\n' % line) - words = line.split(' ', 1) + words = line.split(b' ', 1) cmd = words[0] - rest = len(words)>1 and words[1] or '' - if cmd == 'quit': + rest = len(words)>1 and words[1] or b'' + if cmd == b'quit': break else: cmd = commands.get(cmd) diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py index 35c88cd..0e90780 100644 --- a/lib/bup/helpers.py +++ b/lib/bup/helpers.py @@ -112,7 +112,7 @@ def lines_until_sentinel(f, sentinel, ex_type): # sentinel must end with \n and must contain only one \n while True: line = f.readline() - if not (line and line.endswith('\n')): + if not (line and line.endswith(b'\n')): raise ex_type('Hit EOF while reading line') if line == sentinel: return @@ -483,23 +483,23 @@ class BaseConn: def ok(self): """Indicate end of output from last sent command.""" - self.write('\nok\n') + self.write(b'\nok\n') def error(self, s): """Indicate server error to the client.""" - s = re.sub(r'\s+', ' ', str(s)) - self.write('\nerror %s\n' % s) + s = re.sub(br'\s+', b' ', s) + self.write(b'\nerror %s\n' % s) def _check_ok(self, onempty): self.outp.flush() - rl = '' + rl = b'' for rl in linereader(self): #log('%d got line: %r\n' % (os.getpid(), rl)) if not rl: # empty line continue - elif rl == 'ok': + elif rl == b'ok': return None - elif rl.startswith('error '): + elif rl.startswith(b'error '): #log('client: error: %s\n' % rl[6:]) return NotOk(rl[6:]) else: diff --git a/lib/bup/ssh.py b/lib/bup/ssh.py index 5602921..de0448d 100644 --- a/lib/bup/ssh.py +++ b/lib/bup/ssh.py @@ -4,17 +4,18 @@ Connect to a remote host via SSH and execute a command on the host. from __future__ import absolute_import, print_function import sys, os, re, subprocess -from bup import helpers, path +from bup import helpers, path +from bup.compat import environ def connect(rhost, port, subcmd, stderr=None): """Connect to 'rhost' and execute the bup subcommand 'subcmd' on it.""" - assert(not re.search(r'[^\w-]', subcmd)) - nicedir = re.sub(r':', "_", path.exedir()) - if rhost == '-': + assert not re.search(br'[^\w-]', subcmd) + nicedir = re.sub(b':', b'_', path.exedir()) + if rhost == b'-': rhost = None if not rhost: - argv = ['bup', subcmd] + argv = [b'bup', subcmd] else: # WARNING: shell quoting security holes are possible here, so we # have to be super careful. We have to use 'sh -c' because @@ -23,23 +24,23 @@ def connect(rhost, port, subcmd, stderr=None): # can't exec *safely* using argv, because *both* ssh and 'sh -c' # allow shellquoting. So we end up having to double-shellquote # stuff here. - escapedir = re.sub(r'([^\w/])', r'\\\\\\\1', nicedir) - buglvl = helpers.atoi(os.environ.get('BUP_DEBUG')) - force_tty = helpers.atoi(os.environ.get('BUP_FORCE_TTY')) - cmd = r""" + escapedir = re.sub(br'([^\w/])', br'\\\\\\\1', nicedir) + buglvl = helpers.atoi(environ.get(b'BUP_DEBUG')) + force_tty = helpers.atoi(environ.get(b'BUP_FORCE_TTY')) + cmd = b""" sh -c PATH=%s:'$PATH BUP_DEBUG=%s BUP_FORCE_TTY=%s bup %s' """ % (escapedir, buglvl, force_tty, subcmd) - argv = ['ssh'] + argv = [b'ssh'] if port: - argv.extend(('-p', port)) - argv.extend((rhost, '--', cmd.strip())) + argv.extend((b'-p', port)) + argv.extend((rhost, b'--', cmd.strip())) #helpers.log('argv is: %r\n' % argv) if rhost: - env = os.environ + env = environ else: - envpath = os.environ.get('PATH') - env = os.environ.copy() - env['PATH'] = nicedir if not envpath else nicedir + ':' + envpath + envpath = environ.get(b'PATH') + env = environ.copy() + env[b'PATH'] = nicedir if not envpath else nicedir + b':' + envpath if sys.version_info[0] < 3: return subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/lib/bup/t/tclient.py b/lib/bup/t/tclient.py index 284effc..afbb09f 100644 --- a/lib/bup/t/tclient.py +++ b/lib/bup/t/tclient.py @@ -5,14 +5,15 @@ import sys, os, stat, time, random, subprocess, glob from wvtest import * from bup import client, git, path +from bup.compat import bytes_from_uint, environ, range from bup.helpers import mkdirp from buptest import no_lingering_errors, test_tempdir def randbytes(sz): - s = '' - for i in xrange(sz): - s += chr(random.randrange(0,256)) + s = b'' + for i in range(sz): + s += bytes_from_uint(random.randrange(0,256)) return s @@ -20,14 +21,14 @@ s1 = randbytes(10000) s2 = randbytes(10000) s3 = randbytes(10000) -IDX_PAT = '/*.idx' +IDX_PAT = b'/*.idx' @wvtest def test_server_split_with_indexes(): with no_lingering_errors(): - with test_tempdir('bup-tclient-') as tmpdir: - os.environ['BUP_DIR'] = bupdir = tmpdir + with test_tempdir(b'bup-tclient-') as tmpdir: + environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) lw = git.PackWriter() c = client.Client(bupdir, create=True) @@ -45,8 +46,8 @@ def test_server_split_with_indexes(): @wvtest def test_multiple_suggestions(): with no_lingering_errors(): - with test_tempdir('bup-tclient-') as tmpdir: - os.environ['BUP_DIR'] = bupdir = tmpdir + with test_tempdir(b'bup-tclient-') as tmpdir: + environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) lw = git.PackWriter() @@ -55,7 +56,7 @@ def test_multiple_suggestions(): lw = git.PackWriter() lw.new_blob(s2) lw.close() - WVPASSEQ(len(glob.glob(git.repo('objects/pack'+IDX_PAT))), 2) + WVPASSEQ(len(glob.glob(git.repo(b'objects/pack'+IDX_PAT))), 2) c = client.Client(bupdir, create=True) WVPASSEQ(len(glob.glob(c.cachedir+IDX_PAT)), 0) @@ -80,10 +81,10 @@ def test_multiple_suggestions(): @wvtest def test_dumb_client_server(): with no_lingering_errors(): - with test_tempdir('bup-tclient-') as tmpdir: - os.environ['BUP_DIR'] = bupdir = tmpdir + with test_tempdir(b'bup-tclient-') as tmpdir: + environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) - open(git.repo('bup-dumb-server'), 'w').close() + open(git.repo(b'bup-dumb-server'), 'w').close() lw = git.PackWriter() lw.new_blob(s1) @@ -102,8 +103,8 @@ def test_dumb_client_server(): @wvtest def test_midx_refreshing(): with no_lingering_errors(): - with test_tempdir('bup-tclient-') as tmpdir: - os.environ['BUP_DIR'] = bupdir = tmpdir + with test_tempdir(b'bup-tclient-') as tmpdir: + environ[b'BUP_DIR'] = bupdir = tmpdir git.init_repo(bupdir) c = client.Client(bupdir, create=True) rw = c.new_packwriter() @@ -116,7 +117,7 @@ def test_midx_refreshing(): p2name = os.path.join(c.cachedir, p2base) del rw - pi = git.PackIdxList(bupdir + '/objects/pack') + pi = git.PackIdxList(bupdir + b'/objects/pack') WVPASSEQ(len(pi.packs), 2) pi.refresh() WVPASSEQ(len(pi.packs), 2) @@ -129,7 +130,7 @@ def test_midx_refreshing(): WVFAIL(p2.exists(s1sha)) WVPASS(p2.exists(s2sha)) - subprocess.call([path.exe(), 'midx', '-f']) + subprocess.call([path.exe(), b'midx', b'-f']) pi.refresh() WVPASSEQ(len(pi.packs), 1) pi.refresh(skip_midx=True) @@ -142,18 +143,18 @@ def test_midx_refreshing(): def test_remote_parsing(): with no_lingering_errors(): tests = ( - (':/bup', ('file', None, None, '/bup')), - ('file:///bup', ('file', None, None, '/bup')), - ('192.168.1.1:/bup', ('ssh', '192.168.1.1', None, '/bup')), - ('ssh://192.168.1.1:2222/bup', ('ssh', '192.168.1.1', '2222', '/bup')), - ('ssh://[ff:fe::1]:2222/bup', ('ssh', 'ff:fe::1', '2222', '/bup')), - ('bup://foo.com:1950', ('bup', 'foo.com', '1950', None)), - ('bup://foo.com:1950/bup', ('bup', 'foo.com', '1950', '/bup')), - ('bup://[ff:fe::1]/bup', ('bup', 'ff:fe::1', None, '/bup')),) + (b':/bup', (b'file', None, None, b'/bup')), + (b'file:///bup', (b'file', None, None, b'/bup')), + (b'192.168.1.1:/bup', (b'ssh', b'192.168.1.1', None, b'/bup')), + (b'ssh://192.168.1.1:2222/bup', (b'ssh', b'192.168.1.1', b'2222', b'/bup')), + (b'ssh://[ff:fe::1]:2222/bup', (b'ssh', b'ff:fe::1', b'2222', b'/bup')), + (b'bup://foo.com:1950', (b'bup', b'foo.com', b'1950', None)), + (b'bup://foo.com:1950/bup', (b'bup', b'foo.com', b'1950', b'/bup')), + (b'bup://[ff:fe::1]/bup', (b'bup', b'ff:fe::1', None, b'/bup')),) for remote, values in tests: WVPASSEQ(client.parse_remote(remote), values) try: - client.parse_remote('http://asdf.com/bup') + client.parse_remote(b'http://asdf.com/bup') WVFAIL() except client.ClientError: WVPASS() -- 2.39.2