From: Rob Browning Date: Sat, 28 Dec 2019 01:46:02 +0000 (-0600) Subject: bup.client: accommodate python 3 X-Git-Tag: 0.31~194 X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?p=bup.git;a=commitdiff_plain;h=85edc0f1c133d4866466dfb698af0388ee5917c7 bup.client: accommodate python 3 Signed-off-by: Rob Browning Tested-by: Rob Browning --- diff --git a/lib/bup/client.py b/lib/bup/client.py index 19d52cb..ab8d3b2 100644 --- a/lib/bup/client.py +++ b/lib/bup/client.py @@ -1,12 +1,16 @@ +from __future__ import print_function + from __future__ import absolute_import +from binascii import hexlify, unhexlify import errno, os, re, struct, sys, time, zlib from bup import git, ssh, vfs -from bup.compat import range +from bup.compat import environ, range, reraise from bup.helpers import (Conn, atomically_replaced_file, chunkyreader, debug1, debug2, linereader, lines_until_sentinel, mkdirp, progress, qprogress) +from bup.io import path_msg from bup.vint import read_bvec, read_vuint, write_bvec @@ -40,65 +44,72 @@ def _raw_write_bwlimit(f, buf, bwcount, bwtime): return (bwcount, bwtime) +_protocol_rs = br'([a-z]+)://' +_host_rs = br'(?P\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])' +_port_rs = br'(?::(\d+))?' +_path_rs = br'(/.*)?' +_url_rx = re.compile(br'%s(?:%s%s)?%s' % (_protocol_rs, _host_rs, _port_rs, _path_rs), + re.I) + def parse_remote(remote): - protocol = r'([a-z]+)://' - host = r'(?P\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])' - port = r'(?::(\d+))?' - path = r'(/.*)?' - url_match = re.match( - '%s(?:%s%s)?%s' % (protocol, host, port, path), remote, re.I) + url_match = _url_rx.match(remote) if url_match: - if not url_match.group(1) in ('ssh', 'bup', 'file'): - raise ClientError, 'unexpected protocol: %s' % url_match.group(1) + if not url_match.group(1) in (b'ssh', b'bup', b'file'): + raise ClientError('unexpected protocol: %s' + % url_match.group(1).decode('ascii')) return url_match.group(1,3,4,5) else: - rs = remote.split(':', 1) - if len(rs) == 1 or rs[0] in ('', '-'): - return 'file', None, None, rs[-1] + rs = remote.split(b':', 1) + if len(rs) == 1 or rs[0] in (b'', b'-'): + return b'file', None, None, rs[-1] else: - return 'ssh', rs[0], None, rs[1] + return b'ssh', rs[0], None, rs[1] class Client: def __init__(self, remote, create=False): self._busy = self.conn = None self.sock = self.p = self.pout = self.pin = None - is_reverse = os.environ.get('BUP_SERVER_REVERSE') + is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse: assert(not remote) - remote = '%s:' % is_reverse + remote = b'%s:' % is_reverse (self.protocol, self.host, self.port, self.dir) = parse_remote(remote) - self.cachedir = git.repo('index-cache/%s' - % re.sub(r'[^@\w]', '_', - "%s:%s" % (self.host, self.dir))) + self.cachedir = git.repo(b'index-cache/%s' + % re.sub(br'[^@\w]', + b'_', + # FIXME: the Nones just + # match python 2's behavior + b'%s:%s' % (self.host or b'None', + self.dir or b'None'))) if is_reverse: self.pout = os.fdopen(3, 'rb') self.pin = os.fdopen(4, 'wb') self.conn = Conn(self.pout, self.pin) else: - if self.protocol in ('ssh', 'file'): + if self.protocol in (b'ssh', b'file'): try: # FIXME: ssh and file shouldn't use the same module - self.p = ssh.connect(self.host, self.port, 'server') + self.p = ssh.connect(self.host, self.port, b'server') self.pout = self.p.stdout self.pin = self.p.stdin self.conn = Conn(self.pout, self.pin) except OSError as e: - raise ClientError, 'connect: %s' % e, sys.exc_info()[2] - elif self.protocol == 'bup': + reraise(ClientError('connect: %s' % e)) + elif self.protocol == b'bup': self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((self.host, atoi(self.port) or 1982)) self.sockw = self.sock.makefile('wb') self.conn = DemuxConn(self.sock.fileno(), self.sockw) self._available_commands = self._get_available_commands() - self._require_command('init-dir') - self._require_command('set-dir') + self._require_command(b'init-dir') + self._require_command(b'set-dir') if self.dir: - self.dir = re.sub(r'[\r\n]', ' ', self.dir) + self.dir = re.sub(br'[\r\n]', ' ', self.dir) if create: - self.conn.write('init-dir %s\n' % self.dir) + self.conn.write(b'init-dir %s\n' % self.dir) else: - self.conn.write('set-dir %s\n' % self.dir) + self.conn.write(b'set-dir %s\n' % self.dir) self.check_ok() self.sync_indexes() @@ -113,7 +124,7 @@ class Client: def close(self): if self.conn and not self._busy: - self.conn.write('quit\n') + self.conn.write(b'quit\n') if self.pin: self.pin.close() if self.sock and self.sockw: @@ -142,7 +153,7 @@ class Client: try: return self.conn.check_ok() except Exception as e: - raise ClientError, e, sys.exc_info()[2] + reraise(ClientError(e)) def check_busy(self): if self._busy: @@ -157,18 +168,18 @@ class Client: def _get_available_commands(self): self.check_busy() - self._busy = 'help' + self._busy = b'help' conn = self.conn - conn.write('help\n') + conn.write(b'help\n') result = set() line = self.conn.readline() - if not line == 'Commands:\n': + if not line == b'Commands:\n': raise ClientError('unexpected help header ' + repr(line)) while True: line = self.conn.readline() - if line == '\n': + if line == b'\n': break - if not line.startswith(' '): + if not line.startswith(b' '): raise ClientError('unexpected help line ' + repr(line)) cmd = line.strip() if not cmd: @@ -184,28 +195,28 @@ class Client: def _require_command(self, name): if name not in self._available_commands: raise ClientError('server does not appear to provide %s command' - % name) + % name.encode('ascii')) def sync_indexes(self): - self._require_command('list-indexes') + self._require_command(b'list-indexes') self.check_busy() conn = self.conn mkdirp(self.cachedir) # All cached idxs are extra until proven otherwise extra = set() for f in os.listdir(self.cachedir): - debug1('%s\n' % f) - if f.endswith('.idx'): + debug1(path_msg(f) + '\n') + if f.endswith(b'.idx'): extra.add(f) needed = set() - conn.write('list-indexes\n') + conn.write(b'list-indexes\n') for line in linereader(conn): if not line: break - assert(line.find('/') < 0) - parts = line.split(' ') + assert(line.find(b'/') < 0) + parts = line.split(b' ') idx = parts[0] - if len(parts) == 2 and parts[1] == 'load' and idx not in extra: + if len(parts) == 2 and parts[1] == b'load' and idx not in extra: # If the server requests that we load an idx and we don't # already have a copy of it, it is needed needed.add(idx) @@ -222,18 +233,19 @@ class Client: git.auto_midx(self.cachedir) def sync_index(self, name): - self._require_command('send-index') + self._require_command(b'send-index') #debug1('requesting %r\n' % name) self.check_busy() mkdirp(self.cachedir) fn = os.path.join(self.cachedir, name) if os.path.exists(fn): - msg = "won't request existing .idx, try `bup bloom --check %s`" % fn + msg = ("won't request existing .idx, try `bup bloom --check %s`" + % path_msg(fn)) raise ClientError(msg) - self.conn.write('send-index %s\n' % name) + self.conn.write(b'send-index %s\n' % name) n = struct.unpack('!I', self.conn.read(4))[0] assert(n) - with atomically_replaced_file(fn, 'w') as f: + with atomically_replaced_file(fn, 'wb') as f: count = 0 progress('Receiving index from server: %d/%d\r' % (count, n)) for b in chunkyreader(self.conn, n): @@ -249,22 +261,22 @@ class Client: def _suggest_packs(self): ob = self._busy if ob: - assert(ob == 'receive-objects-v2') - self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2 + assert(ob == b'receive-objects-v2') + self.conn.write(b'\xff\xff\xff\xff') # suspend receive-objects-v2 suggested = [] for line in linereader(self.conn): if not line: break - debug2('%s\n' % line) - if line.startswith('index '): + debug2('%r\n' % line) + if line.startswith(b'index '): idx = line[6:] debug1('client: received index suggestion: %s\n' - % git.shorten_hash(idx)) + % git.shorten_hash(idx).decode('ascii')) suggested.append(idx) else: - assert(line.endswith('.idx')) + assert(line.endswith(b'.idx')) debug1('client: completed writing pack, idx: %s\n' - % git.shorten_hash(line)) + % git.shorten_hash(line).decode('ascii')) suggested.append(line) self.check_ok() if ob: @@ -275,16 +287,16 @@ class Client: git.auto_midx(self.cachedir) if ob: self._busy = ob - self.conn.write('%s\n' % ob) + self.conn.write(b'%s\n' % ob) return idx def new_packwriter(self, compression_level=1, max_pack_size=None, max_pack_objects=None): - self._require_command('receive-objects-v2') + self._require_command(b'receive-objects-v2') self.check_busy() def _set_busy(): - self._busy = 'receive-objects-v2' - self.conn.write('receive-objects-v2\n') + self._busy = b'receive-objects-v2' + self.conn.write(b'receive-objects-v2\n') return PackWriter_Remote(self.conn, objcache_maker = self._make_objcache, suggest_packs = self._suggest_packs, @@ -296,9 +308,9 @@ class Client: max_pack_objects=max_pack_objects) def read_ref(self, refname): - self._require_command('read-ref') + self._require_command(b'read-ref') self.check_busy() - self.conn.write('read-ref %s\n' % refname) + self.conn.write(b'read-ref %s\n' % refname) r = self.conn.readline().strip() self.check_ok() if r: @@ -308,19 +320,19 @@ class Client: return None # nonexistent ref def update_ref(self, refname, newval, oldval): - self._require_command('update-ref') + self._require_command(b'update-ref') self.check_busy() - self.conn.write('update-ref %s\n%s\n%s\n' - % (refname, newval.encode('hex'), - (oldval or '').encode('hex'))) + self.conn.write(b'update-ref %s\n%s\n%s\n' + % (refname, hexlify(newval), + hexlify(oldval) if oldval else b'')) self.check_ok() def join(self, id): - self._require_command('join') + self._require_command(b'join') self.check_busy() - self._busy = 'join' + self._busy = b'join' # Send 'cat' so we'll work fine with older versions - self.conn.write('cat %s\n' % re.sub(r'[\n\r]', '_', id)) + self.conn.write(b'cat %s\n' % re.sub(br'[\n\r]', b'_', id)) while 1: sz = struct.unpack('!I', self.conn.read(4))[0] if not sz: break @@ -332,27 +344,27 @@ class Client: raise KeyError(str(e)) def cat_batch(self, refs): - self._require_command('cat-batch') + self._require_command(b'cat-batch') self.check_busy() - self._busy = 'cat-batch' + self._busy = b'cat-batch' conn = self.conn - conn.write('cat-batch\n') + conn.write(b'cat-batch\n') # FIXME: do we want (only) binary protocol? for ref in refs: assert ref - assert '\n' not in ref + assert b'\n' not in ref conn.write(ref) - conn.write('\n') - conn.write('\n') + conn.write(b'\n') + conn.write(b'\n') for ref in refs: info = conn.readline() - if info == 'missing\n': + if info == b'missing\n': yield None, None, None, None continue - if not (info and info.endswith('\n')): + if not (info and info.endswith(b'\n')): raise ClientError('Hit EOF while looking for object info: %r' % info) - oidx, oid_t, size = info.split(' ') + oidx, oid_t, size = info.split(b' ') size = int(size) cr = chunkyreader(conn, size) yield oidx, oid_t, size, cr @@ -367,25 +379,25 @@ class Client: def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False): patterns = patterns or tuple() - self._require_command('refs') + self._require_command(b'refs') self.check_busy() - self._busy = 'refs' + self._busy = b'refs' conn = self.conn - conn.write('refs %s %s\n' % (1 if limit_to_heads else 0, - 1 if limit_to_tags else 0)) + conn.write(b'refs %d %d\n' % (1 if limit_to_heads else 0, + 1 if limit_to_tags else 0)) for pattern in patterns: - assert '\n' not in pattern + assert b'\n' not in pattern conn.write(pattern) - conn.write('\n') - conn.write('\n') - for line in lines_until_sentinel(conn, '\n', ClientError): + conn.write(b'\n') + conn.write(b'\n') + for line in lines_until_sentinel(conn, b'\n', ClientError): line = line[:-1] - oidx, name = line.split(' ') + oidx, name = line.split(b' ') if len(oidx) != 40: raise ClientError('Invalid object fingerprint in %r' % line) if not name: raise ClientError('Invalid reference name in %r' % line) - yield name, oidx.decode('hex') + yield name, unhexlify(oidx) # FIXME: confusing not_ok = self.check_ok() if not_ok: @@ -400,36 +412,36 @@ class Client: as a terminator for the entire rev-list result. """ - self._require_command('rev-list') + self._require_command(b'rev-list') assert (count is None) or (isinstance(count, Integral)) if format: - assert '\n' not in format + assert b'\n' not in format assert parse for ref in refs: assert ref - assert '\n' not in ref + assert b'\n' not in ref self.check_busy() - self._busy = 'rev-list' + self._busy = b'rev-list' conn = self.conn - conn.write('rev-list\n') + conn.write(b'rev-list\n') if count is not None: - conn.write(str(count)) - conn.write('\n') + conn.write(b'%d' % count) + conn.write(b'\n') if format: conn.write(format) - conn.write('\n') + conn.write(b'\n') for ref in refs: conn.write(ref) - conn.write('\n') - conn.write('\n') + conn.write(b'\n') + conn.write(b'\n') if not format: - for line in lines_until_sentinel(conn, '\n', ClientError): + for line in lines_until_sentinel(conn, b'\n', ClientError): line = line.strip() assert len(line) == 40 yield line else: - for line in lines_until_sentinel(conn, '\n', ClientError): - if not line.startswith('commit '): + for line in lines_until_sentinel(conn, b'\n', ClientError): + if not line.startswith(b'commit '): raise ClientError('unexpected line ' + repr(line)) cmt_oidx = line[7:].strip() assert len(cmt_oidx) == 40 @@ -441,13 +453,13 @@ class Client: self._not_busy() def resolve(self, path, parent=None, want_meta=True, follow=False): - self._require_command('resolve') + self._require_command(b'resolve') self.check_busy() - self._busy = 'resolve' + self._busy = b'resolve' conn = self.conn - conn.write('resolve %d\n' % ((1 if want_meta else 0) - | (2 if follow else 0) - | (4 if parent else 0))) + conn.write(b'resolve %d\n' % ((1 if want_meta else 0) + | (2 if follow else 0) + | (4 if parent else 0))) if parent: vfs.write_resolution(conn, parent) write_bvec(conn, path) @@ -480,7 +492,7 @@ class PackWriter_Remote(git.PackWriter): max_pack_size=max_pack_size, max_pack_objects=max_pack_objects) self.file = conn - self.filename = 'remote socket' + self.filename = b'remote socket' self.suggest_packs = suggest_packs self.onopen = onopen self.onclose = onclose @@ -497,7 +509,7 @@ class PackWriter_Remote(git.PackWriter): def _end(self, run_midx=True): assert(run_midx) # We don't support this via remote yet if self._packopen and self.file: - self.file.write('\0\0\0\0') + self.file.write(b'\0\0\0\0') self._packopen = False self.onclose() # Unbusy self.objcache = None @@ -516,19 +528,19 @@ class PackWriter_Remote(git.PackWriter): if not self._packopen: self._open() self.ensure_busy() - data = ''.join(datalist) + data = b''.join(datalist) assert(data) assert(sha) crc = zlib.crc32(data) & 0xffffffff - outbuf = ''.join((struct.pack('!I', len(data) + 20 + 4), - sha, - struct.pack('!I', crc), - data)) + outbuf = b''.join((struct.pack('!I', len(data) + 20 + 4), + sha, + struct.pack('!I', crc), + data)) try: (self._bwcount, self._bwtime) = _raw_write_bwlimit( self.file, outbuf, self._bwcount, self._bwtime) except IOError as e: - raise ClientError, e, sys.exc_info()[2] + reraise(ClientError(e)) self.outbytes += len(data) self.count += 1