Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>
+from __future__ import print_function
+
from __future__ import absolute_import
from __future__ import absolute_import
+from binascii import hexlify, unhexlify
import errno, os, re, struct, sys, time, zlib
from bup import git, ssh, vfs
import errno, os, re, struct, sys, time, zlib
from bup import git, ssh, vfs
-from bup.compat import range
+from bup.compat import environ, range, reraise
from bup.helpers import (Conn, atomically_replaced_file, chunkyreader, debug1,
debug2, linereader, lines_until_sentinel,
mkdirp, progress, qprogress)
from bup.helpers import (Conn, atomically_replaced_file, chunkyreader, debug1,
debug2, linereader, lines_until_sentinel,
mkdirp, progress, qprogress)
+from bup.io import path_msg
from bup.vint import read_bvec, read_vuint, write_bvec
from bup.vint import read_bvec, read_vuint, write_bvec
+_protocol_rs = br'([a-z]+)://'
+_host_rs = br'(?P<sb>\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])'
+_port_rs = br'(?::(\d+))?'
+_path_rs = br'(/.*)?'
+_url_rx = re.compile(br'%s(?:%s%s)?%s' % (_protocol_rs, _host_rs, _port_rs, _path_rs),
+ re.I)
+
def parse_remote(remote):
def parse_remote(remote):
- protocol = r'([a-z]+)://'
- host = r'(?P<sb>\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])'
- port = r'(?::(\d+))?'
- path = r'(/.*)?'
- url_match = re.match(
- '%s(?:%s%s)?%s' % (protocol, host, port, path), remote, re.I)
+ url_match = _url_rx.match(remote)
- if not url_match.group(1) in ('ssh', 'bup', 'file'):
- raise ClientError, 'unexpected protocol: %s' % url_match.group(1)
+ if not url_match.group(1) in (b'ssh', b'bup', b'file'):
+ raise ClientError('unexpected protocol: %s'
+ % url_match.group(1).decode('ascii'))
return url_match.group(1,3,4,5)
else:
return url_match.group(1,3,4,5)
else:
- rs = remote.split(':', 1)
- if len(rs) == 1 or rs[0] in ('', '-'):
- return 'file', None, None, rs[-1]
+ rs = remote.split(b':', 1)
+ if len(rs) == 1 or rs[0] in (b'', b'-'):
+ return b'file', None, None, rs[-1]
- return 'ssh', rs[0], None, rs[1]
+ return b'ssh', rs[0], None, rs[1]
class Client:
def __init__(self, remote, create=False):
self._busy = self.conn = None
self.sock = self.p = self.pout = self.pin = None
class Client:
def __init__(self, remote, create=False):
self._busy = self.conn = None
self.sock = self.p = self.pout = self.pin = None
- is_reverse = os.environ.get('BUP_SERVER_REVERSE')
+ is_reverse = environ.get(b'BUP_SERVER_REVERSE')
if is_reverse:
assert(not remote)
if is_reverse:
assert(not remote)
- remote = '%s:' % is_reverse
+ remote = b'%s:' % is_reverse
(self.protocol, self.host, self.port, self.dir) = parse_remote(remote)
(self.protocol, self.host, self.port, self.dir) = parse_remote(remote)
- self.cachedir = git.repo('index-cache/%s'
- % re.sub(r'[^@\w]', '_',
- "%s:%s" % (self.host, self.dir)))
+ self.cachedir = git.repo(b'index-cache/%s'
+ % re.sub(br'[^@\w]',
+ b'_',
+ # FIXME: the Nones just
+ # match python 2's behavior
+ b'%s:%s' % (self.host or b'None',
+ self.dir or b'None')))
if is_reverse:
self.pout = os.fdopen(3, 'rb')
self.pin = os.fdopen(4, 'wb')
self.conn = Conn(self.pout, self.pin)
else:
if is_reverse:
self.pout = os.fdopen(3, 'rb')
self.pin = os.fdopen(4, 'wb')
self.conn = Conn(self.pout, self.pin)
else:
- if self.protocol in ('ssh', 'file'):
+ if self.protocol in (b'ssh', b'file'):
try:
# FIXME: ssh and file shouldn't use the same module
try:
# FIXME: ssh and file shouldn't use the same module
- self.p = ssh.connect(self.host, self.port, 'server')
+ self.p = ssh.connect(self.host, self.port, b'server')
self.pout = self.p.stdout
self.pin = self.p.stdin
self.conn = Conn(self.pout, self.pin)
except OSError as e:
self.pout = self.p.stdout
self.pin = self.p.stdin
self.conn = Conn(self.pout, self.pin)
except OSError as e:
- raise ClientError, 'connect: %s' % e, sys.exc_info()[2]
- elif self.protocol == 'bup':
+ reraise(ClientError('connect: %s' % e))
+ elif self.protocol == b'bup':
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host, atoi(self.port) or 1982))
self.sockw = self.sock.makefile('wb')
self.conn = DemuxConn(self.sock.fileno(), self.sockw)
self._available_commands = self._get_available_commands()
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host, atoi(self.port) or 1982))
self.sockw = self.sock.makefile('wb')
self.conn = DemuxConn(self.sock.fileno(), self.sockw)
self._available_commands = self._get_available_commands()
- self._require_command('init-dir')
- self._require_command('set-dir')
+ self._require_command(b'init-dir')
+ self._require_command(b'set-dir')
- self.dir = re.sub(r'[\r\n]', ' ', self.dir)
+ self.dir = re.sub(br'[\r\n]', ' ', self.dir)
- self.conn.write('init-dir %s\n' % self.dir)
+ self.conn.write(b'init-dir %s\n' % self.dir)
- self.conn.write('set-dir %s\n' % self.dir)
+ self.conn.write(b'set-dir %s\n' % self.dir)
self.check_ok()
self.sync_indexes()
self.check_ok()
self.sync_indexes()
def close(self):
if self.conn and not self._busy:
def close(self):
if self.conn and not self._busy:
- self.conn.write('quit\n')
+ self.conn.write(b'quit\n')
if self.pin:
self.pin.close()
if self.sock and self.sockw:
if self.pin:
self.pin.close()
if self.sock and self.sockw:
try:
return self.conn.check_ok()
except Exception as e:
try:
return self.conn.check_ok()
except Exception as e:
- raise ClientError, e, sys.exc_info()[2]
+ reraise(ClientError(e))
def check_busy(self):
if self._busy:
def check_busy(self):
if self._busy:
def _get_available_commands(self):
self.check_busy()
def _get_available_commands(self):
self.check_busy()
result = set()
line = self.conn.readline()
result = set()
line = self.conn.readline()
- if not line == 'Commands:\n':
+ if not line == b'Commands:\n':
raise ClientError('unexpected help header ' + repr(line))
while True:
line = self.conn.readline()
raise ClientError('unexpected help header ' + repr(line))
while True:
line = self.conn.readline()
- if not line.startswith(' '):
+ if not line.startswith(b' '):
raise ClientError('unexpected help line ' + repr(line))
cmd = line.strip()
if not cmd:
raise ClientError('unexpected help line ' + repr(line))
cmd = line.strip()
if not cmd:
def _require_command(self, name):
if name not in self._available_commands:
raise ClientError('server does not appear to provide %s command'
def _require_command(self, name):
if name not in self._available_commands:
raise ClientError('server does not appear to provide %s command'
+ % name.encode('ascii'))
- self._require_command('list-indexes')
+ self._require_command(b'list-indexes')
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
- debug1('%s\n' % f)
- if f.endswith('.idx'):
+ debug1(path_msg(f) + '\n')
+ if f.endswith(b'.idx'):
extra.add(f)
needed = set()
extra.add(f)
needed = set()
- conn.write('list-indexes\n')
+ conn.write(b'list-indexes\n')
for line in linereader(conn):
if not line:
break
for line in linereader(conn):
if not line:
break
- assert(line.find('/') < 0)
- parts = line.split(' ')
+ assert(line.find(b'/') < 0)
+ parts = line.split(b' ')
- if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
+ if len(parts) == 2 and parts[1] == b'load' and idx not in extra:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
git.auto_midx(self.cachedir)
def sync_index(self, name):
git.auto_midx(self.cachedir)
def sync_index(self, name):
- self._require_command('send-index')
+ self._require_command(b'send-index')
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
- msg = "won't request existing .idx, try `bup bloom --check %s`" % fn
+ msg = ("won't request existing .idx, try `bup bloom --check %s`"
+ % path_msg(fn))
- self.conn.write('send-index %s\n' % name)
+ self.conn.write(b'send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert(n)
n = struct.unpack('!I', self.conn.read(4))[0]
assert(n)
- with atomically_replaced_file(fn, 'w') as f:
+ with atomically_replaced_file(fn, 'wb') as f:
count = 0
progress('Receiving index from server: %d/%d\r' % (count, n))
for b in chunkyreader(self.conn, n):
count = 0
progress('Receiving index from server: %d/%d\r' % (count, n))
for b in chunkyreader(self.conn, n):
def _suggest_packs(self):
ob = self._busy
if ob:
def _suggest_packs(self):
ob = self._busy
if ob:
- assert(ob == 'receive-objects-v2')
- self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
+ assert(ob == b'receive-objects-v2')
+ self.conn.write(b'\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
suggested = []
for line in linereader(self.conn):
if not line:
break
- debug2('%s\n' % line)
- if line.startswith('index '):
+ debug2('%r\n' % line)
+ if line.startswith(b'index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
idx = line[6:]
debug1('client: received index suggestion: %s\n'
- % git.shorten_hash(idx))
+ % git.shorten_hash(idx).decode('ascii'))
suggested.append(idx)
else:
suggested.append(idx)
else:
- assert(line.endswith('.idx'))
+ assert(line.endswith(b'.idx'))
debug1('client: completed writing pack, idx: %s\n'
debug1('client: completed writing pack, idx: %s\n'
- % git.shorten_hash(line))
+ % git.shorten_hash(line).decode('ascii'))
suggested.append(line)
self.check_ok()
if ob:
suggested.append(line)
self.check_ok()
if ob:
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
- self.conn.write('%s\n' % ob)
+ self.conn.write(b'%s\n' % ob)
return idx
def new_packwriter(self, compression_level=1,
max_pack_size=None, max_pack_objects=None):
return idx
def new_packwriter(self, compression_level=1,
max_pack_size=None, max_pack_objects=None):
- self._require_command('receive-objects-v2')
+ self._require_command(b'receive-objects-v2')
self.check_busy()
def _set_busy():
self.check_busy()
def _set_busy():
- self._busy = 'receive-objects-v2'
- self.conn.write('receive-objects-v2\n')
+ self._busy = b'receive-objects-v2'
+ self.conn.write(b'receive-objects-v2\n')
return PackWriter_Remote(self.conn,
objcache_maker = self._make_objcache,
suggest_packs = self._suggest_packs,
return PackWriter_Remote(self.conn,
objcache_maker = self._make_objcache,
suggest_packs = self._suggest_packs,
max_pack_objects=max_pack_objects)
def read_ref(self, refname):
max_pack_objects=max_pack_objects)
def read_ref(self, refname):
- self._require_command('read-ref')
+ self._require_command(b'read-ref')
- self.conn.write('read-ref %s\n' % refname)
+ self.conn.write(b'read-ref %s\n' % refname)
r = self.conn.readline().strip()
self.check_ok()
if r:
r = self.conn.readline().strip()
self.check_ok()
if r:
return None # nonexistent ref
def update_ref(self, refname, newval, oldval):
return None # nonexistent ref
def update_ref(self, refname, newval, oldval):
- self._require_command('update-ref')
+ self._require_command(b'update-ref')
- self.conn.write('update-ref %s\n%s\n%s\n'
- % (refname, newval.encode('hex'),
- (oldval or '').encode('hex')))
+ self.conn.write(b'update-ref %s\n%s\n%s\n'
+ % (refname, hexlify(newval),
+ hexlify(oldval) if oldval else b''))
self.check_ok()
def join(self, id):
self.check_ok()
def join(self, id):
- self._require_command('join')
+ self._require_command(b'join')
# Send 'cat' so we'll work fine with older versions
# Send 'cat' so we'll work fine with older versions
- self.conn.write('cat %s\n' % re.sub(r'[\n\r]', '_', id))
+ self.conn.write(b'cat %s\n' % re.sub(br'[\n\r]', b'_', id))
while 1:
sz = struct.unpack('!I', self.conn.read(4))[0]
if not sz: break
while 1:
sz = struct.unpack('!I', self.conn.read(4))[0]
if not sz: break
raise KeyError(str(e))
def cat_batch(self, refs):
raise KeyError(str(e))
def cat_batch(self, refs):
- self._require_command('cat-batch')
+ self._require_command(b'cat-batch')
- self._busy = 'cat-batch'
+ self._busy = b'cat-batch'
- conn.write('cat-batch\n')
+ conn.write(b'cat-batch\n')
# FIXME: do we want (only) binary protocol?
for ref in refs:
assert ref
# FIXME: do we want (only) binary protocol?
for ref in refs:
assert ref
+ assert b'\n' not in ref
- conn.write('\n')
- conn.write('\n')
+ conn.write(b'\n')
+ conn.write(b'\n')
for ref in refs:
info = conn.readline()
for ref in refs:
info = conn.readline()
- if info == 'missing\n':
+ if info == b'missing\n':
yield None, None, None, None
continue
yield None, None, None, None
continue
- if not (info and info.endswith('\n')):
+ if not (info and info.endswith(b'\n')):
raise ClientError('Hit EOF while looking for object info: %r'
% info)
raise ClientError('Hit EOF while looking for object info: %r'
% info)
- oidx, oid_t, size = info.split(' ')
+ oidx, oid_t, size = info.split(b' ')
size = int(size)
cr = chunkyreader(conn, size)
yield oidx, oid_t, size, cr
size = int(size)
cr = chunkyreader(conn, size)
yield oidx, oid_t, size, cr
def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False):
patterns = patterns or tuple()
def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False):
patterns = patterns or tuple()
- self._require_command('refs')
+ self._require_command(b'refs')
- conn.write('refs %s %s\n' % (1 if limit_to_heads else 0,
- 1 if limit_to_tags else 0))
+ conn.write(b'refs %d %d\n' % (1 if limit_to_heads else 0,
+ 1 if limit_to_tags else 0))
- assert '\n' not in pattern
+ assert b'\n' not in pattern
- conn.write('\n')
- conn.write('\n')
- for line in lines_until_sentinel(conn, '\n', ClientError):
+ conn.write(b'\n')
+ conn.write(b'\n')
+ for line in lines_until_sentinel(conn, b'\n', ClientError):
- oidx, name = line.split(' ')
+ oidx, name = line.split(b' ')
if len(oidx) != 40:
raise ClientError('Invalid object fingerprint in %r' % line)
if not name:
raise ClientError('Invalid reference name in %r' % line)
if len(oidx) != 40:
raise ClientError('Invalid object fingerprint in %r' % line)
if not name:
raise ClientError('Invalid reference name in %r' % line)
- yield name, oidx.decode('hex')
+ yield name, unhexlify(oidx)
# FIXME: confusing
not_ok = self.check_ok()
if not_ok:
# FIXME: confusing
not_ok = self.check_ok()
if not_ok:
as a terminator for the entire rev-list result.
"""
as a terminator for the entire rev-list result.
"""
- self._require_command('rev-list')
+ self._require_command(b'rev-list')
assert (count is None) or (isinstance(count, Integral))
if format:
assert (count is None) or (isinstance(count, Integral))
if format:
- assert '\n' not in format
+ assert b'\n' not in format
assert parse
for ref in refs:
assert ref
assert parse
for ref in refs:
assert ref
+ assert b'\n' not in ref
- self._busy = 'rev-list'
+ self._busy = b'rev-list'
- conn.write('rev-list\n')
+ conn.write(b'rev-list\n')
- conn.write(str(count))
- conn.write('\n')
+ conn.write(b'%d' % count)
+ conn.write(b'\n')
if format:
conn.write(format)
if format:
conn.write(format)
for ref in refs:
conn.write(ref)
for ref in refs:
conn.write(ref)
- conn.write('\n')
- conn.write('\n')
+ conn.write(b'\n')
+ conn.write(b'\n')
- for line in lines_until_sentinel(conn, '\n', ClientError):
+ for line in lines_until_sentinel(conn, b'\n', ClientError):
line = line.strip()
assert len(line) == 40
yield line
else:
line = line.strip()
assert len(line) == 40
yield line
else:
- for line in lines_until_sentinel(conn, '\n', ClientError):
- if not line.startswith('commit '):
+ for line in lines_until_sentinel(conn, b'\n', ClientError):
+ if not line.startswith(b'commit '):
raise ClientError('unexpected line ' + repr(line))
cmt_oidx = line[7:].strip()
assert len(cmt_oidx) == 40
raise ClientError('unexpected line ' + repr(line))
cmt_oidx = line[7:].strip()
assert len(cmt_oidx) == 40
self._not_busy()
def resolve(self, path, parent=None, want_meta=True, follow=False):
self._not_busy()
def resolve(self, path, parent=None, want_meta=True, follow=False):
- self._require_command('resolve')
+ self._require_command(b'resolve')
+ self._busy = b'resolve'
- conn.write('resolve %d\n' % ((1 if want_meta else 0)
- | (2 if follow else 0)
- | (4 if parent else 0)))
+ conn.write(b'resolve %d\n' % ((1 if want_meta else 0)
+ | (2 if follow else 0)
+ | (4 if parent else 0)))
if parent:
vfs.write_resolution(conn, parent)
write_bvec(conn, path)
if parent:
vfs.write_resolution(conn, parent)
write_bvec(conn, path)
max_pack_size=max_pack_size,
max_pack_objects=max_pack_objects)
self.file = conn
max_pack_size=max_pack_size,
max_pack_objects=max_pack_objects)
self.file = conn
- self.filename = 'remote socket'
+ self.filename = b'remote socket'
self.suggest_packs = suggest_packs
self.onopen = onopen
self.onclose = onclose
self.suggest_packs = suggest_packs
self.onopen = onopen
self.onclose = onclose
def _end(self, run_midx=True):
assert(run_midx) # We don't support this via remote yet
if self._packopen and self.file:
def _end(self, run_midx=True):
assert(run_midx) # We don't support this via remote yet
if self._packopen and self.file:
- self.file.write('\0\0\0\0')
+ self.file.write(b'\0\0\0\0')
self._packopen = False
self.onclose() # Unbusy
self.objcache = None
self._packopen = False
self.onclose() # Unbusy
self.objcache = None
if not self._packopen:
self._open()
self.ensure_busy()
if not self._packopen:
self._open()
self.ensure_busy()
- data = ''.join(datalist)
+ data = b''.join(datalist)
assert(data)
assert(sha)
crc = zlib.crc32(data) & 0xffffffff
assert(data)
assert(sha)
crc = zlib.crc32(data) & 0xffffffff
- outbuf = ''.join((struct.pack('!I', len(data) + 20 + 4),
- sha,
- struct.pack('!I', crc),
- data))
+ outbuf = b''.join((struct.pack('!I', len(data) + 20 + 4),
+ sha,
+ struct.pack('!I', crc),
+ data))
try:
(self._bwcount, self._bwtime) = _raw_write_bwlimit(
self.file, outbuf, self._bwcount, self._bwtime)
except IOError as e:
try:
(self._bwcount, self._bwtime) = _raw_write_bwlimit(
self.file, outbuf, self._bwcount, self._bwtime)
except IOError as e:
- raise ClientError, e, sys.exc_info()[2]
+ reraise(ClientError(e))
self.outbytes += len(data)
self.count += 1
self.outbytes += len(data)
self.count += 1
projects / bup.git / commitdiff