+from __future__ import print_function
+
from __future__ import absolute_import
+from binascii import hexlify, unhexlify
import errno, os, re, struct, sys, time, zlib
from bup import git, ssh, vfs
-from bup.compat import range
+from bup.compat import environ, range, reraise
from bup.helpers import (Conn, atomically_replaced_file, chunkyreader, debug1,
debug2, linereader, lines_until_sentinel,
mkdirp, progress, qprogress)
+from bup.io import path_msg
from bup.vint import read_bvec, read_vuint, write_bvec
return (bwcount, bwtime)
+_protocol_rs = br'([a-z]+)://'
+_host_rs = br'(?P<sb>\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])'
+_port_rs = br'(?::(\d+))?'
+_path_rs = br'(/.*)?'
+_url_rx = re.compile(br'%s(?:%s%s)?%s' % (_protocol_rs, _host_rs, _port_rs, _path_rs),
+ re.I)
+
def parse_remote(remote):
- protocol = r'([a-z]+)://'
- host = r'(?P<sb>\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])'
- port = r'(?::(\d+))?'
- path = r'(/.*)?'
- url_match = re.match(
- '%s(?:%s%s)?%s' % (protocol, host, port, path), remote, re.I)
+ url_match = _url_rx.match(remote)
if url_match:
- if not url_match.group(1) in ('ssh', 'bup', 'file'):
- raise ClientError, 'unexpected protocol: %s' % url_match.group(1)
+ if not url_match.group(1) in (b'ssh', b'bup', b'file'):
+ raise ClientError('unexpected protocol: %s'
+ % url_match.group(1).decode('ascii'))
return url_match.group(1,3,4,5)
else:
- rs = remote.split(':', 1)
- if len(rs) == 1 or rs[0] in ('', '-'):
- return 'file', None, None, rs[-1]
+ rs = remote.split(b':', 1)
+ if len(rs) == 1 or rs[0] in (b'', b'-'):
+ return b'file', None, None, rs[-1]
else:
- return 'ssh', rs[0], None, rs[1]
+ return b'ssh', rs[0], None, rs[1]
class Client:
def __init__(self, remote, create=False):
self._busy = self.conn = None
self.sock = self.p = self.pout = self.pin = None
- is_reverse = os.environ.get('BUP_SERVER_REVERSE')
+ is_reverse = environ.get(b'BUP_SERVER_REVERSE')
if is_reverse:
assert(not remote)
- remote = '%s:' % is_reverse
+ remote = b'%s:' % is_reverse
(self.protocol, self.host, self.port, self.dir) = parse_remote(remote)
- self.cachedir = git.repo('index-cache/%s'
- % re.sub(r'[^@\w]', '_',
- "%s:%s" % (self.host, self.dir)))
+ self.cachedir = git.repo(b'index-cache/%s'
+ % re.sub(br'[^@\w]',
+ b'_',
+ # FIXME: the Nones just
+ # match python 2's behavior
+ b'%s:%s' % (self.host or b'None',
+ self.dir or b'None')))
if is_reverse:
self.pout = os.fdopen(3, 'rb')
self.pin = os.fdopen(4, 'wb')
self.conn = Conn(self.pout, self.pin)
else:
- if self.protocol in ('ssh', 'file'):
+ if self.protocol in (b'ssh', b'file'):
try:
# FIXME: ssh and file shouldn't use the same module
- self.p = ssh.connect(self.host, self.port, 'server')
+ self.p = ssh.connect(self.host, self.port, b'server')
self.pout = self.p.stdout
self.pin = self.p.stdin
self.conn = Conn(self.pout, self.pin)
except OSError as e:
- raise ClientError, 'connect: %s' % e, sys.exc_info()[2]
- elif self.protocol == 'bup':
+ reraise(ClientError('connect: %s' % e))
+ elif self.protocol == b'bup':
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host, atoi(self.port) or 1982))
self.sockw = self.sock.makefile('wb')
self.conn = DemuxConn(self.sock.fileno(), self.sockw)
self._available_commands = self._get_available_commands()
- self._require_command('init-dir')
- self._require_command('set-dir')
+ self._require_command(b'init-dir')
+ self._require_command(b'set-dir')
if self.dir:
- self.dir = re.sub(r'[\r\n]', ' ', self.dir)
+ self.dir = re.sub(br'[\r\n]', ' ', self.dir)
if create:
- self.conn.write('init-dir %s\n' % self.dir)
+ self.conn.write(b'init-dir %s\n' % self.dir)
else:
- self.conn.write('set-dir %s\n' % self.dir)
+ self.conn.write(b'set-dir %s\n' % self.dir)
self.check_ok()
self.sync_indexes()
def close(self):
if self.conn and not self._busy:
- self.conn.write('quit\n')
+ self.conn.write(b'quit\n')
if self.pin:
self.pin.close()
if self.sock and self.sockw:
try:
return self.conn.check_ok()
except Exception as e:
- raise ClientError, e, sys.exc_info()[2]
+ reraise(ClientError(e))
def check_busy(self):
if self._busy:
def _get_available_commands(self):
self.check_busy()
- self._busy = 'help'
+ self._busy = b'help'
conn = self.conn
- conn.write('help\n')
+ conn.write(b'help\n')
result = set()
line = self.conn.readline()
- if not line == 'Commands:\n':
+ if not line == b'Commands:\n':
raise ClientError('unexpected help header ' + repr(line))
while True:
line = self.conn.readline()
- if line == '\n':
+ if line == b'\n':
break
- if not line.startswith(' '):
+ if not line.startswith(b' '):
raise ClientError('unexpected help line ' + repr(line))
cmd = line.strip()
if not cmd:
def _require_command(self, name):
if name not in self._available_commands:
raise ClientError('server does not appear to provide %s command'
- % name)
+ % name.encode('ascii'))
def sync_indexes(self):
- self._require_command('list-indexes')
+ self._require_command(b'list-indexes')
self.check_busy()
conn = self.conn
mkdirp(self.cachedir)
# All cached idxs are extra until proven otherwise
extra = set()
for f in os.listdir(self.cachedir):
- debug1('%s\n' % f)
- if f.endswith('.idx'):
+ debug1(path_msg(f) + '\n')
+ if f.endswith(b'.idx'):
extra.add(f)
needed = set()
- conn.write('list-indexes\n')
+ conn.write(b'list-indexes\n')
for line in linereader(conn):
if not line:
break
- assert(line.find('/') < 0)
- parts = line.split(' ')
+ assert(line.find(b'/') < 0)
+ parts = line.split(b' ')
idx = parts[0]
- if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
+ if len(parts) == 2 and parts[1] == b'load' and idx not in extra:
# If the server requests that we load an idx and we don't
# already have a copy of it, it is needed
needed.add(idx)
git.auto_midx(self.cachedir)
def sync_index(self, name):
- self._require_command('send-index')
+ self._require_command(b'send-index')
#debug1('requesting %r\n' % name)
self.check_busy()
mkdirp(self.cachedir)
fn = os.path.join(self.cachedir, name)
if os.path.exists(fn):
- msg = "won't request existing .idx, try `bup bloom --check %s`" % fn
+ msg = ("won't request existing .idx, try `bup bloom --check %s`"
+ % path_msg(fn))
raise ClientError(msg)
- self.conn.write('send-index %s\n' % name)
+ self.conn.write(b'send-index %s\n' % name)
n = struct.unpack('!I', self.conn.read(4))[0]
assert(n)
- with atomically_replaced_file(fn, 'w') as f:
+ with atomically_replaced_file(fn, 'wb') as f:
count = 0
progress('Receiving index from server: %d/%d\r' % (count, n))
for b in chunkyreader(self.conn, n):
def _suggest_packs(self):
ob = self._busy
if ob:
- assert(ob == 'receive-objects-v2')
- self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
+ assert(ob == b'receive-objects-v2')
+ self.conn.write(b'\xff\xff\xff\xff') # suspend receive-objects-v2
suggested = []
for line in linereader(self.conn):
if not line:
break
- debug2('%s\n' % line)
- if line.startswith('index '):
+ debug2('%r\n' % line)
+ if line.startswith(b'index '):
idx = line[6:]
debug1('client: received index suggestion: %s\n'
- % git.shorten_hash(idx))
+ % git.shorten_hash(idx).decode('ascii'))
suggested.append(idx)
else:
- assert(line.endswith('.idx'))
+ assert(line.endswith(b'.idx'))
debug1('client: completed writing pack, idx: %s\n'
- % git.shorten_hash(line))
+ % git.shorten_hash(line).decode('ascii'))
suggested.append(line)
self.check_ok()
if ob:
git.auto_midx(self.cachedir)
if ob:
self._busy = ob
- self.conn.write('%s\n' % ob)
+ self.conn.write(b'%s\n' % ob)
return idx
def new_packwriter(self, compression_level=1,
max_pack_size=None, max_pack_objects=None):
- self._require_command('receive-objects-v2')
+ self._require_command(b'receive-objects-v2')
self.check_busy()
def _set_busy():
- self._busy = 'receive-objects-v2'
- self.conn.write('receive-objects-v2\n')
+ self._busy = b'receive-objects-v2'
+ self.conn.write(b'receive-objects-v2\n')
return PackWriter_Remote(self.conn,
objcache_maker = self._make_objcache,
suggest_packs = self._suggest_packs,
max_pack_objects=max_pack_objects)
def read_ref(self, refname):
- self._require_command('read-ref')
+ self._require_command(b'read-ref')
self.check_busy()
- self.conn.write('read-ref %s\n' % refname)
+ self.conn.write(b'read-ref %s\n' % refname)
r = self.conn.readline().strip()
self.check_ok()
if r:
return None # nonexistent ref
def update_ref(self, refname, newval, oldval):
- self._require_command('update-ref')
+ self._require_command(b'update-ref')
self.check_busy()
- self.conn.write('update-ref %s\n%s\n%s\n'
- % (refname, newval.encode('hex'),
- (oldval or '').encode('hex')))
+ self.conn.write(b'update-ref %s\n%s\n%s\n'
+ % (refname, hexlify(newval),
+ hexlify(oldval) if oldval else b''))
self.check_ok()
def join(self, id):
- self._require_command('join')
+ self._require_command(b'join')
self.check_busy()
- self._busy = 'join'
+ self._busy = b'join'
# Send 'cat' so we'll work fine with older versions
- self.conn.write('cat %s\n' % re.sub(r'[\n\r]', '_', id))
+ self.conn.write(b'cat %s\n' % re.sub(br'[\n\r]', b'_', id))
while 1:
sz = struct.unpack('!I', self.conn.read(4))[0]
if not sz: break
raise KeyError(str(e))
def cat_batch(self, refs):
- self._require_command('cat-batch')
+ self._require_command(b'cat-batch')
self.check_busy()
- self._busy = 'cat-batch'
+ self._busy = b'cat-batch'
conn = self.conn
- conn.write('cat-batch\n')
+ conn.write(b'cat-batch\n')
# FIXME: do we want (only) binary protocol?
for ref in refs:
assert ref
- assert '\n' not in ref
+ assert b'\n' not in ref
conn.write(ref)
- conn.write('\n')
- conn.write('\n')
+ conn.write(b'\n')
+ conn.write(b'\n')
for ref in refs:
info = conn.readline()
- if info == 'missing\n':
+ if info == b'missing\n':
yield None, None, None, None
continue
- if not (info and info.endswith('\n')):
+ if not (info and info.endswith(b'\n')):
raise ClientError('Hit EOF while looking for object info: %r'
% info)
- oidx, oid_t, size = info.split(' ')
+ oidx, oid_t, size = info.split(b' ')
size = int(size)
cr = chunkyreader(conn, size)
yield oidx, oid_t, size, cr
def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False):
patterns = patterns or tuple()
- self._require_command('refs')
+ self._require_command(b'refs')
self.check_busy()
- self._busy = 'refs'
+ self._busy = b'refs'
conn = self.conn
- conn.write('refs %s %s\n' % (1 if limit_to_heads else 0,
- 1 if limit_to_tags else 0))
+ conn.write(b'refs %d %d\n' % (1 if limit_to_heads else 0,
+ 1 if limit_to_tags else 0))
for pattern in patterns:
- assert '\n' not in pattern
+ assert b'\n' not in pattern
conn.write(pattern)
- conn.write('\n')
- conn.write('\n')
- for line in lines_until_sentinel(conn, '\n', ClientError):
+ conn.write(b'\n')
+ conn.write(b'\n')
+ for line in lines_until_sentinel(conn, b'\n', ClientError):
line = line[:-1]
- oidx, name = line.split(' ')
+ oidx, name = line.split(b' ')
if len(oidx) != 40:
raise ClientError('Invalid object fingerprint in %r' % line)
if not name:
raise ClientError('Invalid reference name in %r' % line)
- yield name, oidx.decode('hex')
+ yield name, unhexlify(oidx)
# FIXME: confusing
not_ok = self.check_ok()
if not_ok:
as a terminator for the entire rev-list result.
"""
- self._require_command('rev-list')
+ self._require_command(b'rev-list')
assert (count is None) or (isinstance(count, Integral))
if format:
- assert '\n' not in format
+ assert b'\n' not in format
assert parse
for ref in refs:
assert ref
- assert '\n' not in ref
+ assert b'\n' not in ref
self.check_busy()
- self._busy = 'rev-list'
+ self._busy = b'rev-list'
conn = self.conn
- conn.write('rev-list\n')
+ conn.write(b'rev-list\n')
if count is not None:
- conn.write(str(count))
- conn.write('\n')
+ conn.write(b'%d' % count)
+ conn.write(b'\n')
if format:
conn.write(format)
- conn.write('\n')
+ conn.write(b'\n')
for ref in refs:
conn.write(ref)
- conn.write('\n')
- conn.write('\n')
+ conn.write(b'\n')
+ conn.write(b'\n')
if not format:
- for line in lines_until_sentinel(conn, '\n', ClientError):
+ for line in lines_until_sentinel(conn, b'\n', ClientError):
line = line.strip()
assert len(line) == 40
yield line
else:
- for line in lines_until_sentinel(conn, '\n', ClientError):
- if not line.startswith('commit '):
+ for line in lines_until_sentinel(conn, b'\n', ClientError):
+ if not line.startswith(b'commit '):
raise ClientError('unexpected line ' + repr(line))
cmt_oidx = line[7:].strip()
assert len(cmt_oidx) == 40
self._not_busy()
def resolve(self, path, parent=None, want_meta=True, follow=False):
- self._require_command('resolve')
+ self._require_command(b'resolve')
self.check_busy()
- self._busy = 'resolve'
+ self._busy = b'resolve'
conn = self.conn
- conn.write('resolve %d\n' % ((1 if want_meta else 0)
- | (2 if follow else 0)
- | (4 if parent else 0)))
+ conn.write(b'resolve %d\n' % ((1 if want_meta else 0)
+ | (2 if follow else 0)
+ | (4 if parent else 0)))
if parent:
vfs.write_resolution(conn, parent)
write_bvec(conn, path)
max_pack_size=max_pack_size,
max_pack_objects=max_pack_objects)
self.file = conn
- self.filename = 'remote socket'
+ self.filename = b'remote socket'
self.suggest_packs = suggest_packs
self.onopen = onopen
self.onclose = onclose
def _end(self, run_midx=True):
assert(run_midx) # We don't support this via remote yet
if self._packopen and self.file:
- self.file.write('\0\0\0\0')
+ self.file.write(b'\0\0\0\0')
self._packopen = False
self.onclose() # Unbusy
self.objcache = None
if not self._packopen:
self._open()
self.ensure_busy()
- data = ''.join(datalist)
+ data = b''.join(datalist)
assert(data)
assert(sha)
crc = zlib.crc32(data) & 0xffffffff
- outbuf = ''.join((struct.pack('!I', len(data) + 20 + 4),
- sha,
- struct.pack('!I', crc),
- data))
+ outbuf = b''.join((struct.pack('!I', len(data) + 20 + 4),
+ sha,
+ struct.pack('!I', crc),
+ data))
try:
(self._bwcount, self._bwtime) = _raw_write_bwlimit(
self.file, outbuf, self._bwcount, self._bwtime)
except IOError as e:
- raise ClientError, e, sys.exc_info()[2]
+ reraise(ClientError(e))
self.outbytes += len(data)
self.count += 1
projects / bup.git / commitdiff