-import re, struct, errno, time
+import re, struct, errno, time, zlib
from bup import git, ssh
from bup.helpers import *
bwcount = len(sub) # might be less than 4096
bwtime = next
return (bwcount, bwtime)
-
+
+
+def parse_remote(remote):
+ protocol = r'([a-z]+)://'
+ host = r'(?P<sb>\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])'
+ port = r'(?::(\d+))?'
+ path = r'(/.*)?'
+ url_match = re.match(
+ '%s(?:%s%s)?%s' % (protocol, host, port, path), remote, re.I)
+ if url_match:
+ assert(url_match.group(1) in ('ssh', 'bup', 'file'))
+ return url_match.group(1,3,4,5)
+ else:
+ rs = remote.split(':', 1)
+ if len(rs) == 1 or rs[0] in ('', '-'):
+ return 'file', None, None, rs[-1]
+ else:
+ return 'ssh', rs[0], None, rs[1]
+
class Client:
def __init__(self, remote, create=False):
- self._busy = self.conn = self.p = self.pout = self.pin = None
+ self._busy = self.conn = None
+ self.sock = self.p = self.pout = self.pin = None
is_reverse = os.environ.get('BUP_SERVER_REVERSE')
if is_reverse:
assert(not remote)
remote = '%s:' % is_reverse
- rs = remote.split(':', 1)
- if len(rs) == 1:
- (host, dir) = (None, remote)
- else:
- (host, dir) = rs
- (self.host, self.dir) = (host, dir)
+ (self.protocol, self.host, self.port, self.dir) = parse_remote(remote)
self.cachedir = git.repo('index-cache/%s'
% re.sub(r'[^@\w]', '_',
- "%s:%s" % (host, dir)))
+ "%s:%s" % (self.host, self.dir)))
if is_reverse:
self.pout = os.fdopen(3, 'rb')
self.pin = os.fdopen(4, 'wb')
+ self.conn = Conn(self.pout, self.pin)
else:
- try:
- self.p = ssh.connect(host, 'server')
- self.pout = self.p.stdout
- self.pin = self.p.stdin
- except OSError, e:
- raise ClientError, 'connect: %s' % e, sys.exc_info()[2]
- self.conn = Conn(self.pout, self.pin)
- if dir:
- dir = re.sub(r'[\r\n]', ' ', dir)
+ if self.protocol in ('ssh', 'file'):
+ try:
+ # FIXME: ssh and file shouldn't use the same module
+ self.p = ssh.connect(self.host, self.port, 'server')
+ self.pout = self.p.stdout
+ self.pin = self.p.stdin
+ self.conn = Conn(self.pout, self.pin)
+ except OSError, e:
+ raise ClientError, 'connect: %s' % e, sys.exc_info()[2]
+ elif self.protocol == 'bup':
+ self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.sock.connect((self.host, atoi(self.port) or 1982))
+ self.sockw = self.sock.makefile('wb')
+ self.conn = DemuxConn(self.sock.fileno(), self.sockw)
+ if self.dir:
+ self.dir = re.sub(r'[\r\n]', ' ', self.dir)
if create:
- self.conn.write('init-dir %s\n' % dir)
+ self.conn.write('init-dir %s\n' % self.dir)
else:
- self.conn.write('set-dir %s\n' % dir)
+ self.conn.write('set-dir %s\n' % self.dir)
self.check_ok()
- self.sync_indexes_del()
+ self.sync_indexes()
def __del__(self):
try:
def close(self):
if self.conn and not self._busy:
self.conn.write('quit\n')
- if self.pin and self.pout:
+ if self.pin:
self.pin.close()
- while self.pout.read(65536):
- pass
+ if self.sock and self.sockw:
+ self.sockw.close()
+ self.sock.shutdown(socket.SHUT_WR)
+ if self.conn:
+ self.conn.close()
+ if self.pout:
self.pout.close()
+ if self.sock:
+ self.sock.close()
if self.p:
self.p.wait()
rv = self.p.wait()
if rv:
raise ClientError('server tunnel returned exit code %d' % rv)
self.conn = None
- self.p = self.pin = self.pout = None
+ self.sock = self.p = self.pin = self.pout = None
def check_ok(self):
if self.p:
def _not_busy(self):
self._busy = None
- def sync_indexes_del(self):
+ def sync_indexes(self):
self.check_busy()
conn = self.conn
+ mkdirp(self.cachedir)
+ # All cached idxs are extra until proven otherwise
+ extra = set()
+ for f in os.listdir(self.cachedir):
+ debug1('%s\n' % f)
+ if f.endswith('.idx'):
+ extra.add(f)
+ needed = set()
conn.write('list-indexes\n')
- packdir = git.repo('objects/pack')
- all = {}
- needed = {}
for line in linereader(conn):
if not line:
break
- all[line] = 1
assert(line.find('/') < 0)
- if not os.path.exists(os.path.join(self.cachedir, line)):
- needed[line] = 1
+ parts = line.split(' ')
+ idx = parts[0]
+ if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
+ # If the server requests that we load an idx and we don't
+ # already have a copy of it, it is needed
+ needed.add(idx)
+ # Any idx that the server has heard of is proven not extra
+ extra.discard(idx)
+
self.check_ok()
+ debug1('client: removing extra indexes: %s\n' % extra)
+ for idx in extra:
+ os.unlink(os.path.join(self.cachedir, idx))
+ debug1('client: server requested load of: %s\n' % needed)
+ for idx in needed:
+ self.sync_index(idx)
+ git.auto_midx(self.cachedir)
- mkdirp(self.cachedir)
- for f in os.listdir(self.cachedir):
- if f.endswith('.idx') and not f in all:
- debug1('client: pruning old index: %r\n' % f)
- os.unlink(os.path.join(self.cachedir, f))
def sync_index(self, name):
#debug1('requesting %r\n' % name)
self.check_ok()
f.close()
os.rename(fn + '.tmp', fn)
- git.auto_midx(self.cachedir)
def _make_objcache(self):
- ob = self._busy
- self._busy = None
- #self.sync_indexes()
- self._busy = ob
return git.PackIdxList(self.cachedir)
- def _suggest_pack(self, indexname):
- debug1('client: received index suggestion: %s\n' % indexname)
+ def _suggest_packs(self):
ob = self._busy
if ob:
- assert(ob == 'receive-objects')
- self.conn.write('\xff\xff\xff\xff') # suspend receive-objects
+ assert(ob == 'receive-objects-v2')
+ self.conn.write('\xff\xff\xff\xff') # suspend receive-objects-v2
+ suggested = []
+ for line in linereader(self.conn):
+ if not line:
+ break
+ debug2('%s\n' % line)
+ if line.startswith('index '):
+ idx = line[6:]
+ debug1('client: received index suggestion: %s\n' % idx)
+ suggested.append(idx)
+ else:
+ assert(line.endswith('.idx'))
+ debug1('client: completed writing pack, idx: %s\n' % line)
+ suggested.append(line)
+ self.check_ok()
+ if ob:
self._busy = None
- self.conn.drain_and_check_ok()
- self.sync_index(indexname)
+ for idx in suggested:
+ self.sync_index(idx)
+ git.auto_midx(self.cachedir)
if ob:
self._busy = ob
- self.conn.write('receive-objects\n')
+ self.conn.write('%s\n' % ob)
+ return idx
def new_packwriter(self):
self.check_busy()
def _set_busy():
- self._busy = 'receive-objects'
- self.conn.write('receive-objects\n')
+ self._busy = 'receive-objects-v2'
+ self.conn.write('receive-objects-v2\n')
return PackWriter_Remote(self.conn,
objcache_maker = self._make_objcache,
- suggest_pack = self._suggest_pack,
+ suggest_packs = self._suggest_packs,
onopen = _set_busy,
onclose = self._not_busy,
ensure_busy = self.ensure_busy)
class PackWriter_Remote(git.PackWriter):
- def __init__(self, conn, objcache_maker, suggest_pack,
+ def __init__(self, conn, objcache_maker, suggest_packs,
onopen, onclose,
ensure_busy):
git.PackWriter.__init__(self, objcache_maker)
self.file = conn
self.filename = 'remote socket'
- self.suggest_pack = suggest_pack
+ self.suggest_packs = suggest_packs
self.onopen = onopen
self.onclose = onclose
self.ensure_busy = ensure_busy
def _open(self):
if not self._packopen:
- self._make_objcache()
- if self.onopen:
- self.onopen()
+ self.onopen()
self._packopen = True
def _end(self):
if self._packopen and self.file:
self.file.write('\0\0\0\0')
self._packopen = False
- while True:
- line = self.file.readline().strip()
- if line.startswith('index '):
- pass
- else:
- break
- id = line
- self.file.check_ok()
+ self.onclose() # Unbusy
self.objcache = None
- if self.onclose:
- self.onclose()
- if id and self.suggest_pack:
- self.suggest_pack(id)
- return id
+ return self.suggest_packs() # Returns last idx received
def close(self):
id = self._end()
def abort(self):
raise GitError("don't know how to abort remote pack writing")
- def _raw_write(self, datalist):
+ def _raw_write(self, datalist, sha):
assert(self.file)
if not self._packopen:
self._open()
- if self.ensure_busy:
- self.ensure_busy()
+ self.ensure_busy()
data = ''.join(datalist)
- assert(len(data))
- outbuf = struct.pack('!I', len(data)) + data
- (self._bwcount, self._bwtime) = \
- _raw_write_bwlimit(self.file, outbuf, self._bwcount, self._bwtime)
+ assert(data)
+ assert(sha)
+ crc = zlib.crc32(data) & 0xffffffff
+ outbuf = ''.join((struct.pack('!I', len(data) + 20 + 4),
+ sha,
+ struct.pack('!I', crc),
+ data))
+ try:
+ (self._bwcount, self._bwtime) = _raw_write_bwlimit(
+ self.file, outbuf, self._bwcount, self._bwtime)
+ except IOError, e:
+ raise ClientError, e, sys.exc_info()[2]
self.outbytes += len(data)
self.count += 1
if self.file.has_input():
- line = self.file.readline().strip()
- assert(line.startswith('index '))
- idxname = line[6:]
- if self.suggest_pack:
- self.suggest_pack(idxname)
- self.objcache.refresh()
+ self.suggest_packs()
+ self.objcache.refresh()
+
+ return sha, crc
projects / bup.git / blobdiff