self._indexes_synced = 1
+ def _make_objcache(self):
+ ob = self._busy
+ self._busy = None
+ self.sync_indexes()
+ self._busy = ob
+ return git.MultiPackIndex(self.cachedir)
+
def new_packwriter(self):
- assert(self._indexes_synced)
self.check_busy()
self._busy = 'receive-objects'
- self.conn.write('receive-objects\n')
- objcache = git.MultiPackIndex(self.cachedir)
- return git.PackWriter_Remote(self.conn, objcache = objcache,
- onclose = self._not_busy)
+ return PackWriter_Remote(self.conn,
+ objcache_maker = self._make_objcache,
+ onclose = self._not_busy)
def read_ref(self, refname):
self.check_busy()
yield self.conn.read(sz)
self.conn.check_ok()
self._not_busy()
+
+
+class PackWriter_Remote(git.PackWriter):
+ def __init__(self, conn, objcache_maker=None, onclose=None):
+ git.PackWriter.__init__(self, objcache_maker)
+ self.file = conn
+ self.filename = 'remote socket'
+ self.onclose = onclose
+ self._packopen = False
+
+ def _open(self):
+ if not self._packopen:
+ self._make_objcache()
+ self.file.write('receive-objects\n')
+ self._packopen = True
+
+ def _end(self):
+ if self._packopen and self.file:
+ self.file.write('\0\0\0\0')
+ self._packopen = False
+ id = self.file.readline().strip()
+ self.file.check_ok()
+ self.objcache = None
+ return id
+
+ def close(self):
+ id = self._end()
+ self.file = None
+ return id
+
+ def abort(self):
+ raise GitError("don't know how to abort remote pack writing")
+
+ def _raw_write(self, datalist):
+ assert(self.file)
+ if not self._packopen:
+ self._open()
+ data = ''.join(datalist)
+ assert(len(data))
+ self.file.write(struct.pack('!I', len(data)) + data)
+ self.outbytes += len(data)
+ self.count += 1
+
+
if opt.remote:
cli = client.Client(opt.remote)
oldref = refname and cli.read_ref(refname) or None
- cli.sync_indexes()
w = cli.new_packwriter()
else:
cli = None
if not n:
log('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
- w.close()
+ id = w.close()
+ conn.write('%s\n' % id)
+ conn.ok()
return
buf = conn.read(n) # object sizes in bup are reasonably small
#log('read %d bytes\n' % n)
raise Exception('object read: expected %d bytes, got %d\n'
% (n, len(buf)))
w._raw_write([buf])
- w.close()
- conn.ok()
+ # NOTREACHED
def read_ref(conn, refname):
n,name= name of backup set to update (if any)
v,verbose increase log output (can be used more than once)
bench print benchmark timings to stderr
+max-pack-size= maximum bytes in a single pack
+max-pack-objects= maximum number of objects in a single pack
"""
o = options.Options('bup split', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if opt.verbose >= 2:
git.verbose = opt.verbose - 1
opt.bench = 1
+if opt.max_pack_size:
+ hashsplit.max_pack_size = int(opt.max_pack_size)
+if opt.max_pack_objects:
+ hashsplit.max_pack_objects = int(opt.max_pack_objects)
start_time = time.time()
if opt.remote:
cli = client.Client(opt.remote)
oldref = refname and cli.read_ref(refname) or None
- cli.sync_indexes()
w = cli.new_packwriter()
else:
cli = None
class MultiPackIndex:
def __init__(self, dir):
- self.packs = []
+ self.dir = dir
self.also = {}
- for f in os.listdir(dir):
+ self.packs = []
+ for f in os.listdir(self.dir):
if f.endswith('.idx'):
- self.packs.append(PackIndex(os.path.join(dir, f)))
+ self.packs.append(PackIndex(os.path.join(self.dir, f)))
def exists(self, hash):
if hash in self.also:
_typemap = dict(blob=3, tree=2, commit=1, tag=8)
class PackWriter:
- def __init__(self, objcache=None):
+ def __init__(self, objcache_maker=None):
self.count = 0
+ self.outbytes = 0
self.filename = None
self.file = None
- self.objcache = objcache or MultiPackIndex(repo('objects/pack'))
+ self.objcache_maker = objcache_maker
+ self.objcache = None
def __del__(self):
self.close()
+ def _make_objcache(self):
+ if not self.objcache:
+ if self.objcache_maker:
+ self.objcache = self.objcache_maker()
+ else:
+ self.objcache = MultiPackIndex(repo('objects/pack'))
+
def _open(self):
- assert(not self.file)
- self.objcache.zap_also()
- self.filename = repo('objects/bup%d' % os.getpid())
- self.file = open(self.filename + '.pack', 'w+')
- self.file.write('PACK\0\0\0\2\0\0\0\0')
+ if not self.file:
+ self._make_objcache()
+ self.filename = repo('objects/bup%d' % os.getpid())
+ self.file = open(self.filename + '.pack', 'w+')
+ self.file.write('PACK\0\0\0\2\0\0\0\0')
def _raw_write(self, datalist):
- if not self.file:
- self._open()
+ self._open()
f = self.file
for d in datalist:
f.write(d)
+ self.outbytes += len(d)
self.count += 1
def _write(self, bin, type, content):
self._raw_write(out)
return bin
+ def breakpoint(self):
+ id = self._end()
+ self.outbytes = self.count = 0
+ return id
+
def write(self, type, content):
return self._write(calc_hash(type, content), type, content)
def maybe_write(self, type, content):
bin = calc_hash(type, content)
+ if not self.objcache:
+ self._make_objcache()
if not self.objcache.exists(bin):
self._write(bin, type, content)
self.objcache.add(bin)
f.close()
os.unlink(self.filename + '.pack')
- def close(self):
+ def _end(self):
f = self.file
if not f: return None
self.file = None
f.write(sum.digest())
f.close()
+ self.objcache = None
p = subprocess.Popen(['git', 'index-pack', '-v',
'--index-version=2',
os.rename(self.filename + '.idx', nameprefix + '.idx')
return nameprefix
-
-class PackWriter_Remote(PackWriter):
- def __init__(self, conn, objcache=None, onclose=None):
- PackWriter.__init__(self, objcache)
- self.file = conn
- self.filename = 'remote socket'
- self.onclose = onclose
-
- def _open(self):
- assert(not "can't reopen a PackWriter_Remote")
-
def close(self):
- if self.file:
- self.file.write('\0\0\0\0')
- if self.onclose:
- self.onclose()
- self.file = None
-
- def abort(self):
- raise GitError("don't know how to abort remote pack writing")
-
- def _raw_write(self, datalist):
- assert(self.file)
- data = ''.join(datalist)
- assert(len(data))
- self.file.write(struct.pack('!I', len(data)) + data)
+ return self._end()
def _git_date(date):
BLOB_MAX = BLOB_LWM*2
BLOB_HWM = 1024*1024
split_verbosely = 0
+max_pack_size = 1000*1000*1000
+max_pack_objects = 10*1000*1000
class Buf:
def __init__(self):
continue
if blob:
+ if w.outbytes >= max_pack_size or w.count >= max_pack_objects:
+ w.breakpoint()
yield (ofs, len(blob), w.new_blob(blob))
ofs += len(blob)
return self.inp.readline()
def write(self, data):
+ #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
self.outp.write(data)
def ok(self):
self.outp.flush()
rl = ''
for rl in linereader(self.inp):
+ #log('%d got line: %r\n' % (os.getpid(), rl))
if not rl:
continue
elif rl == 'ok':
-import textwrap, getopt
+import textwrap, getopt, re
from helpers import *
class OptDict:
flagl = flags.split(',')
flagl_nice = []
for f in flagl:
+ f_nice = re.sub(r'\W', '_', f)
self._aliases[f] = flagl[0]
+ self._aliases[f_nice] = flagl[0]
self._hasparms[f] = has_parm
if len(f) == 1:
self._shortopts += f + (has_parm and ':' or '')
bup split --bench -b <testfile1 >tags1.tmp
bup split -vvvv -b testfile2 >tags2.tmp
bup split -t testfile2 >tags2t.tmp
-bup split -c testfile2 >tags2c.tmp
+bup split -r "$BUP_DIR" -c testfile2 >tags2c.tmp
diff -u tags1.tmp tags2.tmp || true
wc -c testfile1 testfile2
wc -l tags1.tmp tags2.tmp
bup join $(cat tags1.tmp) >out1.tmp
bup join <tags2.tmp >out2.tmp
bup join <tags2t.tmp >out2t.tmp
-bup join <tags2c.tmp >out2c.tmp
+bup join -r "$BUP_DIR" <tags2c.tmp >out2c.tmp
diff -u testfile1 out1.tmp
diff -u testfile2 out2.tmp
diff -u testfile2 out2t.tmp
projects / bup.git / commitdiff