summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
84f4cf0)
The hierarchy level is a more directly useful measurement than the bit count,
although right now neither is used very heavily.
Signed-off-by: Avery Pennarun <apenwarr@gmail.com>
tree = pack_writer.new_tree(shalist)
else:
last = 0
tree = pack_writer.new_tree(shalist)
else:
last = 0
- for (blob, bits) in hashsplit.hashsplit_iter(files,
- keep_boundaries=opt.keep_boundaries,
- progress=prog):
+ it = hashsplit.hashsplit_iter(files,
+ keep_boundaries=opt.keep_boundaries,
+ progress=prog)
+ for (blob, level) in it:
hashsplit.total_split += len(blob)
if opt.copy:
sys.stdout.write(str(blob))
hashsplit.total_split += len(blob)
if opt.copy:
sys.stdout.write(str(blob))
if (!PyArg_ParseTuple(args, "t#", &buf, &len))
return NULL;
out = bupsplit_find_ofs(buf, len, &bits);
if (!PyArg_ParseTuple(args, "t#", &buf, &len))
return NULL;
out = bupsplit_find_ofs(buf, len, &bits);
+ if (out) assert(bits >= BUP_BLOBBITS);
return Py_BuildValue("ii", out, bits);
}
return Py_BuildValue("ii", out, bits);
}
+def _splitbuf(buf, basebits, fanbits):
while 1:
b = buf.peek(buf.used())
(ofs, bits) = _helpers.splitbuf(b)
while 1:
b = buf.peek(buf.used())
(ofs, bits) = _helpers.splitbuf(b)
ofs = BLOB_MAX
if ofs:
buf.eat(ofs)
ofs = BLOB_MAX
if ofs:
buf.eat(ofs)
- yield buffer(b, 0, ofs), bits
+ level = (bits-basebits)//fanbits # integer division
+ yield buffer(b, 0, ofs), level
else:
break
while buf.used() >= BLOB_MAX:
else:
break
while buf.used() >= BLOB_MAX:
def _hashsplit_iter(files, progress):
assert(BLOB_READ_SIZE > BLOB_MAX)
def _hashsplit_iter(files, progress):
assert(BLOB_READ_SIZE > BLOB_MAX)
+ basebits = _helpers.blobbits()
+ fanbits = int(math.log(fanout or 128, 2))
buf = Buf()
for inblock in readfile_iter(files, progress):
buf.put(inblock)
buf = Buf()
for inblock in readfile_iter(files, progress):
buf.put(inblock)
- for buf_and_bits in _splitbuf(buf):
- yield buf_and_bits
+ for buf_and_level in _splitbuf(buf, basebits, fanbits):
+ yield buf_and_level
if buf.used():
yield buf.get(buf.used()), 0
if buf.used():
yield buf.get(buf.used()), 0
return progress(real_filenum, nbytes)
else:
prog = None
return progress(real_filenum, nbytes)
else:
prog = None
- for buf_and_bits in _hashsplit_iter([f], progress=prog):
- yield buf_and_bits
+ for buf_and_level in _hashsplit_iter([f], progress=prog):
+ yield buf_and_level
def hashsplit_iter(files, keep_boundaries, progress):
def hashsplit_iter(files, keep_boundaries, progress):
total_split = 0
def _split_to_blobs(w, files, keep_boundaries, progress):
global total_split
total_split = 0
def _split_to_blobs(w, files, keep_boundaries, progress):
global total_split
- for (blob, bits) in hashsplit_iter(files, keep_boundaries, progress):
+ for (blob, level) in hashsplit_iter(files, keep_boundaries, progress):
sha = w.new_blob(blob)
total_split += len(blob)
if w.outbytes >= max_pack_size or w.count >= max_pack_objects:
w.breakpoint()
if progress_callback:
progress_callback(len(blob))
sha = w.new_blob(blob)
total_split += len(blob)
if w.outbytes >= max_pack_size or w.count >= max_pack_objects:
w.breakpoint()
if progress_callback:
progress_callback(len(blob))
- yield (sha, len(blob), bits)
+ yield (sha, len(blob), level)
sl = _split_to_blobs(w, files, keep_boundaries, progress)
if not fanout:
shal = []
sl = _split_to_blobs(w, files, keep_boundaries, progress)
if not fanout:
shal = []
- for (sha,size,bits) in sl:
+ for (sha,size,level) in sl:
shal.append(('100644', sha, size))
return _make_shalist(shal)[0]
else:
shal.append(('100644', sha, size))
return _make_shalist(shal)[0]
else:
- base_bits = _helpers.blobbits()
- fanout_bits = int(math.log(fanout, 2))
- def bits_to_idx(n):
- assert(n >= base_bits)
- return (n - base_bits)/fanout_bits
- for (sha,size,bits) in sl:
- assert(bits <= 32)
+ for (sha,size,level) in sl:
stacks[0].append(('100644', sha, size))
stacks[0].append(('100644', sha, size))
- if bits > base_bits:
- _squish(w, stacks, bits_to_idx(bits))
+ if level:
+ _squish(w, stacks, level)
#log('stacks: %r\n' % [len(i) for i in stacks])
_squish(w, stacks, len(stacks)-1)
#log('stacks: %r\n' % [len(i) for i in stacks])
#log('stacks: %r\n' % [len(i) for i in stacks])
_squish(w, stacks, len(stacks)-1)
#log('stacks: %r\n' % [len(i) for i in stacks])