Obviously this is dramatically faster.
Signed-off-by: Brandon Low <lostlogic@lostlogicx.com>
#!/usr/bin/env python
import sys, math, struct, glob, resource
-import tempfile, shutil
-from bup import options, git
+import tempfile
+from bup import options, git, _helpers
from bup.helpers import *
PAGE_SIZE=4096
mf -= 6 # minimum safety margin
return mf
-
-def merge_into(tf_sha, tf_nmap, idxlist, bits, entries, total):
- prefix = 0
- it = git.idxmerge(idxlist, final_progress=False, total=total)
- for i, (e, idx) in enumerate(it):
- new_prefix = git.extract_bits(e, bits)
- if new_prefix != prefix:
- for p in xrange(prefix, new_prefix):
- yield i
- prefix = new_prefix
- tf_sha.write(e)
- tf_nmap.write(struct.pack('!I', idx))
- i += 1
- for p in xrange(prefix, entries):
- yield i
-
+merge_into = _helpers.merge_into
def _do_midx(outdir, outfilename, infilenames, prefixstr):
if not outfilename:
allfilenames = []
for name in infilenames:
ix = git.open_idx(name)
- inp.append(ix.iter_with_idx_i(len(allfilenames)))
+ inp.append((
+ ix.map,
+ len(ix),
+ ix.sha_ofs,
+ isinstance(ix, git.PackMidx) and ix.idxname_ofs or 0,
+ len(allfilenames),
+ ))
for n in ix.idxnames:
allfilenames.append(os.path.basename(n))
total += len(ix)
+ inp.sort(lambda x,y: cmp(str(y[0][y[2]:y[2]+20]),str(x[0][x[2]:x[2]+20])))
log('midx: %screating from %d files (%d objects).\n'
% (prefixstr, len(infilenames), total))
os.unlink(outfilename)
except OSError:
pass
- f = open(outfilename + '.tmp', 'w+')
+ f = open(outfilename + '.tmp', 'w+b')
f.write('MIDX')
f.write(struct.pack('!II', git.MIDX_VERSION, bits))
assert(f.tell() == 12)
- tf_sha = tempfile.TemporaryFile(dir=outdir)
- tf_nmap = tempfile.TemporaryFile(dir=outdir)
- for t in merge_into(tf_sha, tf_nmap, inp, bits, entries, total):
- f.write(struct.pack('!I', t))
- assert(f.tell() == 12 + 4*entries)
+ f.truncate(12 + 4*entries + 20*total + 4*total)
- tf_sha.seek(0)
- shutil.copyfileobj(tf_sha, f)
- tf_sha.close()
- assert(f.tell() == 12 + 4*entries + 20*t) # t may be < total due to dupes
+ fmap = mmap_readwrite(f, close=False)
- tf_nmap.seek(0)
- shutil.copyfileobj(tf_nmap, f)
- tf_nmap.close()
- assert(f.tell() == 12 + 4*entries + 24*t) # t may be < total due to dupes
+ count = merge_into(fmap, bits, total, inp)
+ fmap.flush()
+ fmap.close()
+ f.seek(0, os.SEEK_END)
f.write('\0'.join(allfilenames))
f.close()
os.rename(outfilename + '.tmp', outfilename)
}
+static uint32_t _extract_bits(unsigned char *buf, int nbits)
+{
+ uint32_t v, mask;
+
+ mask = (1<<nbits) - 1;
+ v = ntohl(*(uint32_t *)buf);
+ v = (v >> (32-nbits)) & mask;
+ return v;
+}
static PyObject *extract_bits(PyObject *self, PyObject *args)
{
unsigned char *buf = NULL;
int len = 0, nbits = 0;
- uint32_t v, mask;
if (!PyArg_ParseTuple(args, "t#i", &buf, &len, &nbits))
return NULL;
if (len < 4)
return NULL;
- mask = (1<<nbits) - 1;
- v = ntohl(*(uint32_t *)buf);
- v = (v >> (32-nbits)) & mask;
- return PyLong_FromUnsignedLong(v);
+ return PyLong_FromUnsignedLong(_extract_bits(buf, nbits));
+}
+
+
+struct sha {
+ unsigned char bytes[20];
+};
+struct idx {
+ unsigned char *map;
+ struct sha *cur;
+ struct sha *end;
+ uint32_t *cur_name;
+ Py_ssize_t bytes;
+ long name_base;
+};
+
+
+static int _cmp_sha(const struct sha *sha1, const struct sha *sha2)
+{
+ int i;
+ for (i = 0; i < 20; i++)
+ if (sha1->bytes[i] != sha2->bytes[i])
+ return sha1->bytes[i] - sha2->bytes[i];
+ return 0;
+}
+
+
+static void _fix_idx_order(struct idx **idxs, long *last_i)
+{
+ struct idx *idx;
+ int low, mid, high, c = 0;
+
+ idx = idxs[*last_i];
+ if (idxs[*last_i]->cur >= idxs[*last_i]->end)
+ {
+ idxs[*last_i] = NULL;
+ PyMem_Free(idx);
+ --*last_i;
+ return;
+ }
+ if (*last_i == 0)
+ return;
+
+ low = *last_i-1;
+ mid = *last_i;
+ high = 0;
+ while (low >= high)
+ {
+ mid = (low + high) / 2;
+ c = _cmp_sha(idx->cur, idxs[mid]->cur);
+ if (c < 0)
+ high = mid + 1;
+ else if (c > 0)
+ low = mid - 1;
+ else
+ break;
+ }
+ if (c < 0)
+ ++mid;
+ if (mid == *last_i)
+ return;
+ memmove(&idxs[mid+1], &idxs[mid], (*last_i-mid)*sizeof(struct idx *));
+ idxs[mid] = idx;
+}
+
+
+static uint32_t _get_idx_i(struct idx *idx)
+{
+ if (idx->cur_name == NULL)
+ return idx->name_base;
+ return ntohl(*idx->cur_name) + idx->name_base;
+}
+
+
+static PyObject *merge_into(PyObject *self, PyObject *args)
+{
+ PyObject *ilist = NULL;
+ unsigned char *fmap = NULL;
+ struct sha *sha_ptr, *last = NULL;
+ uint32_t *table_ptr, *name_ptr;
+ struct idx **idxs = NULL;
+ int flen = 0, bits = 0, i;
+ uint32_t total, count, prefix;
+ Py_ssize_t num_i;
+ long last_i;
+
+ if (!PyArg_ParseTuple(args, "w#iIO", &fmap, &flen, &bits, &total, &ilist))
+ return NULL;
+
+ num_i = PyList_Size(ilist);
+ idxs = (struct idx **)PyMem_Malloc(num_i * sizeof(struct idx *));
+
+ for (i = 0; i < num_i; i++)
+ {
+ long len, sha_ofs, name_map_ofs;
+ idxs[i] = (struct idx *)PyMem_Malloc(sizeof(struct idx));
+ PyObject *itup = PyList_GetItem(ilist, i);
+ PyObject_AsReadBuffer(PyTuple_GetItem(itup, 0),
+ (const void **)&idxs[i]->map, &idxs[i]->bytes);
+ len = PyInt_AsLong(PyTuple_GetItem(itup, 1));
+ sha_ofs = PyInt_AsLong(PyTuple_GetItem(itup, 2));
+ name_map_ofs = PyInt_AsLong(PyTuple_GetItem(itup, 3));
+ idxs[i]->cur = (struct sha *)&idxs[i]->map[sha_ofs];
+ idxs[i]->end = &idxs[i]->cur[len];
+ idxs[i]->cur_name = (uint32_t *)&idxs[i]->map[name_map_ofs];
+ idxs[i]->name_base = PyInt_AsLong(PyTuple_GetItem(itup, 4));
+ }
+ table_ptr = (uint32_t *)&fmap[12];
+ sha_ptr = (struct sha *)&table_ptr[1<<bits];
+ name_ptr = (uint32_t *)&sha_ptr[total];
+
+ last_i = num_i-1;
+ count = 0;
+ prefix = 0;
+ while (last_i >= 0)
+ {
+ struct idx *idx;
+ uint32_t new_prefix;
+ if (count % 102424 == 0)
+ PySys_WriteStderr("midx: writing %.2f%% (%d/%d)\r",
+ count*100.0/total, count, total);
+ idx = idxs[last_i];
+ new_prefix = _extract_bits((unsigned char *)idx->cur, bits);
+ while (prefix < new_prefix)
+ table_ptr[prefix++] = htonl(count);
+ if (last == NULL || _cmp_sha(last, idx->cur) != 0)
+ {
+ memcpy(sha_ptr++, idx->cur, 20);
+ *name_ptr++ = htonl(_get_idx_i(idx));
+ last = idx->cur;
+ }
+ ++idx->cur;
+ if (idx->cur_name != NULL)
+ ++idx->cur_name;
+ _fix_idx_order(idxs, &last_i);
+ ++count;
+ }
+ table_ptr[prefix] = htonl(count);
+
+ PyMem_Free(idxs);
+ return PyLong_FromUnsignedLong(count);
}
"Add an object to a bloom filter of 2^nbits bytes" },
{ "extract_bits", extract_bits, METH_VARARGS,
"Take the first 'nbits' bits from 'buf' and return them as an int." },
+ { "merge_into", merge_into, METH_VARARGS,
+ "Merges a bunch of idx and midx files into a single midx." },
{ "write_random", write_random, METH_VARARGS,
"Write random bytes to the given file descriptor" },
{ "random_sha", random_sha, METH_VARARGS,
return mid
return None
- def iter_with_idx_i(self, idx_i):
- for e in self:
- yield e, idx_i
-
class PackIdxV1(PackIdx):
"""Object representation of a Git pack index (version 1) file."""
str(buffer(self.map, 0, 256*4))))
self.fanout.append(0) # entry "-1"
nsha = self.fanout[255]
- self.shatable = buffer(self.map, 256*4, nsha*24)
+ self.sha_ofs = 256*4
+ self.shatable = buffer(self.map, self.sha_ofs, nsha*24)
def _ofs_from_idx(self, idx):
return struct.unpack('!I', str(self.shatable[idx*24 : idx*24+4]))[0]
str(buffer(self.map, 8, 256*4))))
self.fanout.append(0) # entry "-1"
nsha = self.fanout[255]
- self.shatable = buffer(self.map, 8 + 256*4, nsha*20)
+ self.sha_ofs = 8 + 256*4
+ self.shatable = buffer(self.map, self.sha_ofs, nsha*20)
self.ofstable = buffer(self.map,
- 8 + 256*4 + nsha*20 + nsha*4,
+ self.sha_ofs + nsha*20 + nsha*4,
nsha*4)
self.ofs64table = buffer(self.map,
8 + 256*4 + nsha*20 + nsha*4 + nsha*4)
self.bits = _helpers.firstword(self.map[8:12])
self.entries = 2**self.bits
self.fanout = buffer(self.map, 12, self.entries*4)
- shaofs = 12 + self.entries*4
+ self.sha_ofs = 12 + self.entries*4
self.nsha = nsha = self._fanget(self.entries-1)
- self.shatable = buffer(self.map, shaofs, nsha*20)
- self.whichlist = buffer(self.map, shaofs + nsha*20, nsha*4)
- self.idxnames = str(self.map[shaofs + 24*nsha:]).split('\0')
+ self.shatable = buffer(self.map, self.sha_ofs, nsha*20)
+ self.whichlist = buffer(self.map, self.sha_ofs + nsha*20, nsha*4)
+ self.idxname_ofs = self.sha_ofs + 24*nsha
+ self.idxnames = str(self.map[self.idxname_ofs:]).split('\0')
def _init_failed(self):
self.bits = 0
return want_source and self._get_idxname(mid) or True
return None
- def iter_with_idx_i(self, ofs):
- for i in xrange(self._fanget(self.entries-1)):
- yield buffer(self.shatable, i*20, 20), ofs+self._get_idx_i(i)
-
def __iter__(self):
for i in xrange(self._fanget(self.entries-1)):
yield buffer(self.shatable, i*20, 20)
raise GitError('idx filenames must end with .idx or .midx')
-def idxmerge(idxlist, final_progress=True, total=None):
+def idxmerge(idxlist, final_progress=True):
"""Generate a list of all the objects reachable in a PackIdxList."""
def pfunc(count, total):
progress('Reading indexes: %.2f%% (%d/%d)\r'
def pfinal(count, total):
if final_progress:
log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
- return merge_iter(idxlist, 10024, pfunc, pfinal, total=total)
+ return merge_iter(idxlist, 10024, pfunc, pfinal)
def _make_objcache():
return None
-def merge_iter(iters, pfreq, pfunc, pfinal, key=None, total=None):
+def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
if key:
samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
else:
samekey = operator.eq
count = 0
- total = total or sum(len(it) for it in iters)
+ total = sum(len(it) for it in iters)
iters = (iter(it) for it in iters)
heap = ((next(it),it) for it in iters)
heap = [(e,it) for e,it in heap if e]
projects / bup.git / commitdiff