import os, zlib, time, subprocess, struct, stat, re, tempfile
import heapq
from bup.helpers import *
+from bup import _helpers
MIDX_VERSION = 2
return os.path.join(repodir, sub)
+def auto_midx(objdir):
+ main_exe = os.environ.get('BUP_MAIN_EXE') or sys.argv[0]
+ args = [main_exe, 'midx', '--auto', '--dir', objdir]
+ rv = subprocess.call(args, stdout=open('/dev/null', 'w'))
+ if rv:
+ add_error('%r: returned %d' % (args, rv))
+
+
def mangle_name(name, mode, gitmode):
"""Mangle a file name to present an abstract name for segmented files.
Mangled file names will have the ".bup" extension added to them. If a
"""Object representation of a Git pack index file."""
def __init__(self, filename):
self.name = filename
+ self.idxnames = [self.name]
self.map = mmap_read(open(filename))
assert(str(self.map[0:8]) == '\377tOc\0\0\0\2')
self.fanout = list(struct.unpack('!256I',
return int(self.fanout[255])
-def extract_bits(buf, nbits):
- """Take the first 'nbits' bits from 'buf' and return them as an integer."""
- mask = (1<<nbits) - 1
- v = struct.unpack('!I', buf[0:4])[0]
- v = (v >> (32-nbits)) & mask
- return v
+extract_bits = _helpers.extract_bits
class PackMidx:
self.force_keep = True # new stuff is exciting
return self._init_failed()
- self.bits = struct.unpack('!I', self.map[8:12])[0]
+ self.bits = _helpers.firstword(self.map[8:12])
self.entries = 2**self.bits
self.fanout = buffer(self.map, 12, self.entries*4)
shaofs = 12 + self.entries*4
def _fanget(self, i):
start = i*4
s = self.fanout[start:start+4]
- return struct.unpack('!I', s)[0]
+ return _helpers.firstword(s)
+
+ def _get(self, i):
+ return str(self.shalist[i*20:(i+1)*20])
def exists(self, hash):
"""Return nonempty if the object exists in the index files."""
el = extract_bits(want, self.bits)
if el:
start = self._fanget(el-1)
+ startv = el << (32-self.bits)
else:
start = 0
+ startv = 0
end = self._fanget(el)
+ endv = (el+1) << (32-self.bits)
_total_steps += 1 # lookup table is a step
+ hashv = _helpers.firstword(hash)
+ #print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv)
while start < end:
_total_steps += 1
- mid = start + (end-start)/2
- v = str(self.shalist[mid*20:(mid+1)*20])
+ #print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end)
+ mid = start + (hashv-startv)*(end-start-1)/(endv-startv)
+ #print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end)
+ v = self._get(mid)
+ #print ' %08x' % self._num(v)
if v < want:
start = mid+1
+ startv = _helpers.firstword(v)
elif v > want:
end = mid
+ endv = _helpers.firstword(v)
else: # got it!
return True
return None
any += 1
break
if not any and not ix.force_keep:
- log('midx: removing redundant: %s\n'
- % os.path.basename(ix.name))
+ debug1('midx: removing redundant: %s\n'
+ % os.path.basename(ix.name))
unlink(ix.name)
for f in os.listdir(self.dir):
full = os.path.join(self.dir, f)
ix = PackIdx(full)
d[full] = ix
self.packs = list(set(d.values()))
- log('PackIdxList: using %d index%s.\n'
+ debug1('PackIdxList: using %d index%s.\n'
% (len(self.packs), len(self.packs)!=1 and 'es' or ''))
def add(self, hash):
return name
-def idxmerge(idxlist):
+def open_idx(filename):
+ if filename.endswith('.idx'):
+ return PackIdx(filename)
+ elif filename.endswith('.midx'):
+ return PackMidx(filename)
+ else:
+ raise GitError('idx filenames must end with .idx or .midx')
+
+
+def idxmerge(idxlist, final_progress=True):
"""Generate a list of all the objects reachable in a PackIdxList."""
total = sum(len(i) for i in idxlist)
iters = (iter(i) for i in idxlist)
heapq.heapreplace(heap, (e, it))
else:
heapq.heappop(heap)
- log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
+ if final_progress:
+ log('Reading indexes: %.2f%% (%d/%d), done.\n' % (100, total, total))
class PackWriter:
l.append(msg)
return self.maybe_write('commit', '\n'.join(l))
- def new_commit(self, parent, tree, msg):
+ def new_commit(self, parent, tree, date, msg):
"""Create a commit object in the pack."""
- now = time.time()
userline = '%s <%s@%s>' % (userfullname(), username(), hostname())
commit = self._new_commit(tree, parent,
- userline, now, userline, now,
+ userline, date, userline, date,
msg)
return commit
os.unlink(self.filename + '.map')
os.rename(self.filename + '.pack', nameprefix + '.pack')
os.rename(self.filename + '.idx', nameprefix + '.idx')
+
+ auto_midx(repo('objects/pack'))
return nameprefix
def close(self):
assert(not self.inprogress)
assert(id.find('\n') < 0)
assert(id.find('\r') < 0)
- assert(id[0] != '-')
+ assert(not id.startswith('-'))
self.inprogress = id
self.p.stdin.write('%s\n' % id)
hdr = self.p.stdout.readline()
if hdr.endswith(' missing\n'):
+ self.inprogress = None
raise KeyError('blob %r is missing' % id)
spl = hdr.split(' ')
if len(spl) != 3 or len(spl[0]) != 40:
projects / bup.git / blobdiff