-import mmap, struct
+from __future__ import absolute_import, print_function
+import glob, mmap, os, struct
from bup import _helpers
+from bup.compat import range
from bup.helpers import log, mmap_read
+from bup.io import path_msg
MIDX_VERSION = 4
self.name = filename
self.force_keep = False
self.map = None
- assert(filename.endswith('.midx'))
+ assert(filename.endswith(b'.midx'))
self.map = mmap_read(open(filename))
- if str(self.map[0:4]) != 'MIDX':
- log('Warning: skipping: invalid MIDX header in %r\n' % filename)
+ if self.map[0:4] != b'MIDX':
+ log('Warning: skipping: invalid MIDX header in %r\n'
+ % path_msg(filename))
self.force_keep = True
return self._init_failed()
ver = struct.unpack('!I', self.map[4:8])[0]
if ver < MIDX_VERSION:
log('Warning: ignoring old-style (v%d) midx %r\n'
- % (ver, filename))
+ % (ver, path_msg(filename)))
self.force_keep = False # old stuff is boring
return self._init_failed()
if ver > MIDX_VERSION:
log('Warning: ignoring too-new (v%d) midx %r\n'
- % (ver, filename))
+ % (ver, path_msg(filename)))
self.force_keep = True # new stuff is exciting
return self._init_failed()
self.bits = _helpers.firstword(self.map[8:12])
self.entries = 2**self.bits
- self.fanout = buffer(self.map, 12, self.entries*4)
- self.sha_ofs = 12 + self.entries*4
- self.nsha = nsha = self._fanget(self.entries-1)
- self.shatable = buffer(self.map, self.sha_ofs, nsha*20)
- self.which_ofs = self.sha_ofs + 20*nsha
- self.whichlist = buffer(self.map, self.which_ofs, nsha*4)
- self.idxnames = str(self.map[self.which_ofs + 4*nsha:]).split('\0')
+ self.fanout_ofs = 12
+ # fanout len is self.entries * 4
+ self.sha_ofs = self.fanout_ofs + self.entries * 4
+ self.nsha = self._fanget(self.entries - 1)
+ # sha table len is self.nsha * 20
+ self.which_ofs = self.sha_ofs + 20 * self.nsha
+ # which len is self.nsha * 4
+ self.idxnames = self.map[self.which_ofs + 4 * self.nsha:].split(b'\0')
def __del__(self):
self.close()
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
def _init_failed(self):
self.bits = 0
self.entries = 1
- self.fanout = buffer('\0\0\0\0')
- self.shatable = buffer('\0'*20)
self.idxnames = []
def _fanget(self, i):
- start = i*4
- s = self.fanout[start:start+4]
- return _helpers.firstword(s)
+ if i >= self.entries * 4 or i < 0:
+ raise IndexError('invalid midx index %d' % i)
+ ofs = self.fanout_ofs + i * 4
+ return _helpers.firstword(self.map[ofs : ofs + 4])
def _get(self, i):
- return str(self.shatable[i*20:(i+1)*20])
+ if i >= self.nsha or i < 0:
+ raise IndexError('invalid midx index %d' % i)
+ ofs = self.sha_ofs + i * 20
+ return self.map[ofs : ofs + 20]
def _get_idx_i(self, i):
- return struct.unpack('!I', self.whichlist[i*4:(i+1)*4])[0]
+ if i >= self.nsha or i < 0:
+ raise IndexError('invalid midx index %d' % i)
+ ofs = self.which_ofs + i * 4
+ return struct.unpack_from('!I', self.map, offset=ofs)[0]
def _get_idxname(self, i):
return self.idxnames[self._get_idx_i(i)]
def close(self):
if self.map is not None:
+ self.fanout = self.shatable = self.whichlist = self.idxnames = None
self.map.close()
self.map = None
"""Return nonempty if the object exists in the index files."""
global _total_searches, _total_steps
_total_searches += 1
- want = str(hash)
+ want = hash
el = extract_bits(want, self.bits)
if el:
start = self._fanget(el-1)
while start < end:
_total_steps += 1
#print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end)
- mid = start + (hashv-startv)*(end-start-1)/(endv-startv)
+ mid = start + (hashv - startv) * (end - start - 1) // (endv - startv)
#print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end)
v = self._get(mid)
#print ' %08x' % self._num(v)
return None
def __iter__(self):
- for i in xrange(self._fanget(self.entries-1)):
- yield buffer(self.shatable, i*20, 20)
+ start = self.sha_ofs
+ for ofs in range(start, start + self.nsha * 20, 20):
+ yield self.map[ofs : ofs + 20]
def __len__(self):
- return int(self._fanget(self.entries-1))
+ return int(self.nsha)
+def clear_midxes(dir=None):
+ for midx in glob.glob(os.path.join(dir, b'*.midx')):
+ os.unlink(midx)