Split the idx file writing into a separate class, to make that
kind of action available separately. This will be useful for the
next patch where we use it to test some idx/midx code.
In the future, it'll also be useful for encrypted repositories
since the idx format there will be useful for local caching to
take advantage of midx and bloom code as is, but the packwriter
will of course not be useful.
Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
Reviewed-by: Rob Browning <rlb@defaultvalue.org>
assert name.endswith(b'.pack')
self.filename = name[:-5]
self.file.write(b'PACK\0\0\0\2\0\0\0\0')
assert name.endswith(b'.pack')
self.filename = name[:-5]
self.file.write(b'PACK\0\0\0\2\0\0\0\0')
- self.idx = list(list() for i in range(256))
+ self.idx = PackIdxV2Writer()
def _raw_write(self, datalist, sha):
self._open()
def _raw_write(self, datalist, sha):
self._open()
def _update_idx(self, sha, crc, size):
assert(sha)
if self.idx:
def _update_idx(self, sha, crc, size):
assert(sha)
if self.idx:
- self.idx[byte_int(sha[0])].append((sha, crc,
- self.file.tell() - size))
+ self.idx.add(sha, crc, self.file.tell() - size)
def _write(self, sha, type, content):
if verbose:
def _write(self, sha, type, content):
if verbose:
- obj_list_sha = self._write_pack_idx_v2(self.filename + b'.idx', idx,
- packbin)
+ obj_list_sha = idx.write(self.filename + b'.idx', packbin)
nameprefix = os.path.join(self.repo_dir,
b'objects/pack/pack-' + obj_list_sha)
if os.path.exists(self.filename + b'.map'):
nameprefix = os.path.join(self.repo_dir,
b'objects/pack/pack-' + obj_list_sha)
if os.path.exists(self.filename + b'.map'):
"""Close the pack file and move it to its definitive path."""
return self._end(run_midx=run_midx)
"""Close the pack file and move it to its definitive path."""
return self._end(run_midx=run_midx)
- def _write_pack_idx_v2(self, filename, idx, packbin):
+
+class PackIdxV2Writer:
+ def __init__(self):
+ self.idx = list(list() for i in range(256))
+ self.count = 0
+
+ def add(self, sha, crc, offs):
+ assert(sha)
+ self.count += 1
+ self.idx[byte_int(sha[0])].append((sha, crc, offs))
+
+ def write(self, filename, packbin):
+ for section in self.idx:
for entry in section:
if entry[2] >= 2**31:
ofs64_count += 1
for entry in section:
if entry[2] >= 2**31:
ofs64_count += 1
fdatasync(idx_f.fileno())
idx_map = mmap_readwrite(idx_f, close=False)
try:
fdatasync(idx_f.fileno())
idx_map = mmap_readwrite(idx_f, close=False)
try:
- count = _helpers.write_idx(filename, idx_map, idx, self.count)
+ count = _helpers.write_idx(filename, idx_map, self.idx,
+ self.count)
assert(count == self.count)
idx_map.flush()
finally:
assert(count == self.count)
idx_map.flush()
finally:
idx_sum.update(b)
obj_list_sum = Sha1()
idx_sum.update(b)
obj_list_sum = Sha1()
- for b in chunkyreader(idx_f, 20*self.count):
+ for b in chunkyreader(idx_f, 20 * self.count):
idx_sum.update(b)
obj_list_sum.update(b)
namebase = hexlify(obj_list_sum.digest())
idx_sum.update(b)
obj_list_sum.update(b)
namebase = hexlify(obj_list_sum.digest())
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
with test_tempdir(b'bup-tgit-') as tmpdir:
environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
git.init_repo(bupdir)
+ idx = git.PackIdxV2Writer()
obj_bin = struct.pack('!IIIII',
0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
obj2_bin = struct.pack('!IIIII',
obj_bin = struct.pack('!IIIII',
0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
obj2_bin = struct.pack('!IIIII',
0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
pack_bin = struct.pack('!IIIII',
0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
pack_bin = struct.pack('!IIIII',
0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
- idx = list(list() for i in range(256))
- idx[0].append((obj_bin, 1, 0xfffffffff))
- idx[0x11].append((obj2_bin, 2, 0xffffffffff))
- idx[0x22].append((obj3_bin, 3, 0xff))
- w.count = 3
+ idx.add(obj_bin, 1, 0xfffffffff)
+ idx.add(obj2_bin, 2, 0xffffffffff)
+ idx.add(obj3_bin, 3, 0xff)
name = tmpdir + b'/tmp.idx'
name = tmpdir + b'/tmp.idx'
- r = w._write_pack_idx_v2(name, idx, pack_bin)
+ r = idx.write(name, pack_bin)
i = git.PackIdxV2(name, open(name, 'rb'))
WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
i = git.PackIdxV2(name, open(name, 'rb'))
WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)