idx = os.path.join(path, idx)
log('bloom: bloom file: %s\n' % path_msg(rbloomfilename))
log('bloom: checking %s\n' % path_msg(ridx))
- for objsha in git.open_idx(idx):
- if not b.exists(objsha):
- add_error('bloom: ERROR: object %s missing' % hexstr(objsha))
+ with git.open_idx(idx) as oids:
+ for oid in oids:
+ if not b.exists(oid):
+ add_error('bloom: ERROR: object %s missing' % hexstr(oid))
_first = None
rest_count = 0
for i, name in enumerate(glob.glob(b'%s/*.idx' % path)):
progress('bloom: counting: %d\r' % i)
- ix = git.open_idx(name)
- ixbase = os.path.basename(name)
- if b and (ixbase in b.idxnames):
- rest.append(name)
- rest_count += len(ix)
- else:
- add.append(name)
- add_count += len(ix)
+ with git.open_idx(name) as ix:
+ ixbase = os.path.basename(name)
+ if b and (ixbase in b.idxnames):
+ rest.append(name)
+ rest_count += len(ix)
+ else:
+ add.append(name)
+ add_count += len(ix)
if not add:
debug1("bloom: nothing to do.\n")
count = 0
icount = 0
for name in add:
- ix = git.open_idx(name)
- qprogress('bloom: writing %.2f%% (%d/%d objects)\r'
- % (icount*100.0/add_count, icount, add_count))
- b.add_idx(ix)
- count += 1
- icount += len(ix)
+ with git.open_idx(name) as ix:
+ qprogress('bloom: writing %.2f%% (%d/%d objects)\r'
+ % (icount*100.0/add_count, icount, add_count))
+ b.add_idx(ix)
+ count += 1
+ icount += len(ix)
finally: # This won't handle pending exceptions correctly in py2
# Currently, there's an open file object for tfname inside b.
ix = git.open_idx(name)
except git.GitError as e:
add_error('%r: %s' % (name, e))
+ ix.close()
continue
- if len(opt.find) == 40:
- if ix.exists(bin):
- out.write(b'%s %s\n' % (name, find))
- else:
- # slow, exhaustive search
- for _i in ix:
- i = hexlify(_i)
- if i.startswith(find):
- out.write(b'%s %s\n' % (name, i))
- qprogress('Searching: %d\r' % count)
- count += 1
+ with ix:
+ if len(opt.find) == 40:
+ if ix.exists(bin):
+ out.write(b'%s %s\n' % (name, find))
+ else:
+ # slow, exhaustive search
+ for _i in ix:
+ i = hexlify(_i)
+ if i.startswith(find):
+ out.write(b'%s %s\n' % (name, i))
+ qprogress('Searching: %d\r' % count)
+ count += 1
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
import glob, os, math, resource, struct, sys
from bup import options, git, midx, _helpers, xstat
-from bup.compat import argv_bytes, hexstr, range
+from bup.compat import ExitStack, argv_bytes, hexstr, range
from bup.helpers import (Sha1, add_error, atomically_replaced_file, debug1, fdatasync,
log, mmap_readwrite, qprogress,
saved_errors, unlink)
except git.GitError as e:
add_error('%s: %s' % (path_msg(name), e))
return
- for count,subname in enumerate(ix.idxnames):
- sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
- for ecount,e in enumerate(sub):
+ with ix:
+ for count,subname in enumerate(ix.idxnames):
+ with git.open_idx(os.path.join(os.path.dirname(name), subname)) \
+ as sub:
+ for ecount,e in enumerate(sub):
+ if not (ecount % 1234):
+ qprogress(' %d/%d: %s %d/%d\r'
+ % (count, len(ix.idxnames),
+ git.shorten_hash(subname).decode('ascii'),
+ ecount, len(sub)))
+ if not sub.exists(e):
+ add_error("%s: %s: %s missing from idx"
+ % (path_msg(nicename),
+ git.shorten_hash(subname).decode('ascii'),
+ hexstr(e)))
+ if not ix.exists(e):
+ add_error("%s: %s: %s missing from midx"
+ % (path_msg(nicename),
+ git.shorten_hash(subname).decode('ascii'),
+ hexstr(e)))
+ prev = None
+ for ecount,e in enumerate(ix):
if not (ecount % 1234):
- qprogress(' %d/%d: %s %d/%d\r'
- % (count, len(ix.idxnames),
- git.shorten_hash(subname).decode('ascii'),
- ecount, len(sub)))
- if not sub.exists(e):
- add_error("%s: %s: %s missing from idx"
- % (path_msg(nicename),
- git.shorten_hash(subname).decode('ascii'),
- hexstr(e)))
- if not ix.exists(e):
- add_error("%s: %s: %s missing from midx"
- % (path_msg(nicename),
- git.shorten_hash(subname).decode('ascii'),
- hexstr(e)))
- prev = None
- for ecount,e in enumerate(ix):
- if not (ecount % 1234):
- qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
- if e and prev and not e >= prev:
- add_error('%s: ordering error: %s < %s'
- % (nicename, hexstr(e), hexstr(prev)))
- prev = e
+ qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
+ if e and prev and not e >= prev:
+ add_error('%s: ordering error: %s < %s'
+ % (nicename, hexstr(e), hexstr(prev)))
+ prev = e
_first = None
inp = []
total = 0
allfilenames = []
- midxs = []
- try:
+ with ExitStack() as contexts:
for name in infilenames:
ix = git.open_idx(name)
- midxs.append(ix)
+ contexts.enter_context(ix)
inp.append((
ix.map,
len(ix),
f.flush()
fdatasync(f.fileno())
- fmap = mmap_readwrite(f, close=False)
- count = merge_into(fmap, bits, total, inp)
- del fmap # Assume this calls msync() now.
+ with mmap_readwrite(f, close=False) as fmap:
+ count = merge_into(fmap, bits, total, inp)
f.seek(0, os.SEEK_END)
f.write(b'\0'.join(allfilenames))
- finally:
- for ix in midxs:
- if isinstance(ix, midx.PackMidx):
- ix.close()
- midxs = None
- inp = None
-
# This is just for testing (if you enable this, don't clear inp above)
# if 0:
midxs = glob.glob(b'%s/*.midx' % path)
contents = {}
for mname in midxs:
- m = git.open_idx(mname)
- contents[mname] = [(b'%s/%s' % (path,i)) for i in m.idxnames]
- sizes[mname] = len(m)
+ with git.open_idx(mname) as m:
+ contents[mname] = [(b'%s/%s' % (path,i)) for i in m.idxnames]
+ sizes[mname] = len(m)
# sort the biggest+newest midxes first, so that we can eliminate
# smaller (or older) redundant ones that come later in the list
idxs = [k for k in glob.glob(b'%s/*.idx' % path) if not already.get(k)]
for iname in idxs:
- i = git.open_idx(iname)
- sizes[iname] = len(i)
+ with git.open_idx(iname) as i:
+ sizes[iname] = len(i)
all = [(sizes[n],n) for n in (midxs + idxs)]
_init_session()
assert name.find(b'/') < 0
assert name.endswith(b'.idx')
- idx = git.open_idx(git.repo(b'objects/pack/%s' % name))
- conn.write(struct.pack('!I', len(idx.map)))
- conn.write(idx.map)
+ with git.open_idx(git.repo(b'objects/pack/%s' % name)) as idx:
+ conn.write(struct.pack('!I', len(idx.map)))
+ conn.write(idx.map)
conn.ok()
log('found %d objects (%d/%d %s)\r'
% (object_count, i + 1, len(indexes),
path_msg(basename(idx_name))))
- idx = git.open_idx(idx_name)
- object_count += len(idx)
+ with git.open_idx(idx_name) as idx:
+ object_count += len(idx)
return object_count
broken = True
if broken:
mx.close()
- del mx
unlink(full)
else:
midxl.append(mx)
# which len is self.nsha * 4
self.idxnames = self.map[self.which_ofs + 4 * self.nsha:].split(b'\0')
- def __del__(self):
- self.close()
-
def __enter__(self):
return self
assert len(pi.packs) == 2
assert sorted([os.path.basename(i.name) for i in pi.packs]) == sorted([p1base, p2base])
- p1 = git.open_idx(p1name)
- assert p1.exists(s1sha)
- p2 = git.open_idx(p2name)
- assert not p2.exists(s1sha)
- assert p2.exists(s2sha)
+ with git.open_idx(p1name) as p1, \
+ git.open_idx(p2name) as p2:
+ assert p1.exists(s1sha)
+ assert not p2.exists(s1sha)
+ assert p2.exists(s2sha)
subprocess.call([path.exe(), b'midx', b'-f'])
pi.refresh()
WVPASS(os.path.exists(nameprefix + b'.pack'))
WVPASS(os.path.exists(nameprefix + b'.idx'))
- r = git.open_idx(nameprefix + b'.idx')
- print(repr(r.fanout))
+ with git.open_idx(nameprefix + b'.idx') as r:
+ print(repr(r.fanout))
- for i in range(nobj):
- WVPASS(r.find_offset(hashes[i]) > 0)
- WVPASS(r.exists(hashes[99]))
- WVFAIL(r.exists(b'\0'*20))
+ for i in range(nobj):
+ WVPASS(r.find_offset(hashes[i]) > 0)
+ WVPASS(r.exists(hashes[99]))
+ WVFAIL(r.exists(b'\0'*20))
- pi = iter(r)
- for h in sorted(hashes):
- WVPASSEQ(hexlify(next(pi)), hexlify(h))
+ pi = iter(r)
+ for h in sorted(hashes):
+ WVPASSEQ(hexlify(next(pi)), hexlify(h))
- WVFAIL(r.find_offset(b'\0'*20))
+ WVFAIL(r.find_offset(b'\0'*20))
r = git.PackIdxList(bupdir + b'/objects/pack')
WVPASS(r.exists(hashes[5]))
projects / bup.git / commitdiff