from bup.compat import (buffer,
byte_int, bytes_from_byte, bytes_from_uint,
environ,
- items,
+ ExitStack,
pending_raise,
- range,
reraise)
from bup.io import path_msg
from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2,
merge_dict,
merge_iter,
mmap_read, mmap_readwrite,
+ nullcontext_if_not,
progress, qprogress, stat_if_exists,
unlink,
utc_offset_str)
repodir = None # The default repository, once initialized
_typemap = {b'blob': 3, b'tree': 2, b'commit': 1, b'tag': 4}
-_typermap = {v: k for k, v in items(_typemap)}
+_typermap = {v: k for k, v in _typemap.items()}
_total_searches = 0
return (type, zlib.decompress(buf[i+1:]))
-class PackIdx:
- def __init__(self):
- assert(0)
-
+class PackIdx(object):
def find_offset(self, hash):
"""Get the offset of an object inside the index file."""
idx = self._idx_from_hash(hash)
class PackIdxV1(PackIdx):
"""Object representation of a Git pack index (version 1) file."""
def __init__(self, filename, f):
+ super(PackIdxV1, self).__init__()
+ self.closed = False
self.name = filename
self.idxnames = [self.name]
self.map = mmap_read(f)
yield self.map[ofs : ofs + 20]
def close(self):
+ self.closed = True
if self.map is not None:
self.shatable = None
self.map.close()
self.map = None
+ def __del__(self):
+ assert self.closed
+
class PackIdxV2(PackIdx):
"""Object representation of a Git pack index (version 2) file."""
def __init__(self, filename, f):
+ super(PackIdxV2, self).__init__()
+ self.closed = False
self.name = filename
self.idxnames = [self.name]
self.map = mmap_read(f)
yield self.map[ofs : ofs + 20]
def close(self):
+ self.closed = True
if self.map is not None:
self.shatable = None
self.map.close()
self.map = None
+ def __del__(self):
+ assert self.closed
+
_mpi_count = 0
class PackIdxList:
def __init__(self, dir, ignore_midx=False):
global _mpi_count
+ # Q: was this also intended to prevent opening multiple repos?
assert(_mpi_count == 0) # these things suck tons of VM; don't waste it
_mpi_count += 1
+ self.open = True
self.dir = dir
self.also = set()
self.packs = []
self.do_bloom = False
self.bloom = None
self.ignore_midx = ignore_midx
- self.refresh()
+ try:
+ self.refresh()
+ except BaseException as ex:
+ with pending_raise(ex):
+ self.close()
- def __del__(self):
+ def close(self):
global _mpi_count
+ if not self.open:
+ assert _mpi_count == 0
+ return
_mpi_count -= 1
- assert(_mpi_count == 0)
+ assert _mpi_count == 0
+ self.also = None
+ self.bloom, bloom = None, self.bloom
+ self.packs, packs = None, self.packs
+ self.open = False
+ with ExitStack() as stack:
+ for pack in packs:
+ stack.enter_context(pack)
+ if bloom:
+ bloom.close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ with pending_raise(value, rethrow=False):
+ self.close()
+
+ def __del__(self):
+ assert not self.open
def __iter__(self):
return iter(idxmerge(self.packs))
continue
d[full] = ix
bfull = os.path.join(self.dir, b'bup.bloom')
- self.packs = list(set(d.values()))
- self.packs.sort(reverse=True, key=lambda x: len(x))
+ new_packs = set(d.values())
+ for p in self.packs:
+ if not p in new_packs:
+ p.close()
+ new_packs = list(new_packs)
+ new_packs.sort(reverse=True, key=lambda x: len(x))
+ self.packs = new_packs
if self.bloom is None and os.path.exists(bfull):
self.bloom = bloom.ShaBloom(bfull)
try:
l.append(msg)
return b'\n'.join(l)
-
def _make_objcache():
return PackIdxList(repo(b'objects/pack'))
# bup-gc assumes that it can disable all PackWriter activities
# (bloom/midx/cache) via the constructor and close() arguments.
-class PackWriter:
+class PackWriter(object):
"""Writes Git objects inside a pack file."""
def __init__(self, objcache_maker=_make_objcache, compression_level=1,
run_midx=True, on_pack_finish=None,
max_pack_size=None, max_pack_objects=None, repo_dir=None):
+ self.closed = False
self.repo_dir = repo_dir or repo()
self.file = None
self.parentfd = None
def _end(self, run_midx=True, abort=False):
# Ignores run_midx during abort
- if not self.file:
- return None
+ self.parentfd, pfd, = None, self.parentfd
self.file, f = None, self.file
self.idx, idx = None, self.idx
- self.parentfd, pfd, = None, self.parentfd
- self.objcache = None
-
- with finalized(pfd, lambda x: x is not None and os.close(x)), \
- f:
-
- if abort:
- os.unlink(self.filename + b'.pack')
- return None
+ try:
+ with nullcontext_if_not(self.objcache), \
+ finalized(pfd, lambda x: x is not None and os.close(x)), \
+ nullcontext_if_not(f):
+ if not f:
+ return None
+
+ if abort:
+ os.unlink(self.filename + b'.pack')
+ return None
+
+ # update object count
+ f.seek(8)
+ cp = struct.pack('!i', self.count)
+ assert len(cp) == 4
+ f.write(cp)
+
+ # calculate the pack sha1sum
+ f.seek(0)
+ sum = Sha1()
+ for b in chunkyreader(f):
+ sum.update(b)
+ packbin = sum.digest()
+ f.write(packbin)
+ f.flush()
+ fdatasync(f.fileno())
+ f.close()
- # update object count
- f.seek(8)
- cp = struct.pack('!i', self.count)
- assert len(cp) == 4
- f.write(cp)
-
- # calculate the pack sha1sum
- f.seek(0)
- sum = Sha1()
- for b in chunkyreader(f):
- sum.update(b)
- packbin = sum.digest()
- f.write(packbin)
- f.flush()
- fdatasync(f.fileno())
- f.close()
-
- idx.write(self.filename + b'.idx', packbin)
- nameprefix = os.path.join(self.repo_dir,
- b'objects/pack/pack-' + hexlify(packbin))
- if os.path.exists(self.filename + b'.map'):
- os.unlink(self.filename + b'.map')
- os.rename(self.filename + b'.pack', nameprefix + b'.pack')
- os.rename(self.filename + b'.idx', nameprefix + b'.idx')
- os.fsync(pfd)
- if run_midx:
- auto_midx(os.path.join(self.repo_dir, b'objects/pack'))
- if self.on_pack_finish:
- self.on_pack_finish(nameprefix)
- return nameprefix
+ idx.write(self.filename + b'.idx', packbin)
+ nameprefix = os.path.join(self.repo_dir,
+ b'objects/pack/pack-' + hexlify(packbin))
+ if os.path.exists(self.filename + b'.map'):
+ os.unlink(self.filename + b'.map')
+ os.rename(self.filename + b'.pack', nameprefix + b'.pack')
+ os.rename(self.filename + b'.idx', nameprefix + b'.idx')
+ os.fsync(pfd)
+ if run_midx:
+ auto_midx(os.path.join(self.repo_dir, b'objects/pack'))
+ if self.on_pack_finish:
+ self.on_pack_finish(nameprefix)
+ return nameprefix
+ finally:
+ # Must be last -- some of the code above depends on it
+ self.objcache = None
def abort(self):
"""Remove the pack file from disk."""
+ self.closed = True
self._end(abort=True)
def breakpoint(self):
def close(self, run_midx=True):
"""Close the pack file and move it to its definitive path."""
+ self.closed = True
return self._end(run_midx=run_midx)
+ def __del__(self):
+ assert self.closed
+
class PackIdxV2Writer:
def __init__(self):
debug2("resolved from ref: commit = %s\n" % hexlify(head))
return head
- pL = PackIdxList(repo(b'objects/pack', repo_dir=repo_dir))
-
if len(committish) == 40:
try:
hash = unhexlify(committish)
except TypeError:
return None
- if pL.exists(hash):
- return hash
+ with PackIdxList(repo(b'objects/pack', repo_dir=repo_dir)) as pL:
+ if pL.exists(hash):
+ return hash
return None
-def update_ref(refname, newval, oldval, repo_dir=None):
- """Update a repository reference."""
- if not oldval:
- oldval = b''
+def update_ref(refname, newval, oldval, repo_dir=None, force=False):
+ """Update a repository reference.
+
+ With force=True, don't care about the previous ref (oldval);
+ with force=False oldval must be either a sha1 or None (for an
+ entirely new branch)
+ """
+ if force:
+ assert oldval is None
+ oldarg = []
+ elif not oldval:
+ oldarg = [b'']
+ else:
+ oldarg = [hexlify(oldval)]
assert refname.startswith(b'refs/heads/') \
or refname.startswith(b'refs/tags/')
p = subprocess.Popen([b'git', b'update-ref', refname,
- hexlify(newval), hexlify(oldval)],
+ hexlify(newval)] + oldarg,
env=_gitenv(repo_dir),
close_fds=True)
_git_wait(b'git update-ref', p)
projects / bup.git / blobdiff