X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=lib%2Fbup%2Fgit.py;h=d6a745c02d7d9d355370bf23c85a9d5c645273e1;hb=HEAD;hp=c038fd1587b2abeb5ea920db7e476cf32194de92;hpb=0e574aa5760f8511abba6ccaf805f34c2caeb996;p=bup.git diff --git a/lib/bup/git.py b/lib/bup/git.py index c038fd1..d6a745c 100644 --- a/lib/bup/git.py +++ b/lib/bup/git.py @@ -4,20 +4,19 @@ interact with the Git data structures. """ from __future__ import absolute_import, print_function -import os, sys, zlib, subprocess, struct, stat, re, tempfile, glob +import os, sys, zlib, subprocess, struct, stat, re, glob from array import array from binascii import hexlify, unhexlify from collections import namedtuple +from contextlib import ExitStack from itertools import islice +from shutil import rmtree from bup import _helpers, hashsplit, path, midx, bloom, xstat from bup.compat import (buffer, byte_int, bytes_from_byte, bytes_from_uint, environ, - ExitStack, - items, pending_raise, - range, reraise) from bup.io import path_msg from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2, @@ -30,6 +29,7 @@ from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2, mmap_read, mmap_readwrite, nullcontext_if_not, progress, qprogress, stat_if_exists, + temp_dir, unlink, utc_offset_str) @@ -38,7 +38,7 @@ verbose = 0 repodir = None # The default repository, once initialized _typemap = {b'blob': 3, b'tree': 2, b'commit': 1, b'tag': 4} -_typermap = {v: k for k, v in items(_typemap)} +_typermap = {v: k for k, v in _typemap.items()} _total_searches = 0 @@ -372,10 +372,7 @@ def _decode_packobj(buf): return (type, zlib.decompress(buf[i+1:])) -class PackIdx: - def __init__(self): - assert(0) - +class PackIdx(object): def find_offset(self, hash): """Get the offset of an object inside the index file.""" idx = self._idx_from_hash(hash) @@ -414,6 +411,7 @@ class PackIdx: class PackIdxV1(PackIdx): """Object representation of a Git pack index (version 1) file.""" def __init__(self, filename, f): + super(PackIdxV1, self).__init__() self.closed = False self.name = filename self.idxnames = [self.name] @@ -467,6 +465,7 @@ class PackIdxV1(PackIdx): class PackIdxV2(PackIdx): """Object representation of a Git pack index (version 2) file.""" def __init__(self, filename, f): + super(PackIdxV2, self).__init__() self.closed = False self.name = filename self.idxnames = [self.name] @@ -539,7 +538,11 @@ class PackIdxList: self.do_bloom = False self.bloom = None self.ignore_midx = ignore_midx - self.refresh() + try: + self.refresh() + except BaseException as ex: + with pending_raise(ex): + self.close() def close(self): global _mpi_count @@ -678,8 +681,13 @@ class PackIdxList: continue d[full] = ix bfull = os.path.join(self.dir, b'bup.bloom') - self.packs = list(set(d.values())) - self.packs.sort(reverse=True, key=lambda x: len(x)) + new_packs = set(d.values()) + for p in self.packs: + if not p in new_packs: + p.close() + new_packs = list(new_packs) + new_packs.sort(reverse=True, key=lambda x: len(x)) + self.packs = new_packs if self.bloom is None and os.path.exists(bfull): self.bloom = bloom.ShaBloom(bfull) try: @@ -763,7 +771,7 @@ def _make_objcache(): # bup-gc assumes that it can disable all PackWriter activities # (bloom/midx/cache) via the constructor and close() arguments. -class PackWriter: +class PackWriter(object): """Writes Git objects inside a pack file.""" def __init__(self, objcache_maker=_make_objcache, compression_level=1, run_midx=True, on_pack_finish=None, @@ -774,7 +782,7 @@ class PackWriter: self.parentfd = None self.count = 0 self.outbytes = 0 - self.filename = None + self.tmpdir = None self.idx = None self.objcache_maker = objcache_maker self.objcache = None @@ -802,24 +810,15 @@ class PackWriter: def _open(self): if not self.file: - objdir = dir = os.path.join(self.repo_dir, b'objects') - fd, name = tempfile.mkstemp(suffix=b'.pack', dir=objdir) - try: - self.file = os.fdopen(fd, 'w+b') - except: - os.close(fd) - raise - try: - self.parentfd = os.open(objdir, os.O_RDONLY) - except: - f = self.file - self.file = None - f.close() - raise - assert name.endswith(b'.pack') - self.filename = name[:-5] - self.file.write(b'PACK\0\0\0\2\0\0\0\0') - self.idx = PackIdxV2Writer() + with ExitStack() as err_stack: + objdir = dir = os.path.join(self.repo_dir, b'objects') + self.tmpdir = err_stack.enter_context(temp_dir(dir=objdir, prefix=b'pack-tmp-')) + self.file = err_stack.enter_context(open(self.tmpdir + b'/pack', 'w+b')) + self.parentfd = err_stack.enter_context(finalized(os.open(objdir, os.O_RDONLY), + lambda x: os.close(x))) + self.file.write(b'PACK\0\0\0\2\0\0\0\0') + self.idx = PackIdxV2Writer() + err_stack.pop_all() def _raw_write(self, datalist, sha): self._open() @@ -849,8 +848,7 @@ class PackWriter: def _write(self, sha, type, content): if verbose: log('>') - if not sha: - sha = calc_hash(type, content) + assert sha size, crc = self._raw_write(_encode_packobj(type, content, self.compression_level), sha=sha) @@ -909,19 +907,15 @@ class PackWriter: def _end(self, run_midx=True, abort=False): # Ignores run_midx during abort - if not self.file: - return None + self.tmpdir, tmpdir = None, self.tmpdir + self.parentfd, pfd, = None, self.parentfd self.file, f = None, self.file self.idx, idx = None, self.idx - self.parentfd, pfd, = None, self.parentfd - try: with nullcontext_if_not(self.objcache), \ finalized(pfd, lambda x: x is not None and os.close(x)), \ - f: - - if abort: - os.unlink(self.filename + b'.pack') + nullcontext_if_not(f): + if abort or not f: return None # update object count @@ -941,13 +935,11 @@ class PackWriter: fdatasync(f.fileno()) f.close() - idx.write(self.filename + b'.idx', packbin) + idx.write(tmpdir + b'/idx', packbin) nameprefix = os.path.join(self.repo_dir, b'objects/pack/pack-' + hexlify(packbin)) - if os.path.exists(self.filename + b'.map'): - os.unlink(self.filename + b'.map') - os.rename(self.filename + b'.pack', nameprefix + b'.pack') - os.rename(self.filename + b'.idx', nameprefix + b'.idx') + os.rename(tmpdir + b'/pack', nameprefix + b'.pack') + os.rename(tmpdir + b'/idx', nameprefix + b'.idx') os.fsync(pfd) if run_midx: auto_midx(os.path.join(self.repo_dir, b'objects/pack')) @@ -955,6 +947,8 @@ class PackWriter: self.on_pack_finish(nameprefix) return nameprefix finally: + if tmpdir: + rmtree(tmpdir) # Must be last -- some of the code above depends on it self.objcache = None @@ -1148,14 +1142,24 @@ def rev_parse(committish, repo_dir=None): return None -def update_ref(refname, newval, oldval, repo_dir=None): - """Update a repository reference.""" - if not oldval: - oldval = b'' +def update_ref(refname, newval, oldval, repo_dir=None, force=False): + """Update a repository reference. + + With force=True, don't care about the previous ref (oldval); + with force=False oldval must be either a sha1 or None (for an + entirely new branch) + """ + if force: + assert oldval is None + oldarg = [] + elif not oldval: + oldarg = [b''] + else: + oldarg = [hexlify(oldval)] assert refname.startswith(b'refs/heads/') \ or refname.startswith(b'refs/tags/') p = subprocess.Popen([b'git', b'update-ref', refname, - hexlify(newval), hexlify(oldval)], + hexlify(newval)] + oldarg, env=_gitenv(repo_dir), close_fds=True) _git_wait(b'git update-ref', p) @@ -1171,25 +1175,24 @@ def delete_ref(refname, oldvalue=None): _git_wait('git update-ref', p) -def guess_repo(path=None): - """Set the path value in the global variable "repodir". - This makes bup look for an existing bup repository, but not fail if a - repository doesn't exist. Usually, if you are interacting with a bup - repository, you would not be calling this function but using - check_repo_or_die(). +def guess_repo(): + """Return the global repodir or BUP_DIR when either is set, or ~/.bup. + Usually, if you are interacting with a bup repository, you would + not be calling this function but using check_repo_or_die(). + """ - global repodir - if path: - repodir = path - if not repodir: - repodir = environ.get(b'BUP_DIR') - if not repodir: - repodir = os.path.expanduser(b'~/.bup') + if repodir: + return repodir + repo = environ.get(b'BUP_DIR') + if not repo: + repo = os.path.expanduser(b'~/.bup') + return repo def init_repo(path=None): """Create the Git bare repository for bup in a given path.""" - guess_repo(path) + global repodir + repodir = path or guess_repo() d = repo() # appends a / to the path parent = os.path.dirname(os.path.dirname(d)) if parent and not os.path.exists(parent): @@ -1214,7 +1217,8 @@ def init_repo(path=None): def check_repo_or_die(path=None): """Check to see if a bup repository probably exists, and abort if not.""" - guess_repo(path) + global repodir + repodir = path or guess_repo() top = repo() pst = stat_if_exists(top + b'/objects/pack') if pst and stat.S_ISDIR(pst.st_mode):