"""
import os, sys, zlib, time, subprocess, struct, stat, re, tempfile, glob
from bup.helpers import *
-from bup import _helpers, path, midx, bloom
+from bup import _helpers, path, midx, bloom, xstat
max_pack_size = 1000*1000*1000 # larger packs will slow down pruning
max_pack_objects = 200*1000 # cache memory usage is about 83 bytes per object
-SEEK_END=2 # os.SEEK_END is not defined in python 2.4
verbose = 0
ignore_midx = 0
-home_repodir = os.path.expanduser('~/.bup')
repodir = None
_typemap = { 'blob':3, 'tree':2, 'commit':1, 'tag':4 }
return sum.digest()
-def _shalist_sort_key(ent):
+def shalist_item_sort_key(ent):
(mode, name, id) = ent
assert(mode+0 == mode)
if stat.S_ISDIR(mode):
def tree_encode(shalist):
"""Generate a git tree object from (mode,name,hash) tuples."""
- shalist = sorted(shalist, key = _shalist_sort_key)
+ shalist = sorted(shalist, key = shalist_item_sort_key)
l = []
for (mode,name,bin) in shalist:
assert(mode)
"""Generate a list of (mode,name,hash) from the git tree object in buf."""
ofs = 0
while ofs < len(buf):
- z = buf[ofs:].find('\0')
- assert(z > 0)
- spl = buf[ofs:ofs+z].split(' ', 1)
+ z = buf.find('\0', ofs)
+ assert(z > ofs)
+ spl = buf[ofs:z].split(' ', 1)
assert(len(spl) == 2)
mode,name = spl
- sha = buf[ofs+z+1:ofs+z+1+20]
- ofs += z+1+20
+ sha = buf[z+1:z+1+20]
+ ofs = z+1+20
yield (int(mode, 8), name, sha)
-def _encode_packobj(type, content):
+def _encode_packobj(type, content, compression_level=1):
szout = ''
sz = len(content)
szbits = (sz & 0x0f) | (_typemap[type]<<4)
break
szbits = sz & 0x7f
sz >>= 7
- z = zlib.compressobj(1)
+ if compression_level > 9:
+ compression_level = 9
+ elif compression_level < 0:
+ compression_level = 0
+ z = zlib.compressobj(compression_level)
yield szout
yield z.compress(content)
yield z.flush()
-def _encode_looseobj(type, content):
- z = zlib.compressobj(1)
+def _encode_looseobj(type, content, compression_level=1):
+ z = zlib.compressobj(compression_level)
yield z.compress('%s %d\0' % (type, len(content)))
yield z.compress(content)
yield z.flush()
else:
midxl.append(mx)
midxl.sort(key=lambda ix:
- (-len(ix), -os.stat(ix.name).st_mtime))
+ (-len(ix), -xstat.stat(ix.name).st_mtime))
for ix in midxl:
any_needed = False
for sub in ix.idxnames:
class PackWriter:
"""Writes Git objects inside a pack file."""
- def __init__(self, objcache_maker=_make_objcache):
+ def __init__(self, objcache_maker=_make_objcache, compression_level=1):
self.count = 0
self.outbytes = 0
self.filename = None
self.idx = None
self.objcache_maker = objcache_maker
self.objcache = None
+ self.compression_level = compression_level
def __del__(self):
self.close()
log('>')
if not sha:
sha = calc_hash(type, content)
- size, crc = self._raw_write(_encode_packobj(type, content), sha=sha)
+ size, crc = self._raw_write(_encode_packobj(type, content,
+ self.compression_level),
+ sha=sha)
if self.outbytes >= max_pack_size or self.count >= max_pack_objects:
self.breakpoint()
return sha
return self._end(run_midx=run_midx)
def _write_pack_idx_v2(self, filename, idx, packbin):
+ ofs64_count = 0
+ for section in idx:
+ for entry in section:
+ if entry[2] >= 2**31:
+ ofs64_count += 1
+
+ # Length: header + fan-out + shas-and-crcs + overflow-offsets
+ index_len = 8 + (4 * 256) + (28 * self.count) + (8 * ofs64_count)
+ idx_map = None
idx_f = open(filename, 'w+b')
- idx_f.write('\377tOc\0\0\0\2')
-
- ofs64_ofs = 8 + 4*256 + 28*self.count
- idx_f.truncate(ofs64_ofs)
- idx_f.seek(0)
- idx_map = mmap_readwrite(idx_f, close=False)
- idx_f.seek(0, SEEK_END)
- count = _helpers.write_idx(idx_f, idx_map, idx, self.count)
- assert(count == self.count)
- idx_map.close()
- idx_f.write(packbin)
-
- idx_f.seek(0)
- idx_sum = Sha1()
- b = idx_f.read(8 + 4*256)
- idx_sum.update(b)
-
- obj_list_sum = Sha1()
- for b in chunkyreader(idx_f, 20*self.count):
+ try:
+ idx_f.truncate(index_len)
+ idx_map = mmap_readwrite(idx_f, close=False)
+ count = _helpers.write_idx(filename, idx_map, idx, self.count)
+ assert(count == self.count)
+ finally:
+ if idx_map: idx_map.close()
+ idx_f.close()
+
+ idx_f = open(filename, 'a+b')
+ try:
+ idx_f.write(packbin)
+ idx_f.seek(0)
+ idx_sum = Sha1()
+ b = idx_f.read(8 + 4*256)
idx_sum.update(b)
- obj_list_sum.update(b)
- namebase = obj_list_sum.hexdigest()
- for b in chunkyreader(idx_f):
- idx_sum.update(b)
- idx_f.write(idx_sum.digest())
- idx_f.close()
+ obj_list_sum = Sha1()
+ for b in chunkyreader(idx_f, 20*self.count):
+ idx_sum.update(b)
+ obj_list_sum.update(b)
+ namebase = obj_list_sum.hexdigest()
- return namebase
+ for b in chunkyreader(idx_f):
+ idx_sum.update(b)
+ idx_f.write(idx_sum.digest())
+ return namebase
+ finally:
+ idx_f.close()
def _git_date(date):
opts = []
if count:
opts += ['-n', str(atoi(count))]
- argv = ['git', 'rev-list', '--pretty=format:%ct'] + opts + [ref, '--']
+ argv = ['git', 'rev-list', '--pretty=format:%at'] + opts + [ref, '--']
p = subprocess.Popen(argv, preexec_fn = _gitenv, stdout = subprocess.PIPE)
commit = None
for row in p.stdout:
raise GitError, 'git rev-list returned error %d' % rv
-def rev_get_date(ref):
- """Get the date of the latest commit on the specified ref."""
- for (date, commit) in rev_list(ref, count=1):
- return date
- raise GitError, 'no such commit %r' % ref
+def get_commit_dates(refs):
+ """Get the dates for the specified commit refs."""
+ result = []
+ cmd = ['git', 'show', '-s', '--pretty=format:%ct']
+ for chunk in batchpipe(cmd, refs, preexec_fn=_gitenv):
+ result += chunk.splitlines()
+ return result
def rev_parse(committish):
if parent and not os.path.exists(parent):
raise GitError('parent directory "%s" does not exist\n' % parent)
if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
- raise GitError('"%d" exists but is not a directory\n' % d)
+ raise GitError('"%s" exists but is not a directory\n' % d)
p = subprocess.Popen(['git', '--bare', 'init'], stdout=sys.stderr,
preexec_fn = _gitenv)
_git_wait('git init', p)
p = subprocess.Popen(['git', 'config', 'pack.indexVersion', '2'],
stdout=sys.stderr, preexec_fn = _gitenv)
_git_wait('git config', p)
+ # Enable the reflog
+ p = subprocess.Popen(['git', 'config', 'core.logAllRefUpdates', 'true'],
+ stdout=sys.stderr, preexec_fn = _gitenv)
+ _git_wait('git config', p)
def check_repo_or_die(path=None):
os.stat(repo('objects/pack/.'))
except OSError, e:
if e.errno == errno.ENOENT:
- if repodir != home_repodir:
- log('error: %r is not a bup/git repository\n' % repo())
- sys.exit(15)
+ log('error: %r is not a bup repository; run "bup init"\n'
+ % repo())
+ sys.exit(15)
else:
log('error: %s\n' % e)
sys.exit(14)
- init_repo()
-
_ver = None
def ver():
if not self.p or self.p.poll() != None:
self._restart()
assert(self.p)
- assert(self.p.poll() == None)
+ poll_result = self.p.poll()
+ assert(poll_result == None)
if self.inprogress:
- log('_fast_get: opening %r while %r is open'
+ log('_fast_get: opening %r while %r is open\n'
% (id, self.inprogress))
assert(not self.inprogress)
assert(id.find('\n') < 0)
yield type
for blob in it:
yield blob
- assert(self.p.stdout.readline() == '\n')
+ readline_result = self.p.stdout.readline()
+ assert(readline_result == '\n')
self.inprogress = None
except Exception, e:
it.abort()
projects / bup.git / blobdiff