"""
import os, sys, zlib, time, subprocess, struct, stat, re, tempfile, glob
from bup.helpers import *
-from bup import _helpers, path, bloom
+from bup import _helpers, path, midx, bloom, xstat
-MIDX_VERSION = 4
-SEEK_END=2 # os.SEEK_END is not defined in python 2.4
+max_pack_size = 1000*1000*1000 # larger packs will slow down pruning
+max_pack_objects = 200*1000 # cache memory usage is about 83 bytes per object
verbose = 0
ignore_midx = 0
-home_repodir = os.path.expanduser('~/.bup')
repodir = None
_typemap = { 'blob':3, 'tree':2, 'commit':1, 'tag':4 }
return os.path.join(repodir, sub)
+def shorten_hash(s):
+ return re.sub(r'([^0-9a-z]|\b)([0-9a-z]{7})[0-9a-z]{33}([^0-9a-z]|\b)',
+ r'\1\2*\3', s)
+
+
def repo_rel(path):
full = os.path.abspath(path)
fullrepo = os.path.abspath(repo(''))
path = full[len(fullrepo):]
if path.startswith('index-cache/'):
path = path[len('index-cache/'):]
- return path
+ return shorten_hash(path)
def all_packdirs():
return (name, BUP_NORMAL)
-def _encode_packobj(type, content):
+def calc_hash(type, content):
+ """Calculate some content's hash in the Git fashion."""
+ header = '%s %d\0' % (type, len(content))
+ sum = Sha1(header)
+ sum.update(content)
+ return sum.digest()
+
+
+def shalist_item_sort_key(ent):
+ (mode, name, id) = ent
+ assert(mode+0 == mode)
+ if stat.S_ISDIR(mode):
+ return name + '/'
+ else:
+ return name
+
+
+def tree_encode(shalist):
+ """Generate a git tree object from (mode,name,hash) tuples."""
+ shalist = sorted(shalist, key = shalist_item_sort_key)
+ l = []
+ for (mode,name,bin) in shalist:
+ assert(mode)
+ assert(mode+0 == mode)
+ assert(name)
+ assert(len(bin) == 20)
+ s = '%o %s\0%s' % (mode,name,bin)
+ assert(s[0] != '0') # 0-padded octal is not acceptable in a git tree
+ l.append(s)
+ return ''.join(l)
+
+
+def tree_decode(buf):
+ """Generate a list of (mode,name,hash) from the git tree object in buf."""
+ ofs = 0
+ while ofs < len(buf):
+ z = buf.find('\0', ofs)
+ assert(z > ofs)
+ spl = buf[ofs:z].split(' ', 1)
+ assert(len(spl) == 2)
+ mode,name = spl
+ sha = buf[z+1:z+1+20]
+ ofs = z+1+20
+ yield (int(mode, 8), name, sha)
+
+
+def _encode_packobj(type, content, compression_level=1):
szout = ''
sz = len(content)
szbits = (sz & 0x0f) | (_typemap[type]<<4)
break
szbits = sz & 0x7f
sz >>= 7
- z = zlib.compressobj(1)
+ if compression_level > 9:
+ compression_level = 9
+ elif compression_level < 0:
+ compression_level = 0
+ z = zlib.compressobj(compression_level)
yield szout
yield z.compress(content)
yield z.flush()
-def _encode_looseobj(type, content):
- z = zlib.compressobj(1)
+def _encode_looseobj(type, content, compression_level=1):
+ z = zlib.compressobj(compression_level)
yield z.compress('%s %d\0' % (type, len(content)))
yield z.compress(content)
yield z.flush()
yield buffer(self.map, 8 + 256*4 + 20*i, 20)
-extract_bits = _helpers.extract_bits
-
-class PackMidx:
- """Wrapper which contains data from multiple index files.
- Multiple index (.midx) files constitute a wrapper around index (.idx) files
- and make it possible for bup to expand Git's indexing capabilities to vast
- amounts of files.
- """
- def __init__(self, filename):
- self.name = filename
- self.force_keep = False
- assert(filename.endswith('.midx'))
- self.map = mmap_read(open(filename))
- if str(self.map[0:4]) != 'MIDX':
- log('Warning: skipping: invalid MIDX header in %r\n' % filename)
- self.force_keep = True
- return self._init_failed()
- ver = struct.unpack('!I', self.map[4:8])[0]
- if ver < MIDX_VERSION:
- log('Warning: ignoring old-style (v%d) midx %r\n'
- % (ver, filename))
- self.force_keep = False # old stuff is boring
- return self._init_failed()
- if ver > MIDX_VERSION:
- log('Warning: ignoring too-new (v%d) midx %r\n'
- % (ver, filename))
- self.force_keep = True # new stuff is exciting
- return self._init_failed()
-
- self.bits = _helpers.firstword(self.map[8:12])
- self.entries = 2**self.bits
- self.fanout = buffer(self.map, 12, self.entries*4)
- self.sha_ofs = 12 + self.entries*4
- self.nsha = nsha = self._fanget(self.entries-1)
- self.shatable = buffer(self.map, self.sha_ofs, nsha*20)
- self.which_ofs = self.sha_ofs + 20*nsha
- self.whichlist = buffer(self.map, self.which_ofs, nsha*4)
- self.idxnames = str(self.map[self.which_ofs + 4*nsha:]).split('\0')
-
- def _init_failed(self):
- self.bits = 0
- self.entries = 1
- self.fanout = buffer('\0\0\0\0')
- self.shatable = buffer('\0'*20)
- self.idxnames = []
-
- def _fanget(self, i):
- start = i*4
- s = self.fanout[start:start+4]
- return _helpers.firstword(s)
-
- def _get(self, i):
- return str(self.shatable[i*20:(i+1)*20])
-
- def _get_idx_i(self, i):
- return struct.unpack('!I', self.whichlist[i*4:(i+1)*4])[0]
-
- def _get_idxname(self, i):
- return self.idxnames[self._get_idx_i(i)]
-
- def exists(self, hash, want_source=False):
- """Return nonempty if the object exists in the index files."""
- global _total_searches, _total_steps
- _total_searches += 1
- want = str(hash)
- el = extract_bits(want, self.bits)
- if el:
- start = self._fanget(el-1)
- startv = el << (32-self.bits)
- else:
- start = 0
- startv = 0
- end = self._fanget(el)
- endv = (el+1) << (32-self.bits)
- _total_steps += 1 # lookup table is a step
- hashv = _helpers.firstword(hash)
- #print '(%08x) %08x %08x %08x' % (extract_bits(want, 32), startv, hashv, endv)
- while start < end:
- _total_steps += 1
- #print '! %08x %08x %08x %d - %d' % (startv, hashv, endv, start, end)
- mid = start + (hashv-startv)*(end-start-1)/(endv-startv)
- #print ' %08x %08x %08x %d %d %d' % (startv, hashv, endv, start, mid, end)
- v = self._get(mid)
- #print ' %08x' % self._num(v)
- if v < want:
- start = mid+1
- startv = _helpers.firstword(v)
- elif v > want:
- end = mid
- endv = _helpers.firstword(v)
- else: # got it!
- return want_source and self._get_idxname(mid) or True
- return None
-
- def __iter__(self):
- for i in xrange(self._fanget(self.entries-1)):
- yield buffer(self.shatable, i*20, 20)
-
- def __len__(self):
- return int(self._fanget(self.entries-1))
-
-
_mpi_count = 0
class PackIdxList:
def __init__(self, dir):
_total_searches += 1
if hash in self.also:
return True
- if self.do_bloom and self.bloom is not None:
- _total_searches -= 1 # will be incremented by bloom
+ if self.do_bloom and self.bloom:
if self.bloom.exists(hash):
self.do_bloom = False
else:
+ _total_searches -= 1 # was counted by bloom
return None
for i in xrange(len(self.packs)):
p = self.packs[i]
self.do_bloom = False
skip_midx = skip_midx or ignore_midx
d = dict((p.name, p) for p in self.packs
- if not skip_midx or not isinstance(p, PackMidx))
+ if not skip_midx or not isinstance(p, midx.PackMidx))
if os.path.exists(self.dir):
if not skip_midx:
midxl = []
for ix in self.packs:
- if isinstance(ix, PackMidx):
+ if isinstance(ix, midx.PackMidx):
for name in ix.idxnames:
d[os.path.join(self.dir, name)] = ix
for full in glob.glob(os.path.join(self.dir,'*.midx')):
if not d.get(full):
- mx = PackMidx(full)
+ mx = midx.PackMidx(full)
(mxd, mxf) = os.path.split(mx.name)
broken = False
for n in mx.idxnames:
unlink(full)
else:
midxl.append(mx)
- midxl.sort(lambda x,y: -cmp(len(x),len(y)))
+ midxl.sort(key=lambda ix:
+ (-len(ix), -xstat.stat(ix.name).st_mtime))
for ix in midxl:
any_needed = False
for sub in ix.idxnames:
self.also.add(hash)
-def calc_hash(type, content):
- """Calculate some content's hash in the Git fashion."""
- header = '%s %d\0' % (type, len(content))
- sum = Sha1(header)
- sum.update(content)
- return sum.digest()
-
-
-def _shalist_sort_key(ent):
- (mode, name, id) = ent
- if stat.S_ISDIR(int(mode, 8)):
- return name + '/'
- else:
- return name
-
-
def open_idx(filename):
if filename.endswith('.idx'):
f = open(filename, 'rb')
else:
raise GitError('%s: unrecognized idx file header' % filename)
elif filename.endswith('.midx'):
- return PackMidx(filename)
+ return midx.PackMidx(filename)
else:
raise GitError('idx filenames must end with .idx or .midx')
return PackIdxList(repo('objects/pack'))
class PackWriter:
- """Writes Git objects insid a pack file."""
- def __init__(self, objcache_maker=_make_objcache):
+ """Writes Git objects inside a pack file."""
+ def __init__(self, objcache_maker=_make_objcache, compression_level=1):
self.count = 0
self.outbytes = 0
self.filename = None
self.idx = None
self.objcache_maker = objcache_maker
self.objcache = None
+ self.compression_level = compression_level
def __del__(self):
self.close()
log('>')
if not sha:
sha = calc_hash(type, content)
- size, crc = self._raw_write(_encode_packobj(type, content), sha=sha)
+ size, crc = self._raw_write(_encode_packobj(type, content,
+ self.compression_level),
+ sha=sha)
+ if self.outbytes >= max_pack_size or self.count >= max_pack_objects:
+ self.breakpoint()
return sha
def breakpoint(self):
def maybe_write(self, type, content):
"""Write an object to the pack file if not present and return its id."""
- self._require_objcache()
sha = calc_hash(type, content)
if not self.exists(sha):
self._write(sha, type, content)
+ self._require_objcache()
self.objcache.add(sha)
return sha
def new_tree(self, shalist):
"""Create a tree object in the pack."""
- shalist = sorted(shalist, key = _shalist_sort_key)
- l = []
- for (mode,name,bin) in shalist:
- assert(mode)
- assert(mode != '0')
- assert(mode[0] != '0')
- assert(name)
- assert(len(bin) == 20)
- l.append('%s %s\0%s' % (mode,name,bin))
- return self.maybe_write('tree', ''.join(l))
+ content = tree_encode(shalist)
+ return self.maybe_write('tree', content)
def _new_commit(self, tree, parent, author, adate, committer, cdate, msg):
l = []
return self._end(run_midx=run_midx)
def _write_pack_idx_v2(self, filename, idx, packbin):
+ ofs64_count = 0
+ for section in idx:
+ for entry in section:
+ if entry[2] >= 2**31:
+ ofs64_count += 1
+
+ # Length: header + fan-out + shas-and-crcs + overflow-offsets
+ index_len = 8 + (4 * 256) + (28 * self.count) + (8 * ofs64_count)
+ idx_map = None
idx_f = open(filename, 'w+b')
- idx_f.write('\377tOc\0\0\0\2')
-
- ofs64_ofs = 8 + 4*256 + 28*self.count
- idx_f.truncate(ofs64_ofs)
- idx_f.seek(0)
- idx_map = mmap_readwrite(idx_f, close=False)
- idx_f.seek(0, SEEK_END)
- count = _helpers.write_idx(idx_f, idx_map, idx, self.count)
- assert(count == self.count)
- idx_map.close()
- idx_f.write(packbin)
-
- idx_f.seek(0)
- idx_sum = Sha1()
- b = idx_f.read(8 + 4*256)
- idx_sum.update(b)
-
- obj_list_sum = Sha1()
- for b in chunkyreader(idx_f, 20*self.count):
+ try:
+ idx_f.truncate(index_len)
+ idx_map = mmap_readwrite(idx_f, close=False)
+ count = _helpers.write_idx(filename, idx_map, idx, self.count)
+ assert(count == self.count)
+ finally:
+ if idx_map: idx_map.close()
+ idx_f.close()
+
+ idx_f = open(filename, 'a+b')
+ try:
+ idx_f.write(packbin)
+ idx_f.seek(0)
+ idx_sum = Sha1()
+ b = idx_f.read(8 + 4*256)
idx_sum.update(b)
- obj_list_sum.update(b)
- namebase = obj_list_sum.hexdigest()
- for b in chunkyreader(idx_f):
- idx_sum.update(b)
- idx_f.write(idx_sum.digest())
- idx_f.close()
+ obj_list_sum = Sha1()
+ for b in chunkyreader(idx_f, 20*self.count):
+ idx_sum.update(b)
+ obj_list_sum.update(b)
+ namebase = obj_list_sum.hexdigest()
- return namebase
+ for b in chunkyreader(idx_f):
+ idx_sum.update(b)
+ idx_f.write(idx_sum.digest())
+ return namebase
+ finally:
+ idx_f.close()
def _git_date(date):
opts = []
if count:
opts += ['-n', str(atoi(count))]
- argv = ['git', 'rev-list', '--pretty=format:%ct'] + opts + [ref, '--']
+ argv = ['git', 'rev-list', '--pretty=format:%at'] + opts + [ref, '--']
p = subprocess.Popen(argv, preexec_fn = _gitenv, stdout = subprocess.PIPE)
commit = None
for row in p.stdout:
raise GitError, 'git rev-list returned error %d' % rv
-def rev_get_date(ref):
- """Get the date of the latest commit on the specified ref."""
- for (date, commit) in rev_list(ref, count=1):
- return date
- raise GitError, 'no such commit %r' % ref
+def get_commit_dates(refs):
+ """Get the dates for the specified commit refs."""
+ result = []
+ cmd = ['git', 'show', '-s', '--pretty=format:%ct']
+ for chunk in batchpipe(cmd, refs, preexec_fn=_gitenv):
+ result += chunk.splitlines()
+ return result
def rev_parse(committish):
if parent and not os.path.exists(parent):
raise GitError('parent directory "%s" does not exist\n' % parent)
if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
- raise GitError('"%d" exists but is not a directory\n' % d)
+ raise GitError('"%s" exists but is not a directory\n' % d)
p = subprocess.Popen(['git', '--bare', 'init'], stdout=sys.stderr,
preexec_fn = _gitenv)
_git_wait('git init', p)
p = subprocess.Popen(['git', 'config', 'pack.indexVersion', '2'],
stdout=sys.stderr, preexec_fn = _gitenv)
_git_wait('git config', p)
+ # Enable the reflog
+ p = subprocess.Popen(['git', 'config', 'core.logAllRefUpdates', 'true'],
+ stdout=sys.stderr, preexec_fn = _gitenv)
+ _git_wait('git config', p)
def check_repo_or_die(path=None):
initializes the default repository automatically.
"""
guess_repo(path)
- if not os.path.isdir(repo('objects/pack/.')):
- if repodir == home_repodir:
- init_repo()
- else:
- log('error: %r is not a bup/git repository\n' % repo())
+ try:
+ os.stat(repo('objects/pack/.'))
+ except OSError, e:
+ if e.errno == errno.ENOENT:
+ log('error: %r is not a bup repository; run "bup init"\n'
+ % repo())
sys.exit(15)
-
-
-def treeparse(buf):
- """Generate a list of (mode, name, hash) tuples of objects from 'buf'."""
- ofs = 0
- while ofs < len(buf):
- z = buf[ofs:].find('\0')
- assert(z > 0)
- spl = buf[ofs:ofs+z].split(' ', 1)
- assert(len(spl) == 2)
- sha = buf[ofs+z+1:ofs+z+1+20]
- ofs += z+1+20
- yield (spl[0], spl[1], sha)
+ else:
+ log('error: %s\n' % e)
+ sys.exit(14)
_ver = None
if not self.p or self.p.poll() != None:
self._restart()
assert(self.p)
- assert(self.p.poll() == None)
+ poll_result = self.p.poll()
+ assert(poll_result == None)
if self.inprogress:
- log('_fast_get: opening %r while %r is open'
+ log('_fast_get: opening %r while %r is open\n'
% (id, self.inprogress))
assert(not self.inprogress)
assert(id.find('\n') < 0)
yield type
for blob in it:
yield blob
- assert(self.p.stdout.readline() == '\n')
+ readline_result = self.p.stdout.readline()
+ assert(readline_result == '\n')
self.inprogress = None
except Exception, e:
it.abort()
yield blob
elif type == 'tree':
treefile = ''.join(it)
- for (mode, name, sha) in treeparse(treefile):
+ for (mode, name, sha) in tree_decode(treefile):
for blob in self.join(sha.encode('hex')):
yield blob
elif type == 'commit':
projects / bup.git / blobdiff