"""
import os, sys, zlib, time, subprocess, struct, stat, re, tempfile, glob
from bup.helpers import *
-from bup import _helpers, path, midx, bloom
+from bup import _helpers, path, midx, bloom, xstat
-SEEK_END=2 # os.SEEK_END is not defined in python 2.4
+max_pack_size = 1000*1000*1000 # larger packs will slow down pruning
+max_pack_objects = 200*1000 # cache memory usage is about 83 bytes per object
verbose = 0
ignore_midx = 0
return (name, BUP_NORMAL)
-def _encode_packobj(type, content):
+def calc_hash(type, content):
+ """Calculate some content's hash in the Git fashion."""
+ header = '%s %d\0' % (type, len(content))
+ sum = Sha1(header)
+ sum.update(content)
+ return sum.digest()
+
+
+def shalist_item_sort_key(ent):
+ (mode, name, id) = ent
+ assert(mode+0 == mode)
+ if stat.S_ISDIR(mode):
+ return name + '/'
+ else:
+ return name
+
+
+def tree_encode(shalist):
+ """Generate a git tree object from (mode,name,hash) tuples."""
+ shalist = sorted(shalist, key = shalist_item_sort_key)
+ l = []
+ for (mode,name,bin) in shalist:
+ assert(mode)
+ assert(mode+0 == mode)
+ assert(name)
+ assert(len(bin) == 20)
+ s = '%o %s\0%s' % (mode,name,bin)
+ assert(s[0] != '0') # 0-padded octal is not acceptable in a git tree
+ l.append(s)
+ return ''.join(l)
+
+
+def tree_decode(buf):
+ """Generate a list of (mode,name,hash) from the git tree object in buf."""
+ ofs = 0
+ while ofs < len(buf):
+ z = buf.find('\0', ofs)
+ assert(z > ofs)
+ spl = buf[ofs:z].split(' ', 1)
+ assert(len(spl) == 2)
+ mode,name = spl
+ sha = buf[z+1:z+1+20]
+ ofs = z+1+20
+ yield (int(mode, 8), name, sha)
+
+
+def _encode_packobj(type, content, compression_level=1):
szout = ''
sz = len(content)
szbits = (sz & 0x0f) | (_typemap[type]<<4)
break
szbits = sz & 0x7f
sz >>= 7
- z = zlib.compressobj(1)
+ if compression_level > 9:
+ compression_level = 9
+ elif compression_level < 0:
+ compression_level = 0
+ z = zlib.compressobj(compression_level)
yield szout
yield z.compress(content)
yield z.flush()
-def _encode_looseobj(type, content):
- z = zlib.compressobj(1)
+def _encode_looseobj(type, content, compression_level=1):
+ z = zlib.compressobj(compression_level)
yield z.compress('%s %d\0' % (type, len(content)))
yield z.compress(content)
yield z.flush()
unlink(full)
else:
midxl.append(mx)
- midxl.sort(lambda x,y: -cmp(len(x),len(y)))
+ midxl.sort(key=lambda ix:
+ (-len(ix), -xstat.stat(ix.name).st_mtime))
for ix in midxl:
any_needed = False
for sub in ix.idxnames:
self.also.add(hash)
-def calc_hash(type, content):
- """Calculate some content's hash in the Git fashion."""
- header = '%s %d\0' % (type, len(content))
- sum = Sha1(header)
- sum.update(content)
- return sum.digest()
-
-
-def _shalist_sort_key(ent):
- (mode, name, id) = ent
- if stat.S_ISDIR(int(mode, 8)):
- return name + '/'
- else:
- return name
-
-
def open_idx(filename):
if filename.endswith('.idx'):
f = open(filename, 'rb')
class PackWriter:
"""Writes Git objects inside a pack file."""
- def __init__(self, objcache_maker=_make_objcache):
+ def __init__(self, objcache_maker=_make_objcache, compression_level=1):
self.count = 0
self.outbytes = 0
self.filename = None
self.idx = None
self.objcache_maker = objcache_maker
self.objcache = None
+ self.compression_level = compression_level
def __del__(self):
self.close()
log('>')
if not sha:
sha = calc_hash(type, content)
- size, crc = self._raw_write(_encode_packobj(type, content), sha=sha)
+ size, crc = self._raw_write(_encode_packobj(type, content,
+ self.compression_level),
+ sha=sha)
+ if self.outbytes >= max_pack_size or self.count >= max_pack_objects:
+ self.breakpoint()
return sha
def breakpoint(self):
def maybe_write(self, type, content):
"""Write an object to the pack file if not present and return its id."""
- self._require_objcache()
sha = calc_hash(type, content)
if not self.exists(sha):
self._write(sha, type, content)
+ self._require_objcache()
self.objcache.add(sha)
return sha
def new_tree(self, shalist):
"""Create a tree object in the pack."""
- shalist = sorted(shalist, key = _shalist_sort_key)
- l = []
- for (mode,name,bin) in shalist:
- assert(mode)
- assert(mode != '0')
- assert(mode[0] != '0')
- assert(name)
- assert(len(bin) == 20)
- l.append('%s %s\0%s' % (mode,name,bin))
- return self.maybe_write('tree', ''.join(l))
+ content = tree_encode(shalist)
+ return self.maybe_write('tree', content)
def _new_commit(self, tree, parent, author, adate, committer, cdate, msg):
l = []
idx_f.truncate(ofs64_ofs)
idx_f.seek(0)
idx_map = mmap_readwrite(idx_f, close=False)
- idx_f.seek(0, SEEK_END)
+ idx_f.seek(0, os.SEEK_END)
count = _helpers.write_idx(idx_f, idx_map, idx, self.count)
assert(count == self.count)
idx_map.close()
if parent and not os.path.exists(parent):
raise GitError('parent directory "%s" does not exist\n' % parent)
if os.path.exists(d) and not os.path.isdir(os.path.join(d, '.')):
- raise GitError('"%d" exists but is not a directory\n' % d)
+ raise GitError('"%s" exists but is not a directory\n' % d)
p = subprocess.Popen(['git', '--bare', 'init'], stdout=sys.stderr,
preexec_fn = _gitenv)
_git_wait('git init', p)
p = subprocess.Popen(['git', 'config', 'pack.indexVersion', '2'],
stdout=sys.stderr, preexec_fn = _gitenv)
_git_wait('git config', p)
+ # Enable the reflog
+ p = subprocess.Popen(['git', 'config', 'core.logAllRefUpdates', 'true'],
+ stdout=sys.stderr, preexec_fn = _gitenv)
+ _git_wait('git config', p)
def check_repo_or_die(path=None):
initializes the default repository automatically.
"""
guess_repo(path)
- if not os.path.isdir(repo('objects/pack/.')):
- if repodir == home_repodir:
- init_repo()
+ try:
+ os.stat(repo('objects/pack/.'))
+ except OSError, e:
+ if e.errno == errno.ENOENT:
+ if repodir != home_repodir:
+ log('error: %r is not a bup repository; run "bup init"\n'
+ % repo())
+ sys.exit(15)
+ else:
+ init_repo()
else:
- log('error: %r is not a bup/git repository\n' % repo())
- sys.exit(15)
-
-
-def treeparse(buf):
- """Generate a list of (mode, name, hash) tuples of objects from 'buf'."""
- ofs = 0
- while ofs < len(buf):
- z = buf[ofs:].find('\0')
- assert(z > 0)
- spl = buf[ofs:ofs+z].split(' ', 1)
- assert(len(spl) == 2)
- sha = buf[ofs+z+1:ofs+z+1+20]
- ofs += z+1+20
- yield (spl[0], spl[1], sha)
+ log('error: %s\n' % e)
+ sys.exit(14)
_ver = None
assert(self.p)
assert(self.p.poll() == None)
if self.inprogress:
- log('_fast_get: opening %r while %r is open'
+ log('_fast_get: opening %r while %r is open\n'
% (id, self.inprogress))
assert(not self.inprogress)
assert(id.find('\n') < 0)
yield blob
elif type == 'tree':
treefile = ''.join(it)
- for (mode, name, sha) in treeparse(treefile):
+ for (mode, name, sha) in tree_decode(treefile):
for blob in self.join(sha.encode('hex')):
yield blob
elif type == 'commit':
projects / bup.git / blobdiff