bup ls <dirs...>
--
s,hash show hash for each file
+a,all show hidden files
"""
o = options.Options('bup ls', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
n = top.lresolve(d)
if stat.S_ISDIR(n.mode):
for sub in n:
- print_node(sub.name, sub)
+ if opt.all or not sub.name.startswith('.'):
+ print_node(sub.name, sub)
else:
- print_node(d, n)
+ if opt.all or not sub.name.startswith('.'):
+ print_node(d, n)
except vfs.NodeError, e:
log('error: %s\n' % e)
ret = 1
if is_reverse and opt.remote:
o.fatal("don't use -r in reverse mode; it's automatic")
+if opt.name and opt.name.startswith('.'):
+ o.fatal("'%s' is not a valid branch name" % opt.name)
refname = opt.name and 'refs/heads/%s' % opt.name or None
if opt.remote or is_reverse:
cli = client.Client(opt.remote)
print tree.encode('hex')
if opt.commit or opt.name:
msg = 'bup save\n\nGenerated by command:\n%r' % sys.argv
- ref = opt.name and ('refs/heads/%s' % opt.name) or None
commit = w.new_commit(oldref, tree, date, msg)
if opt.commit:
print commit.encode('hex')
conn.ok()
-def receive_objects(conn, junk):
+def receive_objects_v2(conn, junk):
global suspended_w
git.check_repo_or_die()
suggested = {}
conn.ok()
return
+ shar = conn.read(20)
+ crcr = struct.unpack('!I', conn.read(4))[0]
+ n -= 20 + 4
buf = conn.read(n) # object sizes in bup are reasonably small
#debug2('read %d bytes\n' % n)
if len(buf) < n:
w.abort()
raise Exception('object read: expected %d bytes, got %d\n'
% (n, len(buf)))
- (type, content) = git._decode_packobj(buf)
- sha = git.calc_hash(type, content)
- oldpack = w.exists(sha)
+ oldpack = w.exists(shar)
# FIXME: we only suggest a single index per cycle, because the client
# is currently too dumb to download more than one per cycle anyway.
# Actually we should fix the client, but this is a minor optimization
# fix that deficiency of midx files eventually, although it'll
# make the files bigger. This method is certainly not very
# efficient.
- oldpack = w.objcache.packname_containing(sha)
+ oldpack = w.objcache.packname_containing(shar)
debug2('new suggestion: %r\n' % oldpack)
assert(oldpack)
assert(oldpack != True)
conn.write('index %s\n' % name)
suggested[name] = 1
else:
- w._raw_write([buf])
+ nw, crc = w._raw_write([buf], sha=shar)
+ _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
+ _check(w, n, nw, 'object read: expected %d bytes, got %d\n')
# NOTREACHED
+
+
+def _check(w, expected, actual, msg):
+ if expected != actual:
+ w.abort()
+ raise Exception(msg % (expected, actual))
def read_ref(conn, refname):
'set-dir': set_dir,
'list-indexes': list_indexes,
'send-index': send_index,
- 'receive-objects': receive_objects,
+ 'receive-objects-v2': receive_objects_v2,
'read-ref': read_ref,
'update-ref': update_ref,
'cat': cat,
o.fatal("don't use -r in reverse mode; it's automatic")
start_time = time.time()
+if opt.name and opt.name.startswith('.'):
+ o.fatal("'%s' is not a valid branch name." % opt.name)
refname = opt.name and 'refs/heads/%s' % opt.name or None
if opt.noop or opt.copy:
cli = pack_writer = oldref = None
try:
os.unlink(tag_file)
except OSError, e:
- log("bup: error: unable to delete tag: %s" % e)
+ log("bup: error: unable to delete tag '%s': %s" % (opt.delete, e))
sys.exit(1)
sys.exit(0)
-tags = []
-for (t, dummy) in git.list_refs():
- if t.startswith('refs/tags/'):
- tags.append(t[10:])
+tags = [t for sublist in git.tags().values() for t in sublist]
if not extra:
for t in tags:
- log("%s\n" % t)
+ print t
sys.exit(0)
-elif len(extra) != 2:
- log('bup: error: no ref or hash given.')
- sys.exit(1)
+elif len(extra) < 2:
+ o.fatal('no commit ref or hash given.')
-tag_name = extra[0]
-commit = extra[1]
-debug1("from args: tag name = %s; commit = %s\n" % (tag_name, commit))
+(tag_name, commit) = extra[:2]
+if not tag_name:
+ o.fatal("tag name must not be empty.")
+debug1("args: tag name = %s; commit = %s\n" % (tag_name, commit))
if tag_name in tags:
log("bup: error: tag '%s' already exists" % tag_name)
sys.exit(1)
-hash = git.rev_parse(commit)
+if tag_name.startswith('.'):
+ o.fatal("'%s' is not a valid tag name." % tag_name)
+
+try:
+ hash = git.rev_parse(commit)
+except git.GitError, e:
+ log("bup: error: %s" % e)
+ sys.exit(2)
+
if not hash:
log("bup: error: commit %s not found." % commit)
sys.exit(2)
try:
tag = file(tag_file, 'w')
except OSError, e:
- log('bup: error: could not create tag %s: %s' % (tag_name, e))
+ log("bup: error: could not create tag '%s': %s" % (tag_name, e))
sys.exit(3)
tag.write(hash.encode('hex'))
-import re, struct, errno, time
+import re, struct, errno, time, zlib
from bup import git, ssh
from bup.helpers import *
debug1('client: received index suggestion: %s\n' % indexname)
ob = self._busy
if ob:
- assert(ob == 'receive-objects')
+ assert(ob == 'receive-objects-v2')
self.conn.write('\xff\xff\xff\xff') # suspend receive-objects
self._busy = None
self.conn.drain_and_check_ok()
self.sync_index(indexname)
if ob:
self._busy = ob
- self.conn.write('receive-objects\n')
+ self.conn.write('receive-objects-v2\n')
def new_packwriter(self):
self.check_busy()
def _set_busy():
- self._busy = 'receive-objects'
- self.conn.write('receive-objects\n')
+ self._busy = 'receive-objects-v2'
+ self.conn.write('receive-objects-v2\n')
return PackWriter_Remote(self.conn,
objcache_maker = self._make_objcache,
suggest_pack = self._suggest_pack,
def abort(self):
raise GitError("don't know how to abort remote pack writing")
- def _raw_write(self, datalist):
+ def _raw_write(self, datalist, sha):
assert(self.file)
if not self._packopen:
self._open()
if self.ensure_busy:
self.ensure_busy()
data = ''.join(datalist)
- assert(len(data))
- outbuf = struct.pack('!I', len(data)) + data
+ assert(data)
+ assert(sha)
+ crc = zlib.crc32(data) & 0xffffffff
+ outbuf = ''.join((struct.pack('!I', len(data) + 20 + 4),
+ sha,
+ struct.pack('!I', crc),
+ data))
(self._bwcount, self._bwtime) = \
_raw_write_bwlimit(self.file, outbuf, self._bwcount, self._bwtime)
- self.outbytes += len(data)
+ self.outbytes += len(data) - 20 - 4 # Don't count sha1+crc
self.count += 1
if self.file.has_input():
if self.suggest_pack:
self.suggest_pack(idxname)
self.objcache.refresh()
+
+ return sha, crc
bup repositories are in Git format. This library allows us to
interact with the Git data structures.
"""
-import os, zlib, time, subprocess, struct, stat, re, tempfile
-import heapq
+import os, zlib, time, subprocess, struct, stat, re, tempfile, heapq
from bup.helpers import *
from bup import _helpers
class PackIdx:
def __init__(self):
assert(0)
-
+
def find_offset(self, hash):
"""Get the offset of an object inside the index file."""
idx = self._idx_from_hash(hash)
ofs = struct.unpack('!I', str(buffer(self.ofstable, idx*4, 4)))[0]
if ofs & 0x80000000:
idx64 = ofs & 0x7fffffff
- ofs = struct.unpack('!I',
+ ofs = struct.unpack('!Q',
str(buffer(self.ofs64table, idx64*8, 8)))[0]
return ofs
self.outbytes = 0
self.filename = None
self.file = None
+ self.idx = None
self.objcache_maker = objcache_maker
self.objcache = None
assert(name.endswith('.pack'))
self.filename = name[:-5]
self.file.write('PACK\0\0\0\2\0\0\0\0')
+ self.idx = list(list() for i in xrange(256))
- def _raw_write(self, datalist):
+ # the 'sha' parameter is used in client.py's _raw_write(), but not needed
+ # in this basic version.
+ def _raw_write(self, datalist, sha):
self._open()
f = self.file
# in case we get interrupted (eg. KeyboardInterrupt), it's best if
# but that's okay because we'll flush it in _end().
oneblob = ''.join(datalist)
f.write(oneblob)
- self.outbytes += len(oneblob)
+ nw = len(oneblob)
+ crc = zlib.crc32(oneblob) & 0xffffffff
+ self._update_idx(sha, crc, nw)
+ self.outbytes += nw
self.count += 1
+ return nw, crc
+
+ def _update_idx(self, sha, crc, size):
+ assert(sha)
+ if self.idx:
+ self.idx[ord(sha[0])].append((sha, crc, self.file.tell() - size))
- def _write(self, bin, type, content):
+ def _write(self, sha, type, content):
if verbose:
log('>')
- self._raw_write(_encode_packobj(type, content))
- return bin
+ if not sha:
+ sha = calc_hash(type, content)
+ size, crc = self._raw_write(_encode_packobj(type, content), sha=sha)
+ return sha
def breakpoint(self):
"""Clear byte and object counts and return the last processed id."""
def maybe_write(self, type, content):
"""Write an object to the pack file if not present and return its id."""
- bin = calc_hash(type, content)
- if not self.exists(bin):
- self._write(bin, type, content)
- self.objcache.add(bin)
- return bin
+ sha = calc_hash(type, content)
+ if not self.exists(sha):
+ self._write(sha, type, content)
+ self.objcache.add(sha)
+ return sha
def new_blob(self, blob):
"""Create a blob object in the pack with the supplied content."""
"""Remove the pack file from disk."""
f = self.file
if f:
+ self.idx = None
self.file = None
f.close()
os.unlink(self.filename + '.pack')
if not f: return None
self.file = None
self.objcache = None
+ idx = self.idx
+ self.idx = None
# update object count
f.seek(8)
# calculate the pack sha1sum
f.seek(0)
sum = Sha1()
- while 1:
- b = f.read(65536)
+ for b in chunkyreader(f):
sum.update(b)
- if not b: break
- f.write(sum.digest())
-
+ packbin = sum.digest()
+ f.write(packbin)
f.close()
- p = subprocess.Popen(['git', 'index-pack', '-v',
- '--index-version=2',
- self.filename + '.pack'],
- preexec_fn = _gitenv,
- stdout = subprocess.PIPE)
- out = p.stdout.read().strip()
- _git_wait('git index-pack', p)
- if not out:
- raise GitError('git index-pack produced no output')
- nameprefix = repo('objects/pack/%s' % out)
+ idx_f = open(self.filename + '.idx', 'wb')
+ obj_list_sha = self._write_pack_idx_v2(idx_f, idx, packbin)
+ idx_f.close()
+
+ nameprefix = repo('objects/pack/pack-%s' % obj_list_sha)
if os.path.exists(self.filename + '.map'):
os.unlink(self.filename + '.map')
os.rename(self.filename + '.pack', nameprefix + '.pack')
"""Close the pack file and move it to its definitive path."""
return self._end()
+ def _write_pack_idx_v2(self, file, idx, packbin):
+ sum = Sha1()
+
+ def write(data):
+ file.write(data)
+ sum.update(data)
+
+ write('\377tOc\0\0\0\2')
+
+ n = 0
+ for part in idx:
+ n += len(part)
+ write(struct.pack('!i', n))
+ part.sort(key=lambda x: x[0])
+
+ obj_list_sum = Sha1()
+ for part in idx:
+ for entry in part:
+ write(entry[0])
+ obj_list_sum.update(entry[0])
+ for part in idx:
+ for entry in part:
+ write(struct.pack('!I', entry[1]))
+ ofs64_list = []
+ for part in idx:
+ for entry in part:
+ if entry[2] & 0x80000000:
+ write(struct.pack('!I', 0x80000000 | len(ofs64_list)))
+ ofs64_list.append(struct.pack('!Q', entry[2]))
+ else:
+ write(struct.pack('!i', entry[2]))
+ for ofs64 in ofs64_list:
+ write(ofs64)
+
+ write(packbin)
+ file.write(sum.digest())
+ return obj_list_sum.hexdigest()
+
def _git_date(date):
return '%d %s' % (date, time.strftime('%z', time.localtime(date)))
yield d
except StopIteration:
log('booger!\n')
+
+def tags():
+ """Return a dictionary of all tags in the form {hash: [tag_names, ...]}."""
+ tags = {}
+ for (n,c) in list_refs():
+ if n.startswith('refs/tags/'):
+ name = n[10:]
+ if not c in tags:
+ tags[c] = []
+
+ tags[c].append(name) # more than one tag can point at 'c'
+
+ return tags
-import time
+import struct, os, tempfile, time
from bup import git
from bup.helpers import *
from wvtest import *
WVPASS(r.exists(hashes[5]))
WVPASS(r.exists(hashes[6]))
WVFAIL(r.exists('\0'*20))
+
+@wvtest
+def test_long_index():
+ w = git.PackWriter()
+ obj_bin = struct.pack('!IIIII',
+ 0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
+ obj2_bin = struct.pack('!IIIII',
+ 0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
+ obj3_bin = struct.pack('!IIIII',
+ 0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
+ pack_bin = struct.pack('!IIIII',
+ 0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
+ idx = list(list() for i in xrange(256))
+ idx[0].append((obj_bin, 1, 0xfffffffff))
+ idx[0x11].append((obj2_bin, 2, 0xffffffffff))
+ idx[0x22].append((obj3_bin, 3, 0xff))
+ (fd,name) = tempfile.mkstemp(suffix='.idx', dir=git.repo('objects'))
+ f = os.fdopen(fd, 'w+b')
+ r = w._write_pack_idx_v2(f, idx, pack_bin)
+ f.seek(0)
+ i = git.PackIdxV2(name, f)
+ WVPASS(i.find_offset(obj_bin)==0xfffffffff)
+ WVPASS(i.find_offset(obj2_bin)==0xffffffffff)
+ WVPASS(i.find_offset(obj3_bin)==0xff)
+ f.close()
+ os.remove(name)
def _mksubs(self):
self._subs = {}
- tags = {}
- for (n,c) in git.list_refs():
- if n.startswith('refs/tags/'):
- name = n[10:]
- if not c in tags:
- tags[c] = []
-
- tags[c].append(name)
+ tags = git.tags()
revs = list(git.rev_list(self.hash.encode('hex')))
for (date, commit) in revs:
WVSTART "tag"
WVFAIL bup tag -d v0.n 2>/dev/null
WVFAIL bup tag v0.n non-existant 2>/dev/null
-WVPASSEQ "$(bup tag 2>&1)" ""
+WVPASSEQ "$(bup tag)" ""
WVPASS bup tag v0.1 master
-WVPASSEQ "$(bup tag 2>&1)" "v0.1"
+WVPASSEQ "$(bup tag)" "v0.1"
WVPASS bup tag -d v0.1
# This section destroys data in the bup repository, so it is done last.
projects / bup.git / commitdiff