#!/usr/bin/env python
-import sys, os, subprocess, errno, zlib, time
+import sys, time, re
import hashsplit, git, options
from helpers import *
-BLOB_LWM = 8192*2
-BLOB_MAX = BLOB_LWM*2
-BLOB_HWM = 1024*1024
-
-
-class Buf:
- def __init__(self):
- self.data = ''
- self.start = 0
-
- def put(self, s):
- #log('oldsize=%d+%d adding=%d\n' % (len(self.data), self.start, len(s)))
- if s:
- self.data = buffer(self.data, self.start) + s
- self.start = 0
-
- def peek(self, count):
- return buffer(self.data, self.start, count)
-
- def eat(self, count):
- self.start += count
-
- def get(self, count):
- v = buffer(self.data, self.start, count)
- self.start += count
- return v
-
- def used(self):
- return len(self.data) - self.start
-
-
-def splitbuf(buf):
- b = buf.peek(buf.used())
- ofs = hashsplit.splitbuf(b)
- if ofs:
- buf.eat(ofs)
- return buffer(b, 0, ofs)
- return None
-
-
-def hashsplit_iter(f):
- ofs = 0
- buf = Buf()
- blob = 1
-
- eof = 0
- lv = 0
- while blob or not eof:
- if not eof and (buf.used() < BLOB_LWM or not blob):
- bnew = sys.stdin.read(BLOB_HWM)
- if not len(bnew): eof = 1
- #log('got %d, total %d\n' % (len(bnew), buf.used()))
- buf.put(bnew)
-
- blob = splitbuf(buf)
- if eof and not blob:
- blob = buf.get(buf.used())
- if not blob and buf.used() >= BLOB_MAX:
- blob = buf.get(BLOB_MAX) # limit max blob size
- if not blob and not eof:
- continue
-
- if blob:
- yield (ofs, len(blob), git.hash_blob(blob))
- ofs += len(blob)
-
- nv = (ofs + buf.used())/1000000
- if nv != lv:
- log('%d\t' % nv)
- lv = nv
-
optspec = """
-bup split [-t] <filename
+bup split [-tcb] [-n name] [--bench] [filenames...]
--
-t,tree output a tree instead of a series of blobs
-c,commit output a commit instead of a tree or blobs
+r,remote= remote repository path
+b,blobs output a series of blob ids
+t,tree output a tree id
+c,commit output a commit id
+n,name= name of backup set to update (if any)
+v,verbose increase log output (can be used more than once)
bench print benchmark timings to stderr
"""
-(opt, flags, extra) = options.Options('bup split', optspec).parse(sys.argv[1:])
+o = options.Options('bup split', optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+git.check_repo_or_die()
+if not (opt.blobs or opt.tree or opt.commit or opt.name):
+ log("bup split: use one or more of -b, -t, -c, -n\n")
+ o.usage()
+
+hashsplit.split_verbosely = opt.verbose
+if opt.verbose >= 2:
+ git.verbose = opt.verbose - 1
+ opt.bench = 1
start_time = time.time()
-shalist = []
-ofs = 0
-for (ofs, size, sha) in hashsplit_iter(sys.stdin):
- #log('SPLIT @ %-8d size=%-8d\n' % (ofs, size))
- if not opt.tree and not opt.commit:
- print sha
- shalist.append(('100644', '%016x.bupchunk' % ofs, sha))
-if opt.tree or opt.commit:
- tree = git.gen_tree(shalist)
-if opt.commit:
- now = time.time()
- userline = '%s <%s@%s>' % (userfullname(), username(), hostname())
- msg = 'Generated by command:\n%r' % sys.argv
- commit = git.gen_commit(tree, None, userline, now, userline, now, msg)
- print commit
-elif opt.tree:
- print tree
+def server_connect(remote):
+ rs = remote.split(':', 1)
+ if len(rs) == 1:
+ p = subprocess.Popen(['bup', 'server', '-d', opt.remote],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ else:
+ (host, dir) = rs
+ p = subprocess.Popen(['ssh', host, '--', 'bup', 'server'],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ dir = re.sub(r'[\r\n]', ' ', dir)
+ p.stdin.write('set-dir\n%s\n' % dir)
+ return p
+
+if opt.remote:
+ p = server_connect(opt.remote)
+ p.stdin.write('receive-objects\n')
+ w = git.PackWriter_Remote(p.stdin)
+else:
+ w = git.PackWriter()
+
+(shalist,tree) = hashsplit.split_to_tree(w, hashsplit.autofiles(extra))
+
+if opt.verbose:
+ log('\n')
+if opt.blobs:
+ for (mode,name,bin) in shalist:
+ print bin.encode('hex')
+if opt.tree:
+ print tree.encode('hex')
+if opt.commit or opt.name:
+ msg = 'bup split\n\nGenerated by command:\n%r' % sys.argv
+ ref = opt.name and ('refs/heads/%s' % opt.name) or None
+ commit = w.new_commit(ref, tree, msg)
+ if opt.commit:
+ print commit.encode('hex')
+
+if opt.remote:
+ w.close()
+ p.stdin.write('quit\n')
+ p.wait()
secs = time.time() - start_time
+size = hashsplit.total_split
if opt.bench:
log('\nbup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
- % (ofs/1024., secs, ofs/1024./secs))
+ % (size/1024., secs, size/1024./secs))
projects / bup.git / blobdiff