#!/usr/bin/env python
-import sys, os, subprocess, errno, zlib, time
+import sys, time, re
import hashsplit, git, options
from helpers import *
-BLOB_LWM = 8192*2
-BLOB_MAX = BLOB_LWM*2
-BLOB_HWM = 1024*1024
-
-
-class Buf:
- def __init__(self):
- self.data = ''
- self.start = 0
-
- def put(self, s):
- #log('oldsize=%d+%d adding=%d\n' % (len(self.data), self.start, len(s)))
- if s:
- self.data = buffer(self.data, self.start) + s
- self.start = 0
-
- def peek(self, count):
- return buffer(self.data, self.start, count)
-
- def eat(self, count):
- self.start += count
-
- def get(self, count):
- v = buffer(self.data, self.start, count)
- self.start += count
- return v
-
- def used(self):
- return len(self.data) - self.start
-
-
-def splitbuf(buf):
- b = buf.peek(buf.used())
- ofs = hashsplit.splitbuf(b)
- if ofs:
- buf.eat(ofs)
- return buffer(b, 0, ofs)
- return None
-
-
-def hashsplit_iter(f):
- ofs = 0
- buf = Buf()
- blob = 1
-
- eof = 0
- lv = 0
- while blob or not eof:
- if not eof and (buf.used() < BLOB_LWM or not blob):
- bnew = f.read(BLOB_HWM)
- if not len(bnew): eof = 1
- #log('got %d, total %d\n' % (len(bnew), buf.used()))
- buf.put(bnew)
-
- blob = splitbuf(buf)
- if eof and not blob:
- blob = buf.get(buf.used())
- if not blob and buf.used() >= BLOB_MAX:
- blob = buf.get(BLOB_MAX) # limit max blob size
- if not blob and not eof:
- continue
-
- if blob:
- yield (ofs, len(blob), git.hash_blob(blob))
- ofs += len(blob)
-
- nv = (ofs + buf.used())/1000000
- if nv != lv:
- log('%d\t' % nv)
- lv = nv
-
-
-def autofiles(filenames):
- if not filenames:
- yield sys.stdin
- else:
- for n in filenames:
- yield open(n)
-
-
optspec = """
-bup split [-t] [filenames...]
+bup split [-tcb] [-n name] [--bench] [filenames...]
--
+r,remote= remote repository path
b,blobs output a series of blob ids
t,tree output a tree id
c,commit output a commit id
n,name= name of backup set to update (if any)
+v,verbose increase log output (can be used more than once)
bench print benchmark timings to stderr
"""
o = options.Options('bup split', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
+git.check_repo_or_die()
if not (opt.blobs or opt.tree or opt.commit or opt.name):
log("bup split: use one or more of -b, -t, -c, -n\n")
o.usage()
+hashsplit.split_verbosely = opt.verbose
+if opt.verbose >= 2:
+ git.verbose = opt.verbose - 1
+ opt.bench = 1
+
start_time = time.time()
-shalist = []
-ofs = 0
-last_ofs = 0
-for f in autofiles(extra):
- for (xofs, size, sha) in hashsplit_iter(f):
- #log('SPLIT @ %-8d size=%-8d\n' % (ofs, size))
- if opt.blobs:
- print sha
-
- # this silliness keeps chunk filenames "similar" when a file changes
- # slightly.
- bm = BLOB_MAX
- while 1:
- cn = ofs / bm * bm
- #log('%x,%x,%x,%x\n' % (last_ofs,ofs,cn,bm))
- if cn > last_ofs or ofs == last_ofs: break
- bm /= 2
- last_ofs = cn
- shalist.append(('100644', 'bup.chunk.%016x' % cn, sha))
- ofs += size
-tree = git.gen_tree(shalist)
+def server_connect(remote):
+ rs = remote.split(':', 1)
+ if len(rs) == 1:
+ p = subprocess.Popen(['bup', 'server', '-d', opt.remote],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ else:
+ (host, dir) = rs
+ p = subprocess.Popen(['ssh', host, '--', 'bup', 'server'],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ dir = re.sub(r'[\r\n]', ' ', dir)
+ p.stdin.write('set-dir\n%s\n' % dir)
+ return p
+
+if opt.remote:
+ p = server_connect(opt.remote)
+ p.stdin.write('receive-objects\n')
+ w = git.PackWriter_Remote(p.stdin)
+else:
+ w = git.PackWriter()
+
+(shalist,tree) = hashsplit.split_to_tree(w, hashsplit.autofiles(extra))
+
+if opt.verbose:
+ log('\n')
+if opt.blobs:
+ for (mode,name,bin) in shalist:
+ print bin.encode('hex')
if opt.tree:
- print tree
+ print tree.encode('hex')
if opt.commit or opt.name:
- msg = 'Generated by command:\n%r' % sys.argv
+ msg = 'bup split\n\nGenerated by command:\n%r' % sys.argv
ref = opt.name and ('refs/heads/%s' % opt.name) or None
- commit = git.gen_commit_easy(ref, tree, msg)
+ commit = w.new_commit(ref, tree, msg)
if opt.commit:
- print commit
+ print commit.encode('hex')
+
+if opt.remote:
+ w.close()
+ p.stdin.write('quit\n')
+ p.wait()
secs = time.time() - start_time
+size = hashsplit.total_split
if opt.bench:
log('\nbup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
- % (ofs/1024., secs, ofs/1024./secs))
+ % (size/1024., secs, size/1024./secs))
projects / bup.git / blobdiff