#!/usr/bin/env python
-import sys, re, errno, stat, time, math
+import sys, stat, time, math
from bup import hashsplit, git, options, index, client
from bup.helpers import *
optspec = """
bup save [-tc] [-n name] <filenames...>
--
-r,remote= remote repository path
+r,remote= hostname:/path/to/repo of remote repository
t,tree output a tree id
c,commit output a commit id
n,name= name of backup set to update (if any)
+d,date= date for the commit (seconds since the epoch)
v,verbose increase log output (can be used more than once)
q,quiet don't show progress meter
smaller= only back up files smaller than n bytes
+bwlimit= maximum bytes/sec to transmit to server
"""
o = options.Options('bup save', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
o.fatal("no filenames given")
opt.progress = (istty and not opt.quiet)
+opt.smaller = parse_num(opt.smaller or 0)
+if opt.bwlimit:
+ client.bwlimit = parse_num(opt.bwlimit)
+
+if opt.date:
+ date = parse_date_or_fatal(opt.date, o.fatal)
+else:
+ date = time.time()
+
+is_reverse = os.environ.get('BUP_SERVER_REVERSE')
+if is_reverse and opt.remote:
+ o.fatal("don't use -r in reverse mode; it's automatic")
refname = opt.name and 'refs/heads/%s' % opt.name or None
-if opt.remote:
- cli = client.Client(opt.remote)
+if opt.remote or is_reverse:
+ if opt.remote and opt.remote.find(":") == -1:
+ o.fatal("--remote argument must contain a colon")
+ try:
+ cli = client.Client(opt.remote)
+ except client.ClientError:
+ o.fatal("server exited unexpectedly; see errors above")
oldref = refname and cli.read_ref(refname) or None
w = cli.new_packwriter()
else:
oldref = refname and git.read_ref(refname) or None
w = git.PackWriter()
+handle_ctrl_c()
+
def eatslash(dir):
if dir.endswith('/'):
return tree
lastremain = None
+lastprint = 0
def progress_report(n):
- global count, lastremain
- count += n
- pct = total and (count*100.0/total) or 0
+ global count, subcount, lastremain, lastprint
+ subcount += n
+ cc = count + subcount
+ pct = total and (cc*100.0/total) or 0
now = time.time()
elapsed = now - tstart
- kps = elapsed and int(count/1024./elapsed)
+ kps = elapsed and int(cc/1024./elapsed)
kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
kps = int(kps/kps_frac)*kps_frac
- if count:
- remain = elapsed*1.0/count * (total-count)
+ if cc:
+ remain = elapsed*1.0/cc * (total-cc)
else:
remain = 0.0
if (lastremain and (remain > lastremain)
remainstr = '%dm%d' % (mins, secs)
else:
remainstr = '%ds' % secs
- progress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r'
- % (pct, count/1024, total/1024, fcount, ftotal,
- remainstr, kpsstr))
+ if now - lastprint > 0.1:
+ progress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r'
+ % (pct, cc/1024, total/1024, fcount, ftotal,
+ remainstr, kpsstr))
+ lastprint = now
+
+
+def vlog(s):
+ global lastprint
+ lastprint = 0
+ log(s)
r = index.Reader(git.repo('bupindex'))
def already_saved(ent):
return ent.is_valid() and w.exists(ent.sha) and ent.sha
-def wantrecurse(ent):
+def wantrecurse_pre(ent):
return not already_saved(ent)
+def wantrecurse_during(ent):
+ return not already_saved(ent) or ent.sha_missing()
+
total = ftotal = 0
if opt.progress:
- for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse):
+ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
if not (ftotal % 10024):
progress('Reading index: %d\r' % ftotal)
- exists = (ent.flags & index.IX_EXISTS)
+ exists = ent.exists()
hashvalid = already_saved(ent)
- if exists and not hashvalid:
- total += ent.size
+ ent.set_sha_missing(not hashvalid)
+ if not opt.smaller or ent.size < opt.smaller:
+ if exists and not hashvalid:
+ total += ent.size
ftotal += 1
progress('Reading index: %d, done.\n' % ftotal)
hashsplit.progress_callback = progress_report
tstart = time.time()
-count = fcount = 0
-for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse):
+count = subcount = fcount = 0
+lastskip_name = None
+lastdir = ''
+for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
(dir, file) = os.path.split(ent.name)
exists = (ent.flags & index.IX_EXISTS)
hashvalid = already_saved(ent)
+ wasmissing = ent.sha_missing()
+ oldsize = ent.size
if opt.verbose:
if not exists:
status = 'D'
status = 'M'
else:
status = ' '
- if opt.verbose >= 2 or stat.S_ISDIR(ent.mode):
- log('%s %-70s\n' % (status, ent.name))
+ if opt.verbose >= 2:
+ vlog('%s %-70s\n' % (status, ent.name))
+ elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
+ if not lastdir.startswith(dir):
+ vlog('%s %-70s\n' % (status, os.path.join(dir, '')))
+ lastdir = dir
if opt.progress:
progress_report(0)
if not exists:
continue
+ if opt.smaller and ent.size >= opt.smaller:
+ if exists and not hashvalid:
+ add_error('skipping large file "%s"' % ent.name)
+ lastskip_name = ent.name
+ continue
assert(dir.startswith('/'))
dirp = dir.split('/')
_push(part)
if not file:
+ # no filename portion means this is a subdir. But
# sub/parentdirectories already handled in the pop/push() part above.
oldtree = already_saved(ent) # may be None
newtree = _pop(force_tree = oldtree)
if not oldtree:
- ent.validate(040000, newtree)
+ if lastskip_name and lastskip_name.startswith(ent.name):
+ ent.invalidate()
+ else:
+ ent.validate(040000, newtree)
ent.repack()
- count += ent.size
+ if exists and wasmissing:
+ count += oldsize
continue
# it's not a directory
if hashvalid:
mode = '%o' % ent.gitmode
id = ent.sha
- shalists[-1].append((mode, file, id))
- elif opt.smaller and ent.size >= opt.smaller:
- add_error('skipping large file "%s"' % ent.name)
+ shalists[-1].append((mode,
+ git.mangle_name(file, ent.mode, ent.gitmode),
+ id))
else:
if stat.S_ISREG(ent.mode):
try:
- f = open(ent.name)
+ f = hashsplit.open_noatime(ent.name)
except IOError, e:
add_error(e)
+ lastskip_name = ent.name
except OSError, e:
add_error(e)
+ lastskip_name = ent.name
else:
- (mode, id) = hashsplit.split_to_blob_or_tree(w, [f])
+ (mode, id) = hashsplit.split_to_blob_or_tree(w, [f], False)
else:
if stat.S_ISDIR(ent.mode):
assert(0) # handled above
rl = os.readlink(ent.name)
except OSError, e:
add_error(e)
+ lastskip_name = ent.name
except IOError, e:
add_error(e)
+ lastskip_name = ent.name
else:
(mode, id) = ('120000', w.new_blob(rl))
else:
add_error(Exception('skipping special file "%s"' % ent.name))
- count += ent.size
+ lastskip_name = ent.name
if id:
ent.validate(int(mode, 8), id)
ent.repack()
- shalists[-1].append((mode, file, id))
+ shalists[-1].append((mode,
+ git.mangle_name(file, ent.mode, ent.gitmode),
+ id))
+ if exists and wasmissing:
+ count += oldsize
+ subcount = 0
+
if opt.progress:
pct = total and count*100.0/total or 100
if opt.commit or opt.name:
msg = 'bup save\n\nGenerated by command:\n%r' % sys.argv
ref = opt.name and ('refs/heads/%s' % opt.name) or None
- commit = w.new_commit(oldref, tree, msg)
+ commit = w.new_commit(oldref, tree, date, msg)
if opt.commit:
print commit.encode('hex')
if saved_errors:
log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+ sys.exit(1)
projects / bup.git / blobdiff