exec "$bup_python" "$0" ${1+"$@"}
"""
# end of bup preamble
+
+from __future__ import absolute_import, print_function
import sys, os, glob, subprocess
+from shutil import rmtree
+from subprocess import PIPE, Popen
+from tempfile import mkdtemp
+
from bup import options, git
-from bup.helpers import *
+from bup.compat import argv_bytes
+from bup.helpers import Sha1, chunkyreader, istty2, log, progress
+from bup.io import byte_stream
+
par2_ok = 0
-nullf = open('/dev/null')
+nullf = open(os.devnull, 'wb+')
def debug(s):
if opt.verbose > 1:
global par2_ok
rv = 1
try:
- p = subprocess.Popen(['par2', '--help'],
+ p = subprocess.Popen([b'par2', b'--help'],
stdout=nullf, stderr=nullf, stdin=nullf)
rv = p.wait()
except OSError:
else:
par2_ok = 1
-def parv(lvl):
- if opt.verbose >= lvl:
- if istty2:
- return []
- else:
- return ['-q']
+def is_par2_parallel():
+ # A true result means it definitely allows -t1; a false result is
+ # technically inconclusive, but likely means no.
+ tmpdir = mkdtemp(prefix=b'bup-fsck')
+ try:
+ canary = tmpdir + b'/canary'
+ with open(canary, 'wb') as f:
+ f.write(b'canary\n')
+ p = subprocess.Popen((b'par2', b'create', b'-qq', b'-t1', canary),
+ stderr=PIPE, stdin=nullf)
+ _, err = p.communicate()
+ parallel = p.returncode == 0
+ if opt.verbose:
+ if len(err) > 0 and err != b'Invalid option specified: -t1\n':
+ log('Unexpected par2 error output\n')
+ log(repr(err) + '\n')
+ if parallel:
+ log('Assuming par2 supports parallel processing\n')
+ else:
+ log('Assuming par2 does not support parallel processing\n')
+ return parallel
+ finally:
+ rmtree(tmpdir)
+
+_par2_parallel = None
+
+def par2(action, args, verb_floor=0):
+ global _par2_parallel
+ if _par2_parallel is None:
+ _par2_parallel = is_par2_parallel()
+ cmd = [b'par2', action]
+ if opt.verbose >= verb_floor and not istty2:
+ cmd.append(b'-q')
else:
- return ['-qq']
+ cmd.append(b'-qq')
+ if _par2_parallel:
+ cmd.append(b'-t1')
+ cmd.extend(args)
+ return run(cmd)
def par2_generate(base):
- return run(['par2', 'create', '-n1', '-c200'] + parv(2)
- + ['--', base, base+'.pack', base+'.idx'])
+ return par2(b'create',
+ [b'-n1', b'-c200', b'--', base, base + b'.pack', base + b'.idx'],
+ verb_floor=2)
def par2_verify(base):
- return run(['par2', 'verify'] + parv(3) + ['--', base])
+ return par2(b'verify', [b'--', base], verb_floor=3)
def par2_repair(base):
- return run(['par2', 'repair'] + parv(2) + ['--', base])
+ return par2(b'repair', [b'--', base], verb_floor=2)
def quick_verify(base):
- f = open(base + '.pack', 'rb')
+ f = open(base + b'.pack', 'rb')
f.seek(-20, 2)
wantsum = f.read(20)
assert(len(wantsum) == 20)
try:
quick_verify(base)
except Exception as e:
- debug('error: %s\n' % e)
+ log('error: %s\n' % e)
return 1
return 0
else:
- return run(['git', 'verify-pack', '--', base])
+ return run([b'git', b'verify-pack', b'--', base])
-def do_pack(base, last, par2_exists):
+def do_pack(base, last, par2_exists, out):
code = 0
if par2_ok and par2_exists and (opt.repair or not opt.generate):
vresult = par2_verify(base)
if opt.repair:
rresult = par2_repair(base)
if rresult != 0:
- action_result = 'failed'
+ action_result = b'failed'
log('%s par2 repair: failed (%d)\n' % (last, rresult))
code = rresult
else:
- action_result = 'repaired'
+ action_result = b'repaired'
log('%s par2 repair: succeeded (0)\n' % last)
code = 100
else:
- action_result = 'failed'
+ action_result = b'failed'
log('%s par2 verify: failed (%d)\n' % (last, vresult))
code = vresult
else:
- action_result = 'ok'
+ action_result = b'ok'
elif not opt.generate or (par2_ok and not par2_exists):
gresult = git_verify(base)
if gresult != 0:
- action_result = 'failed'
+ action_result = b'failed'
log('%s git verify: failed (%d)\n' % (last, gresult))
code = gresult
else:
if par2_ok and opt.generate:
presult = par2_generate(base)
if presult != 0:
- action_result = 'failed'
+ action_result = b'failed'
log('%s par2 create: failed (%d)\n' % (last, presult))
code = presult
else:
- action_result = 'generated'
+ action_result = b'generated'
else:
- action_result = 'ok'
+ action_result = b'ok'
else:
assert(opt.generate and (not par2_ok or par2_exists))
- action_result = 'exists' if par2_exists else 'skipped'
+ action_result = b'exists' if par2_exists else b'skipped'
if opt.verbose:
- print last, action_result
+ out.write(last + b' ' + action_result + b'\n')
return code
"""
o = options.Options(optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
+opt.verbose = opt.verbose or 0
par2_setup()
if opt.par2_ok:
git.check_repo_or_die()
-if not extra:
+if extra:
+ extra = [argv_byes(x) for x in extra]
+else:
debug('fsck: No filenames given: checking all packs.\n')
- extra = glob.glob(git.repo('objects/pack/*.pack'))
+ extra = glob.glob(git.repo(b'objects/pack/*.pack'))
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
code = 0
count = 0
outstanding = {}
for name in extra:
- if name.endswith('.pack'):
+ if name.endswith(b'.pack'):
base = name[:-5]
- elif name.endswith('.idx'):
+ elif name.endswith(b'.idx'):
base = name[:-4]
- elif name.endswith('.par2'):
+ elif name.endswith(b'.par2'):
base = name[:-5]
- elif os.path.exists(name + '.pack'):
+ elif os.path.exists(name + b'.pack'):
base = name
else:
- raise Exception('%s is not a pack file!' % name)
+ raise Exception('%r is not a pack file!' % name)
(dir,last) = os.path.split(base)
- par2_exists = os.path.exists(base + '.par2')
- if par2_exists and os.stat(base + '.par2').st_size == 0:
+ par2_exists = os.path.exists(base + b'.par2')
+ if par2_exists and os.stat(base + b'.par2').st_size == 0:
par2_exists = 0
- sys.stdout.flush()
- debug('fsck: checking %s (%s)\n'
+ sys.stdout.flush() # Not sure we still need this, but it'll flush out too
+ debug('fsck: checking %r (%s)\n'
% (last, par2_ok and par2_exists and 'par2' or 'git'))
if not opt.verbose:
progress('fsck (%d/%d)\r' % (count, len(extra)))
if not opt.jobs:
- nc = do_pack(base, last, par2_exists)
+ nc = do_pack(base, last, par2_exists, out)
code = code or nc
count += 1
else:
outstanding[pid] = 1
else: # child
try:
- sys.exit(do_pack(base, last, par2_exists))
+ sys.exit(do_pack(base, last, par2_exists, out))
except Exception as e:
log('exception: %r\n' % e)
sys.exit(99)
projects / bup.git / blobdiff