X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=cmd%2Fsplit-cmd.py;h=bb4cf2e619bbb178c9974bed4e166b9776af8ba8;hb=a340ef00288ce67944e295e319680df28e674482;hp=5f52e7086ce5c7172ffc06761354115ef17f9f33;hpb=e0d69acd88fe3d4bc57e30840d9550f66a158776;p=bup.git diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py index 5f52e70..bb4cf2e 100755 --- a/cmd/split-cmd.py +++ b/cmd/split-cmd.py @@ -5,18 +5,25 @@ exec "$bup_python" "$0" ${1+"$@"} """ # end of bup preamble +from __future__ import absolute_import, division, print_function +from binascii import hexlify import os, sys, time from bup import hashsplit, git, options, client +from bup.compat import argv_bytes, environ from bup.helpers import (add_error, handle_ctrl_c, hostname, log, parse_num, qprogress, reprogress, saved_errors, - userfullname, username, valid_save_name) + valid_save_name, + parse_date_or_fatal) +from bup.io import byte_stream +from bup.pwdgrp import userfullname, username optspec = """ bup split [-t] [-c] [-n name] OPTIONS [--git-ids | filenames...] bup split -b OPTIONS [--git-ids | filenames...] -bup split <--noop [--copy]|--copy> OPTIONS [--git-ids | filenames...] +bup split --copy OPTIONS [--git-ids | filenames...] +bup split --noop [-b|-t] OPTIONS [--git-ids | filenames...] -- Modes: b,blobs output a series of blob ids. Implies --fanout=0. @@ -39,17 +46,21 @@ fanout= average number of blobs in a single tree bwlimit= maximum bytes/sec to transmit to server #,compress= set compression level to # (0-9, 9 is highest) [1] """ +handle_ctrl_c() + o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) +if opt.name: opt.name = argv_bytes(opt.name) +if opt.remote: opt.remote = argv_bytes(opt.remote) +if opt.verbose is None: opt.verbose = 0 -handle_ctrl_c() -git.check_repo_or_die() if not (opt.blobs or opt.tree or opt.commit or opt.name or opt.noop or opt.copy): o.fatal("use one or more of -b, -t, -c, -n, --noop, --copy") -if (opt.noop or opt.copy) and (opt.blobs or opt.tree or - opt.commit or opt.name): - o.fatal('--noop and --copy are incompatible with -b, -t, -c, -n') +if opt.copy and (opt.blobs or opt.tree): + o.fatal('--copy is incompatible with -b, -t') +if (opt.noop or opt.copy) and (opt.commit or opt.name): + o.fatal('--noop and --copy are incompatible with -c, -n') if opt.blobs and (opt.tree or opt.commit or opt.name): o.fatal('-b is incompatible with -t, -c, -n') if extra and opt.git_ids: @@ -83,34 +94,39 @@ def prog(filenum, nbytes): total_bytes += nbytes if filenum > 0: qprogress('Splitting: file #%d, %d kbytes\r' - % (filenum+1, total_bytes/1024)) + % (filenum+1, total_bytes // 1024)) else: - qprogress('Splitting: %d kbytes\r' % (total_bytes/1024)) + qprogress('Splitting: %d kbytes\r' % (total_bytes // 1024)) -is_reverse = os.environ.get('BUP_SERVER_REVERSE') +is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse and opt.remote: o.fatal("don't use -r in reverse mode; it's automatic") start_time = time.time() if opt.name and not valid_save_name(opt.name): - o.fatal("'%s' is not a valid branch name." % opt.name) -refname = opt.name and 'refs/heads/%s' % opt.name or None + o.fatal("'%r' is not a valid branch name." % opt.name) +refname = opt.name and b'refs/heads/%s' % opt.name or None + if opt.noop or opt.copy: cli = pack_writer = oldref = None elif opt.remote or is_reverse: + git.check_repo_or_die() cli = client.Client(opt.remote) oldref = refname and cli.read_ref(refname) or None pack_writer = cli.new_packwriter(compression_level=opt.compress, max_pack_size=max_pack_size, max_pack_objects=max_pack_objects) else: + git.check_repo_or_die() cli = None oldref = refname and git.read_ref(refname) or None pack_writer = git.PackWriter(compression_level=opt.compress, max_pack_size=max_pack_size, max_pack_objects=max_pack_objects) +input = byte_stream(sys.stdin) + if opt.git_ids: # the input is actually a series of git object ids that we should retrieve # and split. @@ -126,17 +142,17 @@ if opt.git_ids: self.it = iter(it) def read(self, size): v = next(self.it, None) - return v or '' + return v or b'' def read_ids(): while 1: - line = sys.stdin.readline() + line = input.readline() if not line: break if line: line = line.strip() try: it = cp.get(line.strip()) - next(it, None) # skip the file type + next(it, None) # skip the file info except KeyError as e: add_error('error: %s' % e) continue @@ -144,30 +160,39 @@ if opt.git_ids: files = read_ids() else: # the input either comes from a series of files or from stdin. - files = extra and (open(fn) for fn in extra) or [sys.stdin] + files = extra and (open(argv_bytes(fn), 'rb') for fn in extra) or [input] -if pack_writer and opt.blobs: - shalist = hashsplit.split_to_blobs(pack_writer.new_blob, files, +if pack_writer: + new_blob = pack_writer.new_blob + new_tree = pack_writer.new_tree +elif opt.blobs or opt.tree: + # --noop mode + new_blob = lambda content: git.calc_hash(b'blob', content) + new_tree = lambda shalist: git.calc_hash(b'tree', git.tree_encode(shalist)) + +sys.stdout.flush() +out = byte_stream(sys.stdout) + +if opt.blobs: + shalist = hashsplit.split_to_blobs(new_blob, files, keep_boundaries=opt.keep_boundaries, progress=prog) for (sha, size, level) in shalist: - print sha.encode('hex') + out.write(hexlify(sha) + b'\n') reprogress() -elif pack_writer: # tree or commit or name +elif opt.tree or opt.commit or opt.name: if opt.name: # insert dummy_name which may be used as a restore target mode, sha = \ - hashsplit.split_to_blob_or_tree(pack_writer.new_blob, - pack_writer.new_tree, - files, + hashsplit.split_to_blob_or_tree(new_blob, new_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog) - splitfile_name = git.mangle_name('data', hashsplit.GIT_MODE_FILE, mode) + splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE, mode) shalist = [(mode, splitfile_name, sha)] else: shalist = hashsplit.split_to_shalist( - pack_writer.new_blob, pack_writer.new_tree, files, + new_blob, new_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog) - tree = pack_writer.new_tree(shalist) + tree = new_tree(shalist) else: last = 0 it = hashsplit.hashsplit_iter(files, @@ -177,22 +202,22 @@ else: hashsplit.total_split += len(blob) if opt.copy: sys.stdout.write(str(blob)) - megs = hashsplit.total_split/1024/1024 + megs = hashsplit.total_split // 1024 // 1024 if not opt.quiet and last != megs: last = megs if opt.verbose: log('\n') if opt.tree: - print tree.encode('hex') + out.write(hexlify(tree) + b'\n') if opt.commit or opt.name: - msg = 'bup split\n\nGenerated by command:\n%r\n' % sys.argv - ref = opt.name and ('refs/heads/%s' % opt.name) or None - userline = '%s <%s@%s>' % (userfullname(), username(), hostname()) + msg = b'bup split\n\nGenerated by command:\n%r\n' % sys.argv + ref = opt.name and (b'refs/heads/%s' % opt.name) or None + userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) commit = pack_writer.new_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: - print commit.encode('hex') + out.write(hexlify(commit) + b'\n') if pack_writer: pack_writer.close() # must close before we can update the ref @@ -209,8 +234,8 @@ if cli: secs = time.time() - start_time size = hashsplit.total_split if opt.bench: - log('bup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n' - % (size/1024., secs, size/1024./secs)) + log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n' + % (size / 1024, secs, size / 1024 / secs)) if saved_errors: log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))