From: Rob Browning Date: Wed, 1 Jan 2020 19:52:29 +0000 (-0600) Subject: Adjust split-cmd join-cmd margin-cmd for python 3; test-split-join X-Git-Tag: 0.31~144 X-Git-Url: https://arthur.barton.de/gitweb/?p=bup.git;a=commitdiff_plain;h=a340ef00288ce67944e295e319680df28e674482 Adjust split-cmd join-cmd margin-cmd for python 3; test-split-join Run test-split-join for python 3 after finishing the relevant adjustments. Signed-off-by: Rob Browning Tested-by: Rob Browning --- diff --git a/Makefile b/Makefile index 46a4d33..d2c9ab5 100644 --- a/Makefile +++ b/Makefile @@ -178,6 +178,7 @@ cmdline_tests := \ t/test-list-idx.sh \ t/test-ls \ t/test-ls-remote \ + t/test-split-join.sh \ t/test-tz.sh ifeq "2" "$(bup_python_majver)" @@ -191,7 +192,6 @@ ifeq "2" "$(bup_python_majver)" t/test-gc.sh \ t/test-main.sh \ t/test-index.sh \ - t/test-split-join.sh \ t/test-fuse.sh \ t/test-index-check-device.sh \ t/test-meta.sh \ diff --git a/cmd/join-cmd.py b/cmd/join-cmd.py index 68c31ee..48bebe8 100755 --- a/cmd/join-cmd.py +++ b/cmd/join-cmd.py @@ -9,7 +9,9 @@ from __future__ import absolute_import import sys from bup import git, options +from bup.compat import argv_bytes from bup.helpers import linereader, log +from bup.io import byte_stream from bup.repo import LocalRepo, RemoteRepo @@ -21,11 +23,15 @@ o= output filename """ o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) +if opt.remote: + opt.remote = argv_bytes(opt.remote) git.check_repo_or_die() +stdin = byte_stream(sys.stdin) + if not extra: - extra = linereader(sys.stdin) + extra = linereader(stdin) ret = 0 repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo() @@ -33,9 +39,10 @@ repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo() if opt.o: outfile = open(opt.o, 'wb') else: - outfile = sys.stdout + sys.stdout.flush() + outfile = byte_stream(sys.stdout) -for ref in extra: +for ref in [argv_bytes(x) for x in extra]: try: for blob in repo.join(ref): outfile.write(blob) diff --git a/cmd/margin-cmd.py b/cmd/margin-cmd.py index 7eba448..14e7cd7 100755 --- a/cmd/margin-cmd.py +++ b/cmd/margin-cmd.py @@ -5,11 +5,12 @@ exec "$bup_python" "$0" ${1+"$@"} """ # end of bup preamble -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys, struct, math from bup import options, git, _helpers from bup.helpers import log +from bup.io import byte_stream POPULATION_OF_EARTH=6.7e9 # as of September, 2010 @@ -27,29 +28,33 @@ if extra: git.check_repo_or_die() -mi = git.PackIdxList(git.repo('objects/pack'), ignore_midx=opt.ignore_midx) +mi = git.PackIdxList(git.repo(b'objects/pack'), ignore_midx=opt.ignore_midx) -def do_predict(ix): +def do_predict(ix, out): total = len(ix) maxdiff = 0 for count,i in enumerate(ix): prefix = struct.unpack('!Q', i[:8])[0] - expected = prefix * total / (1<<64) + expected = prefix * total // (1 << 64) diff = count - expected maxdiff = max(maxdiff, abs(diff)) - print('%d of %d (%.3f%%) ' % (maxdiff, len(ix), maxdiff*100.0/len(ix))) - sys.stdout.flush() + out.write(b'%d of %d (%.3f%%) ' + % (maxdiff, len(ix), maxdiff * 100.0 / len(ix))) + out.flush() assert(count+1 == len(ix)) +sys.stdout.flush() +out = byte_stream(sys.stdout) + if opt.predict: if opt.ignore_midx: for pack in mi.packs: - do_predict(pack) + do_predict(pack, out) else: - do_predict(mi) + do_predict(mi, out) else: # default mode: find longest matching prefix - last = '\0'*20 + last = b'\0'*20 longmatch = 0 for i in mi: if i == last: @@ -58,7 +63,7 @@ else: pm = _helpers.bitmatch(last, i) longmatch = max(longmatch, pm) last = i - print(longmatch) + out.write(b'%d\n' % longmatch) log('%d matching prefix bits\n' % longmatch) doublings = math.log(len(mi), 2) bpd = longmatch / doublings diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py index 31950ec..bb4cf2e 100755 --- a/cmd/split-cmd.py +++ b/cmd/split-cmd.py @@ -6,13 +6,16 @@ exec "$bup_python" "$0" ${1+"$@"} # end of bup preamble from __future__ import absolute_import, division, print_function +from binascii import hexlify import os, sys, time from bup import hashsplit, git, options, client +from bup.compat import argv_bytes, environ from bup.helpers import (add_error, handle_ctrl_c, hostname, log, parse_num, qprogress, reprogress, saved_errors, valid_save_name, parse_date_or_fatal) +from bup.io import byte_stream from bup.pwdgrp import userfullname, username @@ -47,6 +50,9 @@ handle_ctrl_c() o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) +if opt.name: opt.name = argv_bytes(opt.name) +if opt.remote: opt.remote = argv_bytes(opt.remote) +if opt.verbose is None: opt.verbose = 0 if not (opt.blobs or opt.tree or opt.commit or opt.name or opt.noop or opt.copy): @@ -93,14 +99,14 @@ def prog(filenum, nbytes): qprogress('Splitting: %d kbytes\r' % (total_bytes // 1024)) -is_reverse = os.environ.get('BUP_SERVER_REVERSE') +is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse and opt.remote: o.fatal("don't use -r in reverse mode; it's automatic") start_time = time.time() if opt.name and not valid_save_name(opt.name): - o.fatal("'%s' is not a valid branch name." % opt.name) -refname = opt.name and 'refs/heads/%s' % opt.name or None + o.fatal("'%r' is not a valid branch name." % opt.name) +refname = opt.name and b'refs/heads/%s' % opt.name or None if opt.noop or opt.copy: cli = pack_writer = oldref = None @@ -119,6 +125,8 @@ else: max_pack_size=max_pack_size, max_pack_objects=max_pack_objects) +input = byte_stream(sys.stdin) + if opt.git_ids: # the input is actually a series of git object ids that we should retrieve # and split. @@ -134,10 +142,10 @@ if opt.git_ids: self.it = iter(it) def read(self, size): v = next(self.it, None) - return v or '' + return v or b'' def read_ids(): while 1: - line = sys.stdin.readline() + line = input.readline() if not line: break if line: @@ -152,22 +160,25 @@ if opt.git_ids: files = read_ids() else: # the input either comes from a series of files or from stdin. - files = extra and (open(fn) for fn in extra) or [sys.stdin] + files = extra and (open(argv_bytes(fn), 'rb') for fn in extra) or [input] if pack_writer: new_blob = pack_writer.new_blob new_tree = pack_writer.new_tree elif opt.blobs or opt.tree: # --noop mode - new_blob = lambda content: git.calc_hash('blob', content) - new_tree = lambda shalist: git.calc_hash('tree', git.tree_encode(shalist)) + new_blob = lambda content: git.calc_hash(b'blob', content) + new_tree = lambda shalist: git.calc_hash(b'tree', git.tree_encode(shalist)) + +sys.stdout.flush() +out = byte_stream(sys.stdout) if opt.blobs: shalist = hashsplit.split_to_blobs(new_blob, files, keep_boundaries=opt.keep_boundaries, progress=prog) for (sha, size, level) in shalist: - print(sha.encode('hex')) + out.write(hexlify(sha) + b'\n') reprogress() elif opt.tree or opt.commit or opt.name: if opt.name: # insert dummy_name which may be used as a restore target @@ -175,7 +186,7 @@ elif opt.tree or opt.commit or opt.name: hashsplit.split_to_blob_or_tree(new_blob, new_tree, files, keep_boundaries=opt.keep_boundaries, progress=prog) - splitfile_name = git.mangle_name('data', hashsplit.GIT_MODE_FILE, mode) + splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE, mode) shalist = [(mode, splitfile_name, sha)] else: shalist = hashsplit.split_to_shalist( @@ -198,15 +209,15 @@ else: if opt.verbose: log('\n') if opt.tree: - print(tree.encode('hex')) + out.write(hexlify(tree) + b'\n') if opt.commit or opt.name: - msg = 'bup split\n\nGenerated by command:\n%r\n' % sys.argv - ref = opt.name and ('refs/heads/%s' % opt.name) or None - userline = '%s <%s@%s>' % (userfullname(), username(), hostname()) + msg = b'bup split\n\nGenerated by command:\n%r\n' % sys.argv + ref = opt.name and (b'refs/heads/%s' % opt.name) or None + userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) commit = pack_writer.new_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: - print(commit.encode('hex')) + out.write(hexlify(commit) + b'\n') if pack_writer: pack_writer.close() # must close before we can update the ref