From f0a4b3b6ef7d00e64b04dee51b8e15db71b693d3 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Sat, 28 Dec 2019 12:49:13 -0600 Subject: [PATCH] bup-save bup.drecurse bup.hlinkdb bup.path: accommodate python 3 Signed-off-by: Rob Browning Tested-by: Rob Browning --- cmd/save-cmd.py | 79 ++++++++++++++++++++++++-------------- lib/bup/drecurse.py | 18 +++++---- lib/bup/hlinkdb.py | 16 +++++--- lib/bup/path.py | 8 ++-- t/test-save-strip-graft.sh | 2 +- 5 files changed, 77 insertions(+), 46 deletions(-) diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py index 04cf55d..5140ac0 100755 --- a/cmd/save-cmd.py +++ b/cmd/save-cmd.py @@ -6,17 +6,20 @@ exec "$bup_python" "$0" ${1+"$@"} # end of bup preamble from __future__ import absolute_import, print_function +from binascii import hexlify from errno import EACCES from io import BytesIO import os, sys, stat, time, math from bup import hashsplit, git, options, index, client, metadata, hlinkdb +from bup.compat import argv_bytes, environ from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c, hostname, istty2, log, parse_date_or_fatal, parse_num, path_components, progress, qprogress, resolve_parent, saved_errors, stripped_path_components, valid_save_name) +from bup.io import byte_stream, path_msg from bup.pwdgrp import userfullname, username @@ -41,12 +44,23 @@ graft= a graft point *old_path*=*new_path* (can be used more than once) o = options.Options(optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) +if opt.indexfile: + opt.indexfile = argv_bytes(opt.indexfile) +if opt.name: + opt.name = argv_bytes(opt.name) +if opt.remote: + opt.remote = argv_bytes(opt.remote) +if opt.strip_path: + opt.strip_path = argv_bytes(opt.strip_path) + git.check_repo_or_die() if not (opt.tree or opt.commit or opt.name): o.fatal("use one or more of -t, -c, -n") if not extra: o.fatal("no filenames given") +extra = [argv_bytes(x) for x in extra] + opt.progress = (istty2 and not opt.quiet) opt.smaller = parse_num(opt.smaller or 0) if opt.bwlimit: @@ -70,7 +84,8 @@ if opt.graft: for (option, parameter) in flags: if option == "--graft": - splitted_parameter = parameter.split('=') + parameter = argv_bytes(parameter) + splitted_parameter = parameter.split(b'=') if len(splitted_parameter) != 2: o.fatal("a graft point must be of the form old_path=new_path") old_path, new_path = splitted_parameter @@ -79,13 +94,14 @@ if opt.graft: graft_points.append((resolve_parent(old_path), resolve_parent(new_path))) -is_reverse = os.environ.get('BUP_SERVER_REVERSE') +is_reverse = environ.get(b'BUP_SERVER_REVERSE') if is_reverse and opt.remote: o.fatal("don't use -r in reverse mode; it's automatic") -if opt.name and not valid_save_name(opt.name): - o.fatal("'%s' is not a valid branch name" % opt.name) -refname = opt.name and 'refs/heads/%s' % opt.name or None +name = opt.name +if name and not valid_save_name(name): + o.fatal("'%s' is not a valid branch name" % path_msg(name)) +refname = name and b'refs/heads/%s' % name or None if opt.remote or is_reverse: try: cli = client.Client(opt.remote) @@ -103,7 +119,7 @@ handle_ctrl_c() def eatslash(dir): - if dir.endswith('/'): + if dir.endswith(b'/'): return dir[:-1] else: return dir @@ -134,7 +150,7 @@ def _push(part, metadata): # Enter a new archive directory -- make it the current directory. parts.append(part) shalists.append([]) - metalists.append([('', metadata)]) # This dir's metadata (no name). + metalists.append([(b'', metadata)]) # This dir's metadata (no name). def _pop(force_tree, dir_metadata=None): @@ -145,14 +161,14 @@ def _pop(force_tree, dir_metadata=None): metalist = metalists.pop() if metalist and not force_tree: if dir_metadata: # Override the original metadata pushed for this dir. - metalist = [('', dir_metadata)] + metalist[1:] + metalist = [(b'', dir_metadata)] + metalist[1:] sorted_metalist = sorted(metalist, key = lambda x : x[0]) - metadata = ''.join([m[1].encode() for m in sorted_metalist]) + metadata = b''.join([m[1].encode() for m in sorted_metalist]) metadata_f = BytesIO(metadata) mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree, [metadata_f], keep_boundaries=False) - shalist.append((mode, '.bupm', id)) + shalist.append((mode, b'.bupm', id)) # FIXME: only test if collision is possible (i.e. given --strip, etc.)? if force_tree: tree = force_tree @@ -162,9 +178,9 @@ def _pop(force_tree, dir_metadata=None): for x in shalist: name = x[1] if name in names_seen: - parent_path = '/'.join(parts) + '/' - add_error('error: ignoring duplicate path %r in %r' - % (name, parent_path)) + parent_path = b'/'.join(parts) + b'/' + add_error('error: ignoring duplicate path %s in %s' + % (path_msg(name), path_msg(parent_path))) else: names_seen.add(name) clean_list.append(x) @@ -216,16 +232,17 @@ def progress_report(n): remainstr, kpsstr)) -indexfile = opt.indexfile or git.repo('bupindex') +indexfile = opt.indexfile or git.repo(b'bupindex') r = index.Reader(indexfile) try: - msr = index.MetaStoreReader(indexfile + '.meta') + msr = index.MetaStoreReader(indexfile + b'.meta') except IOError as ex: if ex.errno != EACCES: raise - log('error: cannot access %r; have you run bup index?' % indexfile) + log('error: cannot access %r; have you run bup index?' + % path_msg(indexfile)) sys.exit(1) -hlink_db = hlinkdb.HLinkDB(indexfile + '.hlink') +hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink') def already_saved(ent): return ent.is_valid() and w.exists(ent.sha) and ent.sha @@ -273,7 +290,7 @@ root_collision = None tstart = time.time() count = subcount = fcount = 0 lastskip_name = None -lastdir = '' +lastdir = b'' for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during): (dir, file) = os.path.split(ent.name) exists = (ent.flags & index.IX_EXISTS) @@ -291,10 +308,10 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during): else: status = ' ' if opt.verbose >= 2: - log('%s %-70s\n' % (status, ent.name)) + log('%s %-70s\n' % (status, path_msg(ent.name))) elif not stat.S_ISDIR(ent.mode) and lastdir != dir: if not lastdir.startswith(dir): - log('%s %-70s\n' % (status, os.path.join(dir, ''))) + log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b'')))) lastdir = dir if opt.progress: @@ -306,11 +323,11 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during): if opt.smaller and ent.size >= opt.smaller: if exists and not hashvalid: if opt.verbose: - log('skipping large file "%s"\n' % ent.name) + log('skipping large file "%s"\n' % path_msg(ent.name)) lastskip_name = ent.name continue - assert(dir.startswith('/')) + assert(dir.startswith(b'/')) if opt.strip: dirp = stripped_path_components(dir, extra) elif opt.strip_path: @@ -415,7 +432,7 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during): # Everything else should be fully described by its # metadata, so just record an empty blob, so the paths # in the tree and .bupm will match up. - (mode, id) = (GIT_MODE_FILE, w.new_blob("")) + (mode, id) = (GIT_MODE_FILE, w.new_blob(b'')) if id: ent.validate(mode, id) @@ -454,15 +471,21 @@ tree = _pop(force_tree = None, # When there's a collision, use empty metadata for the root. dir_metadata = metadata.Metadata() if root_collision else None) +sys.stdout.flush() +out = byte_stream(sys.stdout) + if opt.tree: - print(tree.encode('hex')) -if opt.commit or opt.name: - msg = 'bup save\n\nGenerated by command:\n%r\n' % sys.argv - userline = '%s <%s@%s>' % (userfullname(), username(), hostname()) + out.write(hexlify(tree)) + out.write(b'\n') +if opt.commit or name: + msg = (b'bup save\n\nGenerated by command:\n%r\n' + % [argv_bytes(x) for x in sys.argv]) + userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname())) commit = w.new_commit(tree, oldref, userline, date, None, userline, date, None, msg) if opt.commit: - print(commit.encode('hex')) + out.write(hexlify(commit)) + out.write(b'\n') msr.close() w.close() # must close before we can update the ref diff --git a/lib/bup/drecurse.py b/lib/bup/drecurse.py index 24a0db9..1d9a4e1 100644 --- a/lib/bup/drecurse.py +++ b/lib/bup/drecurse.py @@ -3,6 +3,7 @@ from __future__ import absolute_import import stat, os from bup.helpers import add_error, should_rx_exclude_path, debug1, resolve_parent +from bup.io import path_msg import bup.xstat as xstat @@ -40,14 +41,14 @@ class OsFile: _IFMT = stat.S_IFMT(0xffffffff) # avoid function call in inner loop def _dirlist(): l = [] - for n in os.listdir('.'): + for n in os.listdir(b'.'): try: st = xstat.lstat(n) except OSError as e: add_error(Exception('%s: %s' % (resolve_parent(n), str(e)))) continue if (st.st_mode & _IFMT) == stat.S_IFDIR: - n += '/' + n += b'/' l.append((n,st)) l.sort(reverse=True) return l @@ -61,18 +62,19 @@ def _recursive_dirlist(prepend, xdev, bup_dir=None, path = prepend + name if excluded_paths: if os.path.normpath(path) in excluded_paths: - debug1('Skipping %r: excluded.\n' % path) + debug1('Skipping %r: excluded.\n' % path_msg(path)) continue if exclude_rxs and should_rx_exclude_path(path, exclude_rxs): continue - if name.endswith('/'): + if name.endswith(b'/'): if bup_dir != None: if os.path.normpath(path) == bup_dir: debug1('Skipping BUP_DIR.\n') continue if xdev != None and pst.st_dev != xdev \ and path not in xdev_exceptions: - debug1('Skipping contents of %r: different filesystem.\n' % path) + debug1('Skipping contents of %r: different filesystem.\n' + % path_msg(path)) else: try: OsFile(name).fchdir() @@ -85,7 +87,7 @@ def _recursive_dirlist(prepend, xdev, bup_dir=None, exclude_rxs=exclude_rxs, xdev_exceptions=xdev_exceptions): yield i - os.chdir('..') + os.chdir(b'..') yield (path, pst) @@ -93,7 +95,7 @@ def recursive_dirlist(paths, xdev, bup_dir=None, excluded_paths=None, exclude_rxs=None, xdev_exceptions=frozenset()): - startdir = OsFile('.') + startdir = OsFile(b'.') try: assert(type(paths) != type('')) for path in paths: @@ -117,7 +119,7 @@ def recursive_dirlist(paths, xdev, bup_dir=None, xdev = None if stat.S_ISDIR(pst.st_mode): pfile.fchdir() - prepend = os.path.join(path, '') + prepend = os.path.join(path, b'') for i in _recursive_dirlist(prepend=prepend, xdev=xdev, bup_dir=bup_dir, excluded_paths=excluded_paths, diff --git a/lib/bup/hlinkdb.py b/lib/bup/hlinkdb.py index 17957bd..d4125ae 100644 --- a/lib/bup/hlinkdb.py +++ b/lib/bup/hlinkdb.py @@ -1,9 +1,15 @@ from __future__ import absolute_import -import cPickle, errno, os, tempfile +import errno, os, tempfile from bup import compat +if compat.py_maj > 2: + import pickle +else: + import cPickle as pickle + + class Error(Exception): pass @@ -18,7 +24,7 @@ class HLinkDB: self._tmpname = None f = None try: - f = open(filename, 'r') + f = open(filename, 'rb') except IOError as e: if e.errno == errno.ENOENT: pass @@ -26,7 +32,7 @@ class HLinkDB: raise if f: try: - self._node_paths = cPickle.load(f) + self._node_paths = pickle.load(f) finally: f.close() f = None @@ -42,7 +48,7 @@ class HLinkDB: raise Error('save of %r already in progress' % self._filename) if self._node_paths: (dir, name) = os.path.split(self._filename) - (ffd, self._tmpname) = tempfile.mkstemp('.tmp', name, dir) + (ffd, self._tmpname) = tempfile.mkstemp(b'.tmp', name, dir) try: try: f = os.fdopen(ffd, 'wb', 65536) @@ -50,7 +56,7 @@ class HLinkDB: os.close(ffd) raise try: - cPickle.dump(self._node_paths, f, 2) + pickle.dump(self._node_paths, f, 2) finally: f.close() f = None diff --git a/lib/bup/path.py b/lib/bup/path.py index b6167d0..7dd3905 100644 --- a/lib/bup/path.py +++ b/lib/bup/path.py @@ -6,10 +6,10 @@ import os # Eventually, if we physically move the source tree cmd/ to lib/, then # we could use realpath here and save some stats... -_libdir = os.path.abspath(os.path.dirname(__file__) + '/..') +_libdir = os.path.abspath(os.path.dirname(__file__.encode('iso-8859-1')) + b'/..') _resdir = _libdir -_exedir = os.path.abspath(_libdir + '/cmd') -_exe = os.path.join(_exedir, 'bup') +_exedir = os.path.abspath(_libdir + b'/cmd') +_exe = os.path.join(_exedir, b'bup') def exe(): @@ -23,5 +23,5 @@ cmddir = exedir def libdir(): return _libdir -def resource_path(subdir=''): +def resource_path(subdir=b''): return os.path.join(_resdir, subdir) diff --git a/t/test-save-strip-graft.sh b/t/test-save-strip-graft.sh index 31f06eb..95883dd 100755 --- a/t/test-save-strip-graft.sh +++ b/t/test-save-strip-graft.sh @@ -155,7 +155,7 @@ WVPASS bup init WVPASS mkdir -p src/x/1 src/y/1 WVPASS bup index -u src WVFAIL bup save --strip -n foo src/x src/y 2> tmp-err.log -WVPASS grep -F "error: ignoring duplicate path '1' in '/'" tmp-err.log +WVPASS grep -F "error: ignoring duplicate path 1 in /" tmp-err.log WVPASS rm -rf "$tmpdir" -- 2.39.2