bup-save bup.drecurse bup.hlinkdb bup.path: accommodate python 3

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py
index 04cf55d0e043135697c4467b4b1536a7489ebfe1..5140ac027ecff4cef3b93ff93cd7ab05a6295c00 100755 (executable)
--- a/cmd/save-cmd.py
+++ b/cmd/save-cmd.py
@@ -6,17 +6,20 @@ exec "$bup_python" "$0" ${1+"$@"}
 # end of bup preamble
 
 from __future__ import absolute_import, print_function
 # end of bup preamble
 
 from __future__ import absolute_import, print_function

+from binascii import hexlify

 from errno import EACCES
 from io import BytesIO
 import os, sys, stat, time, math
 
 from bup import hashsplit, git, options, index, client, metadata, hlinkdb
 from errno import EACCES
 from io import BytesIO
 import os, sys, stat, time, math
 
 from bup import hashsplit, git, options, index, client, metadata, hlinkdb

+from bup.compat import argv_bytes, environ

 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
 from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
                          hostname, istty2, log, parse_date_or_fatal, parse_num,
                          path_components, progress, qprogress, resolve_parent,
                          saved_errors, stripped_path_components,
                          valid_save_name)
 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
 from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
                          hostname, istty2, log, parse_date_or_fatal, parse_num,
                          path_components, progress, qprogress, resolve_parent,
                          saved_errors, stripped_path_components,
                          valid_save_name)

+from bup.io import byte_stream, path_msg

 from bup.pwdgrp import userfullname, username
 
 
 from bup.pwdgrp import userfullname, username
 
 


@@ -41,12 +44,23 @@ graft=     a graft point *old_path*=*new_path* (can be used more than once)
 o = options.Options(optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 
 o = options.Options(optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 

+if opt.indexfile:
+    opt.indexfile = argv_bytes(opt.indexfile)
+if opt.name:
+    opt.name = argv_bytes(opt.name)
+if opt.remote:
+    opt.remote = argv_bytes(opt.remote)
+if opt.strip_path:
+    opt.strip_path = argv_bytes(opt.strip_path)
+

 git.check_repo_or_die()
 if not (opt.tree or opt.commit or opt.name):
     o.fatal("use one or more of -t, -c, -n")
 if not extra:
     o.fatal("no filenames given")
 
 git.check_repo_or_die()
 if not (opt.tree or opt.commit or opt.name):
     o.fatal("use one or more of -t, -c, -n")
 if not extra:
     o.fatal("no filenames given")
 

+extra = [argv_bytes(x) for x in extra]
+

 opt.progress = (istty2 and not opt.quiet)
 opt.smaller = parse_num(opt.smaller or 0)
 if opt.bwlimit:
 opt.progress = (istty2 and not opt.quiet)
 opt.smaller = parse_num(opt.smaller or 0)
 if opt.bwlimit:


@@ -70,7 +84,8 @@ if opt.graft:
 
     for (option, parameter) in flags:
         if option == "--graft":
 
     for (option, parameter) in flags:
         if option == "--graft":

-            splitted_parameter = parameter.split('=')
+            parameter = argv_bytes(parameter)
+            splitted_parameter = parameter.split(b'=')

             if len(splitted_parameter) != 2:
                 o.fatal("a graft point must be of the form old_path=new_path")
             old_path, new_path = splitted_parameter
             if len(splitted_parameter) != 2:
                 o.fatal("a graft point must be of the form old_path=new_path")
             old_path, new_path = splitted_parameter


@@ -79,13 +94,14 @@ if opt.graft:
             graft_points.append((resolve_parent(old_path),
                                  resolve_parent(new_path)))
 
             graft_points.append((resolve_parent(old_path),
                                  resolve_parent(new_path)))
 

-is_reverse = os.environ.get('BUP_SERVER_REVERSE')
+is_reverse = environ.get(b'BUP_SERVER_REVERSE')

 if is_reverse and opt.remote:
     o.fatal("don't use -r in reverse mode; it's automatic")
 
 if is_reverse and opt.remote:
     o.fatal("don't use -r in reverse mode; it's automatic")
 

-if opt.name and not valid_save_name(opt.name):
-    o.fatal("'%s' is not a valid branch name" % opt.name)
-refname = opt.name and 'refs/heads/%s' % opt.name or None
+name = opt.name
+if name and not valid_save_name(name):
+    o.fatal("'%s' is not a valid branch name" % path_msg(name))
+refname = name and b'refs/heads/%s' % name or None

 if opt.remote or is_reverse:
     try:
         cli = client.Client(opt.remote)
 if opt.remote or is_reverse:
     try:
         cli = client.Client(opt.remote)


@@ -103,7 +119,7 @@ handle_ctrl_c()
 
 
 def eatslash(dir):
 
 
 def eatslash(dir):

-    if dir.endswith('/'):
+    if dir.endswith(b'/'):

         return dir[:-1]
     else:
         return dir
         return dir[:-1]
     else:
         return dir


@@ -134,7 +150,7 @@ def _push(part, metadata):
     # Enter a new archive directory -- make it the current directory.
     parts.append(part)
     shalists.append([])
     # Enter a new archive directory -- make it the current directory.
     parts.append(part)
     shalists.append([])

-    metalists.append([('', metadata)]) # This dir's metadata (no name).
+    metalists.append([(b'', metadata)]) # This dir's metadata (no name).

 
 
 def _pop(force_tree, dir_metadata=None):
 
 
 def _pop(force_tree, dir_metadata=None):


@@ -145,14 +161,14 @@ def _pop(force_tree, dir_metadata=None):
     metalist = metalists.pop()
     if metalist and not force_tree:
         if dir_metadata: # Override the original metadata pushed for this dir.
     metalist = metalists.pop()
     if metalist and not force_tree:
         if dir_metadata: # Override the original metadata pushed for this dir.

-            metalist = [('', dir_metadata)] + metalist[1:]
+            metalist = [(b'', dir_metadata)] + metalist[1:]

         sorted_metalist = sorted(metalist, key = lambda x : x[0])
         sorted_metalist = sorted(metalist, key = lambda x : x[0])

-        metadata = ''.join([m[1].encode() for m in sorted_metalist])
+        metadata = b''.join([m[1].encode() for m in sorted_metalist])

         metadata_f = BytesIO(metadata)
         mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
                                                    [metadata_f],
                                                    keep_boundaries=False)
         metadata_f = BytesIO(metadata)
         mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
                                                    [metadata_f],
                                                    keep_boundaries=False)

-        shalist.append((mode, '.bupm', id))
+        shalist.append((mode, b'.bupm', id))

     # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
     if force_tree:
         tree = force_tree
     # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
     if force_tree:
         tree = force_tree


@@ -162,9 +178,9 @@ def _pop(force_tree, dir_metadata=None):
         for x in shalist:
             name = x[1]
             if name in names_seen:
         for x in shalist:
             name = x[1]
             if name in names_seen:

-                parent_path = '/'.join(parts) + '/'
-                add_error('error: ignoring duplicate path %r in %r'
-                          % (name, parent_path))
+                parent_path = b'/'.join(parts) + b'/'
+                add_error('error: ignoring duplicate path %s in %s'
+                          % (path_msg(name), path_msg(parent_path)))

             else:
                 names_seen.add(name)
                 clean_list.append(x)
             else:
                 names_seen.add(name)
                 clean_list.append(x)


@@ -216,16 +232,17 @@ def progress_report(n):
                  remainstr, kpsstr))
 
 
                  remainstr, kpsstr))
 
 

-indexfile = opt.indexfile or git.repo('bupindex')
+indexfile = opt.indexfile or git.repo(b'bupindex')

 r = index.Reader(indexfile)
 try:
 r = index.Reader(indexfile)
 try:

-    msr = index.MetaStoreReader(indexfile + '.meta')
+    msr = index.MetaStoreReader(indexfile + b'.meta')

 except IOError as ex:
     if ex.errno != EACCES:
         raise
 except IOError as ex:
     if ex.errno != EACCES:
         raise

-    log('error: cannot access %r; have you run bup index?' % indexfile)
+    log('error: cannot access %r; have you run bup index?'
+        % path_msg(indexfile))

     sys.exit(1)
     sys.exit(1)

-hlink_db = hlinkdb.HLinkDB(indexfile + '.hlink')
+hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink')

 
 def already_saved(ent):
     return ent.is_valid() and w.exists(ent.sha) and ent.sha
 
 def already_saved(ent):
     return ent.is_valid() and w.exists(ent.sha) and ent.sha


@@ -273,7 +290,7 @@ root_collision = None
 tstart = time.time()
 count = subcount = fcount = 0
 lastskip_name = None
 tstart = time.time()
 count = subcount = fcount = 0
 lastskip_name = None

-lastdir = ''
+lastdir = b''

 for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     (dir, file) = os.path.split(ent.name)
     exists = (ent.flags & index.IX_EXISTS)
 for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     (dir, file) = os.path.split(ent.name)
     exists = (ent.flags & index.IX_EXISTS)


@@ -291,10 +308,10 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
         else:
             status = ' '
         if opt.verbose >= 2:
         else:
             status = ' '
         if opt.verbose >= 2:

-            log('%s %-70s\n' % (status, ent.name))
+            log('%s %-70s\n' % (status, path_msg(ent.name)))

         elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
             if not lastdir.startswith(dir):
         elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
             if not lastdir.startswith(dir):

-                log('%s %-70s\n' % (status, os.path.join(dir, '')))
+                log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b''))))

             lastdir = dir
 
     if opt.progress:
             lastdir = dir
 
     if opt.progress:


@@ -306,11 +323,11 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     if opt.smaller and ent.size >= opt.smaller:
         if exists and not hashvalid:
             if opt.verbose:
     if opt.smaller and ent.size >= opt.smaller:
         if exists and not hashvalid:
             if opt.verbose:

-                log('skipping large file "%s"\n' % ent.name)
+                log('skipping large file "%s"\n' % path_msg(ent.name))

             lastskip_name = ent.name
         continue
 
             lastskip_name = ent.name
         continue
 

-    assert(dir.startswith('/'))
+    assert(dir.startswith(b'/'))

     if opt.strip:
         dirp = stripped_path_components(dir, extra)
     elif opt.strip_path:
     if opt.strip:
         dirp = stripped_path_components(dir, extra)
     elif opt.strip_path:


@@ -415,7 +432,7 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
                 # Everything else should be fully described by its
                 # metadata, so just record an empty blob, so the paths
                 # in the tree and .bupm will match up.
                 # Everything else should be fully described by its
                 # metadata, so just record an empty blob, so the paths
                 # in the tree and .bupm will match up.

-                (mode, id) = (GIT_MODE_FILE, w.new_blob(""))
+                (mode, id) = (GIT_MODE_FILE, w.new_blob(b''))

 
         if id:
             ent.validate(mode, id)
 
         if id:
             ent.validate(mode, id)


@@ -454,15 +471,21 @@ tree = _pop(force_tree = None,
             # When there's a collision, use empty metadata for the root.
             dir_metadata = metadata.Metadata() if root_collision else None)
 
             # When there's a collision, use empty metadata for the root.
             dir_metadata = metadata.Metadata() if root_collision else None)
 

+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+

 if opt.tree:
 if opt.tree:

-    print(tree.encode('hex'))
-if opt.commit or opt.name:
-    msg = 'bup save\n\nGenerated by command:\n%r\n' % sys.argv
-    userline = '%s <%s@%s>' % (userfullname(), username(), hostname())
+    out.write(hexlify(tree))
+    out.write(b'\n')
+if opt.commit or name:
+    msg = (b'bup save\n\nGenerated by command:\n%r\n'
+           % [argv_bytes(x) for x in sys.argv])
+    userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))

     commit = w.new_commit(tree, oldref, userline, date, None,
                           userline, date, None, msg)
     if opt.commit:
     commit = w.new_commit(tree, oldref, userline, date, None,
                           userline, date, None, msg)
     if opt.commit:

-        print(commit.encode('hex'))
+        out.write(hexlify(commit))
+        out.write(b'\n')

 
 msr.close()
 w.close()  # must close before we can update the ref
 
 msr.close()
 w.close()  # must close before we can update the ref

diff --git a/lib/bup/drecurse.py b/lib/bup/drecurse.py
index 24a0db941538e68b95d2db061d16b6b0d2b30073..1d9a4e14b5e5d91c3951acd587c88be619955319 100644 (file)
--- a/lib/bup/drecurse.py
+++ b/lib/bup/drecurse.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 import stat, os
 
 from bup.helpers import add_error, should_rx_exclude_path, debug1, resolve_parent
 import stat, os
 
 from bup.helpers import add_error, should_rx_exclude_path, debug1, resolve_parent

+from bup.io import path_msg

 import bup.xstat as xstat
 
 
 import bup.xstat as xstat
 
 


@@ -40,14 +41,14 @@ class OsFile:
 _IFMT = stat.S_IFMT(0xffffffff)  # avoid function call in inner loop
 def _dirlist():
     l = []
 _IFMT = stat.S_IFMT(0xffffffff)  # avoid function call in inner loop
 def _dirlist():
     l = []

-    for n in os.listdir('.'):
+    for n in os.listdir(b'.'):

         try:
             st = xstat.lstat(n)
         except OSError as e:
             add_error(Exception('%s: %s' % (resolve_parent(n), str(e))))
             continue
         if (st.st_mode & _IFMT) == stat.S_IFDIR:
         try:
             st = xstat.lstat(n)
         except OSError as e:
             add_error(Exception('%s: %s' % (resolve_parent(n), str(e))))
             continue
         if (st.st_mode & _IFMT) == stat.S_IFDIR:

-            n += '/'
+            n += b'/'

         l.append((n,st))
     l.sort(reverse=True)
     return l
         l.append((n,st))
     l.sort(reverse=True)
     return l


@@ -61,18 +62,19 @@ def _recursive_dirlist(prepend, xdev, bup_dir=None,
         path = prepend + name
         if excluded_paths:
             if os.path.normpath(path) in excluded_paths:
         path = prepend + name
         if excluded_paths:
             if os.path.normpath(path) in excluded_paths:

-                debug1('Skipping %r: excluded.\n' % path)
+                debug1('Skipping %r: excluded.\n' % path_msg(path))

                 continue
         if exclude_rxs and should_rx_exclude_path(path, exclude_rxs):
             continue
                 continue
         if exclude_rxs and should_rx_exclude_path(path, exclude_rxs):
             continue

-        if name.endswith('/'):
+        if name.endswith(b'/'):

             if bup_dir != None:
                 if os.path.normpath(path) == bup_dir:
                     debug1('Skipping BUP_DIR.\n')
                     continue
             if xdev != None and pst.st_dev != xdev \
                and path not in xdev_exceptions:
             if bup_dir != None:
                 if os.path.normpath(path) == bup_dir:
                     debug1('Skipping BUP_DIR.\n')
                     continue
             if xdev != None and pst.st_dev != xdev \
                and path not in xdev_exceptions:

-                debug1('Skipping contents of %r: different filesystem.\n' % path)
+                debug1('Skipping contents of %r: different filesystem.\n'
+                       % path_msg(path))

             else:
                 try:
                     OsFile(name).fchdir()
             else:
                 try:
                     OsFile(name).fchdir()


@@ -85,7 +87,7 @@ def _recursive_dirlist(prepend, xdev, bup_dir=None,
                                                 exclude_rxs=exclude_rxs,
                                                 xdev_exceptions=xdev_exceptions):
                         yield i
                                                 exclude_rxs=exclude_rxs,
                                                 xdev_exceptions=xdev_exceptions):
                         yield i

-                    os.chdir('..')
+                    os.chdir(b'..')

         yield (path, pst)
 
 
         yield (path, pst)
 
 


@@ -93,7 +95,7 @@ def recursive_dirlist(paths, xdev, bup_dir=None,
                       excluded_paths=None,
                       exclude_rxs=None,
                       xdev_exceptions=frozenset()):
                       excluded_paths=None,
                       exclude_rxs=None,
                       xdev_exceptions=frozenset()):

-    startdir = OsFile('.')
+    startdir = OsFile(b'.')

     try:
         assert(type(paths) != type(''))
         for path in paths:
     try:
         assert(type(paths) != type(''))
         for path in paths:


@@ -117,7 +119,7 @@ def recursive_dirlist(paths, xdev, bup_dir=None,
                 xdev = None
             if stat.S_ISDIR(pst.st_mode):
                 pfile.fchdir()
                 xdev = None
             if stat.S_ISDIR(pst.st_mode):
                 pfile.fchdir()

-                prepend = os.path.join(path, '')
+                prepend = os.path.join(path, b'')

                 for i in _recursive_dirlist(prepend=prepend, xdev=xdev,
                                             bup_dir=bup_dir,
                                             excluded_paths=excluded_paths,
                 for i in _recursive_dirlist(prepend=prepend, xdev=xdev,
                                             bup_dir=bup_dir,
                                             excluded_paths=excluded_paths,

diff --git a/lib/bup/hlinkdb.py b/lib/bup/hlinkdb.py
index 17957bd9a6bb54888c9daef6d4e95bfcae0e6b1d..d4125aef44837084983e712ad67f483a88943f9a 100644 (file)
--- a/lib/bup/hlinkdb.py
+++ b/lib/bup/hlinkdb.py
@@ -1,9 +1,15 @@
 
 from __future__ import absolute_import
 
 from __future__ import absolute_import

-import cPickle, errno, os, tempfile
+import errno, os, tempfile

 
 from bup import compat
 
 
 from bup import compat
 

+if compat.py_maj > 2:
+    import pickle
+else:
+    import cPickle as pickle
+
+

 class Error(Exception):
     pass
 
 class Error(Exception):
     pass
 


@@ -18,7 +24,7 @@ class HLinkDB:
         self._tmpname = None
         f = None
         try:
         self._tmpname = None
         f = None
         try:

-            f = open(filename, 'r')
+            f = open(filename, 'rb')

         except IOError as e:
             if e.errno == errno.ENOENT:
                 pass
         except IOError as e:
             if e.errno == errno.ENOENT:
                 pass


@@ -26,7 +32,7 @@ class HLinkDB:
                 raise
         if f:
             try:
                 raise
         if f:
             try:

-                self._node_paths = cPickle.load(f)
+                self._node_paths = pickle.load(f)

             finally:
                 f.close()
                 f = None
             finally:
                 f.close()
                 f = None


@@ -42,7 +48,7 @@ class HLinkDB:
             raise Error('save of %r already in progress' % self._filename)
         if self._node_paths:
             (dir, name) = os.path.split(self._filename)
             raise Error('save of %r already in progress' % self._filename)
         if self._node_paths:
             (dir, name) = os.path.split(self._filename)

-            (ffd, self._tmpname) = tempfile.mkstemp('.tmp', name, dir)
+            (ffd, self._tmpname) = tempfile.mkstemp(b'.tmp', name, dir)

             try:
                 try:
                     f = os.fdopen(ffd, 'wb', 65536)
             try:
                 try:
                     f = os.fdopen(ffd, 'wb', 65536)


@@ -50,7 +56,7 @@ class HLinkDB:
                     os.close(ffd)
                     raise
                 try:
                     os.close(ffd)
                     raise
                 try:

-                    cPickle.dump(self._node_paths, f, 2)
+                    pickle.dump(self._node_paths, f, 2)

                 finally:
                     f.close()
                     f = None
                 finally:
                     f.close()
                     f = None

diff --git a/lib/bup/path.py b/lib/bup/path.py
index b6167d075fd88689b45b81f1dadb2a45c7b5e1ec..7dd3905a2ecf22196b333362c0d5e20cd22f455a 100644 (file)
--- a/lib/bup/path.py
+++ b/lib/bup/path.py
@@ -6,10 +6,10 @@ import os
 # Eventually, if we physically move the source tree cmd/ to lib/, then
 # we could use realpath here and save some stats...
 
 # Eventually, if we physically move the source tree cmd/ to lib/, then
 # we could use realpath here and save some stats...
 

-_libdir = os.path.abspath(os.path.dirname(__file__) + '/..')
+_libdir = os.path.abspath(os.path.dirname(__file__.encode('iso-8859-1')) + b'/..')

 _resdir = _libdir
 _resdir = _libdir

-_exedir = os.path.abspath(_libdir + '/cmd')
-_exe = os.path.join(_exedir, 'bup')
+_exedir = os.path.abspath(_libdir + b'/cmd')
+_exe = os.path.join(_exedir, b'bup')

 
 
 def exe():
 
 
 def exe():


@@ -23,5 +23,5 @@ cmddir = exedir
 def libdir():
     return _libdir
 
 def libdir():
     return _libdir
 

-def resource_path(subdir=''):
+def resource_path(subdir=b''):

     return os.path.join(_resdir, subdir)
     return os.path.join(_resdir, subdir)

diff --git a/t/test-save-strip-graft.sh b/t/test-save-strip-graft.sh
index 31f06ebe90058609b24bfe202e3a6f81fc099787..95883ddd54fabfd8be7016223e78bc638dbf4bac 100755 (executable)
--- a/t/test-save-strip-graft.sh
+++ b/t/test-save-strip-graft.sh
@@ -155,7 +155,7 @@ WVPASS bup init
 WVPASS mkdir -p src/x/1 src/y/1
 WVPASS bup index -u src
 WVFAIL bup save --strip -n foo src/x src/y 2> tmp-err.log
 WVPASS mkdir -p src/x/1 src/y/1
 WVPASS bup index -u src
 WVFAIL bup save --strip -n foo src/x src/y 2> tmp-err.log

-WVPASS grep -F "error: ignoring duplicate path '1' in '/'" tmp-err.log
+WVPASS grep -F "error: ignoring duplicate path 1 in /" tmp-err.log

 
 
 WVPASS rm -rf "$tmpdir"
 
 
 WVPASS rm -rf "$tmpdir"