helpers: use float for format_filesize()

[bup.git] / lib / bup / helpers.py

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index a4dc3376c67aac5b0b9c94a7c10822eecbed912a..5b0c458f0a8a3abcaf748856a5f89660a90eb971 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -1,16 +1,19 @@
 """Helper functions and classes for bup."""
 
 """Helper functions and classes for bup."""
 

+from __future__ import absolute_import, division

 from collections import namedtuple
 from contextlib import contextmanager
 from ctypes import sizeof, c_void_p
 from collections import namedtuple
 from contextlib import contextmanager
 from ctypes import sizeof, c_void_p

+from math import floor

 from os import environ
 from os import environ

-from pipes import quote

 from subprocess import PIPE, Popen
 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
 import hashlib, heapq, math, operator, time, grp, tempfile
 
 from bup import _helpers
 from bup import compat
 from subprocess import PIPE, Popen
 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
 import hashlib, heapq, math, operator, time, grp, tempfile
 
 from bup import _helpers
 from bup import compat

+from bup.compat import argv_bytes, byte_int
+from bup.io import byte_stream, path_msg

 # This function should really be in helpers, not in bup.options.  But we
 # want options.py to be standalone so people can include it in other projects.
 from bup.options import _tty_width as tty_width
 # This function should really be in helpers, not in bup.options.  But we
 # want options.py to be standalone so people can include it in other projects.
 from bup.options import _tty_width as tty_width


@@ -28,7 +31,6 @@ sc_arg_max = os.sysconf('SC_ARG_MAX')
 if sc_arg_max == -1:  # "no definite limit" - let's choose 2M
     sc_arg_max = 2 * 1024 * 1024
 
 if sc_arg_max == -1:  # "no definite limit" - let's choose 2M
     sc_arg_max = 2 * 1024 * 1024
 

-

 def last(iterable):
     result = None
     for result in iterable:
 def last(iterable):
     result = None
     for result in iterable:


@@ -37,17 +39,17 @@ def last(iterable):
 
 
 def atoi(s):
 
 
 def atoi(s):

-    """Convert the string 's' to an integer. Return 0 if s is not a number."""
+    """Convert s (ascii bytes) to an integer. Return 0 if s is not a number."""

     try:
     try:

-        return int(s or '0')
+        return int(s or b'0')

     except ValueError:
         return 0
 
 
 def atof(s):
     except ValueError:
         return 0
 
 
 def atof(s):

-    """Convert the string 's' to a float. Return 0 if s is not a number."""
+    """Convert s (ascii bytes) to a float. Return 0 if s is not a number."""

     try:
     try:

-        return float(s or '0')
+        return float(s or b'0')

     except ValueError:
         return 0
 
     except ValueError:
         return 0
 


@@ -99,11 +101,18 @@ def partition(predicate, stream):
     return (leading_matches(), rest())
 
 
     return (leading_matches(), rest())
 
 

+def merge_dict(*xs):
+    result = {}
+    for x in xs:
+        result.update(x)
+    return result
+
+

 def lines_until_sentinel(f, sentinel, ex_type):
     # sentinel must end with \n and must contain only one \n
     while True:
         line = f.readline()
 def lines_until_sentinel(f, sentinel, ex_type):
     # sentinel must end with \n and must contain only one \n
     while True:
         line = f.readline()

-        if not (line and line.endswith('\n')):
+        if not (line and line.endswith(b'\n')):

             raise ex_type('Hit EOF while reading line')
         if line == sentinel:
             return
             raise ex_type('Hit EOF while reading line')
         if line == sentinel:
             return


@@ -142,7 +151,7 @@ def log(s):
     """Print a log message to stderr."""
     global _last_prog
     sys.stdout.flush()
     """Print a log message to stderr."""
     global _last_prog
     sys.stdout.flush()

-    _hard_write(sys.stderr.fileno(), s)
+    _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode())

     _last_prog = 0
 
 
     _last_prog = 0
 
 


@@ -207,6 +216,13 @@ def mkdirp(d, mode=None):
             raise
 
 
             raise
 
 

+class MergeIterItem:
+    def __init__(self, entry, read_it):
+        self.entry = entry
+        self.read_it = read_it
+    def __lt__(self, x):
+        return self.entry < x.entry
+

 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
     if key:
         samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
     if key:
         samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)


@@ -216,14 +232,14 @@ def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
     total = sum(len(it) for it in iters)
     iters = (iter(it) for it in iters)
     heap = ((next(it, None),it) for it in iters)
     total = sum(len(it) for it in iters)
     iters = (iter(it) for it in iters)
     heap = ((next(it, None),it) for it in iters)

-    heap = [(e,it) for e,it in heap if e]
+    heap = [MergeIterItem(e, it) for e, it in heap if e]

 
     heapq.heapify(heap)
     pe = None
     while heap:
         if not count % pfreq:
             pfunc(count, total)
 
     heapq.heapify(heap)
     pe = None
     while heap:
         if not count % pfreq:
             pfunc(count, total)

-        e, it = heap[0]
+        e, it = heap[0].entry, heap[0].read_it

         if not samekey(e, pe):
             pe = e
             yield e
         if not samekey(e, pe):
             pe = e
             yield e


@@ -233,7 +249,8 @@ def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
         except StopIteration:
             heapq.heappop(heap) # remove current
         else:
         except StopIteration:
             heapq.heappop(heap) # remove current
         else:

-            heapq.heapreplace(heap, (e, it)) # shift current to new location
+            # shift current to new location
+            heapq.heapreplace(heap, MergeIterItem(e, it))

     pfinal(count, total)
 
 
     pfinal(count, total)
 
 


@@ -250,11 +267,47 @@ def unlink(f):
             raise
 
 
             raise
 
 

+_bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
+_sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
+
+def bquote(x):
+    if x == b'':
+        return b"''"
+    if _bq_simple_id_rx.match(x):
+        return x
+    return b"'%s'" % x.replace(b"'", b"'\"'\"'")
+
+def squote(x):
+    if x == '':
+        return "''"
+    if _sq_simple_id_rx.match(x):
+        return x
+    return "'%s'" % x.replace("'", "'\"'\"'")
+
+def quote(x):
+    if isinstance(x, bytes):
+        return bquote(x)
+    if isinstance(x, compat.str_type):
+        return squote(x)
+    assert False
+

 def shstr(cmd):
 def shstr(cmd):

-    if isinstance(cmd, compat.str_type):
+    """Return a shell quoted string for cmd if it's a sequence, else cmd.
+
+    cmd must be a string, bytes, or a sequence of one or the other,
+    and the assumption is that if cmd is a string or bytes, then it's
+    already quoted (because it's what's actually being passed to
+    call() and friends.  e.g. log(shstr(cmd)); call(cmd)
+
+    """
+    if isinstance(cmd, (bytes, compat.str_type)):

         return cmd
         return cmd

-    else:
-        return ' '.join(map(quote, cmd))
+    elif all(isinstance(x, bytes) for x in cmd):
+        return b' '.join(map(bquote, cmd))
+    elif all(isinstance(x, compat.str_type) for x in cmd):
+        return ' '.join(map(squote, cmd))
+    raise TypeError('unsupported shstr argument: ' + repr(cmd))
+

 
 exc = subprocess.check_call
 
 
 exc = subprocess.check_call
 


@@ -264,36 +317,33 @@ def exo(cmd,
         stderr=None,
         shell=False,
         check=True,
         stderr=None,
         shell=False,
         check=True,

-        preexec_fn=None):
+        preexec_fn=None,
+        close_fds=True):

     if input:
         assert stdin in (None, PIPE)
         stdin = PIPE
     p = Popen(cmd,
               stdin=stdin, stdout=PIPE, stderr=stderr,
               shell=shell,
     if input:
         assert stdin in (None, PIPE)
         stdin = PIPE
     p = Popen(cmd,
               stdin=stdin, stdout=PIPE, stderr=stderr,
               shell=shell,

-              preexec_fn=preexec_fn)
+              preexec_fn=preexec_fn,
+              close_fds=close_fds)

     out, err = p.communicate(input)
     if check and p.returncode != 0:
     out, err = p.communicate(input)
     if check and p.returncode != 0:

-        raise Exception('subprocess %r failed with status %d, stderr: %r'
-                        % (' '.join(map(quote, cmd)), p.returncode, err))
+        raise Exception('subprocess %r failed with status %d%s'
+                        % (b' '.join(map(quote, cmd)), p.returncode,
+                           ', stderr: %r' % err if err else ''))

     return out, err, p
 
 def readpipe(argv, preexec_fn=None, shell=False):
     """Run a subprocess and return its output."""
     return out, err, p
 
 def readpipe(argv, preexec_fn=None, shell=False):
     """Run a subprocess and return its output."""

-    p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
-                         shell=shell)
-    out, err = p.communicate()
-    if p.returncode != 0:
-        raise Exception('subprocess %r failed with status %d'
-                        % (' '.join(argv), p.returncode))
-    return out
+    return exo(argv, preexec_fn=preexec_fn, shell=shell)[0]

 
 
 def _argmax_base(command):
     base_size = 2048
     for c in command:
         base_size += len(command) + 1
 
 
 def _argmax_base(command):
     base_size = 2048
     for c in command:
         base_size += len(command) + 1

-    for k, v in environ.iteritems():
+    for k, v in compat.items(environ):

         base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
     return base_size
 
         base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
     return base_size
 


@@ -350,23 +400,17 @@ def detect_fakeroot():
     return os.getenv("FAKEROOTKEY") != None
 
 
     return os.getenv("FAKEROOTKEY") != None
 
 

-_warned_about_superuser_detection = None
-def is_superuser():
-    if sys.platform.startswith('cygwin'):
-        if sys.getwindowsversion()[0] > 5:
-            # Sounds like situation is much more complicated here
-            global _warned_about_superuser_detection
-            if not _warned_about_superuser_detection:
-                log("can't detect root status for OS version > 5; assuming not root")
-                _warned_about_superuser_detection = True
-            return False
-        import ctypes
-        return ctypes.cdll.shell32.IsUserAnAdmin()
-    else:
+if sys.platform.startswith('cygwin'):
+    def is_superuser():
+        # https://cygwin.com/ml/cygwin/2015-02/msg00057.html
+        groups = os.getgroups()
+        return 544 in groups or 0 in groups
+else:
+    def is_superuser():

         return os.geteuid() == 0
 
 
         return os.geteuid() == 0
 
 

-def _cache_key_value(get_value, key, cache):
+def cache_key_value(get_value, key, cache):

     """Return (value, was_cached).  If there is a value in the cache
     for key, use that, otherwise, call get_value(key) which should
     throw a KeyError if there is no value -- in which case the cached
     """Return (value, was_cached).  If there is a value in the cache
     for key, use that, otherwise, call get_value(key) which should
     throw a KeyError if there is no value -- in which case the cached


@@ -385,102 +429,21 @@ def _cache_key_value(get_value, key, cache):
     return value, False
 
 
     return value, False
 
 

-_uid_to_pwd_cache = {}
-_name_to_pwd_cache = {}
-
-def pwd_from_uid(uid):
-    """Return password database entry for uid (may be a cached value).
-    Return None if no entry is found.
-    """
-    global _uid_to_pwd_cache, _name_to_pwd_cache
-    entry, cached = _cache_key_value(pwd.getpwuid, uid, _uid_to_pwd_cache)
-    if entry and not cached:
-        _name_to_pwd_cache[entry.pw_name] = entry
-    return entry
-
-
-def pwd_from_name(name):
-    """Return password database entry for name (may be a cached value).
-    Return None if no entry is found.
-    """
-    global _uid_to_pwd_cache, _name_to_pwd_cache
-    entry, cached = _cache_key_value(pwd.getpwnam, name, _name_to_pwd_cache)
-    if entry and not cached:
-        _uid_to_pwd_cache[entry.pw_uid] = entry
-    return entry
-
-
-_gid_to_grp_cache = {}
-_name_to_grp_cache = {}
-
-def grp_from_gid(gid):
-    """Return password database entry for gid (may be a cached value).
-    Return None if no entry is found.
-    """
-    global _gid_to_grp_cache, _name_to_grp_cache
-    entry, cached = _cache_key_value(grp.getgrgid, gid, _gid_to_grp_cache)
-    if entry and not cached:
-        _name_to_grp_cache[entry.gr_name] = entry
-    return entry
-
-
-def grp_from_name(name):
-    """Return password database entry for name (may be a cached value).
-    Return None if no entry is found.
-    """
-    global _gid_to_grp_cache, _name_to_grp_cache
-    entry, cached = _cache_key_value(grp.getgrnam, name, _name_to_grp_cache)
-    if entry and not cached:
-        _gid_to_grp_cache[entry.gr_gid] = entry
-    return entry
-
-
-_username = None
-def username():
-    """Get the user's login name."""
-    global _username
-    if not _username:
-        uid = os.getuid()
-        _username = pwd_from_uid(uid)[0] or 'user%d' % uid
-    return _username
-
-
-_userfullname = None
-def userfullname():
-    """Get the user's full name."""
-    global _userfullname
-    if not _userfullname:
-        uid = os.getuid()
-        entry = pwd_from_uid(uid)
-        if entry:
-            _userfullname = entry[4].split(',')[0] or entry[0]
-        if not _userfullname:
-            _userfullname = 'user%d' % uid
-    return _userfullname
-
-

 _hostname = None
 def hostname():
     """Get the FQDN of this machine."""
     global _hostname
     if not _hostname:
 _hostname = None
 def hostname():
     """Get the FQDN of this machine."""
     global _hostname
     if not _hostname:

-        _hostname = socket.getfqdn()
+        _hostname = socket.getfqdn().encode('iso-8859-1')

     return _hostname
 
 
     return _hostname
 
 

-_resource_path = None
-def resource_path(subdir=''):
-    global _resource_path
-    if not _resource_path:
-        _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
-    return os.path.join(_resource_path, subdir)
-

 def format_filesize(size):
     unit = 1024.0
     size = float(size)
     if size < unit:
         return "%d" % (size)
 def format_filesize(size):
     unit = 1024.0
     size = float(size)
     if size < unit:
         return "%d" % (size)

-    exponent = int(math.log(size) / math.log(unit))
+    exponent = int(math.log(size) // math.log(unit))

     size_prefix = "KMGTPE"[exponent - 1]
     return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
 
     size_prefix = "KMGTPE"[exponent - 1]
     return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
 


@@ -496,11 +459,17 @@ class BaseConn:
     def close(self):
         while self._read(65536): pass
 
     def close(self):
         while self._read(65536): pass
 

+    def _read(self, size):
+        raise NotImplementedError("Subclasses must implement _read")
+

     def read(self, size):
         """Read 'size' bytes from input stream."""
         self.outp.flush()
         return self._read(size)
 
     def read(self, size):
         """Read 'size' bytes from input stream."""
         self.outp.flush()
         return self._read(size)
 

+    def _readline(self, size):
+        raise NotImplementedError("Subclasses must implement _readline")
+

     def readline(self):
         """Read from input stream until a newline is found."""
         self.outp.flush()
     def readline(self):
         """Read from input stream until a newline is found."""
         self.outp.flush()


@@ -513,27 +482,27 @@ class BaseConn:
 
     def has_input(self):
         """Return true if input stream is readable."""
 
     def has_input(self):
         """Return true if input stream is readable."""

-        raise NotImplemented("Subclasses must implement has_input")
+        raise NotImplementedError("Subclasses must implement has_input")

 
     def ok(self):
         """Indicate end of output from last sent command."""
 
     def ok(self):
         """Indicate end of output from last sent command."""

-        self.write('\nok\n')
+        self.write(b'\nok\n')

 
     def error(self, s):
         """Indicate server error to the client."""
 
     def error(self, s):
         """Indicate server error to the client."""

-        s = re.sub(r'\s+', ' ', str(s))
-        self.write('\nerror %s\n' % s)
+        s = re.sub(br'\s+', b' ', s)
+        self.write(b'\nerror %s\n' % s)

 
     def _check_ok(self, onempty):
         self.outp.flush()
 
     def _check_ok(self, onempty):
         self.outp.flush()

-        rl = ''
+        rl = b''

         for rl in linereader(self):
             #log('%d got line: %r\n' % (os.getpid(), rl))
             if not rl:  # empty line
                 continue
         for rl in linereader(self):
             #log('%d got line: %r\n' % (os.getpid(), rl))
             if not rl:  # empty line
                 continue

-            elif rl == 'ok':
+            elif rl == b'ok':

                 return None
                 return None

-            elif rl.startswith('error '):
+            elif rl.startswith(b'error '):

                 #log('client: error: %s\n' % rl[6:])
                 return NotOk(rl[6:])
             else:
                 #log('client: error: %s\n' % rl[6:])
                 return NotOk(rl[6:])
             else:


@@ -608,13 +577,13 @@ class DemuxConn(BaseConn):
         BaseConn.__init__(self, outp)
         # Anything that comes through before the sync string was not
         # multiplexed and can be assumed to be debug/log before mux init.
         BaseConn.__init__(self, outp)
         # Anything that comes through before the sync string was not
         # multiplexed and can be assumed to be debug/log before mux init.

-        tail = ''
-        while tail != 'BUPMUX':
+        tail = b''
+        while tail != b'BUPMUX':

             b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
             if not b:
                 raise IOError('demux: unexpected EOF during initialization')
             tail += b
             b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
             if not b:
                 raise IOError('demux: unexpected EOF during initialization')
             tail += b

-            sys.stderr.write(tail[:-6])  # pre-mux log messages
+            byte_stream(sys.stderr).write(tail[:-6])  # pre-mux log messages

             tail = tail[-6:]
         self.infd = infd
         self.reader = None
             tail = tail[-6:]
         self.infd = infd
         self.reader = None


@@ -630,14 +599,14 @@ class DemuxConn(BaseConn):
         rl, wl, xl = select.select([self.infd], [], [], timeout)
         if not rl: return False
         assert(rl[0] == self.infd)
         rl, wl, xl = select.select([self.infd], [], [], timeout)
         if not rl: return False
         assert(rl[0] == self.infd)

-        ns = ''.join(checked_reader(self.infd, 5))
+        ns = b''.join(checked_reader(self.infd, 5))

         n, fdw = struct.unpack('!IB', ns)
         assert(n <= MAX_PACKET)
         if fdw == 1:
             self.reader = checked_reader(self.infd, n)
         elif fdw == 2:
             for buf in checked_reader(self.infd, n):
         n, fdw = struct.unpack('!IB', ns)
         assert(n <= MAX_PACKET)
         if fdw == 1:
             self.reader = checked_reader(self.infd, n)
         elif fdw == 2:
             for buf in checked_reader(self.infd, n):

-                sys.stderr.write(buf)
+                byte_stream(sys.stderr).write(buf)

         elif fdw == 3:
             self.closed = True
             debug2("DemuxConn: marked closed\n")
         elif fdw == 3:
             self.closed = True
             debug2("DemuxConn: marked closed\n")


@@ -674,10 +643,10 @@ class DemuxConn(BaseConn):
     def _readline(self):
         def find_eol(buf):
             try:
     def _readline(self):
         def find_eol(buf):
             try:

-                return buf.index('\n')+1
+                return buf.index(b'\n')+1

             except ValueError:
                 return None
             except ValueError:
                 return None

-        return ''.join(self._read_parts(find_eol))
+        return b''.join(self._read_parts(find_eol))

 
     def _read(self, size):
         csize = [size]
 
     def _read(self, size):
         csize = [size]


@@ -687,7 +656,7 @@ class DemuxConn(BaseConn):
                 return None
             else:
                 return csize[0]
                 return None
             else:
                 return csize[0]

-        return ''.join(self._read_parts(until_size))
+        return b''.join(self._read_parts(until_size))

 
     def has_input(self):
         return self._load_buf(0)
 
     def has_input(self):
         return self._load_buf(0)


@@ -762,8 +731,9 @@ def atomically_replaced_file(name, mode='w', buffering=-1):
 
 def slashappend(s):
     """Append "/" to 's' if it doesn't aleady end in "/"."""
 
 def slashappend(s):
     """Append "/" to 's' if it doesn't aleady end in "/"."""

-    if s and not s.endswith('/'):
-        return s + '/'
+    assert isinstance(s, bytes)
+    if s and not s.endswith(b'/'):
+        return s + b'/'

     else:
         return s
 
     else:
         return s
 


@@ -819,7 +789,7 @@ if _mincore:
         pref_chunk_size = 64 * 1024 * 1024
         chunk_size = sc_page_size
         if (sc_page_size < pref_chunk_size):
         pref_chunk_size = 64 * 1024 * 1024
         chunk_size = sc_page_size
         if (sc_page_size < pref_chunk_size):

-            chunk_size = sc_page_size * (pref_chunk_size / sc_page_size)
+            chunk_size = sc_page_size * (pref_chunk_size // sc_page_size)

         _fmincore_chunk_size = chunk_size
 
     def fmincore(fd):
         _fmincore_chunk_size = chunk_size
 
     def fmincore(fd):


@@ -831,13 +801,11 @@ if _mincore:
             return bytearray(0)
         if not _fmincore_chunk_size:
             _set_fmincore_chunk_size()
             return bytearray(0)
         if not _fmincore_chunk_size:
             _set_fmincore_chunk_size()

-        pages_per_chunk = _fmincore_chunk_size / sc_page_size;
-        page_count = (st.st_size + sc_page_size - 1) / sc_page_size;
-        chunk_count = page_count / _fmincore_chunk_size
-        if chunk_count < 1:
-            chunk_count = 1
+        pages_per_chunk = _fmincore_chunk_size // sc_page_size;
+        page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
+        chunk_count = (st.st_size + _fmincore_chunk_size - 1) // _fmincore_chunk_size

         result = bytearray(page_count)
         result = bytearray(page_count)

-        for ci in xrange(chunk_count):
+        for ci in compat.range(chunk_count):

             pos = _fmincore_chunk_size * ci;
             msize = min(_fmincore_chunk_size, st.st_size - pos)
             try:
             pos = _fmincore_chunk_size * ci;
             msize = min(_fmincore_chunk_size, st.st_size - pos)
             try:


@@ -877,13 +845,17 @@ throw a ValueError that may contain additional information."""
 
 
 def parse_num(s):
 
 
 def parse_num(s):

-    """Parse data size information into a float number.
+    """Parse string or bytes as a possibly unit suffixed number.

 
 

-    Here are some examples of conversions:
+    For example:

         199.2k means 203981 bytes
         1GB means 1073741824 bytes
         2.1 tb means 2199023255552 bytes
     """
         199.2k means 203981 bytes
         1GB means 1073741824 bytes
         2.1 tb means 2199023255552 bytes
     """

+    if isinstance(s, bytes):
+        # FIXME: should this raise a ValueError for UnicodeDecodeError
+        # (perhaps with the latter as the context).
+        s = s.decode('ascii')

     g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
     if not g:
         raise ValueError("can't parse %r as a number" % s)
     g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
     if not g:
         raise ValueError("can't parse %r as a number" % s)


@@ -905,11 +877,6 @@ def parse_num(s):
     return int(num*mult)
 
 
     return int(num*mult)
 
 

-def count(l):
-    """Count the number of elements in an iterator. (consumes the iterator)"""
-    return reduce(lambda x,y: x+1, l)
-
-

 saved_errors = []
 def add_error(e):
     """Append an error message to the list of saved errors.
 saved_errors = []
 def add_error(e):
     """Append an error message to the list of saved errors.


@@ -956,8 +923,11 @@ def columnate(l, prefix):
     The number of columns is determined automatically based on the string
     lengths.
     """
     The number of columns is determined automatically based on the string
     lengths.
     """

+    binary = isinstance(prefix, bytes)
+    nothing = b'' if binary else ''
+    nl = b'\n' if binary else '\n'

     if not l:
     if not l:

-        return ""
+        return nothing

     l = l[:]
     clen = max(len(s) for s in l)
     ncols = (tty_width() - len(prefix)) // (clen + 2)
     l = l[:]
     clen = max(len(s) for s in l)
     ncols = (tty_width() - len(prefix)) // (clen + 2)


@@ -966,13 +936,14 @@ def columnate(l, prefix):
         clen = 0
     cols = []
     while len(l) % ncols:
         clen = 0
     cols = []
     while len(l) % ncols:

-        l.append('')
+        l.append(nothing)

     rows = len(l) // ncols
     rows = len(l) // ncols

-    for s in range(0, len(l), rows):
+    for s in compat.range(0, len(l), rows):

         cols.append(l[s:s+rows])
         cols.append(l[s:s+rows])

-    out = ''
+    out = nothing
+    fmt = b'%-*s' if binary else '%-*s'

     for row in zip(*cols):
     for row in zip(*cols):

-        out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
+        out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl

     return out
 
 
     return out
 
 


@@ -994,12 +965,12 @@ def parse_excludes(options, fatal):
     for flag in options:
         (option, parameter) = flag
         if option == '--exclude':
     for flag in options:
         (option, parameter) = flag
         if option == '--exclude':

-            excluded_paths.append(resolve_parent(parameter))
+            excluded_paths.append(resolve_parent(argv_bytes(parameter)))

         elif option == '--exclude-from':
             try:
         elif option == '--exclude-from':
             try:

-                f = open(resolve_parent(parameter))
+                f = open(resolve_parent(argv_bytes(parameter)), 'rb')

             except IOError as e:
             except IOError as e:

-                raise fatal("couldn't read %s" % parameter)
+                raise fatal("couldn't read %r" % parameter)

             for exclude_path in f.readlines():
                 # FIXME: perhaps this should be rstrip('\n')
                 exclude_path = resolve_parent(exclude_path.strip())
             for exclude_path in f.readlines():
                 # FIXME: perhaps this should be rstrip('\n')
                 exclude_path = resolve_parent(exclude_path.strip())


@@ -1017,22 +988,22 @@ def parse_rx_excludes(options, fatal):
         (option, parameter) = flag
         if option == '--exclude-rx':
             try:
         (option, parameter) = flag
         if option == '--exclude-rx':
             try:

-                excluded_patterns.append(re.compile(parameter))
+                excluded_patterns.append(re.compile(argv_bytes(parameter)))

             except re.error as ex:
             except re.error as ex:

-                fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex))
+                fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))

         elif option == '--exclude-rx-from':
             try:
         elif option == '--exclude-rx-from':
             try:

-                f = open(resolve_parent(parameter))
+                f = open(resolve_parent(parameter), 'rb')

             except IOError as e:
             except IOError as e:

-                raise fatal("couldn't read %s" % parameter)
+                raise fatal("couldn't read %r" % parameter)

             for pattern in f.readlines():
             for pattern in f.readlines():

-                spattern = pattern.rstrip('\n')
+                spattern = pattern.rstrip(b'\n')

                 if not spattern:
                     continue
                 try:
                     excluded_patterns.append(re.compile(spattern))
                 except re.error as ex:
                 if not spattern:
                     continue
                 try:
                     excluded_patterns.append(re.compile(spattern))
                 except re.error as ex:

-                    fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex))
+                    fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))

     return excluded_patterns
 
 
     return excluded_patterns
 
 


@@ -1059,16 +1030,16 @@ def path_components(path):
     full_path_to_name).  Path must start with '/'.
     Example:
       '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
     full_path_to_name).  Path must start with '/'.
     Example:
       '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""

-    if not path.startswith('/'):
-        raise Exception('path must start with "/": %s' % path)
+    if not path.startswith(b'/'):
+        raise Exception('path must start with "/": %s' % path_msg(path))

     # Since we assume path startswith('/'), we can skip the first element.
     # Since we assume path startswith('/'), we can skip the first element.

-    result = [('', '/')]
+    result = [(b'', b'/')]

     norm_path = os.path.abspath(path)
     norm_path = os.path.abspath(path)

-    if norm_path == '/':
+    if norm_path == b'/':

         return result
         return result

-    full_path = ''
-    for p in norm_path.split('/')[1:]:
-        full_path += '/' + p
+    full_path = b''
+    for p in norm_path.split(b'/')[1:]:
+        full_path += b'/' + p

         result.append((p, full_path))
     return result
 
         result.append((p, full_path))
     return result
 


@@ -1082,14 +1053,14 @@ def stripped_path_components(path, strip_prefixes):
     sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
     for bp in sorted_strip_prefixes:
         normalized_bp = os.path.abspath(bp)
     sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
     for bp in sorted_strip_prefixes:
         normalized_bp = os.path.abspath(bp)

-        if normalized_bp == '/':
+        if normalized_bp == b'/':

             continue
         if normalized_path.startswith(normalized_bp):
             prefix = normalized_path[:len(normalized_bp)]
             result = []
             continue
         if normalized_path.startswith(normalized_bp):
             prefix = normalized_path[:len(normalized_bp)]
             result = []

-            for p in normalized_path[len(normalized_bp):].split('/'):
+            for p in normalized_path[len(normalized_bp):].split(b'/'):

                 if p: # not root
                 if p: # not root

-                    prefix += '/'
+                    prefix += b'/'

                 prefix += p
                 result.append((p, prefix))
             return result
                 prefix += p
                 result.append((p, prefix))
             return result


@@ -1120,21 +1091,21 @@ def grafted_path_components(graft_points, path):
         new_prefix = os.path.normpath(new_prefix)
         if clean_path.startswith(old_prefix):
             escaped_prefix = re.escape(old_prefix)
         new_prefix = os.path.normpath(new_prefix)
         if clean_path.startswith(old_prefix):
             escaped_prefix = re.escape(old_prefix)

-            grafted_path = re.sub(r'^' + escaped_prefix, new_prefix, clean_path)
+            grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)

             # Handle /foo=/ (at least) -- which produces //whatever.
             # Handle /foo=/ (at least) -- which produces //whatever.

-            grafted_path = '/' + grafted_path.lstrip('/')
+            grafted_path = b'/' + grafted_path.lstrip(b'/')

             clean_path_components = path_components(clean_path)
             # Count the components that were stripped.
             clean_path_components = path_components(clean_path)
             # Count the components that were stripped.

-            strip_count = 0 if old_prefix == '/' else old_prefix.count('/')
-            new_prefix_parts = new_prefix.split('/')
-            result_prefix = grafted_path.split('/')[:new_prefix.count('/')]
+            strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
+            new_prefix_parts = new_prefix.split(b'/')
+            result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]

             result = [(p, None) for p in result_prefix] \
                 + clean_path_components[strip_count:]
             # Now set the graft point name to match the end of new_prefix.
             graft_point = len(result_prefix)
             result[graft_point] = \
                 (new_prefix_parts[-1], clean_path_components[strip_count][1])
             result = [(p, None) for p in result_prefix] \
                 + clean_path_components[strip_count:]
             # Now set the graft point name to match the end of new_prefix.
             graft_point = len(result_prefix)
             result[graft_point] = \
                 (new_prefix_parts[-1], clean_path_components[strip_count][1])

-            if new_prefix == '/': # --graft ...=/ is a special case.
+            if new_prefix == b'/': # --graft ...=/ is a special case.

                 return result[1:]
             return result
     return path_components(clean_path)
                 return result[1:]
             return result
     return path_components(clean_path)


@@ -1157,7 +1128,7 @@ if _localtime:
 # module, which doesn't appear willing to ignore the extra items.
 if _localtime:
     def localtime(time):
 # module, which doesn't appear willing to ignore the extra items.
 if _localtime:
     def localtime(time):

-        return bup_time(*_helpers.localtime(time))
+        return bup_time(*_helpers.localtime(floor(time)))

     def utc_offset_str(t):
         """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
         If the current UTC offset does not represent an integer number
     def utc_offset_str(t):
         """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
         If the current UTC offset does not represent an integer number


@@ -1167,7 +1138,7 @@ if _localtime:
         offmin = abs(off) // 60
         m = offmin % 60
         h = (offmin - m) // 60
         offmin = abs(off) // 60
         m = offmin % 60
         h = (offmin - m) // 60

-        return "%+03d%02d" % (-h if off < 0 else h, m)
+        return b'%+03d%02d' % (-h if off < 0 else h, m)

     def to_py_time(x):
         if isinstance(x, time.struct_time):
             return x
     def to_py_time(x):
         if isinstance(x, time.struct_time):
             return x


@@ -1175,44 +1146,44 @@ if _localtime:
 else:
     localtime = time.localtime
     def utc_offset_str(t):
 else:
     localtime = time.localtime
     def utc_offset_str(t):

-        return time.strftime('%z', localtime(t))
+        return time.strftime(b'%z', localtime(t))

     def to_py_time(x):
         return x
 
 
     def to_py_time(x):
         return x
 
 

-_some_invalid_save_parts_rx = re.compile(r'[[ ~^:?*\\]|\.\.|//|@{')
+_some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')

 
 def valid_save_name(name):
     # Enforce a superset of the restrictions in git-check-ref-format(1)
 
 def valid_save_name(name):
     # Enforce a superset of the restrictions in git-check-ref-format(1)

-    if name == '@' \
-       or name.startswith('/') or name.endswith('/') \
-       or name.endswith('.'):
+    if name == b'@' \
+       or name.startswith(b'/') or name.endswith(b'/') \
+       or name.endswith(b'.'):

         return False
     if _some_invalid_save_parts_rx.search(name):
         return False
     for c in name:
         return False
     if _some_invalid_save_parts_rx.search(name):
         return False
     for c in name:

-        if ord(c) < 0x20 or ord(c) == 0x7f:
+        if byte_int(c) < 0x20 or byte_int(c) == 0x7f:

             return False
             return False

-    for part in name.split('/'):
-        if part.startswith('.') or part.endswith('.lock'):
+    for part in name.split(b'/'):
+        if part.startswith(b'.') or part.endswith(b'.lock'):

             return False
     return True
 
 
             return False
     return True
 
 

-_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+_period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')

 
 def period_as_secs(s):
 
 def period_as_secs(s):

-    if s == 'forever':
+    if s == b'forever':

         return float('inf')
     match = _period_rx.match(s)
     if not match:
         return None
     mag = int(match.group(1))
     scale = match.group(2)
         return float('inf')
     match = _period_rx.match(s)
     if not match:
         return None
     mag = int(match.group(1))
     scale = match.group(2)

-    return mag * {'s': 1,
-                  'min': 60,
-                  'h': 60 * 60,
-                  'd': 60 * 60 * 24,
-                  'w': 60 * 60 * 24 * 7,
-                  'm': 60 * 60 * 24 * 31,
-                  'y': 60 * 60 * 24 * 366}[scale]
+    return mag * {b's': 1,
+                  b'min': 60,
+                  b'h': 60 * 60,
+                  b'd': 60 * 60 * 24,
+                  b'w': 60 * 60 * 24 * 7,
+                  b'm': 60 * 60 * 24 * 31,
+                  b'y': 60 * 60 * 24 * 366}[scale]