"""Helper functions and classes for bup."""
+from __future__ import absolute_import, division
+from collections import namedtuple
+from contextlib import contextmanager
from ctypes import sizeof, c_void_p
+from math import floor
from os import environ
+from subprocess import PIPE, Popen
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
-import hashlib, heapq, operator, time, grp
+import hashlib, heapq, math, operator, time, grp, tempfile
from bup import _helpers
-import bup._helpers as _helpers
-import math
-
+from bup import compat
+from bup.compat import argv_bytes, byte_int
+from bup.io import byte_stream, path_msg
# This function should really be in helpers, not in bup.options. But we
# want options.py to be standalone so people can include it in other projects.
-from bup.options import _tty_width
-tty_width = _tty_width
+from bup.options import _tty_width as tty_width
+
+
+class Nonlocal:
+ """Helper to deal with Python scoping issues"""
+ pass
+
+
+sc_page_size = os.sysconf('SC_PAGE_SIZE')
+assert(sc_page_size > 0)
+
+sc_arg_max = os.sysconf('SC_ARG_MAX')
+if sc_arg_max == -1: # "no definite limit" - let's choose 2M
+ sc_arg_max = 2 * 1024 * 1024
+
+def last(iterable):
+ result = None
+ for result in iterable:
+ pass
+ return result
def atoi(s):
- """Convert the string 's' to an integer. Return 0 if s is not a number."""
+ """Convert s (ascii bytes) to an integer. Return 0 if s is not a number."""
try:
- return int(s or '0')
+ return int(s or b'0')
except ValueError:
return 0
def atof(s):
- """Convert the string 's' to a float. Return 0 if s is not a number."""
+ """Convert s (ascii bytes) to a float. Return 0 if s is not a number."""
try:
- return float(s or '0')
+ return float(s or b'0')
except ValueError:
return 0
buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
-# If the platform doesn't have fdatasync (OS X), fall back to fsync.
try:
- fdatasync = os.fdatasync
+ _fdatasync = os.fdatasync
except AttributeError:
- fdatasync = os.fsync
+ _fdatasync = os.fsync
+
+if sys.platform.startswith('darwin'):
+ # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
+ import fcntl
+ def fdatasync(fd):
+ try:
+ return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
+ except IOError as e:
+ # Fallback for file systems (SMB) that do not support F_FULLFSYNC
+ if e.errno == errno.ENOTSUP:
+ return _fdatasync(fd)
+ else:
+ raise
+else:
+ fdatasync = _fdatasync
+
+
+def partition(predicate, stream):
+ """Returns (leading_matches_it, rest_it), where leading_matches_it
+ must be completely exhausted before traversing rest_it.
+
+ """
+ stream = iter(stream)
+ ns = Nonlocal()
+ ns.first_nonmatch = None
+ def leading_matches():
+ for x in stream:
+ if predicate(x):
+ yield x
+ else:
+ ns.first_nonmatch = (x,)
+ break
+ def rest():
+ if ns.first_nonmatch:
+ yield ns.first_nonmatch[0]
+ for x in stream:
+ yield x
+ return (leading_matches(), rest())
+
+
+def merge_dict(*xs):
+ result = {}
+ for x in xs:
+ result.update(x)
+ return result
+
+
+def lines_until_sentinel(f, sentinel, ex_type):
+ # sentinel must end with \n and must contain only one \n
+ while True:
+ line = f.readline()
+ if not (line and line.endswith(b'\n')):
+ raise ex_type('Hit EOF while reading line')
+ if line == sentinel:
+ return
+ yield line
+
+
+def stat_if_exists(path):
+ try:
+ return os.stat(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ return None
# Write (blockingly) to sockets that may or may not be in blocking mode.
raise IOError('select(fd) returned without being writable')
try:
sz = os.write(fd, buf)
- except OSError, e:
+ except OSError as e:
if e.errno != errno.EAGAIN:
raise
assert(sz >= 0)
"""Print a log message to stderr."""
global _last_prog
sys.stdout.flush()
- _hard_write(sys.stderr.fileno(), s)
+ _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode())
_last_prog = 0
os.makedirs(d, mode)
else:
os.makedirs(d)
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
-_unspecified_next_default = object()
-
-def _fallback_next(it, default=_unspecified_next_default):
- """Retrieve the next item from the iterator by calling its
- next() method. If default is given, it is returned if the
- iterator is exhausted, otherwise StopIteration is raised."""
-
- if default is _unspecified_next_default:
- return it.next()
- else:
- try:
- return it.next()
- except StopIteration:
- return default
-
-if sys.version_info < (2, 6):
- next = _fallback_next
-
+class MergeIterItem:
+ def __init__(self, entry, read_it):
+ self.entry = entry
+ self.read_it = read_it
+ def __lt__(self, x):
+ return self.entry < x.entry
def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
if key:
total = sum(len(it) for it in iters)
iters = (iter(it) for it in iters)
heap = ((next(it, None),it) for it in iters)
- heap = [(e,it) for e,it in heap if e]
+ heap = [MergeIterItem(e, it) for e, it in heap if e]
heapq.heapify(heap)
pe = None
while heap:
if not count % pfreq:
pfunc(count, total)
- e, it = heap[0]
+ e, it = heap[0].entry, heap[0].read_it
if not samekey(e, pe):
pe = e
yield e
count += 1
try:
- e = it.next() # Don't use next() function, it's too expensive
+ e = next(it)
except StopIteration:
heapq.heappop(heap) # remove current
else:
- heapq.heapreplace(heap, (e, it)) # shift current to new location
+ # shift current to new location
+ heapq.heapreplace(heap, MergeIterItem(e, it))
pfinal(count, total)
"""
try:
os.unlink(f)
- except OSError, e:
- if e.errno == errno.ENOENT:
- pass # it doesn't exist, that's what you asked for
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
-def readpipe(argv, preexec_fn=None):
+_bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
+_sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
+
+def bquote(x):
+ if x == b'':
+ return b"''"
+ if _bq_simple_id_rx.match(x):
+ return x
+ return b"'%s'" % x.replace(b"'", b"'\"'\"'")
+
+def squote(x):
+ if x == '':
+ return "''"
+ if _sq_simple_id_rx.match(x):
+ return x
+ return "'%s'" % x.replace("'", "'\"'\"'")
+
+def quote(x):
+ if isinstance(x, bytes):
+ return bquote(x)
+ if isinstance(x, compat.str_type):
+ return squote(x)
+ assert False
+
+def shstr(cmd):
+ """Return a shell quoted string for cmd if it's a sequence, else cmd.
+
+ cmd must be a string, bytes, or a sequence of one or the other,
+ and the assumption is that if cmd is a string or bytes, then it's
+ already quoted (because it's what's actually being passed to
+ call() and friends. e.g. log(shstr(cmd)); call(cmd)
+
+ """
+ if isinstance(cmd, (bytes, compat.str_type)):
+ return cmd
+ elif all(isinstance(x, bytes) for x in cmd):
+ return b' '.join(map(bquote, cmd))
+ elif all(isinstance(x, compat.str_type) for x in cmd):
+ return ' '.join(map(squote, cmd))
+ raise TypeError('unsupported shstr argument: ' + repr(cmd))
+
+
+exc = subprocess.check_call
+
+def exo(cmd,
+ input=None,
+ stdin=None,
+ stderr=None,
+ shell=False,
+ check=True,
+ preexec_fn=None):
+ if input:
+ assert stdin in (None, PIPE)
+ stdin = PIPE
+ p = Popen(cmd,
+ stdin=stdin, stdout=PIPE, stderr=stderr,
+ shell=shell,
+ preexec_fn=preexec_fn)
+ out, err = p.communicate(input)
+ if check and p.returncode != 0:
+ raise Exception('subprocess %r failed with status %d%s'
+ % (b' '.join(map(quote, cmd)), p.returncode,
+ ', stderr: %r' % err if err else ''))
+ return out, err, p
+
+def readpipe(argv, preexec_fn=None, shell=False):
"""Run a subprocess and return its output."""
- p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn)
+ p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
+ shell=shell)
out, err = p.communicate()
if p.returncode != 0:
raise Exception('subprocess %r failed with status %d'
- % (' '.join(argv), p.returncode))
+ % (b' '.join(argv), p.returncode))
return out
base_size = 2048
for c in command:
base_size += len(command) + 1
- for k, v in environ.iteritems():
+ for k, v in compat.items(environ):
base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
return base_size
return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
-def batchpipe(command, args, preexec_fn=None, arg_max=_helpers.SC_ARG_MAX):
+def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
"""If args is not empty, yield the output produced by calling the
command list with args as a sequence of strings (It may be necessary
to return multiple strings in order to respect ARG_MAX)."""
yield readpipe(command + sub_args, preexec_fn=preexec_fn)
-def realpath(p):
- """Get the absolute path of a file.
+def resolve_parent(p):
+ """Return the absolute path of a file without following any final symlink.
Behaves like os.path.realpath, but doesn't follow a symlink for the last
element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
return os.getenv("FAKEROOTKEY") != None
-def is_superuser():
- if sys.platform.startswith('cygwin'):
- import ctypes
- return ctypes.cdll.shell32.IsUserAnAdmin()
- else:
+if sys.platform.startswith('cygwin'):
+ def is_superuser():
+ # https://cygwin.com/ml/cygwin/2015-02/msg00057.html
+ groups = os.getgroups()
+ return 544 in groups or 0 in groups
+else:
+ def is_superuser():
return os.geteuid() == 0
-def _cache_key_value(get_value, key, cache):
+def cache_key_value(get_value, key, cache):
"""Return (value, was_cached). If there is a value in the cache
for key, use that, otherwise, call get_value(key) which should
throw a KeyError if there is no value -- in which case the cached
return value, False
-_uid_to_pwd_cache = {}
-_name_to_pwd_cache = {}
-
-def pwd_from_uid(uid):
- """Return password database entry for uid (may be a cached value).
- Return None if no entry is found.
- """
- global _uid_to_pwd_cache, _name_to_pwd_cache
- entry, cached = _cache_key_value(pwd.getpwuid, uid, _uid_to_pwd_cache)
- if entry and not cached:
- _name_to_pwd_cache[entry.pw_name] = entry
- return entry
-
-
-def pwd_from_name(name):
- """Return password database entry for name (may be a cached value).
- Return None if no entry is found.
- """
- global _uid_to_pwd_cache, _name_to_pwd_cache
- entry, cached = _cache_key_value(pwd.getpwnam, name, _name_to_pwd_cache)
- if entry and not cached:
- _uid_to_pwd_cache[entry.pw_uid] = entry
- return entry
-
-
-_gid_to_grp_cache = {}
-_name_to_grp_cache = {}
-
-def grp_from_gid(gid):
- """Return password database entry for gid (may be a cached value).
- Return None if no entry is found.
- """
- global _gid_to_grp_cache, _name_to_grp_cache
- entry, cached = _cache_key_value(grp.getgrgid, gid, _gid_to_grp_cache)
- if entry and not cached:
- _name_to_grp_cache[entry.gr_name] = entry
- return entry
-
-
-def grp_from_name(name):
- """Return password database entry for name (may be a cached value).
- Return None if no entry is found.
- """
- global _gid_to_grp_cache, _name_to_grp_cache
- entry, cached = _cache_key_value(grp.getgrnam, name, _name_to_grp_cache)
- if entry and not cached:
- _gid_to_grp_cache[entry.gr_gid] = entry
- return entry
-
-
-_username = None
-def username():
- """Get the user's login name."""
- global _username
- if not _username:
- uid = os.getuid()
- _username = pwd_from_uid(uid)[0] or 'user%d' % uid
- return _username
-
-
-_userfullname = None
-def userfullname():
- """Get the user's full name."""
- global _userfullname
- if not _userfullname:
- uid = os.getuid()
- entry = pwd_from_uid(uid)
- if entry:
- _userfullname = entry[4].split(',')[0] or entry[0]
- if not _userfullname:
- _userfullname = 'user%d' % uid
- return _userfullname
-
-
_hostname = None
def hostname():
"""Get the FQDN of this machine."""
global _hostname
if not _hostname:
- _hostname = socket.getfqdn()
+ _hostname = socket.getfqdn().encode('iso-8859-1')
return _hostname
-_resource_path = None
-def resource_path(subdir=''):
- global _resource_path
- if not _resource_path:
- _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
- return os.path.join(_resource_path, subdir)
-
def format_filesize(size):
unit = 1024.0
size = float(size)
if size < unit:
return "%d" % (size)
- exponent = int(math.log(size) / math.log(unit))
+ exponent = int(math.log(size) // math.log(unit))
size_prefix = "KMGTPE"[exponent - 1]
- return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
+ return "%.1f%s" % (size // math.pow(unit, exponent), size_prefix)
class NotOk(Exception):
def ok(self):
"""Indicate end of output from last sent command."""
- self.write('\nok\n')
+ self.write(b'\nok\n')
def error(self, s):
"""Indicate server error to the client."""
- s = re.sub(r'\s+', ' ', str(s))
- self.write('\nerror %s\n' % s)
+ s = re.sub(br'\s+', b' ', s)
+ self.write(b'\nerror %s\n' % s)
def _check_ok(self, onempty):
self.outp.flush()
- rl = ''
+ rl = b''
for rl in linereader(self):
#log('%d got line: %r\n' % (os.getpid(), rl))
if not rl: # empty line
continue
- elif rl == 'ok':
+ elif rl == b'ok':
return None
- elif rl.startswith('error '):
+ elif rl.startswith(b'error '):
#log('client: error: %s\n' % rl[6:])
return NotOk(rl[6:])
else:
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
- tail = ''
- while tail != 'BUPMUX':
+ tail = b''
+ while tail != b'BUPMUX':
b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
if not b:
raise IOError('demux: unexpected EOF during initialization')
tail += b
- sys.stderr.write(tail[:-6]) # pre-mux log messages
+ byte_stream(sys.stderr).write(tail[:-6]) # pre-mux log messages
tail = tail[-6:]
self.infd = infd
self.reader = None
rl, wl, xl = select.select([self.infd], [], [], timeout)
if not rl: return False
assert(rl[0] == self.infd)
- ns = ''.join(checked_reader(self.infd, 5))
+ ns = b''.join(checked_reader(self.infd, 5))
n, fdw = struct.unpack('!IB', ns)
assert(n <= MAX_PACKET)
if fdw == 1:
self.reader = checked_reader(self.infd, n)
elif fdw == 2:
for buf in checked_reader(self.infd, n):
- sys.stderr.write(buf)
+ byte_stream(sys.stderr).write(buf)
elif fdw == 3:
self.closed = True
debug2("DemuxConn: marked closed\n")
if not self._next_packet(timeout):
return False
try:
- self.buf = self.reader.next()
+ self.buf = next(self.reader)
return True
except StopIteration:
self.reader = None
def _readline(self):
def find_eol(buf):
try:
- return buf.index('\n')+1
+ return buf.index(b'\n')+1
except ValueError:
return None
- return ''.join(self._read_parts(find_eol))
+ return b''.join(self._read_parts(find_eol))
def _read(self, size):
csize = [size]
return None
else:
return csize[0]
- return ''.join(self._read_parts(until_size))
+ return b''.join(self._read_parts(until_size))
def has_input(self):
return self._load_buf(0)
yield b
+@contextmanager
+def atomically_replaced_file(name, mode='w', buffering=-1):
+ """Yield a file that will be atomically renamed name when leaving the block.
+
+ This contextmanager yields an open file object that is backed by a
+ temporary file which will be renamed (atomically) to the target
+ name if everything succeeds.
+
+ The mode and buffering arguments are handled exactly as with open,
+ and the yielded file will have very restrictive permissions, as
+ per mkstemp.
+
+ E.g.::
+
+ with atomically_replaced_file('foo.txt', 'w') as f:
+ f.write('hello jack.')
+
+ """
+
+ (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
+ text=('b' not in mode))
+ try:
+ try:
+ f = os.fdopen(ffd, mode, buffering)
+ except:
+ os.close(ffd)
+ raise
+ try:
+ yield f
+ finally:
+ f.close()
+ os.rename(tempname, name)
+ finally:
+ unlink(tempname) # nonexistant file is ignored
+
+
def slashappend(s):
"""Append "/" to 's' if it doesn't aleady end in "/"."""
- if s and not s.endswith('/'):
- return s + '/'
+ assert isinstance(s, bytes)
+ if s and not s.endswith(b'/'):
+ return s + b'/'
else:
return s
close)
+_mincore = getattr(_helpers, 'mincore', None)
+if _mincore:
+ # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
+ MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
+
+ _fmincore_chunk_size = None
+ def _set_fmincore_chunk_size():
+ global _fmincore_chunk_size
+ pref_chunk_size = 64 * 1024 * 1024
+ chunk_size = sc_page_size
+ if (sc_page_size < pref_chunk_size):
+ chunk_size = sc_page_size * (pref_chunk_size // sc_page_size)
+ _fmincore_chunk_size = chunk_size
+
+ def fmincore(fd):
+ """Return the mincore() data for fd as a bytearray whose values can be
+ tested via MINCORE_INCORE, or None if fd does not fully
+ support the operation."""
+ st = os.fstat(fd)
+ if (st.st_size == 0):
+ return bytearray(0)
+ if not _fmincore_chunk_size:
+ _set_fmincore_chunk_size()
+ pages_per_chunk = _fmincore_chunk_size // sc_page_size;
+ page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
+ chunk_count = page_count // _fmincore_chunk_size
+ if chunk_count < 1:
+ chunk_count = 1
+ result = bytearray(page_count)
+ for ci in compat.range(chunk_count):
+ pos = _fmincore_chunk_size * ci;
+ msize = min(_fmincore_chunk_size, st.st_size - pos)
+ try:
+ m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
+ except mmap.error as ex:
+ if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
+ # Perhaps the file was a pipe, i.e. "... | bup split ..."
+ return None
+ raise ex
+ try:
+ _mincore(m, msize, 0, result, ci * pages_per_chunk)
+ except OSError as ex:
+ if ex.errno == errno.ENOSYS:
+ return None
+ raise
+ return result
+
+
def parse_timestamp(epoch_str):
"""Return the number of nanoseconds since the epoch that are described
by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
def parse_num(s):
- """Parse data size information into a float number.
+ """Parse string or bytes as a possibly unit suffixed number.
- Here are some examples of conversions:
+ For example:
199.2k means 203981 bytes
1GB means 1073741824 bytes
2.1 tb means 2199023255552 bytes
"""
+ if isinstance(s, bytes):
+ # FIXME: should this raise a ValueError for UnicodeDecodeError
+ # (perhaps with the latter as the context).
+ s = s.decode('ascii')
g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
if not g:
raise ValueError("can't parse %r as a number" % s)
return int(num*mult)
-def count(l):
- """Count the number of elements in an iterator. (consumes the iterator)"""
- return reduce(lambda x,y: x+1, l)
-
-
saved_errors = []
def add_error(e):
"""Append an error message to the list of saved errors.
saved_errors = []
+def die_if_errors(msg=None, status=1):
+ global saved_errors
+ if saved_errors:
+ if not msg:
+ msg = 'warning: %d errors encountered\n' % len(saved_errors)
+ log(msg)
+ sys.exit(status)
+
+
def handle_ctrl_c():
"""Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
The number of columns is determined automatically based on the string
lengths.
"""
+ binary = isinstance(prefix, bytes)
+ nothing = b'' if binary else ''
+ nl = b'\n' if binary else '\n'
if not l:
- return ""
+ return nothing
l = l[:]
clen = max(len(s) for s in l)
- ncols = (tty_width() - len(prefix)) / (clen + 2)
+ ncols = (tty_width() - len(prefix)) // (clen + 2)
if ncols <= 1:
ncols = 1
clen = 0
cols = []
while len(l) % ncols:
- l.append('')
- rows = len(l)/ncols
- for s in range(0, len(l), rows):
+ l.append(nothing)
+ rows = len(l) // ncols
+ for s in compat.range(0, len(l), rows):
cols.append(l[s:s+rows])
- out = ''
+ out = nothing
+ fmt = b'%-*s' if binary else '%-*s'
for row in zip(*cols):
- out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
+ out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl
return out
"""Parses the given date or calls Option.fatal().
For now we expect a string that contains a float."""
try:
- date = atof(str)
- except ValueError, e:
+ date = float(str)
+ except ValueError as e:
raise fatal('invalid date format (should be a float): %r' % e)
else:
return date
for flag in options:
(option, parameter) = flag
if option == '--exclude':
- excluded_paths.append(realpath(parameter))
+ excluded_paths.append(resolve_parent(argv_bytes(parameter)))
elif option == '--exclude-from':
try:
- f = open(realpath(parameter))
- except IOError, e:
- raise fatal("couldn't read %s" % parameter)
+ f = open(resolve_parent(argv_bytes(parameter)), 'rb')
+ except IOError as e:
+ raise fatal("couldn't read %r" % parameter)
for exclude_path in f.readlines():
- excluded_paths.append(realpath(exclude_path.strip()))
+ # FIXME: perhaps this should be rstrip('\n')
+ exclude_path = resolve_parent(exclude_path.strip())
+ if exclude_path:
+ excluded_paths.append(exclude_path)
return sorted(frozenset(excluded_paths))
(option, parameter) = flag
if option == '--exclude-rx':
try:
- excluded_patterns.append(re.compile(parameter))
- except re.error, ex:
- fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex))
+ excluded_patterns.append(re.compile(argv_bytes(parameter)))
+ except re.error as ex:
+ fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))
elif option == '--exclude-rx-from':
try:
- f = open(realpath(parameter))
- except IOError, e:
- raise fatal("couldn't read %s" % parameter)
+ f = open(resolve_parent(parameter), 'rb')
+ except IOError as e:
+ raise fatal("couldn't read %r" % parameter)
for pattern in f.readlines():
- spattern = pattern.rstrip('\n')
+ spattern = pattern.rstrip(b'\n')
+ if not spattern:
+ continue
try:
excluded_patterns.append(re.compile(spattern))
- except re.error, ex:
- fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex))
+ except re.error as ex:
+ fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))
return excluded_patterns
full_path_to_name). Path must start with '/'.
Example:
'/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
- if not path.startswith('/'):
- raise Exception, 'path must start with "/": %s' % path
+ if not path.startswith(b'/'):
+ raise Exception('path must start with "/": %s' % path_msg(path))
# Since we assume path startswith('/'), we can skip the first element.
- result = [('', '/')]
+ result = [(b'', b'/')]
norm_path = os.path.abspath(path)
- if norm_path == '/':
+ if norm_path == b'/':
return result
- full_path = ''
- for p in norm_path.split('/')[1:]:
- full_path += '/' + p
+ full_path = b''
+ for p in norm_path.split(b'/')[1:]:
+ full_path += b'/' + p
result.append((p, full_path))
return result
sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
for bp in sorted_strip_prefixes:
normalized_bp = os.path.abspath(bp)
+ if normalized_bp == b'/':
+ continue
if normalized_path.startswith(normalized_bp):
prefix = normalized_path[:len(normalized_bp)]
result = []
- for p in normalized_path[len(normalized_bp):].split('/'):
+ for p in normalized_path[len(normalized_bp):].split(b'/'):
if p: # not root
- prefix += '/'
+ prefix += b'/'
prefix += p
result.append((p, prefix))
return result
new_prefix = os.path.normpath(new_prefix)
if clean_path.startswith(old_prefix):
escaped_prefix = re.escape(old_prefix)
- grafted_path = re.sub(r'^' + escaped_prefix, new_prefix, clean_path)
+ grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)
# Handle /foo=/ (at least) -- which produces //whatever.
- grafted_path = '/' + grafted_path.lstrip('/')
+ grafted_path = b'/' + grafted_path.lstrip(b'/')
clean_path_components = path_components(clean_path)
# Count the components that were stripped.
- strip_count = 0 if old_prefix == '/' else old_prefix.count('/')
- new_prefix_parts = new_prefix.split('/')
- result_prefix = grafted_path.split('/')[:new_prefix.count('/')]
+ strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
+ new_prefix_parts = new_prefix.split(b'/')
+ result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]
result = [(p, None) for p in result_prefix] \
+ clean_path_components[strip_count:]
# Now set the graft point name to match the end of new_prefix.
graft_point = len(result_prefix)
result[graft_point] = \
(new_prefix_parts[-1], clean_path_components[strip_count][1])
- if new_prefix == '/': # --graft ...=/ is a special case.
+ if new_prefix == b'/': # --graft ...=/ is a special case.
return result[1:]
return result
return path_components(clean_path)
+
Sha1 = hashlib.sha1
+
+
+_localtime = getattr(_helpers, 'localtime', None)
+
+if _localtime:
+ bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
+ 'tm_hour', 'tm_min', 'tm_sec',
+ 'tm_wday', 'tm_yday',
+ 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
+
+# Define a localtime() that returns bup_time when possible. Note:
+# this means that any helpers.localtime() results may need to be
+# passed through to_py_time() before being passed to python's time
+# module, which doesn't appear willing to ignore the extra items.
+if _localtime:
+ def localtime(time):
+ return bup_time(*_helpers.localtime(floor(time)))
+ def utc_offset_str(t):
+ """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
+ If the current UTC offset does not represent an integer number
+ of minutes, the fractional component will be truncated."""
+ off = localtime(t).tm_gmtoff
+ # Note: // doesn't truncate like C for negative values, it rounds down.
+ offmin = abs(off) // 60
+ m = offmin % 60
+ h = (offmin - m) // 60
+ return b'%+03d%02d' % (-h if off < 0 else h, m)
+ def to_py_time(x):
+ if isinstance(x, time.struct_time):
+ return x
+ return time.struct_time(x[:9])
+else:
+ localtime = time.localtime
+ def utc_offset_str(t):
+ return time.strftime(b'%z', localtime(t))
+ def to_py_time(x):
+ return x
+
+
+_some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')
+
+def valid_save_name(name):
+ # Enforce a superset of the restrictions in git-check-ref-format(1)
+ if name == b'@' \
+ or name.startswith(b'/') or name.endswith(b'/') \
+ or name.endswith(b'.'):
+ return False
+ if _some_invalid_save_parts_rx.search(name):
+ return False
+ for c in name:
+ if byte_int(c) < 0x20 or byte_int(c) == 0x7f:
+ return False
+ for part in name.split(b'/'):
+ if part.startswith(b'.') or part.endswith(b'.lock'):
+ return False
+ return True
+
+
+_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+
+def period_as_secs(s):
+ if s == 'forever':
+ return float('inf')
+ match = _period_rx.match(s)
+ if not match:
+ return None
+ mag = int(match.group(1))
+ scale = match.group(2)
+ return mag * {'s': 1,
+ 'min': 60,
+ 'h': 60 * 60,
+ 'd': 60 * 60 * 24,
+ 'w': 60 * 60 * 24 * 7,
+ 'm': 60 * 60 * 24 * 31,
+ 'y': 60 * 60 * 24 * 366}[scale]
projects / bup.git / blobdiff