X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?p=bup.git;a=blobdiff_plain;f=lib%2Fbup%2Fhelpers.py;h=a5c5165bc5d92bd3274e755886a1f055c46d87a2;hp=1b390c51dd51ea3f11481f2e55531a86cf0a62c4;hb=b26eaeb08c3c497e9b65122f676bb5f7ef495d21;hpb=16cd7cae8107e837447b6466f62cc9ce3b4fe635 diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py index 1b390c5..a5c5165 100644 --- a/lib/bup/helpers.py +++ b/lib/bup/helpers.py @@ -1,32 +1,55 @@ """Helper functions and classes for bup.""" +from __future__ import absolute_import, division +from collections import namedtuple +from contextlib import contextmanager from ctypes import sizeof, c_void_p +from math import floor from os import environ +from subprocess import PIPE, Popen import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct -import hashlib, heapq, operator, time, grp +import hashlib, heapq, math, operator, time, grp, tempfile from bup import _helpers -import bup._helpers as _helpers -import math - +from bup import compat +from bup.compat import argv_bytes, byte_int +from bup.io import byte_stream, path_msg # This function should really be in helpers, not in bup.options. But we # want options.py to be standalone so people can include it in other projects. -from bup.options import _tty_width -tty_width = _tty_width +from bup.options import _tty_width as tty_width + + +class Nonlocal: + """Helper to deal with Python scoping issues""" + pass + + +sc_page_size = os.sysconf('SC_PAGE_SIZE') +assert(sc_page_size > 0) + +sc_arg_max = os.sysconf('SC_ARG_MAX') +if sc_arg_max == -1: # "no definite limit" - let's choose 2M + sc_arg_max = 2 * 1024 * 1024 + +def last(iterable): + result = None + for result in iterable: + pass + return result def atoi(s): - """Convert the string 's' to an integer. Return 0 if s is not a number.""" + """Convert s (ascii bytes) to an integer. Return 0 if s is not a number.""" try: - return int(s or '0') + return int(s or b'0') except ValueError: return 0 def atof(s): - """Convert the string 's' to a float. Return 0 if s is not a number.""" + """Convert s (ascii bytes) to a float. Return 0 if s is not a number.""" try: - return float(s or '0') + return float(s or b'0') except ValueError: return 0 @@ -34,11 +57,75 @@ def atof(s): buglvl = atoi(os.environ.get('BUP_DEBUG', 0)) -# If the platform doesn't have fdatasync (OS X), fall back to fsync. try: - fdatasync = os.fdatasync + _fdatasync = os.fdatasync except AttributeError: - fdatasync = os.fsync + _fdatasync = os.fsync + +if sys.platform.startswith('darwin'): + # Apparently os.fsync on OS X doesn't guarantee to sync all the way down + import fcntl + def fdatasync(fd): + try: + return fcntl.fcntl(fd, fcntl.F_FULLFSYNC) + except IOError as e: + # Fallback for file systems (SMB) that do not support F_FULLFSYNC + if e.errno == errno.ENOTSUP: + return _fdatasync(fd) + else: + raise +else: + fdatasync = _fdatasync + + +def partition(predicate, stream): + """Returns (leading_matches_it, rest_it), where leading_matches_it + must be completely exhausted before traversing rest_it. + + """ + stream = iter(stream) + ns = Nonlocal() + ns.first_nonmatch = None + def leading_matches(): + for x in stream: + if predicate(x): + yield x + else: + ns.first_nonmatch = (x,) + break + def rest(): + if ns.first_nonmatch: + yield ns.first_nonmatch[0] + for x in stream: + yield x + return (leading_matches(), rest()) + + +def merge_dict(*xs): + result = {} + for x in xs: + result.update(x) + return result + + +def lines_until_sentinel(f, sentinel, ex_type): + # sentinel must end with \n and must contain only one \n + while True: + line = f.readline() + if not (line and line.endswith(b'\n')): + raise ex_type('Hit EOF while reading line') + if line == sentinel: + return + yield line + + +def stat_if_exists(path): + try: + return os.stat(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + return None # Write (blockingly) to sockets that may or may not be in blocking mode. @@ -52,7 +139,7 @@ def _hard_write(fd, buf): raise IOError('select(fd) returned without being writable') try: sz = os.write(fd, buf) - except OSError, e: + except OSError as e: if e.errno != errno.EAGAIN: raise assert(sz >= 0) @@ -64,7 +151,7 @@ def log(s): """Print a log message to stderr.""" global _last_prog sys.stdout.flush() - _hard_write(sys.stderr.fileno(), s) + _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode()) _last_prog = 0 @@ -122,31 +209,19 @@ def mkdirp(d, mode=None): os.makedirs(d, mode) else: os.makedirs(d) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass else: raise -_unspecified_next_default = object() - -def _fallback_next(it, default=_unspecified_next_default): - """Retrieve the next item from the iterator by calling its - next() method. If default is given, it is returned if the - iterator is exhausted, otherwise StopIteration is raised.""" - - if default is _unspecified_next_default: - return it.next() - else: - try: - return it.next() - except StopIteration: - return default - -if sys.version_info < (2, 6): - next = _fallback_next - +class MergeIterItem: + def __init__(self, entry, read_it): + self.entry = entry + self.read_it = read_it + def __lt__(self, x): + return self.entry < x.entry def merge_iter(iters, pfreq, pfunc, pfinal, key=None): if key: @@ -157,24 +232,25 @@ def merge_iter(iters, pfreq, pfunc, pfinal, key=None): total = sum(len(it) for it in iters) iters = (iter(it) for it in iters) heap = ((next(it, None),it) for it in iters) - heap = [(e,it) for e,it in heap if e] + heap = [MergeIterItem(e, it) for e, it in heap if e] heapq.heapify(heap) pe = None while heap: if not count % pfreq: pfunc(count, total) - e, it = heap[0] + e, it = heap[0].entry, heap[0].read_it if not samekey(e, pe): pe = e yield e count += 1 try: - e = it.next() # Don't use next() function, it's too expensive + e = next(it) except StopIteration: heapq.heappop(heap) # remove current else: - heapq.heapreplace(heap, (e, it)) # shift current to new location + # shift current to new location + heapq.heapreplace(heap, MergeIterItem(e, it)) pfinal(count, total) @@ -186,18 +262,84 @@ def unlink(f): """ try: os.unlink(f) - except OSError, e: - if e.errno == errno.ENOENT: - pass # it doesn't exist, that's what you asked for + except OSError as e: + if e.errno != errno.ENOENT: + raise -def readpipe(argv, preexec_fn=None): +_bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$') +_sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$') + +def bquote(x): + if x == b'': + return b"''" + if _bq_simple_id_rx.match(x): + return x + return b"'%s'" % x.replace(b"'", b"'\"'\"'") + +def squote(x): + if x == '': + return "''" + if _sq_simple_id_rx.match(x): + return x + return "'%s'" % x.replace("'", "'\"'\"'") + +def quote(x): + if isinstance(x, bytes): + return bquote(x) + if isinstance(x, compat.str_type): + return squote(x) + assert False + +def shstr(cmd): + """Return a shell quoted string for cmd if it's a sequence, else cmd. + + cmd must be a string, bytes, or a sequence of one or the other, + and the assumption is that if cmd is a string or bytes, then it's + already quoted (because it's what's actually being passed to + call() and friends. e.g. log(shstr(cmd)); call(cmd) + + """ + if isinstance(cmd, (bytes, compat.str_type)): + return cmd + elif all(isinstance(x, bytes) for x in cmd): + return b' '.join(map(bquote, cmd)) + elif all(isinstance(x, compat.str_type) for x in cmd): + return ' '.join(map(squote, cmd)) + raise TypeError('unsupported shstr argument: ' + repr(cmd)) + + +exc = subprocess.check_call + +def exo(cmd, + input=None, + stdin=None, + stderr=None, + shell=False, + check=True, + preexec_fn=None): + if input: + assert stdin in (None, PIPE) + stdin = PIPE + p = Popen(cmd, + stdin=stdin, stdout=PIPE, stderr=stderr, + shell=shell, + preexec_fn=preexec_fn) + out, err = p.communicate(input) + if check and p.returncode != 0: + raise Exception('subprocess %r failed with status %d%s' + % (b' '.join(map(quote, cmd)), p.returncode, + ', stderr: %r' % err if err else '')) + return out, err, p + +def readpipe(argv, preexec_fn=None, shell=False): """Run a subprocess and return its output.""" - p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn) + p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn, + shell=shell) out, err = p.communicate() if p.returncode != 0: raise Exception('subprocess %r failed with status %d' - % (' '.join(argv), p.returncode)) + % (b' '.join(argv), p.returncode)) return out @@ -205,7 +347,7 @@ def _argmax_base(command): base_size = 2048 for c in command: base_size += len(command) + 1 - for k, v in environ.iteritems(): + for k, v in compat.items(environ): base_size += len(k) + len(v) + 2 + sizeof(c_void_p) return base_size @@ -214,7 +356,7 @@ def _argmax_args_size(args): return sum(len(x) + 1 + sizeof(c_void_p) for x in args) -def batchpipe(command, args, preexec_fn=None, arg_max=_helpers.SC_ARG_MAX): +def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max): """If args is not empty, yield the output produced by calling the command list with args as a sequence of strings (It may be necessary to return multiple strings in order to respect ARG_MAX).""" @@ -236,8 +378,8 @@ to return multiple strings in order to respect ARG_MAX).""" yield readpipe(command + sub_args, preexec_fn=preexec_fn) -def realpath(p): - """Get the absolute path of a file. +def resolve_parent(p): + """Return the absolute path of a file without following any final symlink. Behaves like os.path.realpath, but doesn't follow a symlink for the last element. (ie. if 'p' itself is a symlink, this one won't follow it, but it @@ -262,15 +404,17 @@ def detect_fakeroot(): return os.getenv("FAKEROOTKEY") != None -def is_superuser(): - if sys.platform.startswith('cygwin'): - import ctypes - return ctypes.cdll.shell32.IsUserAnAdmin() - else: +if sys.platform.startswith('cygwin'): + def is_superuser(): + # https://cygwin.com/ml/cygwin/2015-02/msg00057.html + groups = os.getgroups() + return 544 in groups or 0 in groups +else: + def is_superuser(): return os.geteuid() == 0 -def _cache_key_value(get_value, key, cache): +def cache_key_value(get_value, key, cache): """Return (value, was_cached). If there is a value in the cache for key, use that, otherwise, call get_value(key) which should throw a KeyError if there is no value -- in which case the cached @@ -289,104 +433,23 @@ def _cache_key_value(get_value, key, cache): return value, False -_uid_to_pwd_cache = {} -_name_to_pwd_cache = {} - -def pwd_from_uid(uid): - """Return password database entry for uid (may be a cached value). - Return None if no entry is found. - """ - global _uid_to_pwd_cache, _name_to_pwd_cache - entry, cached = _cache_key_value(pwd.getpwuid, uid, _uid_to_pwd_cache) - if entry and not cached: - _name_to_pwd_cache[entry.pw_name] = entry - return entry - - -def pwd_from_name(name): - """Return password database entry for name (may be a cached value). - Return None if no entry is found. - """ - global _uid_to_pwd_cache, _name_to_pwd_cache - entry, cached = _cache_key_value(pwd.getpwnam, name, _name_to_pwd_cache) - if entry and not cached: - _uid_to_pwd_cache[entry.pw_uid] = entry - return entry - - -_gid_to_grp_cache = {} -_name_to_grp_cache = {} - -def grp_from_gid(gid): - """Return password database entry for gid (may be a cached value). - Return None if no entry is found. - """ - global _gid_to_grp_cache, _name_to_grp_cache - entry, cached = _cache_key_value(grp.getgrgid, gid, _gid_to_grp_cache) - if entry and not cached: - _name_to_grp_cache[entry.gr_name] = entry - return entry - - -def grp_from_name(name): - """Return password database entry for name (may be a cached value). - Return None if no entry is found. - """ - global _gid_to_grp_cache, _name_to_grp_cache - entry, cached = _cache_key_value(grp.getgrnam, name, _name_to_grp_cache) - if entry and not cached: - _gid_to_grp_cache[entry.gr_gid] = entry - return entry - - -_username = None -def username(): - """Get the user's login name.""" - global _username - if not _username: - uid = os.getuid() - _username = pwd_from_uid(uid)[0] or 'user%d' % uid - return _username - - -_userfullname = None -def userfullname(): - """Get the user's full name.""" - global _userfullname - if not _userfullname: - uid = os.getuid() - entry = pwd_from_uid(uid) - if entry: - _userfullname = entry[4].split(',')[0] or entry[0] - if not _userfullname: - _userfullname = 'user%d' % uid - return _userfullname - - _hostname = None def hostname(): """Get the FQDN of this machine.""" global _hostname if not _hostname: - _hostname = socket.getfqdn() + _hostname = socket.getfqdn().encode('iso-8859-1') return _hostname -_resource_path = None -def resource_path(subdir=''): - global _resource_path - if not _resource_path: - _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.' - return os.path.join(_resource_path, subdir) - def format_filesize(size): unit = 1024.0 size = float(size) if size < unit: return "%d" % (size) - exponent = int(math.log(size) / math.log(unit)) + exponent = int(math.log(size) // math.log(unit)) size_prefix = "KMGTPE"[exponent - 1] - return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix) + return "%.1f%s" % (size // math.pow(unit, exponent), size_prefix) class NotOk(Exception): @@ -421,23 +484,23 @@ class BaseConn: def ok(self): """Indicate end of output from last sent command.""" - self.write('\nok\n') + self.write(b'\nok\n') def error(self, s): """Indicate server error to the client.""" - s = re.sub(r'\s+', ' ', str(s)) - self.write('\nerror %s\n' % s) + s = re.sub(br'\s+', b' ', s) + self.write(b'\nerror %s\n' % s) def _check_ok(self, onempty): self.outp.flush() - rl = '' + rl = b'' for rl in linereader(self): #log('%d got line: %r\n' % (os.getpid(), rl)) if not rl: # empty line continue - elif rl == 'ok': + elif rl == b'ok': return None - elif rl.startswith('error '): + elif rl.startswith(b'error '): #log('client: error: %s\n' % rl[6:]) return NotOk(rl[6:]) else: @@ -512,13 +575,13 @@ class DemuxConn(BaseConn): BaseConn.__init__(self, outp) # Anything that comes through before the sync string was not # multiplexed and can be assumed to be debug/log before mux init. - tail = '' - while tail != 'BUPMUX': + tail = b'' + while tail != b'BUPMUX': b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1) if not b: raise IOError('demux: unexpected EOF during initialization') tail += b - sys.stderr.write(tail[:-6]) # pre-mux log messages + byte_stream(sys.stderr).write(tail[:-6]) # pre-mux log messages tail = tail[-6:] self.infd = infd self.reader = None @@ -534,14 +597,14 @@ class DemuxConn(BaseConn): rl, wl, xl = select.select([self.infd], [], [], timeout) if not rl: return False assert(rl[0] == self.infd) - ns = ''.join(checked_reader(self.infd, 5)) + ns = b''.join(checked_reader(self.infd, 5)) n, fdw = struct.unpack('!IB', ns) assert(n <= MAX_PACKET) if fdw == 1: self.reader = checked_reader(self.infd, n) elif fdw == 2: for buf in checked_reader(self.infd, n): - sys.stderr.write(buf) + byte_stream(sys.stderr).write(buf) elif fdw == 3: self.closed = True debug2("DemuxConn: marked closed\n") @@ -555,7 +618,7 @@ class DemuxConn(BaseConn): if not self._next_packet(timeout): return False try: - self.buf = self.reader.next() + self.buf = next(self.reader) return True except StopIteration: self.reader = None @@ -578,10 +641,10 @@ class DemuxConn(BaseConn): def _readline(self): def find_eol(buf): try: - return buf.index('\n')+1 + return buf.index(b'\n')+1 except ValueError: return None - return ''.join(self._read_parts(find_eol)) + return b''.join(self._read_parts(find_eol)) def _read(self, size): csize = [size] @@ -591,7 +654,7 @@ class DemuxConn(BaseConn): return None else: return csize[0] - return ''.join(self._read_parts(until_size)) + return b''.join(self._read_parts(until_size)) def has_input(self): return self._load_buf(0) @@ -628,10 +691,47 @@ def chunkyreader(f, count = None): yield b +@contextmanager +def atomically_replaced_file(name, mode='w', buffering=-1): + """Yield a file that will be atomically renamed name when leaving the block. + + This contextmanager yields an open file object that is backed by a + temporary file which will be renamed (atomically) to the target + name if everything succeeds. + + The mode and buffering arguments are handled exactly as with open, + and the yielded file will have very restrictive permissions, as + per mkstemp. + + E.g.:: + + with atomically_replaced_file('foo.txt', 'w') as f: + f.write('hello jack.') + + """ + + (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name), + text=('b' not in mode)) + try: + try: + f = os.fdopen(ffd, mode, buffering) + except: + os.close(ffd) + raise + try: + yield f + finally: + f.close() + os.rename(tempname, name) + finally: + unlink(tempname) # nonexistant file is ignored + + def slashappend(s): """Append "/" to 's' if it doesn't aleady end in "/".""" - if s and not s.endswith('/'): - return s + '/' + assert isinstance(s, bytes) + if s and not s.endswith(b'/'): + return s + b'/' else: return s @@ -676,6 +776,54 @@ def mmap_readwrite_private(f, sz = 0, close=True): close) +_mincore = getattr(_helpers, 'mincore', None) +if _mincore: + # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined. + MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1) + + _fmincore_chunk_size = None + def _set_fmincore_chunk_size(): + global _fmincore_chunk_size + pref_chunk_size = 64 * 1024 * 1024 + chunk_size = sc_page_size + if (sc_page_size < pref_chunk_size): + chunk_size = sc_page_size * (pref_chunk_size // sc_page_size) + _fmincore_chunk_size = chunk_size + + def fmincore(fd): + """Return the mincore() data for fd as a bytearray whose values can be + tested via MINCORE_INCORE, or None if fd does not fully + support the operation.""" + st = os.fstat(fd) + if (st.st_size == 0): + return bytearray(0) + if not _fmincore_chunk_size: + _set_fmincore_chunk_size() + pages_per_chunk = _fmincore_chunk_size // sc_page_size; + page_count = (st.st_size + sc_page_size - 1) // sc_page_size; + chunk_count = page_count // _fmincore_chunk_size + if chunk_count < 1: + chunk_count = 1 + result = bytearray(page_count) + for ci in compat.range(chunk_count): + pos = _fmincore_chunk_size * ci; + msize = min(_fmincore_chunk_size, st.st_size - pos) + try: + m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos) + except mmap.error as ex: + if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV: + # Perhaps the file was a pipe, i.e. "... | bup split ..." + return None + raise ex + try: + _mincore(m, msize, 0, result, ci * pages_per_chunk) + except OSError as ex: + if ex.errno == errno.ENOSYS: + return None + raise + return result + + def parse_timestamp(epoch_str): """Return the number of nanoseconds since the epoch that are described by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed, @@ -697,13 +845,17 @@ throw a ValueError that may contain additional information.""" def parse_num(s): - """Parse data size information into a float number. + """Parse string or bytes as a possibly unit suffixed number. - Here are some examples of conversions: + For example: 199.2k means 203981 bytes 1GB means 1073741824 bytes 2.1 tb means 2199023255552 bytes """ + if isinstance(s, bytes): + # FIXME: should this raise a ValueError for UnicodeDecodeError + # (perhaps with the latter as the context). + s = s.decode('ascii') g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s)) if not g: raise ValueError("can't parse %r as a number" % s) @@ -725,11 +877,6 @@ def parse_num(s): return int(num*mult) -def count(l): - """Count the number of elements in an iterator. (consumes the iterator)""" - return reduce(lambda x,y: x+1, l) - - saved_errors = [] def add_error(e): """Append an error message to the list of saved errors. @@ -746,6 +893,15 @@ def clear_errors(): saved_errors = [] +def die_if_errors(msg=None, status=1): + global saved_errors + if saved_errors: + if not msg: + msg = 'warning: %d errors encountered\n' % len(saved_errors) + log(msg) + sys.exit(status) + + def handle_ctrl_c(): """Replace the default exception handler for KeyboardInterrupt (Ctrl-C). @@ -767,23 +923,27 @@ def columnate(l, prefix): The number of columns is determined automatically based on the string lengths. """ + binary = isinstance(prefix, bytes) + nothing = b'' if binary else '' + nl = b'\n' if binary else '\n' if not l: - return "" + return nothing l = l[:] clen = max(len(s) for s in l) - ncols = (tty_width() - len(prefix)) / (clen + 2) + ncols = (tty_width() - len(prefix)) // (clen + 2) if ncols <= 1: ncols = 1 clen = 0 cols = [] while len(l) % ncols: - l.append('') - rows = len(l)/ncols - for s in range(0, len(l), rows): + l.append(nothing) + rows = len(l) // ncols + for s in compat.range(0, len(l), rows): cols.append(l[s:s+rows]) - out = '' + out = nothing + fmt = b'%-*s' if binary else '%-*s' for row in zip(*cols): - out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n' + out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl return out @@ -791,8 +951,8 @@ def parse_date_or_fatal(str, fatal): """Parses the given date or calls Option.fatal(). For now we expect a string that contains a float.""" try: - date = atof(str) - except ValueError, e: + date = float(str) + except ValueError as e: raise fatal('invalid date format (should be a float): %r' % e) else: return date @@ -805,14 +965,17 @@ def parse_excludes(options, fatal): for flag in options: (option, parameter) = flag if option == '--exclude': - excluded_paths.append(realpath(parameter)) + excluded_paths.append(resolve_parent(argv_bytes(parameter))) elif option == '--exclude-from': try: - f = open(realpath(parameter)) - except IOError, e: - raise fatal("couldn't read %s" % parameter) + f = open(resolve_parent(argv_bytes(parameter)), 'rb') + except IOError as e: + raise fatal("couldn't read %r" % parameter) for exclude_path in f.readlines(): - excluded_paths.append(realpath(exclude_path.strip())) + # FIXME: perhaps this should be rstrip('\n') + exclude_path = resolve_parent(exclude_path.strip()) + if exclude_path: + excluded_paths.append(exclude_path) return sorted(frozenset(excluded_paths)) @@ -825,20 +988,22 @@ def parse_rx_excludes(options, fatal): (option, parameter) = flag if option == '--exclude-rx': try: - excluded_patterns.append(re.compile(parameter)) - except re.error, ex: - fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex)) + excluded_patterns.append(re.compile(argv_bytes(parameter))) + except re.error as ex: + fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex)) elif option == '--exclude-rx-from': try: - f = open(realpath(parameter)) - except IOError, e: - raise fatal("couldn't read %s" % parameter) + f = open(resolve_parent(parameter), 'rb') + except IOError as e: + raise fatal("couldn't read %r" % parameter) for pattern in f.readlines(): - spattern = pattern.rstrip('\n') + spattern = pattern.rstrip(b'\n') + if not spattern: + continue try: excluded_patterns.append(re.compile(spattern)) - except re.error, ex: - fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex)) + except re.error as ex: + fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex)) return excluded_patterns @@ -865,16 +1030,16 @@ def path_components(path): full_path_to_name). Path must start with '/'. Example: '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]""" - if not path.startswith('/'): - raise Exception, 'path must start with "/": %s' % path + if not path.startswith(b'/'): + raise Exception('path must start with "/": %s' % path_msg(path)) # Since we assume path startswith('/'), we can skip the first element. - result = [('', '/')] + result = [(b'', b'/')] norm_path = os.path.abspath(path) - if norm_path == '/': + if norm_path == b'/': return result - full_path = '' - for p in norm_path.split('/')[1:]: - full_path += '/' + p + full_path = b'' + for p in norm_path.split(b'/')[1:]: + full_path += b'/' + p result.append((p, full_path)) return result @@ -888,12 +1053,14 @@ def stripped_path_components(path, strip_prefixes): sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True) for bp in sorted_strip_prefixes: normalized_bp = os.path.abspath(bp) + if normalized_bp == b'/': + continue if normalized_path.startswith(normalized_bp): prefix = normalized_path[:len(normalized_bp)] result = [] - for p in normalized_path[len(normalized_bp):].split('/'): + for p in normalized_path[len(normalized_bp):].split(b'/'): if p: # not root - prefix += '/' + prefix += b'/' prefix += p result.append((p, prefix)) return result @@ -924,23 +1091,99 @@ def grafted_path_components(graft_points, path): new_prefix = os.path.normpath(new_prefix) if clean_path.startswith(old_prefix): escaped_prefix = re.escape(old_prefix) - grafted_path = re.sub(r'^' + escaped_prefix, new_prefix, clean_path) + grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path) # Handle /foo=/ (at least) -- which produces //whatever. - grafted_path = '/' + grafted_path.lstrip('/') + grafted_path = b'/' + grafted_path.lstrip(b'/') clean_path_components = path_components(clean_path) # Count the components that were stripped. - strip_count = 0 if old_prefix == '/' else old_prefix.count('/') - new_prefix_parts = new_prefix.split('/') - result_prefix = grafted_path.split('/')[:new_prefix.count('/')] + strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/') + new_prefix_parts = new_prefix.split(b'/') + result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')] result = [(p, None) for p in result_prefix] \ + clean_path_components[strip_count:] # Now set the graft point name to match the end of new_prefix. graft_point = len(result_prefix) result[graft_point] = \ (new_prefix_parts[-1], clean_path_components[strip_count][1]) - if new_prefix == '/': # --graft ...=/ is a special case. + if new_prefix == b'/': # --graft ...=/ is a special case. return result[1:] return result return path_components(clean_path) + Sha1 = hashlib.sha1 + + +_localtime = getattr(_helpers, 'localtime', None) + +if _localtime: + bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday', + 'tm_hour', 'tm_min', 'tm_sec', + 'tm_wday', 'tm_yday', + 'tm_isdst', 'tm_gmtoff', 'tm_zone']) + +# Define a localtime() that returns bup_time when possible. Note: +# this means that any helpers.localtime() results may need to be +# passed through to_py_time() before being passed to python's time +# module, which doesn't appear willing to ignore the extra items. +if _localtime: + def localtime(time): + return bup_time(*_helpers.localtime(floor(time))) + def utc_offset_str(t): + """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t. + If the current UTC offset does not represent an integer number + of minutes, the fractional component will be truncated.""" + off = localtime(t).tm_gmtoff + # Note: // doesn't truncate like C for negative values, it rounds down. + offmin = abs(off) // 60 + m = offmin % 60 + h = (offmin - m) // 60 + return b'%+03d%02d' % (-h if off < 0 else h, m) + def to_py_time(x): + if isinstance(x, time.struct_time): + return x + return time.struct_time(x[:9]) +else: + localtime = time.localtime + def utc_offset_str(t): + return time.strftime(b'%z', localtime(t)) + def to_py_time(x): + return x + + +_some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{') + +def valid_save_name(name): + # Enforce a superset of the restrictions in git-check-ref-format(1) + if name == b'@' \ + or name.startswith(b'/') or name.endswith(b'/') \ + or name.endswith(b'.'): + return False + if _some_invalid_save_parts_rx.search(name): + return False + for c in name: + if byte_int(c) < 0x20 or byte_int(c) == 0x7f: + return False + for part in name.split(b'/'): + if part.startswith(b'.') or part.endswith(b'.lock'): + return False + return True + + +_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$') + +def period_as_secs(s): + if s == 'forever': + return float('inf') + match = _period_rx.match(s) + if not match: + return None + mag = int(match.group(1)) + scale = match.group(2) + return mag * {'s': 1, + 'min': 60, + 'h': 60 * 60, + 'd': 60 * 60 * 24, + 'w': 60 * 60 * 24 * 7, + 'm': 60 * 60 * 24 * 31, + 'y': 60 * 60 * 24 * 366}[scale]