1 """Helper functions and classes for bup."""
3 from __future__ import absolute_import, division
4 from collections import namedtuple
5 from contextlib import contextmanager
6 from ctypes import sizeof, c_void_p
9 from subprocess import PIPE, Popen
10 import sys, os, subprocess, errno, select, mmap, stat, re, struct
11 import hashlib, heapq, math, operator, time, tempfile
13 from bup import _helpers
14 from bup import compat
15 from bup.compat import argv_bytes, byte_int, nullcontext, pending_raise
16 from bup.io import byte_stream, path_msg
17 # This function should really be in helpers, not in bup.options. But we
18 # want options.py to be standalone so people can include it in other projects.
19 from bup.options import _tty_width as tty_width
22 buglvl = int(os.environ.get('BUP_DEBUG', 0))
26 """Helper to deal with Python scoping issues"""
30 def nullcontext_if_not(manager):
31 return manager if manager is not None else nullcontext()
35 def finalized(enter_result=None, finalize=None):
39 except BaseException as ex:
40 with pending_raise(ex):
41 finalize(enter_result)
42 finalize(enter_result)
45 sc_page_size = os.sysconf('SC_PAGE_SIZE')
46 assert(sc_page_size > 0)
48 sc_arg_max = os.sysconf('SC_ARG_MAX')
49 if sc_arg_max == -1: # "no definite limit" - let's choose 2M
50 sc_arg_max = 2 * 1024 * 1024
54 for result in iterable:
59 _fdatasync = os.fdatasync
60 except AttributeError:
63 if sys.platform.startswith('darwin'):
64 # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
68 return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
70 # Fallback for file systems (SMB) that do not support F_FULLFSYNC
71 if e.errno == errno.ENOTSUP:
76 fdatasync = _fdatasync
79 def partition(predicate, stream):
80 """Returns (leading_matches_it, rest_it), where leading_matches_it
81 must be completely exhausted before traversing rest_it.
86 ns.first_nonmatch = None
87 def leading_matches():
92 ns.first_nonmatch = (x,)
96 yield ns.first_nonmatch[0]
99 return (leading_matches(), rest())
109 def lines_until_sentinel(f, sentinel, ex_type):
110 # sentinel must end with \n and must contain only one \n
113 if not (line and line.endswith(b'\n')):
114 raise ex_type('Hit EOF while reading line')
120 def stat_if_exists(path):
124 if e.errno != errno.ENOENT:
129 # Write (blockingly) to sockets that may or may not be in blocking mode.
130 # We need this because our stderr is sometimes eaten by subprocesses
131 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
132 # leading to race conditions. Ick. We'll do it the hard way.
133 def _hard_write(fd, buf):
135 (r,w,x) = select.select([], [fd], [], None)
137 raise IOError('select(fd) returned without being writable')
139 sz = os.write(fd, buf)
141 if e.errno != errno.EAGAIN:
149 """Print a log message to stderr."""
152 _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode())
166 istty1 = os.isatty(1) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 1)
167 istty2 = os.isatty(2) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 2)
170 """Calls log() if stderr is a TTY. Does nothing otherwise."""
171 global _last_progress
178 """Calls progress() only if we haven't printed progress in a while.
180 This avoids overloading the stderr buffer with excess junk.
184 if now - _last_prog > 0.1:
190 """Calls progress() to redisplay the most recent progress message.
192 Useful after you've printed some other message that wipes out the
195 if _last_progress and _last_progress.endswith('\r'):
196 progress(_last_progress)
199 def mkdirp(d, mode=None):
200 """Recursively create directories on path 'd'.
202 Unlike os.makedirs(), it doesn't raise an exception if the last element of
203 the path already exists.
211 if e.errno == errno.EEXIST:
218 def __init__(self, entry, read_it):
220 self.read_it = read_it
222 return self.entry < x.entry
224 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
226 samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
228 samekey = operator.eq
230 total = sum(len(it) for it in iters)
231 iters = (iter(it) for it in iters)
232 heap = ((next(it, None),it) for it in iters)
233 heap = [MergeIterItem(e, it) for e, it in heap if e]
238 if not count % pfreq:
240 e, it = heap[0].entry, heap[0].read_it
241 if not samekey(e, pe):
247 except StopIteration:
248 heapq.heappop(heap) # remove current
250 # shift current to new location
251 heapq.heapreplace(heap, MergeIterItem(e, it))
256 """Delete a file at path 'f' if it currently exists.
258 Unlike os.unlink(), does not throw an exception if the file didn't already
264 if e.errno != errno.ENOENT:
268 _bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
269 _sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
274 if _bq_simple_id_rx.match(x):
276 return b"'%s'" % x.replace(b"'", b"'\"'\"'")
281 if _sq_simple_id_rx.match(x):
283 return "'%s'" % x.replace("'", "'\"'\"'")
286 if isinstance(x, bytes):
288 if isinstance(x, compat.str_type):
291 # some versions of pylint get confused
295 """Return a shell quoted string for cmd if it's a sequence, else cmd.
297 cmd must be a string, bytes, or a sequence of one or the other,
298 and the assumption is that if cmd is a string or bytes, then it's
299 already quoted (because it's what's actually being passed to
300 call() and friends. e.g. log(shstr(cmd)); call(cmd)
303 if isinstance(cmd, (bytes, compat.str_type)):
305 elif all(isinstance(x, bytes) for x in cmd):
306 return b' '.join(map(bquote, cmd))
307 elif all(isinstance(x, compat.str_type) for x in cmd):
308 return ' '.join(map(squote, cmd))
309 raise TypeError('unsupported shstr argument: ' + repr(cmd))
312 exc = subprocess.check_call
323 assert stdin in (None, PIPE)
326 stdin=stdin, stdout=PIPE, stderr=stderr,
328 preexec_fn=preexec_fn,
330 out, err = p.communicate(input)
331 if check and p.returncode != 0:
332 raise Exception('subprocess %r failed with status %d%s'
333 % (b' '.join(map(quote, cmd)), p.returncode,
334 ', stderr: %r' % err if err else ''))
337 def readpipe(argv, preexec_fn=None, shell=False):
338 """Run a subprocess and return its output."""
339 return exo(argv, preexec_fn=preexec_fn, shell=shell)[0]
342 def _argmax_base(command):
345 base_size += len(command) + 1
346 for k, v in compat.items(environ):
347 base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
351 def _argmax_args_size(args):
352 return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
355 def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
356 """If args is not empty, yield the output produced by calling the
357 command list with args as a sequence of strings (It may be necessary
358 to return multiple strings in order to respect ARG_MAX)."""
359 # The optional arg_max arg is a workaround for an issue with the
360 # current wvtest behavior.
361 base_size = _argmax_base(command)
363 room = arg_max - base_size
366 next_size = _argmax_args_size(args[i:i+1])
367 if room - next_size < 0:
373 assert(len(sub_args))
374 yield readpipe(command + sub_args, preexec_fn=preexec_fn)
377 def resolve_parent(p):
378 """Return the absolute path of a file without following any final symlink.
380 Behaves like os.path.realpath, but doesn't follow a symlink for the last
381 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
382 will follow symlinks in p's directory)
388 if st and stat.S_ISLNK(st.st_mode):
389 (dir, name) = os.path.split(p)
390 dir = os.path.realpath(dir)
391 out = os.path.join(dir, name)
393 out = os.path.realpath(p)
394 #log('realpathing:%r,%r\n' % (p, out))
398 def detect_fakeroot():
399 "Return True if we appear to be running under fakeroot."
400 return os.getenv("FAKEROOTKEY") != None
403 if sys.platform.startswith('cygwin'):
405 # https://cygwin.com/ml/cygwin/2015-02/msg00057.html
406 groups = os.getgroups()
407 return 544 in groups or 0 in groups
410 return os.geteuid() == 0
413 def cache_key_value(get_value, key, cache):
414 """Return (value, was_cached). If there is a value in the cache
415 for key, use that, otherwise, call get_value(key) which should
416 throw a KeyError if there is no value -- in which case the cached
417 and returned value will be None.
419 try: # Do we already have it (or know there wasn't one)?
426 cache[key] = value = get_value(key)
434 """Get the FQDN of this machine."""
437 _hostname = _helpers.gethostname()
441 def format_filesize(size):
446 exponent = int(math.log(size) // math.log(unit))
447 size_prefix = "KMGTPE"[exponent - 1]
448 return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
451 class NotOk(Exception):
456 def __init__(self, outp):
457 self._base_closed = False
461 self._base_closed = True
466 def __exit__(self, exc_type, exc_value, tb):
467 with pending_raise(exc_value, rethrow=False):
471 assert self._base_closed
473 def _read(self, size):
474 raise NotImplementedError("Subclasses must implement _read")
476 def read(self, size):
477 """Read 'size' bytes from input stream."""
479 return self._read(size)
481 def _readline(self, size):
482 raise NotImplementedError("Subclasses must implement _readline")
485 """Read from input stream until a newline is found."""
487 return self._readline()
489 def write(self, data):
490 """Write 'data' to output stream."""
491 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
492 self.outp.write(data)
495 """Return true if input stream is readable."""
496 raise NotImplementedError("Subclasses must implement has_input")
499 """Indicate end of output from last sent command."""
500 self.write(b'\nok\n')
503 """Indicate server error to the client."""
504 s = re.sub(br'\s+', b' ', s)
505 self.write(b'\nerror %s\n' % s)
507 def _check_ok(self, onempty):
510 for rl in linereader(self):
511 #log('%d got line: %r\n' % (os.getpid(), rl))
512 if not rl: # empty line
516 elif rl.startswith(b'error '):
517 #log('client: error: %s\n' % rl[6:])
521 raise Exception('server exited unexpectedly; see errors above')
523 def drain_and_check_ok(self):
524 """Remove all data for the current command from input stream."""
527 return self._check_ok(onempty)
530 """Verify that server action completed successfully."""
532 raise Exception('expected "ok", got %r' % rl)
533 return self._check_ok(onempty)
536 class Conn(BaseConn):
537 def __init__(self, inp, outp):
538 BaseConn.__init__(self, outp)
541 def _read(self, size):
542 return self.inp.read(size)
545 return self.inp.readline()
548 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
550 assert(rl[0] == self.inp.fileno())
556 def checked_reader(fd, n):
558 rl, _, _ = select.select([fd], [], [])
561 if not buf: raise Exception("Unexpected EOF reading %d more bytes" % n)
566 MAX_PACKET = 128 * 1024
567 def mux(p, outfd, outr, errr):
570 while p.poll() is None:
571 rl, _, _ = select.select(fds, [], [])
574 buf = os.read(outr, MAX_PACKET)
576 os.write(outfd, struct.pack('!IB', len(buf), 1) + buf)
578 buf = os.read(errr, 1024)
580 os.write(outfd, struct.pack('!IB', len(buf), 2) + buf)
582 os.write(outfd, struct.pack('!IB', 0, 3))
585 class DemuxConn(BaseConn):
586 """A helper class for bup's client-server protocol."""
587 def __init__(self, infd, outp):
588 BaseConn.__init__(self, outp)
589 # Anything that comes through before the sync string was not
590 # multiplexed and can be assumed to be debug/log before mux init.
592 stderr = byte_stream(sys.stderr)
593 while tail != b'BUPMUX':
594 # Make sure to write all pre-BUPMUX output to stderr
595 b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
597 ex = IOError('demux: unexpected EOF during initialization')
598 with pending_raise(ex):
602 stderr.write(tail[:-6])
610 def write(self, data):
612 BaseConn.write(self, data)
614 def _next_packet(self, timeout):
615 if self.closed: return False
616 rl, wl, xl = select.select([self.infd], [], [], timeout)
617 if not rl: return False
618 assert(rl[0] == self.infd)
619 ns = b''.join(checked_reader(self.infd, 5))
620 n, fdw = struct.unpack('!IB', ns)
622 # assume that something went wrong and print stuff
623 ns += os.read(self.infd, 1024)
624 stderr = byte_stream(sys.stderr)
627 raise Exception("Connection broken")
629 self.reader = checked_reader(self.infd, n)
631 for buf in checked_reader(self.infd, n):
632 byte_stream(sys.stderr).write(buf)
635 debug2("DemuxConn: marked closed\n")
638 def _load_buf(self, timeout):
639 if self.buf is not None:
641 while not self.closed:
642 while not self.reader:
643 if not self._next_packet(timeout):
646 self.buf = next(self.reader)
648 except StopIteration:
652 def _read_parts(self, ix_fn):
653 while self._load_buf(None):
654 assert(self.buf is not None)
656 if i is None or i == len(self.buf):
661 self.buf = self.buf[i:]
669 return buf.index(b'\n')+1
672 return b''.join(self._read_parts(find_eol))
674 def _read(self, size):
676 def until_size(buf): # Closes on csize
677 if len(buf) < csize[0]:
682 return b''.join(self._read_parts(until_size))
685 return self._load_buf(0)
689 """Generate a list of input lines from 'f' without terminating newlines."""
697 def chunkyreader(f, count = None):
698 """Generate a list of chunks of data read from 'f'.
700 If count is None, read until EOF is reached.
702 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
703 reached while reading, raise IOError.
707 b = f.read(min(count, 65536))
709 raise IOError('EOF with %d bytes remaining' % count)
720 def atomically_replaced_file(name, mode='w', buffering=-1):
721 """Yield a file that will be atomically renamed name when leaving the block.
723 This contextmanager yields an open file object that is backed by a
724 temporary file which will be renamed (atomically) to the target
725 name if everything succeeds.
727 The mode and buffering arguments are handled exactly as with open,
728 and the yielded file will have very restrictive permissions, as
733 with atomically_replaced_file('foo.txt', 'w') as f:
734 f.write('hello jack.')
738 (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
739 text=('b' not in mode))
742 f = os.fdopen(ffd, mode, buffering)
750 os.rename(tempname, name)
752 unlink(tempname) # nonexistant file is ignored
756 """Append "/" to 's' if it doesn't aleady end in "/"."""
757 assert isinstance(s, bytes)
758 if s and not s.endswith(b'/'):
764 def _mmap_do(f, sz, flags, prot, close):
766 st = os.fstat(f.fileno())
769 # trying to open a zero-length map gives an error, but an empty
770 # string has all the same behaviour of a zero-length map, ie. it has
773 map = compat.mmap(f.fileno(), sz, flags, prot)
775 f.close() # map will persist beyond file close
779 def mmap_read(f, sz = 0, close=True):
780 """Create a read-only memory mapped region on file 'f'.
781 If sz is 0, the region will cover the entire file.
783 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
786 def mmap_readwrite(f, sz = 0, close=True):
787 """Create a read-write memory mapped region on file 'f'.
788 If sz is 0, the region will cover the entire file.
790 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
794 def mmap_readwrite_private(f, sz = 0, close=True):
795 """Create a read-write memory mapped region on file 'f'.
796 If sz is 0, the region will cover the entire file.
797 The map is private, which means the changes are never flushed back to the
800 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
804 _mincore = getattr(_helpers, 'mincore', None)
806 # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
807 MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
809 _fmincore_chunk_size = None
810 def _set_fmincore_chunk_size():
811 global _fmincore_chunk_size
812 pref_chunk_size = 64 * 1024 * 1024
813 chunk_size = sc_page_size
814 if (sc_page_size < pref_chunk_size):
815 chunk_size = sc_page_size * (pref_chunk_size // sc_page_size)
816 _fmincore_chunk_size = chunk_size
819 """Return the mincore() data for fd as a bytearray whose values can be
820 tested via MINCORE_INCORE, or None if fd does not fully
821 support the operation."""
823 if (st.st_size == 0):
825 if not _fmincore_chunk_size:
826 _set_fmincore_chunk_size()
827 pages_per_chunk = _fmincore_chunk_size // sc_page_size;
828 page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
829 chunk_count = (st.st_size + _fmincore_chunk_size - 1) // _fmincore_chunk_size
830 result = bytearray(page_count)
831 for ci in compat.range(chunk_count):
832 pos = _fmincore_chunk_size * ci;
833 msize = min(_fmincore_chunk_size, st.st_size - pos)
835 m = compat.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
836 except mmap.error as ex:
837 if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
838 # Perhaps the file was a pipe, i.e. "... | bup split ..."
843 _mincore(m, msize, 0, result, ci * pages_per_chunk)
844 except OSError as ex:
845 if ex.errno == errno.ENOSYS:
851 def parse_timestamp(epoch_str):
852 """Return the number of nanoseconds since the epoch that are described
853 by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
854 throw a ValueError that may contain additional information."""
855 ns_per = {'s' : 1000000000,
859 match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
861 if re.match(r'^([-+]?[0-9]+)$', epoch_str):
862 raise ValueError('must include units, i.e. 100ns, 100ms, ...')
864 (n, units) = match.group(1, 2)
868 return n * ns_per[units]
872 """Parse string or bytes as a possibly unit suffixed number.
875 199.2k means 203981 bytes
876 1GB means 1073741824 bytes
877 2.1 tb means 2199023255552 bytes
879 if isinstance(s, bytes):
880 # FIXME: should this raise a ValueError for UnicodeDecodeError
881 # (perhaps with the latter as the context).
882 s = s.decode('ascii')
883 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
885 raise ValueError("can't parse %r as a number" % s)
886 (val, unit) = g.groups()
889 if unit in ['t', 'tb']:
890 mult = 1024*1024*1024*1024
891 elif unit in ['g', 'gb']:
892 mult = 1024*1024*1024
893 elif unit in ['m', 'mb']:
895 elif unit in ['k', 'kb']:
897 elif unit in ['', 'b']:
900 raise ValueError("invalid unit %r in number %r" % (unit, s))
906 """Append an error message to the list of saved errors.
908 Once processing is able to stop and output the errors, the saved errors are
909 accessible in the module variable helpers.saved_errors.
911 saved_errors.append(e)
920 def die_if_errors(msg=None, status=1):
924 msg = 'warning: %d errors encountered\n' % len(saved_errors)
930 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
932 The new exception handler will make sure that bup will exit without an ugly
933 stacktrace when Ctrl-C is hit.
935 oldhook = sys.excepthook
936 def newhook(exctype, value, traceback):
937 if exctype == KeyboardInterrupt:
938 log('\nInterrupted.\n')
940 oldhook(exctype, value, traceback)
941 sys.excepthook = newhook
944 def columnate(l, prefix):
945 """Format elements of 'l' in columns with 'prefix' leading each line.
947 The number of columns is determined automatically based on the string
950 binary = isinstance(prefix, bytes)
951 nothing = b'' if binary else ''
952 nl = b'\n' if binary else '\n'
956 clen = max(len(s) for s in l)
957 ncols = (tty_width() - len(prefix)) // (clen + 2)
962 while len(l) % ncols:
964 rows = len(l) // ncols
965 for s in compat.range(0, len(l), rows):
966 cols.append(l[s:s+rows])
968 fmt = b'%-*s' if binary else '%-*s'
969 for row in zip(*cols):
970 out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl
974 def parse_date_or_fatal(str, fatal):
975 """Parses the given date or calls Option.fatal().
976 For now we expect a string that contains a float."""
979 except ValueError as e:
980 raise fatal('invalid date format (should be a float): %r' % e)
985 def parse_excludes(options, fatal):
986 """Traverse the options and extract all excludes, or call Option.fatal()."""
990 (option, parameter) = flag
991 if option == '--exclude':
992 excluded_paths.append(resolve_parent(argv_bytes(parameter)))
993 elif option == '--exclude-from':
995 f = open(resolve_parent(argv_bytes(parameter)), 'rb')
997 raise fatal("couldn't read %r" % parameter)
998 for exclude_path in f.readlines():
999 # FIXME: perhaps this should be rstrip('\n')
1000 exclude_path = resolve_parent(exclude_path.strip())
1002 excluded_paths.append(exclude_path)
1003 return sorted(frozenset(excluded_paths))
1006 def parse_rx_excludes(options, fatal):
1007 """Traverse the options and extract all rx excludes, or call
1009 excluded_patterns = []
1011 for flag in options:
1012 (option, parameter) = flag
1013 if option == '--exclude-rx':
1015 excluded_patterns.append(re.compile(argv_bytes(parameter)))
1016 except re.error as ex:
1017 fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))
1018 elif option == '--exclude-rx-from':
1020 f = open(resolve_parent(parameter), 'rb')
1021 except IOError as e:
1022 raise fatal("couldn't read %r" % parameter)
1023 for pattern in f.readlines():
1024 spattern = pattern.rstrip(b'\n')
1028 excluded_patterns.append(re.compile(spattern))
1029 except re.error as ex:
1030 fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))
1031 return excluded_patterns
1034 def should_rx_exclude_path(path, exclude_rxs):
1035 """Return True if path matches a regular expression in exclude_rxs."""
1036 for rx in exclude_rxs:
1038 debug1('Skipping %r: excluded by rx pattern %r.\n'
1039 % (path, rx.pattern))
1044 # FIXME: Carefully consider the use of functions (os.path.*, etc.)
1045 # that resolve against the current filesystem in the strip/graft
1046 # functions for example, but elsewhere as well. I suspect bup's not
1047 # always being careful about that. For some cases, the contents of
1048 # the current filesystem should be irrelevant, and consulting it might
1049 # produce the wrong result, perhaps via unintended symlink resolution,
1052 def path_components(path):
1053 """Break path into a list of pairs of the form (name,
1054 full_path_to_name). Path must start with '/'.
1056 '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
1057 if not path.startswith(b'/'):
1058 raise Exception('path must start with "/": %s' % path_msg(path))
1059 # Since we assume path startswith('/'), we can skip the first element.
1060 result = [(b'', b'/')]
1061 norm_path = os.path.abspath(path)
1062 if norm_path == b'/':
1065 for p in norm_path.split(b'/')[1:]:
1066 full_path += b'/' + p
1067 result.append((p, full_path))
1071 def stripped_path_components(path, strip_prefixes):
1072 """Strip any prefix in strip_prefixes from path and return a list
1073 of path components where each component is (name,
1074 none_or_full_fs_path_to_name). Assume path startswith('/').
1075 See thelpers.py for examples."""
1076 normalized_path = os.path.abspath(path)
1077 sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
1078 for bp in sorted_strip_prefixes:
1079 normalized_bp = os.path.abspath(bp)
1080 if normalized_bp == b'/':
1082 if normalized_path.startswith(normalized_bp):
1083 prefix = normalized_path[:len(normalized_bp)]
1085 for p in normalized_path[len(normalized_bp):].split(b'/'):
1089 result.append((p, prefix))
1092 return path_components(path)
1095 def grafted_path_components(graft_points, path):
1096 # Create a result that consists of some number of faked graft
1097 # directories before the graft point, followed by all of the real
1098 # directories from path that are after the graft point. Arrange
1099 # for the directory at the graft point in the result to correspond
1100 # to the "orig" directory in --graft orig=new. See t/thelpers.py
1101 # for some examples.
1103 # Note that given --graft orig=new, orig and new have *nothing* to
1104 # do with each other, even if some of their component names
1105 # match. i.e. --graft /foo/bar/baz=/foo/bar/bax is semantically
1106 # equivalent to --graft /foo/bar/baz=/x/y/z, or even
1109 # FIXME: This can't be the best solution...
1110 clean_path = os.path.abspath(path)
1111 for graft_point in graft_points:
1112 old_prefix, new_prefix = graft_point
1113 # Expand prefixes iff not absolute paths.
1114 old_prefix = os.path.normpath(old_prefix)
1115 new_prefix = os.path.normpath(new_prefix)
1116 if clean_path.startswith(old_prefix):
1117 escaped_prefix = re.escape(old_prefix)
1118 grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)
1119 # Handle /foo=/ (at least) -- which produces //whatever.
1120 grafted_path = b'/' + grafted_path.lstrip(b'/')
1121 clean_path_components = path_components(clean_path)
1122 # Count the components that were stripped.
1123 strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
1124 new_prefix_parts = new_prefix.split(b'/')
1125 result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]
1126 result = [(p, None) for p in result_prefix] \
1127 + clean_path_components[strip_count:]
1128 # Now set the graft point name to match the end of new_prefix.
1129 graft_point = len(result_prefix)
1130 result[graft_point] = \
1131 (new_prefix_parts[-1], clean_path_components[strip_count][1])
1132 if new_prefix == b'/': # --graft ...=/ is a special case.
1135 return path_components(clean_path)
1141 _localtime = getattr(_helpers, 'localtime', None)
1144 bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
1145 'tm_hour', 'tm_min', 'tm_sec',
1146 'tm_wday', 'tm_yday',
1147 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
1149 # Define a localtime() that returns bup_time when possible. Note:
1150 # this means that any helpers.localtime() results may need to be
1151 # passed through to_py_time() before being passed to python's time
1152 # module, which doesn't appear willing to ignore the extra items.
1154 def localtime(time):
1155 return bup_time(*_helpers.localtime(int(floor(time))))
1156 def utc_offset_str(t):
1157 """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
1158 If the current UTC offset does not represent an integer number
1159 of minutes, the fractional component will be truncated."""
1160 off = localtime(t).tm_gmtoff
1161 # Note: // doesn't truncate like C for negative values, it rounds down.
1162 offmin = abs(off) // 60
1164 h = (offmin - m) // 60
1165 return b'%+03d%02d' % (-h if off < 0 else h, m)
1167 if isinstance(x, time.struct_time):
1169 return time.struct_time(x[:9])
1171 localtime = time.localtime
1172 def utc_offset_str(t):
1173 return time.strftime(b'%z', localtime(t))
1178 _some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')
1180 def valid_save_name(name):
1181 # Enforce a superset of the restrictions in git-check-ref-format(1)
1183 or name.startswith(b'/') or name.endswith(b'/') \
1184 or name.endswith(b'.'):
1186 if _some_invalid_save_parts_rx.search(name):
1189 if byte_int(c) < 0x20 or byte_int(c) == 0x7f:
1191 for part in name.split(b'/'):
1192 if part.startswith(b'.') or part.endswith(b'.lock'):
1197 _period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')
1199 def period_as_secs(s):
1202 match = _period_rx.match(s)
1205 mag = int(match.group(1))
1206 scale = match.group(2)
1207 return mag * {b's': 1,
1211 b'w': 60 * 60 * 24 * 7,
1212 b'm': 60 * 60 * 24 * 31,
1213 b'y': 60 * 60 * 24 * 366}[scale]