1 """Helper functions and classes for bup."""
3 from __future__ import absolute_import, division
4 from collections import namedtuple
5 from contextlib import contextmanager
6 from ctypes import sizeof, c_void_p
9 from subprocess import PIPE, Popen
10 import sys, os, subprocess, errno, select, mmap, stat, re, struct
11 import hashlib, heapq, math, operator, time, tempfile
13 from bup import _helpers
14 from bup import compat
16 from bup.compat import argv_bytes, byte_int, nullcontext, pending_raise
17 from bup.io import byte_stream, path_msg
18 # This function should really be in helpers, not in bup.options. But we
19 # want options.py to be standalone so people can include it in other projects.
20 from bup.options import _tty_width as tty_width
23 buglvl = int(os.environ.get('BUP_DEBUG', 0))
27 """Helper to deal with Python scoping issues"""
31 def nullcontext_if_not(manager):
32 return manager if manager is not None else nullcontext()
36 def finalized(enter_result=None, finalize=None):
40 except BaseException as ex:
41 with pending_raise(ex):
42 finalize(enter_result)
43 finalize(enter_result)
46 sc_page_size = os.sysconf('SC_PAGE_SIZE')
47 assert(sc_page_size > 0)
49 sc_arg_max = os.sysconf('SC_ARG_MAX')
50 if sc_arg_max == -1: # "no definite limit" - let's choose 2M
51 sc_arg_max = 2 * 1024 * 1024
55 for result in iterable:
60 _fdatasync = os.fdatasync
61 except AttributeError:
64 if sys.platform.startswith('darwin'):
65 # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
69 return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
71 # Fallback for file systems (SMB) that do not support F_FULLFSYNC
72 if e.errno == errno.ENOTSUP:
77 fdatasync = _fdatasync
80 def partition(predicate, stream):
81 """Returns (leading_matches_it, rest_it), where leading_matches_it
82 must be completely exhausted before traversing rest_it.
87 ns.first_nonmatch = None
88 def leading_matches():
93 ns.first_nonmatch = (x,)
97 yield ns.first_nonmatch[0]
100 return (leading_matches(), rest())
110 def lines_until_sentinel(f, sentinel, ex_type):
111 # sentinel must end with \n and must contain only one \n
114 if not (line and line.endswith(b'\n')):
115 raise ex_type('Hit EOF while reading line')
121 def stat_if_exists(path):
125 if e.errno != errno.ENOENT:
130 # Write (blockingly) to sockets that may or may not be in blocking mode.
131 # We need this because our stderr is sometimes eaten by subprocesses
132 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
133 # leading to race conditions. Ick. We'll do it the hard way.
134 def _hard_write(fd, buf):
136 (r,w,x) = select.select([], [fd], [], None)
138 raise IOError('select(fd) returned without being writable')
140 sz = os.write(fd, buf)
142 if e.errno != errno.EAGAIN:
150 """Print a log message to stderr."""
153 _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode())
167 istty1 = os.isatty(1) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 1)
168 istty2 = os.isatty(2) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 2)
171 """Calls log() if stderr is a TTY. Does nothing otherwise."""
172 global _last_progress
179 """Calls progress() only if we haven't printed progress in a while.
181 This avoids overloading the stderr buffer with excess junk.
185 if now - _last_prog > 0.1:
191 """Calls progress() to redisplay the most recent progress message.
193 Useful after you've printed some other message that wipes out the
196 if _last_progress and _last_progress.endswith('\r'):
197 progress(_last_progress)
200 def mkdirp(d, mode=None):
201 """Recursively create directories on path 'd'.
203 Unlike os.makedirs(), it doesn't raise an exception if the last element of
204 the path already exists.
212 if e.errno == errno.EEXIST:
219 def __init__(self, entry, read_it):
221 self.read_it = read_it
223 return self.entry < x.entry
225 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
227 samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
229 samekey = operator.eq
231 total = sum(len(it) for it in iters)
232 iters = (iter(it) for it in iters)
233 heap = ((next(it, None),it) for it in iters)
234 heap = [MergeIterItem(e, it) for e, it in heap if e]
239 if not count % pfreq:
241 e, it = heap[0].entry, heap[0].read_it
242 if not samekey(e, pe):
248 except StopIteration:
249 heapq.heappop(heap) # remove current
251 # shift current to new location
252 heapq.heapreplace(heap, MergeIterItem(e, it))
257 """Delete a file at path 'f' if it currently exists.
259 Unlike os.unlink(), does not throw an exception if the file didn't already
265 if e.errno != errno.ENOENT:
269 _bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
270 _sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
275 if _bq_simple_id_rx.match(x):
277 return b"'%s'" % x.replace(b"'", b"'\"'\"'")
282 if _sq_simple_id_rx.match(x):
284 return "'%s'" % x.replace("'", "'\"'\"'")
287 if isinstance(x, bytes):
289 if isinstance(x, compat.str_type):
292 # some versions of pylint get confused
296 """Return a shell quoted string for cmd if it's a sequence, else cmd.
298 cmd must be a string, bytes, or a sequence of one or the other,
299 and the assumption is that if cmd is a string or bytes, then it's
300 already quoted (because it's what's actually being passed to
301 call() and friends. e.g. log(shstr(cmd)); call(cmd)
304 if isinstance(cmd, (bytes, compat.str_type)):
306 elif all(isinstance(x, bytes) for x in cmd):
307 return b' '.join(map(bquote, cmd))
308 elif all(isinstance(x, compat.str_type) for x in cmd):
309 return ' '.join(map(squote, cmd))
310 raise TypeError('unsupported shstr argument: ' + repr(cmd))
313 exc = subprocess.check_call
324 assert stdin in (None, PIPE)
327 stdin=stdin, stdout=PIPE, stderr=stderr,
329 preexec_fn=preexec_fn,
331 out, err = p.communicate(input)
332 if check and p.returncode != 0:
333 raise Exception('subprocess %r failed with status %d%s'
334 % (b' '.join(map(quote, cmd)), p.returncode,
335 ', stderr: %r' % err if err else ''))
338 def readpipe(argv, preexec_fn=None, shell=False):
339 """Run a subprocess and return its output."""
340 return exo(argv, preexec_fn=preexec_fn, shell=shell)[0]
343 def _argmax_base(command):
346 base_size += len(command) + 1
347 for k, v in compat.items(environ):
348 base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
352 def _argmax_args_size(args):
353 return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
356 def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
357 """If args is not empty, yield the output produced by calling the
358 command list with args as a sequence of strings (It may be necessary
359 to return multiple strings in order to respect ARG_MAX)."""
360 # The optional arg_max arg is a workaround for an issue with the
361 # current wvtest behavior.
362 base_size = _argmax_base(command)
364 room = arg_max - base_size
367 next_size = _argmax_args_size(args[i:i+1])
368 if room - next_size < 0:
374 assert(len(sub_args))
375 yield readpipe(command + sub_args, preexec_fn=preexec_fn)
378 def resolve_parent(p):
379 """Return the absolute path of a file without following any final symlink.
381 Behaves like os.path.realpath, but doesn't follow a symlink for the last
382 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
383 will follow symlinks in p's directory)
389 if st and stat.S_ISLNK(st.st_mode):
390 (dir, name) = os.path.split(p)
391 dir = os.path.realpath(dir)
392 out = os.path.join(dir, name)
394 out = os.path.realpath(p)
395 #log('realpathing:%r,%r\n' % (p, out))
399 def detect_fakeroot():
400 "Return True if we appear to be running under fakeroot."
401 return os.getenv("FAKEROOTKEY") != None
404 if sys.platform.startswith('cygwin'):
406 # https://cygwin.com/ml/cygwin/2015-02/msg00057.html
407 groups = os.getgroups()
408 return 544 in groups or 0 in groups
411 return os.geteuid() == 0
414 def cache_key_value(get_value, key, cache):
415 """Return (value, was_cached). If there is a value in the cache
416 for key, use that, otherwise, call get_value(key) which should
417 throw a KeyError if there is no value -- in which case the cached
418 and returned value will be None.
420 try: # Do we already have it (or know there wasn't one)?
427 cache[key] = value = get_value(key)
435 """Get the FQDN of this machine."""
438 _hostname = _helpers.gethostname()
442 def format_filesize(size):
447 exponent = int(math.log(size) // math.log(unit))
448 size_prefix = "KMGTPE"[exponent - 1]
449 return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
452 class NotOk(Exception):
457 def __init__(self, outp):
458 self._base_closed = False
462 self._base_closed = True
467 def __exit__(self, exc_type, exc_value, tb):
468 with pending_raise(exc_value, rethrow=False):
472 assert self._base_closed
474 def _read(self, size):
475 raise NotImplementedError("Subclasses must implement _read")
477 def read(self, size):
478 """Read 'size' bytes from input stream."""
480 return self._read(size)
482 def _readline(self, size):
483 raise NotImplementedError("Subclasses must implement _readline")
486 """Read from input stream until a newline is found."""
488 return self._readline()
490 def write(self, data):
491 """Write 'data' to output stream."""
492 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
493 self.outp.write(data)
496 """Return true if input stream is readable."""
497 raise NotImplementedError("Subclasses must implement has_input")
500 """Indicate end of output from last sent command."""
501 self.write(b'\nok\n')
504 """Indicate server error to the client."""
505 s = re.sub(br'\s+', b' ', s)
506 self.write(b'\nerror %s\n' % s)
508 def _check_ok(self, onempty):
511 for rl in linereader(self):
512 #log('%d got line: %r\n' % (os.getpid(), rl))
513 if not rl: # empty line
517 elif rl.startswith(b'error '):
518 #log('client: error: %s\n' % rl[6:])
522 raise Exception('server exited unexpectedly; see errors above')
524 def drain_and_check_ok(self):
525 """Remove all data for the current command from input stream."""
528 return self._check_ok(onempty)
531 """Verify that server action completed successfully."""
533 raise Exception('expected "ok", got %r' % rl)
534 return self._check_ok(onempty)
537 class Conn(BaseConn):
538 def __init__(self, inp, outp):
539 BaseConn.__init__(self, outp)
542 def _read(self, size):
543 return self.inp.read(size)
546 return self.inp.readline()
549 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
551 assert(rl[0] == self.inp.fileno())
557 def checked_reader(fd, n):
559 rl, _, _ = select.select([fd], [], [])
562 if not buf: raise Exception("Unexpected EOF reading %d more bytes" % n)
567 MAX_PACKET = 128 * 1024
568 def mux(p, outfd, outr, errr):
571 while p.poll() is None:
572 rl, _, _ = select.select(fds, [], [])
575 buf = os.read(outr, MAX_PACKET)
577 os.write(outfd, struct.pack('!IB', len(buf), 1) + buf)
579 buf = os.read(errr, 1024)
581 os.write(outfd, struct.pack('!IB', len(buf), 2) + buf)
583 os.write(outfd, struct.pack('!IB', 0, 3))
586 class DemuxConn(BaseConn):
587 """A helper class for bup's client-server protocol."""
588 def __init__(self, infd, outp):
589 BaseConn.__init__(self, outp)
590 # Anything that comes through before the sync string was not
591 # multiplexed and can be assumed to be debug/log before mux init.
593 stderr = byte_stream(sys.stderr)
594 while tail != b'BUPMUX':
595 # Make sure to write all pre-BUPMUX output to stderr
596 b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
598 ex = IOError('demux: unexpected EOF during initialization')
599 with pending_raise(ex):
603 stderr.write(tail[:-6])
611 def write(self, data):
613 BaseConn.write(self, data)
615 def _next_packet(self, timeout):
616 if self.closed: return False
617 rl, wl, xl = select.select([self.infd], [], [], timeout)
618 if not rl: return False
619 assert(rl[0] == self.infd)
620 ns = b''.join(checked_reader(self.infd, 5))
621 n, fdw = struct.unpack('!IB', ns)
623 # assume that something went wrong and print stuff
624 ns += os.read(self.infd, 1024)
625 stderr = byte_stream(sys.stderr)
628 raise Exception("Connection broken")
630 self.reader = checked_reader(self.infd, n)
632 for buf in checked_reader(self.infd, n):
633 byte_stream(sys.stderr).write(buf)
636 debug2("DemuxConn: marked closed\n")
639 def _load_buf(self, timeout):
640 if self.buf is not None:
642 while not self.closed:
643 while not self.reader:
644 if not self._next_packet(timeout):
647 self.buf = next(self.reader)
649 except StopIteration:
653 def _read_parts(self, ix_fn):
654 while self._load_buf(None):
655 assert(self.buf is not None)
657 if i is None or i == len(self.buf):
662 self.buf = self.buf[i:]
670 return buf.index(b'\n')+1
673 return b''.join(self._read_parts(find_eol))
675 def _read(self, size):
677 def until_size(buf): # Closes on csize
678 if len(buf) < csize[0]:
683 return b''.join(self._read_parts(until_size))
686 return self._load_buf(0)
690 """Generate a list of input lines from 'f' without terminating newlines."""
698 def chunkyreader(f, count = None):
699 """Generate a list of chunks of data read from 'f'.
701 If count is None, read until EOF is reached.
703 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
704 reached while reading, raise IOError.
708 b = f.read(min(count, 65536))
710 raise IOError('EOF with %d bytes remaining' % count)
721 def atomically_replaced_file(name, mode='w', buffering=-1):
722 """Yield a file that will be atomically renamed name when leaving the block.
724 This contextmanager yields an open file object that is backed by a
725 temporary file which will be renamed (atomically) to the target
726 name if everything succeeds.
728 The mode and buffering arguments are handled exactly as with open,
729 and the yielded file will have very restrictive permissions, as
734 with atomically_replaced_file('foo.txt', 'w') as f:
735 f.write('hello jack.')
739 (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
740 text=('b' not in mode))
743 f = os.fdopen(ffd, mode, buffering)
751 os.rename(tempname, name)
753 unlink(tempname) # nonexistant file is ignored
757 """Append "/" to 's' if it doesn't aleady end in "/"."""
758 assert isinstance(s, bytes)
759 if s and not s.endswith(b'/'):
765 def _mmap_do(f, sz, flags, prot, close):
767 st = os.fstat(f.fileno())
770 # trying to open a zero-length map gives an error, but an empty
771 # string has all the same behaviour of a zero-length map, ie. it has
774 map = io.mmap(f.fileno(), sz, flags, prot)
776 f.close() # map will persist beyond file close
780 def mmap_read(f, sz = 0, close=True):
781 """Create a read-only memory mapped region on file 'f'.
782 If sz is 0, the region will cover the entire file.
784 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
787 def mmap_readwrite(f, sz = 0, close=True):
788 """Create a read-write memory mapped region on file 'f'.
789 If sz is 0, the region will cover the entire file.
791 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
795 def mmap_readwrite_private(f, sz = 0, close=True):
796 """Create a read-write memory mapped region on file 'f'.
797 If sz is 0, the region will cover the entire file.
798 The map is private, which means the changes are never flushed back to the
801 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
805 _mincore = getattr(_helpers, 'mincore', None)
807 # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
808 MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
810 _fmincore_chunk_size = None
811 def _set_fmincore_chunk_size():
812 global _fmincore_chunk_size
813 pref_chunk_size = 64 * 1024 * 1024
814 chunk_size = sc_page_size
815 if (sc_page_size < pref_chunk_size):
816 chunk_size = sc_page_size * (pref_chunk_size // sc_page_size)
817 _fmincore_chunk_size = chunk_size
820 """Return the mincore() data for fd as a bytearray whose values can be
821 tested via MINCORE_INCORE, or None if fd does not fully
822 support the operation."""
824 if (st.st_size == 0):
826 if not _fmincore_chunk_size:
827 _set_fmincore_chunk_size()
828 pages_per_chunk = _fmincore_chunk_size // sc_page_size;
829 page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
830 chunk_count = (st.st_size + _fmincore_chunk_size - 1) // _fmincore_chunk_size
831 result = bytearray(page_count)
832 for ci in compat.range(chunk_count):
833 pos = _fmincore_chunk_size * ci;
834 msize = min(_fmincore_chunk_size, st.st_size - pos)
836 m = io.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
837 except mmap.error as ex:
838 if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
839 # Perhaps the file was a pipe, i.e. "... | bup split ..."
844 _mincore(m, msize, 0, result, ci * pages_per_chunk)
845 except OSError as ex:
846 if ex.errno == errno.ENOSYS:
852 def parse_timestamp(epoch_str):
853 """Return the number of nanoseconds since the epoch that are described
854 by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
855 throw a ValueError that may contain additional information."""
856 ns_per = {'s' : 1000000000,
860 match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
862 if re.match(r'^([-+]?[0-9]+)$', epoch_str):
863 raise ValueError('must include units, i.e. 100ns, 100ms, ...')
865 (n, units) = match.group(1, 2)
869 return n * ns_per[units]
873 """Parse string or bytes as a possibly unit suffixed number.
876 199.2k means 203981 bytes
877 1GB means 1073741824 bytes
878 2.1 tb means 2199023255552 bytes
880 if isinstance(s, bytes):
881 # FIXME: should this raise a ValueError for UnicodeDecodeError
882 # (perhaps with the latter as the context).
883 s = s.decode('ascii')
884 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
886 raise ValueError("can't parse %r as a number" % s)
887 (val, unit) = g.groups()
890 if unit in ['t', 'tb']:
891 mult = 1024*1024*1024*1024
892 elif unit in ['g', 'gb']:
893 mult = 1024*1024*1024
894 elif unit in ['m', 'mb']:
896 elif unit in ['k', 'kb']:
898 elif unit in ['', 'b']:
901 raise ValueError("invalid unit %r in number %r" % (unit, s))
907 """Append an error message to the list of saved errors.
909 Once processing is able to stop and output the errors, the saved errors are
910 accessible in the module variable helpers.saved_errors.
912 saved_errors.append(e)
921 def die_if_errors(msg=None, status=1):
925 msg = 'warning: %d errors encountered\n' % len(saved_errors)
931 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
933 The new exception handler will make sure that bup will exit without an ugly
934 stacktrace when Ctrl-C is hit.
936 oldhook = sys.excepthook
937 def newhook(exctype, value, traceback):
938 if exctype == KeyboardInterrupt:
939 log('\nInterrupted.\n')
941 oldhook(exctype, value, traceback)
942 sys.excepthook = newhook
945 def columnate(l, prefix):
946 """Format elements of 'l' in columns with 'prefix' leading each line.
948 The number of columns is determined automatically based on the string
951 binary = isinstance(prefix, bytes)
952 nothing = b'' if binary else ''
953 nl = b'\n' if binary else '\n'
957 clen = max(len(s) for s in l)
958 ncols = (tty_width() - len(prefix)) // (clen + 2)
963 while len(l) % ncols:
965 rows = len(l) // ncols
966 for s in compat.range(0, len(l), rows):
967 cols.append(l[s:s+rows])
969 fmt = b'%-*s' if binary else '%-*s'
970 for row in zip(*cols):
971 out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl
975 def parse_date_or_fatal(str, fatal):
976 """Parses the given date or calls Option.fatal().
977 For now we expect a string that contains a float."""
980 except ValueError as e:
981 raise fatal('invalid date format (should be a float): %r' % e)
986 def parse_excludes(options, fatal):
987 """Traverse the options and extract all excludes, or call Option.fatal()."""
991 (option, parameter) = flag
992 if option == '--exclude':
993 excluded_paths.append(resolve_parent(argv_bytes(parameter)))
994 elif option == '--exclude-from':
996 f = open(resolve_parent(argv_bytes(parameter)), 'rb')
998 raise fatal("couldn't read %r" % parameter)
999 for exclude_path in f.readlines():
1000 # FIXME: perhaps this should be rstrip('\n')
1001 exclude_path = resolve_parent(exclude_path.strip())
1003 excluded_paths.append(exclude_path)
1004 return sorted(frozenset(excluded_paths))
1007 def parse_rx_excludes(options, fatal):
1008 """Traverse the options and extract all rx excludes, or call
1010 excluded_patterns = []
1012 for flag in options:
1013 (option, parameter) = flag
1014 if option == '--exclude-rx':
1016 excluded_patterns.append(re.compile(argv_bytes(parameter)))
1017 except re.error as ex:
1018 fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))
1019 elif option == '--exclude-rx-from':
1021 f = open(resolve_parent(parameter), 'rb')
1022 except IOError as e:
1023 raise fatal("couldn't read %r" % parameter)
1024 for pattern in f.readlines():
1025 spattern = pattern.rstrip(b'\n')
1029 excluded_patterns.append(re.compile(spattern))
1030 except re.error as ex:
1031 fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))
1032 return excluded_patterns
1035 def should_rx_exclude_path(path, exclude_rxs):
1036 """Return True if path matches a regular expression in exclude_rxs."""
1037 for rx in exclude_rxs:
1039 debug1('Skipping %r: excluded by rx pattern %r.\n'
1040 % (path, rx.pattern))
1045 # FIXME: Carefully consider the use of functions (os.path.*, etc.)
1046 # that resolve against the current filesystem in the strip/graft
1047 # functions for example, but elsewhere as well. I suspect bup's not
1048 # always being careful about that. For some cases, the contents of
1049 # the current filesystem should be irrelevant, and consulting it might
1050 # produce the wrong result, perhaps via unintended symlink resolution,
1053 def path_components(path):
1054 """Break path into a list of pairs of the form (name,
1055 full_path_to_name). Path must start with '/'.
1057 '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
1058 if not path.startswith(b'/'):
1059 raise Exception('path must start with "/": %s' % path_msg(path))
1060 # Since we assume path startswith('/'), we can skip the first element.
1061 result = [(b'', b'/')]
1062 norm_path = os.path.abspath(path)
1063 if norm_path == b'/':
1066 for p in norm_path.split(b'/')[1:]:
1067 full_path += b'/' + p
1068 result.append((p, full_path))
1072 def stripped_path_components(path, strip_prefixes):
1073 """Strip any prefix in strip_prefixes from path and return a list
1074 of path components where each component is (name,
1075 none_or_full_fs_path_to_name). Assume path startswith('/').
1076 See thelpers.py for examples."""
1077 normalized_path = os.path.abspath(path)
1078 sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
1079 for bp in sorted_strip_prefixes:
1080 normalized_bp = os.path.abspath(bp)
1081 if normalized_bp == b'/':
1083 if normalized_path.startswith(normalized_bp):
1084 prefix = normalized_path[:len(normalized_bp)]
1086 for p in normalized_path[len(normalized_bp):].split(b'/'):
1090 result.append((p, prefix))
1093 return path_components(path)
1096 def grafted_path_components(graft_points, path):
1097 # Create a result that consists of some number of faked graft
1098 # directories before the graft point, followed by all of the real
1099 # directories from path that are after the graft point. Arrange
1100 # for the directory at the graft point in the result to correspond
1101 # to the "orig" directory in --graft orig=new. See t/thelpers.py
1102 # for some examples.
1104 # Note that given --graft orig=new, orig and new have *nothing* to
1105 # do with each other, even if some of their component names
1106 # match. i.e. --graft /foo/bar/baz=/foo/bar/bax is semantically
1107 # equivalent to --graft /foo/bar/baz=/x/y/z, or even
1110 # FIXME: This can't be the best solution...
1111 clean_path = os.path.abspath(path)
1112 for graft_point in graft_points:
1113 old_prefix, new_prefix = graft_point
1114 # Expand prefixes iff not absolute paths.
1115 old_prefix = os.path.normpath(old_prefix)
1116 new_prefix = os.path.normpath(new_prefix)
1117 if clean_path.startswith(old_prefix):
1118 escaped_prefix = re.escape(old_prefix)
1119 grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)
1120 # Handle /foo=/ (at least) -- which produces //whatever.
1121 grafted_path = b'/' + grafted_path.lstrip(b'/')
1122 clean_path_components = path_components(clean_path)
1123 # Count the components that were stripped.
1124 strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
1125 new_prefix_parts = new_prefix.split(b'/')
1126 result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]
1127 result = [(p, None) for p in result_prefix] \
1128 + clean_path_components[strip_count:]
1129 # Now set the graft point name to match the end of new_prefix.
1130 graft_point = len(result_prefix)
1131 result[graft_point] = \
1132 (new_prefix_parts[-1], clean_path_components[strip_count][1])
1133 if new_prefix == b'/': # --graft ...=/ is a special case.
1136 return path_components(clean_path)
1142 _localtime = getattr(_helpers, 'localtime', None)
1145 bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
1146 'tm_hour', 'tm_min', 'tm_sec',
1147 'tm_wday', 'tm_yday',
1148 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
1150 # Define a localtime() that returns bup_time when possible. Note:
1151 # this means that any helpers.localtime() results may need to be
1152 # passed through to_py_time() before being passed to python's time
1153 # module, which doesn't appear willing to ignore the extra items.
1155 def localtime(time):
1156 return bup_time(*_helpers.localtime(int(floor(time))))
1157 def utc_offset_str(t):
1158 """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
1159 If the current UTC offset does not represent an integer number
1160 of minutes, the fractional component will be truncated."""
1161 off = localtime(t).tm_gmtoff
1162 # Note: // doesn't truncate like C for negative values, it rounds down.
1163 offmin = abs(off) // 60
1165 h = (offmin - m) // 60
1166 return b'%+03d%02d' % (-h if off < 0 else h, m)
1168 if isinstance(x, time.struct_time):
1170 return time.struct_time(x[:9])
1172 localtime = time.localtime
1173 def utc_offset_str(t):
1174 return time.strftime(b'%z', localtime(t))
1179 _some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')
1181 def valid_save_name(name):
1182 # Enforce a superset of the restrictions in git-check-ref-format(1)
1184 or name.startswith(b'/') or name.endswith(b'/') \
1185 or name.endswith(b'.'):
1187 if _some_invalid_save_parts_rx.search(name):
1190 if byte_int(c) < 0x20 or byte_int(c) == 0x7f:
1192 for part in name.split(b'/'):
1193 if part.startswith(b'.') or part.endswith(b'.lock'):
1198 _period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')
1200 def period_as_secs(s):
1203 match = _period_rx.match(s)
1206 mag = int(match.group(1))
1207 scale = match.group(2)
1208 return mag * {b's': 1,
1212 b'w': 60 * 60 * 24 * 7,
1213 b'm': 60 * 60 * 24 * 31,
1214 b'y': 60 * 60 * 24 * 366}[scale]