1 """Helper functions and classes for bup."""
3 from __future__ import absolute_import, division
4 from collections import namedtuple
5 from contextlib import contextmanager
6 from ctypes import sizeof, c_void_p
9 from subprocess import PIPE, Popen
10 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
11 import hashlib, heapq, math, operator, time, grp, tempfile
13 from bup import _helpers
14 from bup import compat
15 from bup.compat import argv_bytes, byte_int
16 from bup.io import byte_stream, path_msg
17 # This function should really be in helpers, not in bup.options. But we
18 # want options.py to be standalone so people can include it in other projects.
19 from bup.options import _tty_width as tty_width
23 """Helper to deal with Python scoping issues"""
27 sc_page_size = os.sysconf('SC_PAGE_SIZE')
28 assert(sc_page_size > 0)
30 sc_arg_max = os.sysconf('SC_ARG_MAX')
31 if sc_arg_max == -1: # "no definite limit" - let's choose 2M
32 sc_arg_max = 2 * 1024 * 1024
36 for result in iterable:
42 """Convert s (ascii bytes) to an integer. Return 0 if s is not a number."""
50 """Convert s (ascii bytes) to a float. Return 0 if s is not a number."""
52 return float(s or b'0')
57 buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
61 _fdatasync = os.fdatasync
62 except AttributeError:
65 if sys.platform.startswith('darwin'):
66 # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
70 return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
72 # Fallback for file systems (SMB) that do not support F_FULLFSYNC
73 if e.errno == errno.ENOTSUP:
78 fdatasync = _fdatasync
81 def partition(predicate, stream):
82 """Returns (leading_matches_it, rest_it), where leading_matches_it
83 must be completely exhausted before traversing rest_it.
88 ns.first_nonmatch = None
89 def leading_matches():
94 ns.first_nonmatch = (x,)
98 yield ns.first_nonmatch[0]
101 return (leading_matches(), rest())
111 def lines_until_sentinel(f, sentinel, ex_type):
112 # sentinel must end with \n and must contain only one \n
115 if not (line and line.endswith(b'\n')):
116 raise ex_type('Hit EOF while reading line')
122 def stat_if_exists(path):
126 if e.errno != errno.ENOENT:
131 # Write (blockingly) to sockets that may or may not be in blocking mode.
132 # We need this because our stderr is sometimes eaten by subprocesses
133 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
134 # leading to race conditions. Ick. We'll do it the hard way.
135 def _hard_write(fd, buf):
137 (r,w,x) = select.select([], [fd], [], None)
139 raise IOError('select(fd) returned without being writable')
141 sz = os.write(fd, buf)
143 if e.errno != errno.EAGAIN:
151 """Print a log message to stderr."""
154 _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode())
168 istty1 = os.isatty(1) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 1)
169 istty2 = os.isatty(2) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 2)
172 """Calls log() if stderr is a TTY. Does nothing otherwise."""
173 global _last_progress
180 """Calls progress() only if we haven't printed progress in a while.
182 This avoids overloading the stderr buffer with excess junk.
186 if now - _last_prog > 0.1:
192 """Calls progress() to redisplay the most recent progress message.
194 Useful after you've printed some other message that wipes out the
197 if _last_progress and _last_progress.endswith('\r'):
198 progress(_last_progress)
201 def mkdirp(d, mode=None):
202 """Recursively create directories on path 'd'.
204 Unlike os.makedirs(), it doesn't raise an exception if the last element of
205 the path already exists.
213 if e.errno == errno.EEXIST:
220 def __init__(self, entry, read_it):
222 self.read_it = read_it
224 return self.entry < x.entry
226 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
228 samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
230 samekey = operator.eq
232 total = sum(len(it) for it in iters)
233 iters = (iter(it) for it in iters)
234 heap = ((next(it, None),it) for it in iters)
235 heap = [MergeIterItem(e, it) for e, it in heap if e]
240 if not count % pfreq:
242 e, it = heap[0].entry, heap[0].read_it
243 if not samekey(e, pe):
249 except StopIteration:
250 heapq.heappop(heap) # remove current
252 # shift current to new location
253 heapq.heapreplace(heap, MergeIterItem(e, it))
258 """Delete a file at path 'f' if it currently exists.
260 Unlike os.unlink(), does not throw an exception if the file didn't already
266 if e.errno != errno.ENOENT:
270 _bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
271 _sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
276 if _bq_simple_id_rx.match(x):
278 return b"'%s'" % x.replace(b"'", b"'\"'\"'")
283 if _sq_simple_id_rx.match(x):
285 return "'%s'" % x.replace("'", "'\"'\"'")
288 if isinstance(x, bytes):
290 if isinstance(x, compat.str_type):
295 """Return a shell quoted string for cmd if it's a sequence, else cmd.
297 cmd must be a string, bytes, or a sequence of one or the other,
298 and the assumption is that if cmd is a string or bytes, then it's
299 already quoted (because it's what's actually being passed to
300 call() and friends. e.g. log(shstr(cmd)); call(cmd)
303 if isinstance(cmd, (bytes, compat.str_type)):
305 elif all(isinstance(x, bytes) for x in cmd):
306 return b' '.join(map(bquote, cmd))
307 elif all(isinstance(x, compat.str_type) for x in cmd):
308 return ' '.join(map(squote, cmd))
309 raise TypeError('unsupported shstr argument: ' + repr(cmd))
312 exc = subprocess.check_call
322 assert stdin in (None, PIPE)
325 stdin=stdin, stdout=PIPE, stderr=stderr,
327 preexec_fn=preexec_fn)
328 out, err = p.communicate(input)
329 if check and p.returncode != 0:
330 raise Exception('subprocess %r failed with status %d%s'
331 % (b' '.join(map(quote, cmd)), p.returncode,
332 ', stderr: %r' % err if err else ''))
335 def readpipe(argv, preexec_fn=None, shell=False):
336 """Run a subprocess and return its output."""
337 p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
339 out, err = p.communicate()
340 if p.returncode != 0:
341 raise Exception('subprocess %r failed with status %d'
342 % (b' '.join(argv), p.returncode))
346 def _argmax_base(command):
349 base_size += len(command) + 1
350 for k, v in compat.items(environ):
351 base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
355 def _argmax_args_size(args):
356 return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
359 def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
360 """If args is not empty, yield the output produced by calling the
361 command list with args as a sequence of strings (It may be necessary
362 to return multiple strings in order to respect ARG_MAX)."""
363 # The optional arg_max arg is a workaround for an issue with the
364 # current wvtest behavior.
365 base_size = _argmax_base(command)
367 room = arg_max - base_size
370 next_size = _argmax_args_size(args[i:i+1])
371 if room - next_size < 0:
377 assert(len(sub_args))
378 yield readpipe(command + sub_args, preexec_fn=preexec_fn)
381 def resolve_parent(p):
382 """Return the absolute path of a file without following any final symlink.
384 Behaves like os.path.realpath, but doesn't follow a symlink for the last
385 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
386 will follow symlinks in p's directory)
392 if st and stat.S_ISLNK(st.st_mode):
393 (dir, name) = os.path.split(p)
394 dir = os.path.realpath(dir)
395 out = os.path.join(dir, name)
397 out = os.path.realpath(p)
398 #log('realpathing:%r,%r\n' % (p, out))
402 def detect_fakeroot():
403 "Return True if we appear to be running under fakeroot."
404 return os.getenv("FAKEROOTKEY") != None
407 if sys.platform.startswith('cygwin'):
409 # https://cygwin.com/ml/cygwin/2015-02/msg00057.html
410 groups = os.getgroups()
411 return 544 in groups or 0 in groups
414 return os.geteuid() == 0
417 def cache_key_value(get_value, key, cache):
418 """Return (value, was_cached). If there is a value in the cache
419 for key, use that, otherwise, call get_value(key) which should
420 throw a KeyError if there is no value -- in which case the cached
421 and returned value will be None.
423 try: # Do we already have it (or know there wasn't one)?
430 cache[key] = value = get_value(key)
438 """Get the FQDN of this machine."""
441 _hostname = socket.getfqdn().encode('iso-8859-1')
445 def format_filesize(size):
450 exponent = int(math.log(size) // math.log(unit))
451 size_prefix = "KMGTPE"[exponent - 1]
452 return "%.1f%s" % (size // math.pow(unit, exponent), size_prefix)
455 class NotOk(Exception):
460 def __init__(self, outp):
464 while self._read(65536): pass
466 def read(self, size):
467 """Read 'size' bytes from input stream."""
469 return self._read(size)
472 """Read from input stream until a newline is found."""
474 return self._readline()
476 def write(self, data):
477 """Write 'data' to output stream."""
478 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
479 self.outp.write(data)
482 """Return true if input stream is readable."""
483 raise NotImplemented("Subclasses must implement has_input")
486 """Indicate end of output from last sent command."""
487 self.write(b'\nok\n')
490 """Indicate server error to the client."""
491 s = re.sub(br'\s+', b' ', s)
492 self.write(b'\nerror %s\n' % s)
494 def _check_ok(self, onempty):
497 for rl in linereader(self):
498 #log('%d got line: %r\n' % (os.getpid(), rl))
499 if not rl: # empty line
503 elif rl.startswith(b'error '):
504 #log('client: error: %s\n' % rl[6:])
508 raise Exception('server exited unexpectedly; see errors above')
510 def drain_and_check_ok(self):
511 """Remove all data for the current command from input stream."""
514 return self._check_ok(onempty)
517 """Verify that server action completed successfully."""
519 raise Exception('expected "ok", got %r' % rl)
520 return self._check_ok(onempty)
523 class Conn(BaseConn):
524 def __init__(self, inp, outp):
525 BaseConn.__init__(self, outp)
528 def _read(self, size):
529 return self.inp.read(size)
532 return self.inp.readline()
535 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
537 assert(rl[0] == self.inp.fileno())
543 def checked_reader(fd, n):
545 rl, _, _ = select.select([fd], [], [])
548 if not buf: raise Exception("Unexpected EOF reading %d more bytes" % n)
553 MAX_PACKET = 128 * 1024
554 def mux(p, outfd, outr, errr):
557 while p.poll() is None:
558 rl, _, _ = select.select(fds, [], [])
561 buf = os.read(outr, MAX_PACKET)
563 os.write(outfd, struct.pack('!IB', len(buf), 1) + buf)
565 buf = os.read(errr, 1024)
567 os.write(outfd, struct.pack('!IB', len(buf), 2) + buf)
569 os.write(outfd, struct.pack('!IB', 0, 3))
572 class DemuxConn(BaseConn):
573 """A helper class for bup's client-server protocol."""
574 def __init__(self, infd, outp):
575 BaseConn.__init__(self, outp)
576 # Anything that comes through before the sync string was not
577 # multiplexed and can be assumed to be debug/log before mux init.
579 while tail != b'BUPMUX':
580 b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
582 raise IOError('demux: unexpected EOF during initialization')
584 byte_stream(sys.stderr).write(tail[:-6]) # pre-mux log messages
591 def write(self, data):
593 BaseConn.write(self, data)
595 def _next_packet(self, timeout):
596 if self.closed: return False
597 rl, wl, xl = select.select([self.infd], [], [], timeout)
598 if not rl: return False
599 assert(rl[0] == self.infd)
600 ns = b''.join(checked_reader(self.infd, 5))
601 n, fdw = struct.unpack('!IB', ns)
602 assert(n <= MAX_PACKET)
604 self.reader = checked_reader(self.infd, n)
606 for buf in checked_reader(self.infd, n):
607 byte_stream(sys.stderr).write(buf)
610 debug2("DemuxConn: marked closed\n")
613 def _load_buf(self, timeout):
614 if self.buf is not None:
616 while not self.closed:
617 while not self.reader:
618 if not self._next_packet(timeout):
621 self.buf = next(self.reader)
623 except StopIteration:
627 def _read_parts(self, ix_fn):
628 while self._load_buf(None):
629 assert(self.buf is not None)
631 if i is None or i == len(self.buf):
636 self.buf = self.buf[i:]
644 return buf.index(b'\n')+1
647 return b''.join(self._read_parts(find_eol))
649 def _read(self, size):
651 def until_size(buf): # Closes on csize
652 if len(buf) < csize[0]:
657 return b''.join(self._read_parts(until_size))
660 return self._load_buf(0)
664 """Generate a list of input lines from 'f' without terminating newlines."""
672 def chunkyreader(f, count = None):
673 """Generate a list of chunks of data read from 'f'.
675 If count is None, read until EOF is reached.
677 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
678 reached while reading, raise IOError.
682 b = f.read(min(count, 65536))
684 raise IOError('EOF with %d bytes remaining' % count)
695 def atomically_replaced_file(name, mode='w', buffering=-1):
696 """Yield a file that will be atomically renamed name when leaving the block.
698 This contextmanager yields an open file object that is backed by a
699 temporary file which will be renamed (atomically) to the target
700 name if everything succeeds.
702 The mode and buffering arguments are handled exactly as with open,
703 and the yielded file will have very restrictive permissions, as
708 with atomically_replaced_file('foo.txt', 'w') as f:
709 f.write('hello jack.')
713 (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
714 text=('b' not in mode))
717 f = os.fdopen(ffd, mode, buffering)
725 os.rename(tempname, name)
727 unlink(tempname) # nonexistant file is ignored
731 """Append "/" to 's' if it doesn't aleady end in "/"."""
732 assert isinstance(s, bytes)
733 if s and not s.endswith(b'/'):
739 def _mmap_do(f, sz, flags, prot, close):
741 st = os.fstat(f.fileno())
744 # trying to open a zero-length map gives an error, but an empty
745 # string has all the same behaviour of a zero-length map, ie. it has
748 map = mmap.mmap(f.fileno(), sz, flags, prot)
750 f.close() # map will persist beyond file close
754 def mmap_read(f, sz = 0, close=True):
755 """Create a read-only memory mapped region on file 'f'.
756 If sz is 0, the region will cover the entire file.
758 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
761 def mmap_readwrite(f, sz = 0, close=True):
762 """Create a read-write memory mapped region on file 'f'.
763 If sz is 0, the region will cover the entire file.
765 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
769 def mmap_readwrite_private(f, sz = 0, close=True):
770 """Create a read-write memory mapped region on file 'f'.
771 If sz is 0, the region will cover the entire file.
772 The map is private, which means the changes are never flushed back to the
775 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
779 _mincore = getattr(_helpers, 'mincore', None)
781 # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
782 MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
784 _fmincore_chunk_size = None
785 def _set_fmincore_chunk_size():
786 global _fmincore_chunk_size
787 pref_chunk_size = 64 * 1024 * 1024
788 chunk_size = sc_page_size
789 if (sc_page_size < pref_chunk_size):
790 chunk_size = sc_page_size * (pref_chunk_size // sc_page_size)
791 _fmincore_chunk_size = chunk_size
794 """Return the mincore() data for fd as a bytearray whose values can be
795 tested via MINCORE_INCORE, or None if fd does not fully
796 support the operation."""
798 if (st.st_size == 0):
800 if not _fmincore_chunk_size:
801 _set_fmincore_chunk_size()
802 pages_per_chunk = _fmincore_chunk_size // sc_page_size;
803 page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
804 chunk_count = page_count // _fmincore_chunk_size
807 result = bytearray(page_count)
808 for ci in compat.range(chunk_count):
809 pos = _fmincore_chunk_size * ci;
810 msize = min(_fmincore_chunk_size, st.st_size - pos)
812 m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
813 except mmap.error as ex:
814 if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
815 # Perhaps the file was a pipe, i.e. "... | bup split ..."
819 _mincore(m, msize, 0, result, ci * pages_per_chunk)
820 except OSError as ex:
821 if ex.errno == errno.ENOSYS:
827 def parse_timestamp(epoch_str):
828 """Return the number of nanoseconds since the epoch that are described
829 by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
830 throw a ValueError that may contain additional information."""
831 ns_per = {'s' : 1000000000,
835 match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
837 if re.match(r'^([-+]?[0-9]+)$', epoch_str):
838 raise ValueError('must include units, i.e. 100ns, 100ms, ...')
840 (n, units) = match.group(1, 2)
844 return n * ns_per[units]
848 """Parse string or bytes as a possibly unit suffixed number.
851 199.2k means 203981 bytes
852 1GB means 1073741824 bytes
853 2.1 tb means 2199023255552 bytes
855 if isinstance(s, bytes):
856 # FIXME: should this raise a ValueError for UnicodeDecodeError
857 # (perhaps with the latter as the context).
858 s = s.decode('ascii')
859 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
861 raise ValueError("can't parse %r as a number" % s)
862 (val, unit) = g.groups()
865 if unit in ['t', 'tb']:
866 mult = 1024*1024*1024*1024
867 elif unit in ['g', 'gb']:
868 mult = 1024*1024*1024
869 elif unit in ['m', 'mb']:
871 elif unit in ['k', 'kb']:
873 elif unit in ['', 'b']:
876 raise ValueError("invalid unit %r in number %r" % (unit, s))
882 """Append an error message to the list of saved errors.
884 Once processing is able to stop and output the errors, the saved errors are
885 accessible in the module variable helpers.saved_errors.
887 saved_errors.append(e)
896 def die_if_errors(msg=None, status=1):
900 msg = 'warning: %d errors encountered\n' % len(saved_errors)
906 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
908 The new exception handler will make sure that bup will exit without an ugly
909 stacktrace when Ctrl-C is hit.
911 oldhook = sys.excepthook
912 def newhook(exctype, value, traceback):
913 if exctype == KeyboardInterrupt:
914 log('\nInterrupted.\n')
916 return oldhook(exctype, value, traceback)
917 sys.excepthook = newhook
920 def columnate(l, prefix):
921 """Format elements of 'l' in columns with 'prefix' leading each line.
923 The number of columns is determined automatically based on the string
926 binary = isinstance(prefix, bytes)
927 nothing = b'' if binary else ''
928 nl = b'\n' if binary else '\n'
932 clen = max(len(s) for s in l)
933 ncols = (tty_width() - len(prefix)) // (clen + 2)
938 while len(l) % ncols:
940 rows = len(l) // ncols
941 for s in compat.range(0, len(l), rows):
942 cols.append(l[s:s+rows])
944 fmt = b'%-*s' if binary else '%-*s'
945 for row in zip(*cols):
946 out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl
950 def parse_date_or_fatal(str, fatal):
951 """Parses the given date or calls Option.fatal().
952 For now we expect a string that contains a float."""
955 except ValueError as e:
956 raise fatal('invalid date format (should be a float): %r' % e)
961 def parse_excludes(options, fatal):
962 """Traverse the options and extract all excludes, or call Option.fatal()."""
966 (option, parameter) = flag
967 if option == '--exclude':
968 excluded_paths.append(resolve_parent(argv_bytes(parameter)))
969 elif option == '--exclude-from':
971 f = open(resolve_parent(argv_bytes(parameter)), 'rb')
973 raise fatal("couldn't read %r" % parameter)
974 for exclude_path in f.readlines():
975 # FIXME: perhaps this should be rstrip('\n')
976 exclude_path = resolve_parent(exclude_path.strip())
978 excluded_paths.append(exclude_path)
979 return sorted(frozenset(excluded_paths))
982 def parse_rx_excludes(options, fatal):
983 """Traverse the options and extract all rx excludes, or call
985 excluded_patterns = []
988 (option, parameter) = flag
989 if option == '--exclude-rx':
991 excluded_patterns.append(re.compile(argv_bytes(parameter)))
992 except re.error as ex:
993 fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))
994 elif option == '--exclude-rx-from':
996 f = open(resolve_parent(parameter), 'rb')
998 raise fatal("couldn't read %r" % parameter)
999 for pattern in f.readlines():
1000 spattern = pattern.rstrip(b'\n')
1004 excluded_patterns.append(re.compile(spattern))
1005 except re.error as ex:
1006 fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))
1007 return excluded_patterns
1010 def should_rx_exclude_path(path, exclude_rxs):
1011 """Return True if path matches a regular expression in exclude_rxs."""
1012 for rx in exclude_rxs:
1014 debug1('Skipping %r: excluded by rx pattern %r.\n'
1015 % (path, rx.pattern))
1020 # FIXME: Carefully consider the use of functions (os.path.*, etc.)
1021 # that resolve against the current filesystem in the strip/graft
1022 # functions for example, but elsewhere as well. I suspect bup's not
1023 # always being careful about that. For some cases, the contents of
1024 # the current filesystem should be irrelevant, and consulting it might
1025 # produce the wrong result, perhaps via unintended symlink resolution,
1028 def path_components(path):
1029 """Break path into a list of pairs of the form (name,
1030 full_path_to_name). Path must start with '/'.
1032 '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
1033 if not path.startswith(b'/'):
1034 raise Exception('path must start with "/": %s' % path_msg(path))
1035 # Since we assume path startswith('/'), we can skip the first element.
1036 result = [(b'', b'/')]
1037 norm_path = os.path.abspath(path)
1038 if norm_path == b'/':
1041 for p in norm_path.split(b'/')[1:]:
1042 full_path += b'/' + p
1043 result.append((p, full_path))
1047 def stripped_path_components(path, strip_prefixes):
1048 """Strip any prefix in strip_prefixes from path and return a list
1049 of path components where each component is (name,
1050 none_or_full_fs_path_to_name). Assume path startswith('/').
1051 See thelpers.py for examples."""
1052 normalized_path = os.path.abspath(path)
1053 sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
1054 for bp in sorted_strip_prefixes:
1055 normalized_bp = os.path.abspath(bp)
1056 if normalized_bp == b'/':
1058 if normalized_path.startswith(normalized_bp):
1059 prefix = normalized_path[:len(normalized_bp)]
1061 for p in normalized_path[len(normalized_bp):].split(b'/'):
1065 result.append((p, prefix))
1068 return path_components(path)
1071 def grafted_path_components(graft_points, path):
1072 # Create a result that consists of some number of faked graft
1073 # directories before the graft point, followed by all of the real
1074 # directories from path that are after the graft point. Arrange
1075 # for the directory at the graft point in the result to correspond
1076 # to the "orig" directory in --graft orig=new. See t/thelpers.py
1077 # for some examples.
1079 # Note that given --graft orig=new, orig and new have *nothing* to
1080 # do with each other, even if some of their component names
1081 # match. i.e. --graft /foo/bar/baz=/foo/bar/bax is semantically
1082 # equivalent to --graft /foo/bar/baz=/x/y/z, or even
1085 # FIXME: This can't be the best solution...
1086 clean_path = os.path.abspath(path)
1087 for graft_point in graft_points:
1088 old_prefix, new_prefix = graft_point
1089 # Expand prefixes iff not absolute paths.
1090 old_prefix = os.path.normpath(old_prefix)
1091 new_prefix = os.path.normpath(new_prefix)
1092 if clean_path.startswith(old_prefix):
1093 escaped_prefix = re.escape(old_prefix)
1094 grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)
1095 # Handle /foo=/ (at least) -- which produces //whatever.
1096 grafted_path = b'/' + grafted_path.lstrip(b'/')
1097 clean_path_components = path_components(clean_path)
1098 # Count the components that were stripped.
1099 strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
1100 new_prefix_parts = new_prefix.split(b'/')
1101 result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]
1102 result = [(p, None) for p in result_prefix] \
1103 + clean_path_components[strip_count:]
1104 # Now set the graft point name to match the end of new_prefix.
1105 graft_point = len(result_prefix)
1106 result[graft_point] = \
1107 (new_prefix_parts[-1], clean_path_components[strip_count][1])
1108 if new_prefix == b'/': # --graft ...=/ is a special case.
1111 return path_components(clean_path)
1117 _localtime = getattr(_helpers, 'localtime', None)
1120 bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
1121 'tm_hour', 'tm_min', 'tm_sec',
1122 'tm_wday', 'tm_yday',
1123 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
1125 # Define a localtime() that returns bup_time when possible. Note:
1126 # this means that any helpers.localtime() results may need to be
1127 # passed through to_py_time() before being passed to python's time
1128 # module, which doesn't appear willing to ignore the extra items.
1130 def localtime(time):
1131 return bup_time(*_helpers.localtime(floor(time)))
1132 def utc_offset_str(t):
1133 """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
1134 If the current UTC offset does not represent an integer number
1135 of minutes, the fractional component will be truncated."""
1136 off = localtime(t).tm_gmtoff
1137 # Note: // doesn't truncate like C for negative values, it rounds down.
1138 offmin = abs(off) // 60
1140 h = (offmin - m) // 60
1141 return b'%+03d%02d' % (-h if off < 0 else h, m)
1143 if isinstance(x, time.struct_time):
1145 return time.struct_time(x[:9])
1147 localtime = time.localtime
1148 def utc_offset_str(t):
1149 return time.strftime(b'%z', localtime(t))
1154 _some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')
1156 def valid_save_name(name):
1157 # Enforce a superset of the restrictions in git-check-ref-format(1)
1159 or name.startswith(b'/') or name.endswith(b'/') \
1160 or name.endswith(b'.'):
1162 if _some_invalid_save_parts_rx.search(name):
1165 if byte_int(c) < 0x20 or byte_int(c) == 0x7f:
1167 for part in name.split(b'/'):
1168 if part.startswith(b'.') or part.endswith(b'.lock'):
1173 _period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
1175 def period_as_secs(s):
1178 match = _period_rx.match(s)
1181 mag = int(match.group(1))
1182 scale = match.group(2)
1183 return mag * {'s': 1,
1187 'w': 60 * 60 * 24 * 7,
1188 'm': 60 * 60 * 24 * 31,
1189 'y': 60 * 60 * 24 * 366}[scale]