1 """Helper functions and classes for bup."""
3 from __future__ import absolute_import, division
4 from collections import namedtuple
5 from contextlib import contextmanager
6 from ctypes import sizeof, c_void_p
9 from subprocess import PIPE, Popen
10 import sys, os, subprocess, errno, select, mmap, stat, re, struct
11 import hashlib, heapq, math, operator, time, tempfile
13 from bup import _helpers
14 from bup import compat
15 from bup.compat import argv_bytes, byte_int, pending_raise
16 from bup.io import byte_stream, path_msg
17 # This function should really be in helpers, not in bup.options. But we
18 # want options.py to be standalone so people can include it in other projects.
19 from bup.options import _tty_width as tty_width
22 buglvl = int(os.environ.get('BUP_DEBUG', 0))
26 """Helper to deal with Python scoping issues"""
30 sc_page_size = os.sysconf('SC_PAGE_SIZE')
31 assert(sc_page_size > 0)
33 sc_arg_max = os.sysconf('SC_ARG_MAX')
34 if sc_arg_max == -1: # "no definite limit" - let's choose 2M
35 sc_arg_max = 2 * 1024 * 1024
39 for result in iterable:
44 _fdatasync = os.fdatasync
45 except AttributeError:
48 if sys.platform.startswith('darwin'):
49 # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
53 return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
55 # Fallback for file systems (SMB) that do not support F_FULLFSYNC
56 if e.errno == errno.ENOTSUP:
61 fdatasync = _fdatasync
64 def partition(predicate, stream):
65 """Returns (leading_matches_it, rest_it), where leading_matches_it
66 must be completely exhausted before traversing rest_it.
71 ns.first_nonmatch = None
72 def leading_matches():
77 ns.first_nonmatch = (x,)
81 yield ns.first_nonmatch[0]
84 return (leading_matches(), rest())
94 def lines_until_sentinel(f, sentinel, ex_type):
95 # sentinel must end with \n and must contain only one \n
98 if not (line and line.endswith(b'\n')):
99 raise ex_type('Hit EOF while reading line')
105 def stat_if_exists(path):
109 if e.errno != errno.ENOENT:
114 # Write (blockingly) to sockets that may or may not be in blocking mode.
115 # We need this because our stderr is sometimes eaten by subprocesses
116 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
117 # leading to race conditions. Ick. We'll do it the hard way.
118 def _hard_write(fd, buf):
120 (r,w,x) = select.select([], [fd], [], None)
122 raise IOError('select(fd) returned without being writable')
124 sz = os.write(fd, buf)
126 if e.errno != errno.EAGAIN:
134 """Print a log message to stderr."""
137 _hard_write(sys.stderr.fileno(), s if isinstance(s, bytes) else s.encode())
151 istty1 = os.isatty(1) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 1)
152 istty2 = os.isatty(2) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 2)
155 """Calls log() if stderr is a TTY. Does nothing otherwise."""
156 global _last_progress
163 """Calls progress() only if we haven't printed progress in a while.
165 This avoids overloading the stderr buffer with excess junk.
169 if now - _last_prog > 0.1:
175 """Calls progress() to redisplay the most recent progress message.
177 Useful after you've printed some other message that wipes out the
180 if _last_progress and _last_progress.endswith('\r'):
181 progress(_last_progress)
184 def mkdirp(d, mode=None):
185 """Recursively create directories on path 'd'.
187 Unlike os.makedirs(), it doesn't raise an exception if the last element of
188 the path already exists.
196 if e.errno == errno.EEXIST:
203 def __init__(self, entry, read_it):
205 self.read_it = read_it
207 return self.entry < x.entry
209 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
211 samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
213 samekey = operator.eq
215 total = sum(len(it) for it in iters)
216 iters = (iter(it) for it in iters)
217 heap = ((next(it, None),it) for it in iters)
218 heap = [MergeIterItem(e, it) for e, it in heap if e]
223 if not count % pfreq:
225 e, it = heap[0].entry, heap[0].read_it
226 if not samekey(e, pe):
232 except StopIteration:
233 heapq.heappop(heap) # remove current
235 # shift current to new location
236 heapq.heapreplace(heap, MergeIterItem(e, it))
241 """Delete a file at path 'f' if it currently exists.
243 Unlike os.unlink(), does not throw an exception if the file didn't already
249 if e.errno != errno.ENOENT:
253 _bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
254 _sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
259 if _bq_simple_id_rx.match(x):
261 return b"'%s'" % x.replace(b"'", b"'\"'\"'")
266 if _sq_simple_id_rx.match(x):
268 return "'%s'" % x.replace("'", "'\"'\"'")
271 if isinstance(x, bytes):
273 if isinstance(x, compat.str_type):
276 # some versions of pylint get confused
280 """Return a shell quoted string for cmd if it's a sequence, else cmd.
282 cmd must be a string, bytes, or a sequence of one or the other,
283 and the assumption is that if cmd is a string or bytes, then it's
284 already quoted (because it's what's actually being passed to
285 call() and friends. e.g. log(shstr(cmd)); call(cmd)
288 if isinstance(cmd, (bytes, compat.str_type)):
290 elif all(isinstance(x, bytes) for x in cmd):
291 return b' '.join(map(bquote, cmd))
292 elif all(isinstance(x, compat.str_type) for x in cmd):
293 return ' '.join(map(squote, cmd))
294 raise TypeError('unsupported shstr argument: ' + repr(cmd))
297 exc = subprocess.check_call
308 assert stdin in (None, PIPE)
311 stdin=stdin, stdout=PIPE, stderr=stderr,
313 preexec_fn=preexec_fn,
315 out, err = p.communicate(input)
316 if check and p.returncode != 0:
317 raise Exception('subprocess %r failed with status %d%s'
318 % (b' '.join(map(quote, cmd)), p.returncode,
319 ', stderr: %r' % err if err else ''))
322 def readpipe(argv, preexec_fn=None, shell=False):
323 """Run a subprocess and return its output."""
324 return exo(argv, preexec_fn=preexec_fn, shell=shell)[0]
327 def _argmax_base(command):
330 base_size += len(command) + 1
331 for k, v in compat.items(environ):
332 base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
336 def _argmax_args_size(args):
337 return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
340 def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
341 """If args is not empty, yield the output produced by calling the
342 command list with args as a sequence of strings (It may be necessary
343 to return multiple strings in order to respect ARG_MAX)."""
344 # The optional arg_max arg is a workaround for an issue with the
345 # current wvtest behavior.
346 base_size = _argmax_base(command)
348 room = arg_max - base_size
351 next_size = _argmax_args_size(args[i:i+1])
352 if room - next_size < 0:
358 assert(len(sub_args))
359 yield readpipe(command + sub_args, preexec_fn=preexec_fn)
362 def resolve_parent(p):
363 """Return the absolute path of a file without following any final symlink.
365 Behaves like os.path.realpath, but doesn't follow a symlink for the last
366 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
367 will follow symlinks in p's directory)
373 if st and stat.S_ISLNK(st.st_mode):
374 (dir, name) = os.path.split(p)
375 dir = os.path.realpath(dir)
376 out = os.path.join(dir, name)
378 out = os.path.realpath(p)
379 #log('realpathing:%r,%r\n' % (p, out))
383 def detect_fakeroot():
384 "Return True if we appear to be running under fakeroot."
385 return os.getenv("FAKEROOTKEY") != None
388 if sys.platform.startswith('cygwin'):
390 # https://cygwin.com/ml/cygwin/2015-02/msg00057.html
391 groups = os.getgroups()
392 return 544 in groups or 0 in groups
395 return os.geteuid() == 0
398 def cache_key_value(get_value, key, cache):
399 """Return (value, was_cached). If there is a value in the cache
400 for key, use that, otherwise, call get_value(key) which should
401 throw a KeyError if there is no value -- in which case the cached
402 and returned value will be None.
404 try: # Do we already have it (or know there wasn't one)?
411 cache[key] = value = get_value(key)
419 """Get the FQDN of this machine."""
422 _hostname = _helpers.gethostname()
426 def format_filesize(size):
431 exponent = int(math.log(size) // math.log(unit))
432 size_prefix = "KMGTPE"[exponent - 1]
433 return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
436 class NotOk(Exception):
441 def __init__(self, outp):
445 while self._read(65536): pass
447 def _read(self, size):
448 raise NotImplementedError("Subclasses must implement _read")
450 def read(self, size):
451 """Read 'size' bytes from input stream."""
453 return self._read(size)
455 def _readline(self, size):
456 raise NotImplementedError("Subclasses must implement _readline")
459 """Read from input stream until a newline is found."""
461 return self._readline()
463 def write(self, data):
464 """Write 'data' to output stream."""
465 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
466 self.outp.write(data)
469 """Return true if input stream is readable."""
470 raise NotImplementedError("Subclasses must implement has_input")
473 """Indicate end of output from last sent command."""
474 self.write(b'\nok\n')
477 """Indicate server error to the client."""
478 s = re.sub(br'\s+', b' ', s)
479 self.write(b'\nerror %s\n' % s)
481 def _check_ok(self, onempty):
484 for rl in linereader(self):
485 #log('%d got line: %r\n' % (os.getpid(), rl))
486 if not rl: # empty line
490 elif rl.startswith(b'error '):
491 #log('client: error: %s\n' % rl[6:])
495 raise Exception('server exited unexpectedly; see errors above')
497 def drain_and_check_ok(self):
498 """Remove all data for the current command from input stream."""
501 return self._check_ok(onempty)
504 """Verify that server action completed successfully."""
506 raise Exception('expected "ok", got %r' % rl)
507 return self._check_ok(onempty)
510 class Conn(BaseConn):
511 def __init__(self, inp, outp):
512 BaseConn.__init__(self, outp)
515 def _read(self, size):
516 return self.inp.read(size)
519 return self.inp.readline()
522 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
524 assert(rl[0] == self.inp.fileno())
530 def checked_reader(fd, n):
532 rl, _, _ = select.select([fd], [], [])
535 if not buf: raise Exception("Unexpected EOF reading %d more bytes" % n)
540 MAX_PACKET = 128 * 1024
541 def mux(p, outfd, outr, errr):
544 while p.poll() is None:
545 rl, _, _ = select.select(fds, [], [])
548 buf = os.read(outr, MAX_PACKET)
550 os.write(outfd, struct.pack('!IB', len(buf), 1) + buf)
552 buf = os.read(errr, 1024)
554 os.write(outfd, struct.pack('!IB', len(buf), 2) + buf)
556 os.write(outfd, struct.pack('!IB', 0, 3))
559 class DemuxConn(BaseConn):
560 """A helper class for bup's client-server protocol."""
561 def __init__(self, infd, outp):
562 BaseConn.__init__(self, outp)
563 # Anything that comes through before the sync string was not
564 # multiplexed and can be assumed to be debug/log before mux init.
566 stderr = byte_stream(sys.stderr)
567 while tail != b'BUPMUX':
568 # Make sure to write all pre-BUPMUX output to stderr
569 b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
571 ex = IOError('demux: unexpected EOF during initialization')
572 with pending_raise(ex):
576 stderr.write(tail[:-6])
584 def write(self, data):
586 BaseConn.write(self, data)
588 def _next_packet(self, timeout):
589 if self.closed: return False
590 rl, wl, xl = select.select([self.infd], [], [], timeout)
591 if not rl: return False
592 assert(rl[0] == self.infd)
593 ns = b''.join(checked_reader(self.infd, 5))
594 n, fdw = struct.unpack('!IB', ns)
596 # assume that something went wrong and print stuff
597 ns += os.read(self.infd, 1024)
598 stderr = byte_stream(sys.stderr)
601 raise Exception("Connection broken")
603 self.reader = checked_reader(self.infd, n)
605 for buf in checked_reader(self.infd, n):
606 byte_stream(sys.stderr).write(buf)
609 debug2("DemuxConn: marked closed\n")
612 def _load_buf(self, timeout):
613 if self.buf is not None:
615 while not self.closed:
616 while not self.reader:
617 if not self._next_packet(timeout):
620 self.buf = next(self.reader)
622 except StopIteration:
626 def _read_parts(self, ix_fn):
627 while self._load_buf(None):
628 assert(self.buf is not None)
630 if i is None or i == len(self.buf):
635 self.buf = self.buf[i:]
643 return buf.index(b'\n')+1
646 return b''.join(self._read_parts(find_eol))
648 def _read(self, size):
650 def until_size(buf): # Closes on csize
651 if len(buf) < csize[0]:
656 return b''.join(self._read_parts(until_size))
659 return self._load_buf(0)
663 """Generate a list of input lines from 'f' without terminating newlines."""
671 def chunkyreader(f, count = None):
672 """Generate a list of chunks of data read from 'f'.
674 If count is None, read until EOF is reached.
676 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
677 reached while reading, raise IOError.
681 b = f.read(min(count, 65536))
683 raise IOError('EOF with %d bytes remaining' % count)
694 def atomically_replaced_file(name, mode='w', buffering=-1):
695 """Yield a file that will be atomically renamed name when leaving the block.
697 This contextmanager yields an open file object that is backed by a
698 temporary file which will be renamed (atomically) to the target
699 name if everything succeeds.
701 The mode and buffering arguments are handled exactly as with open,
702 and the yielded file will have very restrictive permissions, as
707 with atomically_replaced_file('foo.txt', 'w') as f:
708 f.write('hello jack.')
712 (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
713 text=('b' not in mode))
716 f = os.fdopen(ffd, mode, buffering)
724 os.rename(tempname, name)
726 unlink(tempname) # nonexistant file is ignored
730 """Append "/" to 's' if it doesn't aleady end in "/"."""
731 assert isinstance(s, bytes)
732 if s and not s.endswith(b'/'):
738 def _mmap_do(f, sz, flags, prot, close):
740 st = os.fstat(f.fileno())
743 # trying to open a zero-length map gives an error, but an empty
744 # string has all the same behaviour of a zero-length map, ie. it has
747 map = mmap.mmap(f.fileno(), sz, flags, prot)
749 f.close() # map will persist beyond file close
753 def mmap_read(f, sz = 0, close=True):
754 """Create a read-only memory mapped region on file 'f'.
755 If sz is 0, the region will cover the entire file.
757 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
760 def mmap_readwrite(f, sz = 0, close=True):
761 """Create a read-write memory mapped region on file 'f'.
762 If sz is 0, the region will cover the entire file.
764 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
768 def mmap_readwrite_private(f, sz = 0, close=True):
769 """Create a read-write memory mapped region on file 'f'.
770 If sz is 0, the region will cover the entire file.
771 The map is private, which means the changes are never flushed back to the
774 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
778 _mincore = getattr(_helpers, 'mincore', None)
780 # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
781 MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
783 _fmincore_chunk_size = None
784 def _set_fmincore_chunk_size():
785 global _fmincore_chunk_size
786 pref_chunk_size = 64 * 1024 * 1024
787 chunk_size = sc_page_size
788 if (sc_page_size < pref_chunk_size):
789 chunk_size = sc_page_size * (pref_chunk_size // sc_page_size)
790 _fmincore_chunk_size = chunk_size
793 """Return the mincore() data for fd as a bytearray whose values can be
794 tested via MINCORE_INCORE, or None if fd does not fully
795 support the operation."""
797 if (st.st_size == 0):
799 if not _fmincore_chunk_size:
800 _set_fmincore_chunk_size()
801 pages_per_chunk = _fmincore_chunk_size // sc_page_size;
802 page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
803 chunk_count = (st.st_size + _fmincore_chunk_size - 1) // _fmincore_chunk_size
804 result = bytearray(page_count)
805 for ci in compat.range(chunk_count):
806 pos = _fmincore_chunk_size * ci;
807 msize = min(_fmincore_chunk_size, st.st_size - pos)
809 m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
810 except mmap.error as ex:
811 if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
812 # Perhaps the file was a pipe, i.e. "... | bup split ..."
816 _mincore(m, msize, 0, result, ci * pages_per_chunk)
817 except OSError as ex:
818 if ex.errno == errno.ENOSYS:
824 def parse_timestamp(epoch_str):
825 """Return the number of nanoseconds since the epoch that are described
826 by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
827 throw a ValueError that may contain additional information."""
828 ns_per = {'s' : 1000000000,
832 match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
834 if re.match(r'^([-+]?[0-9]+)$', epoch_str):
835 raise ValueError('must include units, i.e. 100ns, 100ms, ...')
837 (n, units) = match.group(1, 2)
841 return n * ns_per[units]
845 """Parse string or bytes as a possibly unit suffixed number.
848 199.2k means 203981 bytes
849 1GB means 1073741824 bytes
850 2.1 tb means 2199023255552 bytes
852 if isinstance(s, bytes):
853 # FIXME: should this raise a ValueError for UnicodeDecodeError
854 # (perhaps with the latter as the context).
855 s = s.decode('ascii')
856 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
858 raise ValueError("can't parse %r as a number" % s)
859 (val, unit) = g.groups()
862 if unit in ['t', 'tb']:
863 mult = 1024*1024*1024*1024
864 elif unit in ['g', 'gb']:
865 mult = 1024*1024*1024
866 elif unit in ['m', 'mb']:
868 elif unit in ['k', 'kb']:
870 elif unit in ['', 'b']:
873 raise ValueError("invalid unit %r in number %r" % (unit, s))
879 """Append an error message to the list of saved errors.
881 Once processing is able to stop and output the errors, the saved errors are
882 accessible in the module variable helpers.saved_errors.
884 saved_errors.append(e)
893 def die_if_errors(msg=None, status=1):
897 msg = 'warning: %d errors encountered\n' % len(saved_errors)
903 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
905 The new exception handler will make sure that bup will exit without an ugly
906 stacktrace when Ctrl-C is hit.
908 oldhook = sys.excepthook
909 def newhook(exctype, value, traceback):
910 if exctype == KeyboardInterrupt:
911 log('\nInterrupted.\n')
913 oldhook(exctype, value, traceback)
914 sys.excepthook = newhook
917 def columnate(l, prefix):
918 """Format elements of 'l' in columns with 'prefix' leading each line.
920 The number of columns is determined automatically based on the string
923 binary = isinstance(prefix, bytes)
924 nothing = b'' if binary else ''
925 nl = b'\n' if binary else '\n'
929 clen = max(len(s) for s in l)
930 ncols = (tty_width() - len(prefix)) // (clen + 2)
935 while len(l) % ncols:
937 rows = len(l) // ncols
938 for s in compat.range(0, len(l), rows):
939 cols.append(l[s:s+rows])
941 fmt = b'%-*s' if binary else '%-*s'
942 for row in zip(*cols):
943 out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl
947 def parse_date_or_fatal(str, fatal):
948 """Parses the given date or calls Option.fatal().
949 For now we expect a string that contains a float."""
952 except ValueError as e:
953 raise fatal('invalid date format (should be a float): %r' % e)
958 def parse_excludes(options, fatal):
959 """Traverse the options and extract all excludes, or call Option.fatal()."""
963 (option, parameter) = flag
964 if option == '--exclude':
965 excluded_paths.append(resolve_parent(argv_bytes(parameter)))
966 elif option == '--exclude-from':
968 f = open(resolve_parent(argv_bytes(parameter)), 'rb')
970 raise fatal("couldn't read %r" % parameter)
971 for exclude_path in f.readlines():
972 # FIXME: perhaps this should be rstrip('\n')
973 exclude_path = resolve_parent(exclude_path.strip())
975 excluded_paths.append(exclude_path)
976 return sorted(frozenset(excluded_paths))
979 def parse_rx_excludes(options, fatal):
980 """Traverse the options and extract all rx excludes, or call
982 excluded_patterns = []
985 (option, parameter) = flag
986 if option == '--exclude-rx':
988 excluded_patterns.append(re.compile(argv_bytes(parameter)))
989 except re.error as ex:
990 fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))
991 elif option == '--exclude-rx-from':
993 f = open(resolve_parent(parameter), 'rb')
995 raise fatal("couldn't read %r" % parameter)
996 for pattern in f.readlines():
997 spattern = pattern.rstrip(b'\n')
1001 excluded_patterns.append(re.compile(spattern))
1002 except re.error as ex:
1003 fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))
1004 return excluded_patterns
1007 def should_rx_exclude_path(path, exclude_rxs):
1008 """Return True if path matches a regular expression in exclude_rxs."""
1009 for rx in exclude_rxs:
1011 debug1('Skipping %r: excluded by rx pattern %r.\n'
1012 % (path, rx.pattern))
1017 # FIXME: Carefully consider the use of functions (os.path.*, etc.)
1018 # that resolve against the current filesystem in the strip/graft
1019 # functions for example, but elsewhere as well. I suspect bup's not
1020 # always being careful about that. For some cases, the contents of
1021 # the current filesystem should be irrelevant, and consulting it might
1022 # produce the wrong result, perhaps via unintended symlink resolution,
1025 def path_components(path):
1026 """Break path into a list of pairs of the form (name,
1027 full_path_to_name). Path must start with '/'.
1029 '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
1030 if not path.startswith(b'/'):
1031 raise Exception('path must start with "/": %s' % path_msg(path))
1032 # Since we assume path startswith('/'), we can skip the first element.
1033 result = [(b'', b'/')]
1034 norm_path = os.path.abspath(path)
1035 if norm_path == b'/':
1038 for p in norm_path.split(b'/')[1:]:
1039 full_path += b'/' + p
1040 result.append((p, full_path))
1044 def stripped_path_components(path, strip_prefixes):
1045 """Strip any prefix in strip_prefixes from path and return a list
1046 of path components where each component is (name,
1047 none_or_full_fs_path_to_name). Assume path startswith('/').
1048 See thelpers.py for examples."""
1049 normalized_path = os.path.abspath(path)
1050 sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
1051 for bp in sorted_strip_prefixes:
1052 normalized_bp = os.path.abspath(bp)
1053 if normalized_bp == b'/':
1055 if normalized_path.startswith(normalized_bp):
1056 prefix = normalized_path[:len(normalized_bp)]
1058 for p in normalized_path[len(normalized_bp):].split(b'/'):
1062 result.append((p, prefix))
1065 return path_components(path)
1068 def grafted_path_components(graft_points, path):
1069 # Create a result that consists of some number of faked graft
1070 # directories before the graft point, followed by all of the real
1071 # directories from path that are after the graft point. Arrange
1072 # for the directory at the graft point in the result to correspond
1073 # to the "orig" directory in --graft orig=new. See t/thelpers.py
1074 # for some examples.
1076 # Note that given --graft orig=new, orig and new have *nothing* to
1077 # do with each other, even if some of their component names
1078 # match. i.e. --graft /foo/bar/baz=/foo/bar/bax is semantically
1079 # equivalent to --graft /foo/bar/baz=/x/y/z, or even
1082 # FIXME: This can't be the best solution...
1083 clean_path = os.path.abspath(path)
1084 for graft_point in graft_points:
1085 old_prefix, new_prefix = graft_point
1086 # Expand prefixes iff not absolute paths.
1087 old_prefix = os.path.normpath(old_prefix)
1088 new_prefix = os.path.normpath(new_prefix)
1089 if clean_path.startswith(old_prefix):
1090 escaped_prefix = re.escape(old_prefix)
1091 grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)
1092 # Handle /foo=/ (at least) -- which produces //whatever.
1093 grafted_path = b'/' + grafted_path.lstrip(b'/')
1094 clean_path_components = path_components(clean_path)
1095 # Count the components that were stripped.
1096 strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
1097 new_prefix_parts = new_prefix.split(b'/')
1098 result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]
1099 result = [(p, None) for p in result_prefix] \
1100 + clean_path_components[strip_count:]
1101 # Now set the graft point name to match the end of new_prefix.
1102 graft_point = len(result_prefix)
1103 result[graft_point] = \
1104 (new_prefix_parts[-1], clean_path_components[strip_count][1])
1105 if new_prefix == b'/': # --graft ...=/ is a special case.
1108 return path_components(clean_path)
1114 _localtime = getattr(_helpers, 'localtime', None)
1117 bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
1118 'tm_hour', 'tm_min', 'tm_sec',
1119 'tm_wday', 'tm_yday',
1120 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
1122 # Define a localtime() that returns bup_time when possible. Note:
1123 # this means that any helpers.localtime() results may need to be
1124 # passed through to_py_time() before being passed to python's time
1125 # module, which doesn't appear willing to ignore the extra items.
1127 def localtime(time):
1128 return bup_time(*_helpers.localtime(int(floor(time))))
1129 def utc_offset_str(t):
1130 """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
1131 If the current UTC offset does not represent an integer number
1132 of minutes, the fractional component will be truncated."""
1133 off = localtime(t).tm_gmtoff
1134 # Note: // doesn't truncate like C for negative values, it rounds down.
1135 offmin = abs(off) // 60
1137 h = (offmin - m) // 60
1138 return b'%+03d%02d' % (-h if off < 0 else h, m)
1140 if isinstance(x, time.struct_time):
1142 return time.struct_time(x[:9])
1144 localtime = time.localtime
1145 def utc_offset_str(t):
1146 return time.strftime(b'%z', localtime(t))
1151 _some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')
1153 def valid_save_name(name):
1154 # Enforce a superset of the restrictions in git-check-ref-format(1)
1156 or name.startswith(b'/') or name.endswith(b'/') \
1157 or name.endswith(b'.'):
1159 if _some_invalid_save_parts_rx.search(name):
1162 if byte_int(c) < 0x20 or byte_int(c) == 0x7f:
1164 for part in name.split(b'/'):
1165 if part.startswith(b'.') or part.endswith(b'.lock'):
1170 _period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')
1172 def period_as_secs(s):
1175 match = _period_rx.match(s)
1178 mag = int(match.group(1))
1179 scale = match.group(2)
1180 return mag * {b's': 1,
1184 b'w': 60 * 60 * 24 * 7,
1185 b'm': 60 * 60 * 24 * 31,
1186 b'y': 60 * 60 * 24 * 366}[scale]