1 """Helper functions and classes for bup."""
3 from __future__ import absolute_import
4 from collections import namedtuple
5 from contextlib import contextmanager
6 from ctypes import sizeof, c_void_p
8 from pipes import quote
9 from subprocess import PIPE, Popen
10 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
11 import hashlib, heapq, math, operator, time, grp, tempfile
13 from bup import _helpers
14 from bup import compat
15 # This function should really be in helpers, not in bup.options. But we
16 # want options.py to be standalone so people can include it in other projects.
17 from bup.options import _tty_width as tty_width
21 """Helper to deal with Python scoping issues"""
25 sc_page_size = os.sysconf('SC_PAGE_SIZE')
26 assert(sc_page_size > 0)
28 sc_arg_max = os.sysconf('SC_ARG_MAX')
29 if sc_arg_max == -1: # "no definite limit" - let's choose 2M
30 sc_arg_max = 2 * 1024 * 1024
34 for result in iterable:
40 """Convert the string 's' to an integer. Return 0 if s is not a number."""
48 """Convert the string 's' to a float. Return 0 if s is not a number."""
50 return float(s or '0')
55 buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
59 _fdatasync = os.fdatasync
60 except AttributeError:
63 if sys.platform.startswith('darwin'):
64 # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
68 return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
70 # Fallback for file systems (SMB) that do not support F_FULLFSYNC
71 if e.errno == errno.ENOTSUP:
76 fdatasync = _fdatasync
79 def partition(predicate, stream):
80 """Returns (leading_matches_it, rest_it), where leading_matches_it
81 must be completely exhausted before traversing rest_it.
86 ns.first_nonmatch = None
87 def leading_matches():
92 ns.first_nonmatch = (x,)
96 yield ns.first_nonmatch[0]
99 return (leading_matches(), rest())
102 def lines_until_sentinel(f, sentinel, ex_type):
103 # sentinel must end with \n and must contain only one \n
106 if not (line and line.endswith('\n')):
107 raise ex_type('Hit EOF while reading line')
113 def stat_if_exists(path):
117 if e.errno != errno.ENOENT:
122 # Write (blockingly) to sockets that may or may not be in blocking mode.
123 # We need this because our stderr is sometimes eaten by subprocesses
124 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
125 # leading to race conditions. Ick. We'll do it the hard way.
126 def _hard_write(fd, buf):
128 (r,w,x) = select.select([], [fd], [], None)
130 raise IOError('select(fd) returned without being writable')
132 sz = os.write(fd, buf)
134 if e.errno != errno.EAGAIN:
142 """Print a log message to stderr."""
145 _hard_write(sys.stderr.fileno(), s)
159 istty1 = os.isatty(1) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 1)
160 istty2 = os.isatty(2) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 2)
163 """Calls log() if stderr is a TTY. Does nothing otherwise."""
164 global _last_progress
171 """Calls progress() only if we haven't printed progress in a while.
173 This avoids overloading the stderr buffer with excess junk.
177 if now - _last_prog > 0.1:
183 """Calls progress() to redisplay the most recent progress message.
185 Useful after you've printed some other message that wipes out the
188 if _last_progress and _last_progress.endswith('\r'):
189 progress(_last_progress)
192 def mkdirp(d, mode=None):
193 """Recursively create directories on path 'd'.
195 Unlike os.makedirs(), it doesn't raise an exception if the last element of
196 the path already exists.
204 if e.errno == errno.EEXIST:
210 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
212 samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
214 samekey = operator.eq
216 total = sum(len(it) for it in iters)
217 iters = (iter(it) for it in iters)
218 heap = ((next(it, None),it) for it in iters)
219 heap = [(e,it) for e,it in heap if e]
224 if not count % pfreq:
227 if not samekey(e, pe):
233 except StopIteration:
234 heapq.heappop(heap) # remove current
236 heapq.heapreplace(heap, (e, it)) # shift current to new location
241 """Delete a file at path 'f' if it currently exists.
243 Unlike os.unlink(), does not throw an exception if the file didn't already
249 if e.errno != errno.ENOENT:
254 if isinstance(cmd, compat.str_type):
257 return ' '.join(map(quote, cmd))
259 exc = subprocess.check_call
269 assert stdin in (None, PIPE)
272 stdin=stdin, stdout=PIPE, stderr=stderr,
274 preexec_fn=preexec_fn)
275 out, err = p.communicate(input)
276 if check and p.returncode != 0:
277 raise Exception('subprocess %r failed with status %d, stderr: %r'
278 % (' '.join(map(quote, cmd)), p.returncode, err))
281 def readpipe(argv, preexec_fn=None, shell=False):
282 """Run a subprocess and return its output."""
283 p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
285 out, err = p.communicate()
286 if p.returncode != 0:
287 raise Exception('subprocess %r failed with status %d'
288 % (' '.join(argv), p.returncode))
292 def _argmax_base(command):
295 base_size += len(command) + 1
296 for k, v in compat.items(environ):
297 base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
301 def _argmax_args_size(args):
302 return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
305 def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
306 """If args is not empty, yield the output produced by calling the
307 command list with args as a sequence of strings (It may be necessary
308 to return multiple strings in order to respect ARG_MAX)."""
309 # The optional arg_max arg is a workaround for an issue with the
310 # current wvtest behavior.
311 base_size = _argmax_base(command)
313 room = arg_max - base_size
316 next_size = _argmax_args_size(args[i:i+1])
317 if room - next_size < 0:
323 assert(len(sub_args))
324 yield readpipe(command + sub_args, preexec_fn=preexec_fn)
327 def resolve_parent(p):
328 """Return the absolute path of a file without following any final symlink.
330 Behaves like os.path.realpath, but doesn't follow a symlink for the last
331 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
332 will follow symlinks in p's directory)
338 if st and stat.S_ISLNK(st.st_mode):
339 (dir, name) = os.path.split(p)
340 dir = os.path.realpath(dir)
341 out = os.path.join(dir, name)
343 out = os.path.realpath(p)
344 #log('realpathing:%r,%r\n' % (p, out))
348 def detect_fakeroot():
349 "Return True if we appear to be running under fakeroot."
350 return os.getenv("FAKEROOTKEY") != None
353 _warned_about_superuser_detection = None
355 if sys.platform.startswith('cygwin'):
356 if sys.getwindowsversion()[0] > 5:
357 # Sounds like situation is much more complicated here
358 global _warned_about_superuser_detection
359 if not _warned_about_superuser_detection:
360 log("can't detect root status for OS version > 5; assuming not root")
361 _warned_about_superuser_detection = True
364 return ctypes.cdll.shell32.IsUserAnAdmin()
366 return os.geteuid() == 0
369 def _cache_key_value(get_value, key, cache):
370 """Return (value, was_cached). If there is a value in the cache
371 for key, use that, otherwise, call get_value(key) which should
372 throw a KeyError if there is no value -- in which case the cached
373 and returned value will be None.
375 try: # Do we already have it (or know there wasn't one)?
382 cache[key] = value = get_value(key)
388 _uid_to_pwd_cache = {}
389 _name_to_pwd_cache = {}
391 def pwd_from_uid(uid):
392 """Return password database entry for uid (may be a cached value).
393 Return None if no entry is found.
395 global _uid_to_pwd_cache, _name_to_pwd_cache
396 entry, cached = _cache_key_value(pwd.getpwuid, uid, _uid_to_pwd_cache)
397 if entry and not cached:
398 _name_to_pwd_cache[entry.pw_name] = entry
402 def pwd_from_name(name):
403 """Return password database entry for name (may be a cached value).
404 Return None if no entry is found.
406 global _uid_to_pwd_cache, _name_to_pwd_cache
407 entry, cached = _cache_key_value(pwd.getpwnam, name, _name_to_pwd_cache)
408 if entry and not cached:
409 _uid_to_pwd_cache[entry.pw_uid] = entry
413 _gid_to_grp_cache = {}
414 _name_to_grp_cache = {}
416 def grp_from_gid(gid):
417 """Return password database entry for gid (may be a cached value).
418 Return None if no entry is found.
420 global _gid_to_grp_cache, _name_to_grp_cache
421 entry, cached = _cache_key_value(grp.getgrgid, gid, _gid_to_grp_cache)
422 if entry and not cached:
423 _name_to_grp_cache[entry.gr_name] = entry
427 def grp_from_name(name):
428 """Return password database entry for name (may be a cached value).
429 Return None if no entry is found.
431 global _gid_to_grp_cache, _name_to_grp_cache
432 entry, cached = _cache_key_value(grp.getgrnam, name, _name_to_grp_cache)
433 if entry and not cached:
434 _gid_to_grp_cache[entry.gr_gid] = entry
440 """Get the user's login name."""
444 _username = pwd_from_uid(uid)[0] or 'user%d' % uid
450 """Get the user's full name."""
452 if not _userfullname:
454 entry = pwd_from_uid(uid)
456 _userfullname = entry[4].split(',')[0] or entry[0]
457 if not _userfullname:
458 _userfullname = 'user%d' % uid
464 """Get the FQDN of this machine."""
467 _hostname = socket.getfqdn()
471 _resource_path = None
472 def resource_path(subdir=''):
473 global _resource_path
474 if not _resource_path:
475 _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
476 return os.path.join(_resource_path, subdir)
478 def format_filesize(size):
483 exponent = int(math.log(size) / math.log(unit))
484 size_prefix = "KMGTPE"[exponent - 1]
485 return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
488 class NotOk(Exception):
493 def __init__(self, outp):
497 while self._read(65536): pass
499 def read(self, size):
500 """Read 'size' bytes from input stream."""
502 return self._read(size)
505 """Read from input stream until a newline is found."""
507 return self._readline()
509 def write(self, data):
510 """Write 'data' to output stream."""
511 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
512 self.outp.write(data)
515 """Return true if input stream is readable."""
516 raise NotImplemented("Subclasses must implement has_input")
519 """Indicate end of output from last sent command."""
523 """Indicate server error to the client."""
524 s = re.sub(r'\s+', ' ', str(s))
525 self.write('\nerror %s\n' % s)
527 def _check_ok(self, onempty):
530 for rl in linereader(self):
531 #log('%d got line: %r\n' % (os.getpid(), rl))
532 if not rl: # empty line
536 elif rl.startswith('error '):
537 #log('client: error: %s\n' % rl[6:])
541 raise Exception('server exited unexpectedly; see errors above')
543 def drain_and_check_ok(self):
544 """Remove all data for the current command from input stream."""
547 return self._check_ok(onempty)
550 """Verify that server action completed successfully."""
552 raise Exception('expected "ok", got %r' % rl)
553 return self._check_ok(onempty)
556 class Conn(BaseConn):
557 def __init__(self, inp, outp):
558 BaseConn.__init__(self, outp)
561 def _read(self, size):
562 return self.inp.read(size)
565 return self.inp.readline()
568 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
570 assert(rl[0] == self.inp.fileno())
576 def checked_reader(fd, n):
578 rl, _, _ = select.select([fd], [], [])
581 if not buf: raise Exception("Unexpected EOF reading %d more bytes" % n)
586 MAX_PACKET = 128 * 1024
587 def mux(p, outfd, outr, errr):
590 while p.poll() is None:
591 rl, _, _ = select.select(fds, [], [])
594 buf = os.read(outr, MAX_PACKET)
596 os.write(outfd, struct.pack('!IB', len(buf), 1) + buf)
598 buf = os.read(errr, 1024)
600 os.write(outfd, struct.pack('!IB', len(buf), 2) + buf)
602 os.write(outfd, struct.pack('!IB', 0, 3))
605 class DemuxConn(BaseConn):
606 """A helper class for bup's client-server protocol."""
607 def __init__(self, infd, outp):
608 BaseConn.__init__(self, outp)
609 # Anything that comes through before the sync string was not
610 # multiplexed and can be assumed to be debug/log before mux init.
612 while tail != 'BUPMUX':
613 b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
615 raise IOError('demux: unexpected EOF during initialization')
617 sys.stderr.write(tail[:-6]) # pre-mux log messages
624 def write(self, data):
626 BaseConn.write(self, data)
628 def _next_packet(self, timeout):
629 if self.closed: return False
630 rl, wl, xl = select.select([self.infd], [], [], timeout)
631 if not rl: return False
632 assert(rl[0] == self.infd)
633 ns = ''.join(checked_reader(self.infd, 5))
634 n, fdw = struct.unpack('!IB', ns)
635 assert(n <= MAX_PACKET)
637 self.reader = checked_reader(self.infd, n)
639 for buf in checked_reader(self.infd, n):
640 sys.stderr.write(buf)
643 debug2("DemuxConn: marked closed\n")
646 def _load_buf(self, timeout):
647 if self.buf is not None:
649 while not self.closed:
650 while not self.reader:
651 if not self._next_packet(timeout):
654 self.buf = next(self.reader)
656 except StopIteration:
660 def _read_parts(self, ix_fn):
661 while self._load_buf(None):
662 assert(self.buf is not None)
664 if i is None or i == len(self.buf):
669 self.buf = self.buf[i:]
677 return buf.index('\n')+1
680 return ''.join(self._read_parts(find_eol))
682 def _read(self, size):
684 def until_size(buf): # Closes on csize
685 if len(buf) < csize[0]:
690 return ''.join(self._read_parts(until_size))
693 return self._load_buf(0)
697 """Generate a list of input lines from 'f' without terminating newlines."""
705 def chunkyreader(f, count = None):
706 """Generate a list of chunks of data read from 'f'.
708 If count is None, read until EOF is reached.
710 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
711 reached while reading, raise IOError.
715 b = f.read(min(count, 65536))
717 raise IOError('EOF with %d bytes remaining' % count)
728 def atomically_replaced_file(name, mode='w', buffering=-1):
729 """Yield a file that will be atomically renamed name when leaving the block.
731 This contextmanager yields an open file object that is backed by a
732 temporary file which will be renamed (atomically) to the target
733 name if everything succeeds.
735 The mode and buffering arguments are handled exactly as with open,
736 and the yielded file will have very restrictive permissions, as
741 with atomically_replaced_file('foo.txt', 'w') as f:
742 f.write('hello jack.')
746 (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
747 text=('b' not in mode))
750 f = os.fdopen(ffd, mode, buffering)
758 os.rename(tempname, name)
760 unlink(tempname) # nonexistant file is ignored
764 """Append "/" to 's' if it doesn't aleady end in "/"."""
765 if s and not s.endswith('/'):
771 def _mmap_do(f, sz, flags, prot, close):
773 st = os.fstat(f.fileno())
776 # trying to open a zero-length map gives an error, but an empty
777 # string has all the same behaviour of a zero-length map, ie. it has
780 map = mmap.mmap(f.fileno(), sz, flags, prot)
782 f.close() # map will persist beyond file close
786 def mmap_read(f, sz = 0, close=True):
787 """Create a read-only memory mapped region on file 'f'.
788 If sz is 0, the region will cover the entire file.
790 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
793 def mmap_readwrite(f, sz = 0, close=True):
794 """Create a read-write memory mapped region on file 'f'.
795 If sz is 0, the region will cover the entire file.
797 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
801 def mmap_readwrite_private(f, sz = 0, close=True):
802 """Create a read-write memory mapped region on file 'f'.
803 If sz is 0, the region will cover the entire file.
804 The map is private, which means the changes are never flushed back to the
807 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
811 _mincore = getattr(_helpers, 'mincore', None)
813 # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
814 MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
816 _fmincore_chunk_size = None
817 def _set_fmincore_chunk_size():
818 global _fmincore_chunk_size
819 pref_chunk_size = 64 * 1024 * 1024
820 chunk_size = sc_page_size
821 if (sc_page_size < pref_chunk_size):
822 chunk_size = sc_page_size * (pref_chunk_size / sc_page_size)
823 _fmincore_chunk_size = chunk_size
826 """Return the mincore() data for fd as a bytearray whose values can be
827 tested via MINCORE_INCORE, or None if fd does not fully
828 support the operation."""
830 if (st.st_size == 0):
832 if not _fmincore_chunk_size:
833 _set_fmincore_chunk_size()
834 pages_per_chunk = _fmincore_chunk_size / sc_page_size;
835 page_count = (st.st_size + sc_page_size - 1) / sc_page_size;
836 chunk_count = page_count / _fmincore_chunk_size
839 result = bytearray(page_count)
840 for ci in xrange(chunk_count):
841 pos = _fmincore_chunk_size * ci;
842 msize = min(_fmincore_chunk_size, st.st_size - pos)
844 m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
845 except mmap.error as ex:
846 if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
847 # Perhaps the file was a pipe, i.e. "... | bup split ..."
851 _mincore(m, msize, 0, result, ci * pages_per_chunk)
852 except OSError as ex:
853 if ex.errno == errno.ENOSYS:
859 def parse_timestamp(epoch_str):
860 """Return the number of nanoseconds since the epoch that are described
861 by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
862 throw a ValueError that may contain additional information."""
863 ns_per = {'s' : 1000000000,
867 match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
869 if re.match(r'^([-+]?[0-9]+)$', epoch_str):
870 raise ValueError('must include units, i.e. 100ns, 100ms, ...')
872 (n, units) = match.group(1, 2)
876 return n * ns_per[units]
880 """Parse data size information into a float number.
882 Here are some examples of conversions:
883 199.2k means 203981 bytes
884 1GB means 1073741824 bytes
885 2.1 tb means 2199023255552 bytes
887 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
889 raise ValueError("can't parse %r as a number" % s)
890 (val, unit) = g.groups()
893 if unit in ['t', 'tb']:
894 mult = 1024*1024*1024*1024
895 elif unit in ['g', 'gb']:
896 mult = 1024*1024*1024
897 elif unit in ['m', 'mb']:
899 elif unit in ['k', 'kb']:
901 elif unit in ['', 'b']:
904 raise ValueError("invalid unit %r in number %r" % (unit, s))
909 """Count the number of elements in an iterator. (consumes the iterator)"""
910 return reduce(lambda x,y: x+1, l)
915 """Append an error message to the list of saved errors.
917 Once processing is able to stop and output the errors, the saved errors are
918 accessible in the module variable helpers.saved_errors.
920 saved_errors.append(e)
929 def die_if_errors(msg=None, status=1):
933 msg = 'warning: %d errors encountered\n' % len(saved_errors)
939 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
941 The new exception handler will make sure that bup will exit without an ugly
942 stacktrace when Ctrl-C is hit.
944 oldhook = sys.excepthook
945 def newhook(exctype, value, traceback):
946 if exctype == KeyboardInterrupt:
947 log('\nInterrupted.\n')
949 return oldhook(exctype, value, traceback)
950 sys.excepthook = newhook
953 def columnate(l, prefix):
954 """Format elements of 'l' in columns with 'prefix' leading each line.
956 The number of columns is determined automatically based on the string
962 clen = max(len(s) for s in l)
963 ncols = (tty_width() - len(prefix)) // (clen + 2)
968 while len(l) % ncols:
970 rows = len(l) // ncols
971 for s in range(0, len(l), rows):
972 cols.append(l[s:s+rows])
974 for row in zip(*cols):
975 out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
979 def parse_date_or_fatal(str, fatal):
980 """Parses the given date or calls Option.fatal().
981 For now we expect a string that contains a float."""
984 except ValueError as e:
985 raise fatal('invalid date format (should be a float): %r' % e)
990 def parse_excludes(options, fatal):
991 """Traverse the options and extract all excludes, or call Option.fatal()."""
995 (option, parameter) = flag
996 if option == '--exclude':
997 excluded_paths.append(resolve_parent(parameter))
998 elif option == '--exclude-from':
1000 f = open(resolve_parent(parameter))
1001 except IOError as e:
1002 raise fatal("couldn't read %s" % parameter)
1003 for exclude_path in f.readlines():
1004 # FIXME: perhaps this should be rstrip('\n')
1005 exclude_path = resolve_parent(exclude_path.strip())
1007 excluded_paths.append(exclude_path)
1008 return sorted(frozenset(excluded_paths))
1011 def parse_rx_excludes(options, fatal):
1012 """Traverse the options and extract all rx excludes, or call
1014 excluded_patterns = []
1016 for flag in options:
1017 (option, parameter) = flag
1018 if option == '--exclude-rx':
1020 excluded_patterns.append(re.compile(parameter))
1021 except re.error as ex:
1022 fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex))
1023 elif option == '--exclude-rx-from':
1025 f = open(resolve_parent(parameter))
1026 except IOError as e:
1027 raise fatal("couldn't read %s" % parameter)
1028 for pattern in f.readlines():
1029 spattern = pattern.rstrip('\n')
1033 excluded_patterns.append(re.compile(spattern))
1034 except re.error as ex:
1035 fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex))
1036 return excluded_patterns
1039 def should_rx_exclude_path(path, exclude_rxs):
1040 """Return True if path matches a regular expression in exclude_rxs."""
1041 for rx in exclude_rxs:
1043 debug1('Skipping %r: excluded by rx pattern %r.\n'
1044 % (path, rx.pattern))
1049 # FIXME: Carefully consider the use of functions (os.path.*, etc.)
1050 # that resolve against the current filesystem in the strip/graft
1051 # functions for example, but elsewhere as well. I suspect bup's not
1052 # always being careful about that. For some cases, the contents of
1053 # the current filesystem should be irrelevant, and consulting it might
1054 # produce the wrong result, perhaps via unintended symlink resolution,
1057 def path_components(path):
1058 """Break path into a list of pairs of the form (name,
1059 full_path_to_name). Path must start with '/'.
1061 '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
1062 if not path.startswith('/'):
1063 raise Exception('path must start with "/": %s' % path)
1064 # Since we assume path startswith('/'), we can skip the first element.
1065 result = [('', '/')]
1066 norm_path = os.path.abspath(path)
1067 if norm_path == '/':
1070 for p in norm_path.split('/')[1:]:
1071 full_path += '/' + p
1072 result.append((p, full_path))
1076 def stripped_path_components(path, strip_prefixes):
1077 """Strip any prefix in strip_prefixes from path and return a list
1078 of path components where each component is (name,
1079 none_or_full_fs_path_to_name). Assume path startswith('/').
1080 See thelpers.py for examples."""
1081 normalized_path = os.path.abspath(path)
1082 sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
1083 for bp in sorted_strip_prefixes:
1084 normalized_bp = os.path.abspath(bp)
1085 if normalized_bp == '/':
1087 if normalized_path.startswith(normalized_bp):
1088 prefix = normalized_path[:len(normalized_bp)]
1090 for p in normalized_path[len(normalized_bp):].split('/'):
1094 result.append((p, prefix))
1097 return path_components(path)
1100 def grafted_path_components(graft_points, path):
1101 # Create a result that consists of some number of faked graft
1102 # directories before the graft point, followed by all of the real
1103 # directories from path that are after the graft point. Arrange
1104 # for the directory at the graft point in the result to correspond
1105 # to the "orig" directory in --graft orig=new. See t/thelpers.py
1106 # for some examples.
1108 # Note that given --graft orig=new, orig and new have *nothing* to
1109 # do with each other, even if some of their component names
1110 # match. i.e. --graft /foo/bar/baz=/foo/bar/bax is semantically
1111 # equivalent to --graft /foo/bar/baz=/x/y/z, or even
1114 # FIXME: This can't be the best solution...
1115 clean_path = os.path.abspath(path)
1116 for graft_point in graft_points:
1117 old_prefix, new_prefix = graft_point
1118 # Expand prefixes iff not absolute paths.
1119 old_prefix = os.path.normpath(old_prefix)
1120 new_prefix = os.path.normpath(new_prefix)
1121 if clean_path.startswith(old_prefix):
1122 escaped_prefix = re.escape(old_prefix)
1123 grafted_path = re.sub(r'^' + escaped_prefix, new_prefix, clean_path)
1124 # Handle /foo=/ (at least) -- which produces //whatever.
1125 grafted_path = '/' + grafted_path.lstrip('/')
1126 clean_path_components = path_components(clean_path)
1127 # Count the components that were stripped.
1128 strip_count = 0 if old_prefix == '/' else old_prefix.count('/')
1129 new_prefix_parts = new_prefix.split('/')
1130 result_prefix = grafted_path.split('/')[:new_prefix.count('/')]
1131 result = [(p, None) for p in result_prefix] \
1132 + clean_path_components[strip_count:]
1133 # Now set the graft point name to match the end of new_prefix.
1134 graft_point = len(result_prefix)
1135 result[graft_point] = \
1136 (new_prefix_parts[-1], clean_path_components[strip_count][1])
1137 if new_prefix == '/': # --graft ...=/ is a special case.
1140 return path_components(clean_path)
1146 _localtime = getattr(_helpers, 'localtime', None)
1149 bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
1150 'tm_hour', 'tm_min', 'tm_sec',
1151 'tm_wday', 'tm_yday',
1152 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
1154 # Define a localtime() that returns bup_time when possible. Note:
1155 # this means that any helpers.localtime() results may need to be
1156 # passed through to_py_time() before being passed to python's time
1157 # module, which doesn't appear willing to ignore the extra items.
1159 def localtime(time):
1160 return bup_time(*_helpers.localtime(time))
1161 def utc_offset_str(t):
1162 """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
1163 If the current UTC offset does not represent an integer number
1164 of minutes, the fractional component will be truncated."""
1165 off = localtime(t).tm_gmtoff
1166 # Note: // doesn't truncate like C for negative values, it rounds down.
1167 offmin = abs(off) // 60
1169 h = (offmin - m) // 60
1170 return "%+03d%02d" % (-h if off < 0 else h, m)
1172 if isinstance(x, time.struct_time):
1174 return time.struct_time(x[:9])
1176 localtime = time.localtime
1177 def utc_offset_str(t):
1178 return time.strftime('%z', localtime(t))
1183 _some_invalid_save_parts_rx = re.compile(r'[[ ~^:?*\\]|\.\.|//|@{')
1185 def valid_save_name(name):
1186 # Enforce a superset of the restrictions in git-check-ref-format(1)
1188 or name.startswith('/') or name.endswith('/') \
1189 or name.endswith('.'):
1191 if _some_invalid_save_parts_rx.search(name):
1194 if ord(c) < 0x20 or ord(c) == 0x7f:
1196 for part in name.split('/'):
1197 if part.startswith('.') or part.endswith('.lock'):
1202 _period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
1204 def period_as_secs(s):
1207 match = _period_rx.match(s)
1210 mag = int(match.group(1))
1211 scale = match.group(2)
1212 return mag * {'s': 1,
1216 'w': 60 * 60 * 24 * 7,
1217 'm': 60 * 60 * 24 * 31,
1218 'y': 60 * 60 * 24 * 366}[scale]