1 """Helper functions and classes for bup."""
3 from collections import namedtuple
4 from contextlib import contextmanager
5 from ctypes import sizeof, c_void_p
7 from pipes import quote
8 from subprocess import PIPE, Popen
9 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
10 import hashlib, heapq, math, operator, time, grp, tempfile
12 from bup import _helpers
13 from bup import compat
14 # This function should really be in helpers, not in bup.options. But we
15 # want options.py to be standalone so people can include it in other projects.
16 from bup.options import _tty_width as tty_width
20 """Helper to deal with Python scoping issues"""
24 sc_page_size = os.sysconf('SC_PAGE_SIZE')
25 assert(sc_page_size > 0)
27 sc_arg_max = os.sysconf('SC_ARG_MAX')
28 if sc_arg_max == -1: # "no definite limit" - let's choose 2M
29 sc_arg_max = 2 * 1024 * 1024
33 for result in iterable:
39 """Convert the string 's' to an integer. Return 0 if s is not a number."""
47 """Convert the string 's' to a float. Return 0 if s is not a number."""
49 return float(s or '0')
54 buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
58 _fdatasync = os.fdatasync
59 except AttributeError:
62 if sys.platform.startswith('darwin'):
63 # Apparently os.fsync on OS X doesn't guarantee to sync all the way down
67 return fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
69 # Fallback for file systems (SMB) that do not support F_FULLFSYNC
70 if e.errno == errno.ENOTSUP:
75 fdatasync = _fdatasync
78 def partition(predicate, stream):
79 """Returns (leading_matches_it, rest_it), where leading_matches_it
80 must be completely exhausted before traversing rest_it.
85 ns.first_nonmatch = None
86 def leading_matches():
91 ns.first_nonmatch = (x,)
95 yield ns.first_nonmatch[0]
98 return (leading_matches(), rest())
101 def lines_until_sentinel(f, sentinel, ex_type):
102 # sentinel must end with \n and must contain only one \n
105 if not (line and line.endswith('\n')):
106 raise ex_type('Hit EOF while reading line')
112 def stat_if_exists(path):
116 if e.errno != errno.ENOENT:
121 # Write (blockingly) to sockets that may or may not be in blocking mode.
122 # We need this because our stderr is sometimes eaten by subprocesses
123 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
124 # leading to race conditions. Ick. We'll do it the hard way.
125 def _hard_write(fd, buf):
127 (r,w,x) = select.select([], [fd], [], None)
129 raise IOError('select(fd) returned without being writable')
131 sz = os.write(fd, buf)
133 if e.errno != errno.EAGAIN:
141 """Print a log message to stderr."""
144 _hard_write(sys.stderr.fileno(), s)
158 istty1 = os.isatty(1) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 1)
159 istty2 = os.isatty(2) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 2)
162 """Calls log() if stderr is a TTY. Does nothing otherwise."""
163 global _last_progress
170 """Calls progress() only if we haven't printed progress in a while.
172 This avoids overloading the stderr buffer with excess junk.
176 if now - _last_prog > 0.1:
182 """Calls progress() to redisplay the most recent progress message.
184 Useful after you've printed some other message that wipes out the
187 if _last_progress and _last_progress.endswith('\r'):
188 progress(_last_progress)
191 def mkdirp(d, mode=None):
192 """Recursively create directories on path 'd'.
194 Unlike os.makedirs(), it doesn't raise an exception if the last element of
195 the path already exists.
203 if e.errno == errno.EEXIST:
209 def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
211 samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
213 samekey = operator.eq
215 total = sum(len(it) for it in iters)
216 iters = (iter(it) for it in iters)
217 heap = ((next(it, None),it) for it in iters)
218 heap = [(e,it) for e,it in heap if e]
223 if not count % pfreq:
226 if not samekey(e, pe):
232 except StopIteration:
233 heapq.heappop(heap) # remove current
235 heapq.heapreplace(heap, (e, it)) # shift current to new location
240 """Delete a file at path 'f' if it currently exists.
242 Unlike os.unlink(), does not throw an exception if the file didn't already
248 if e.errno != errno.ENOENT:
253 if isinstance(cmd, compat.str_type):
256 return ' '.join(map(quote, cmd))
258 exc = subprocess.check_call
268 assert stdin in (None, PIPE)
271 stdin=stdin, stdout=PIPE, stderr=stderr,
273 preexec_fn=preexec_fn)
274 out, err = p.communicate(input)
275 if check and p.returncode != 0:
276 raise Exception('subprocess %r failed with status %d, stderr: %r'
277 % (' '.join(map(quote, cmd)), p.returncode, err))
280 def readpipe(argv, preexec_fn=None, shell=False):
281 """Run a subprocess and return its output."""
282 p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
284 out, err = p.communicate()
285 if p.returncode != 0:
286 raise Exception('subprocess %r failed with status %d'
287 % (' '.join(argv), p.returncode))
291 def _argmax_base(command):
294 base_size += len(command) + 1
295 for k, v in compat.items(environ):
296 base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
300 def _argmax_args_size(args):
301 return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
304 def batchpipe(command, args, preexec_fn=None, arg_max=sc_arg_max):
305 """If args is not empty, yield the output produced by calling the
306 command list with args as a sequence of strings (It may be necessary
307 to return multiple strings in order to respect ARG_MAX)."""
308 # The optional arg_max arg is a workaround for an issue with the
309 # current wvtest behavior.
310 base_size = _argmax_base(command)
312 room = arg_max - base_size
315 next_size = _argmax_args_size(args[i:i+1])
316 if room - next_size < 0:
322 assert(len(sub_args))
323 yield readpipe(command + sub_args, preexec_fn=preexec_fn)
326 def resolve_parent(p):
327 """Return the absolute path of a file without following any final symlink.
329 Behaves like os.path.realpath, but doesn't follow a symlink for the last
330 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
331 will follow symlinks in p's directory)
337 if st and stat.S_ISLNK(st.st_mode):
338 (dir, name) = os.path.split(p)
339 dir = os.path.realpath(dir)
340 out = os.path.join(dir, name)
342 out = os.path.realpath(p)
343 #log('realpathing:%r,%r\n' % (p, out))
347 def detect_fakeroot():
348 "Return True if we appear to be running under fakeroot."
349 return os.getenv("FAKEROOTKEY") != None
352 _warned_about_superuser_detection = None
354 if sys.platform.startswith('cygwin'):
355 if sys.getwindowsversion()[0] > 5:
356 # Sounds like situation is much more complicated here
357 global _warned_about_superuser_detection
358 if not _warned_about_superuser_detection:
359 log("can't detect root status for OS version > 5; assuming not root")
360 _warned_about_superuser_detection = True
363 return ctypes.cdll.shell32.IsUserAnAdmin()
365 return os.geteuid() == 0
368 def _cache_key_value(get_value, key, cache):
369 """Return (value, was_cached). If there is a value in the cache
370 for key, use that, otherwise, call get_value(key) which should
371 throw a KeyError if there is no value -- in which case the cached
372 and returned value will be None.
374 try: # Do we already have it (or know there wasn't one)?
381 cache[key] = value = get_value(key)
387 _uid_to_pwd_cache = {}
388 _name_to_pwd_cache = {}
390 def pwd_from_uid(uid):
391 """Return password database entry for uid (may be a cached value).
392 Return None if no entry is found.
394 global _uid_to_pwd_cache, _name_to_pwd_cache
395 entry, cached = _cache_key_value(pwd.getpwuid, uid, _uid_to_pwd_cache)
396 if entry and not cached:
397 _name_to_pwd_cache[entry.pw_name] = entry
401 def pwd_from_name(name):
402 """Return password database entry for name (may be a cached value).
403 Return None if no entry is found.
405 global _uid_to_pwd_cache, _name_to_pwd_cache
406 entry, cached = _cache_key_value(pwd.getpwnam, name, _name_to_pwd_cache)
407 if entry and not cached:
408 _uid_to_pwd_cache[entry.pw_uid] = entry
412 _gid_to_grp_cache = {}
413 _name_to_grp_cache = {}
415 def grp_from_gid(gid):
416 """Return password database entry for gid (may be a cached value).
417 Return None if no entry is found.
419 global _gid_to_grp_cache, _name_to_grp_cache
420 entry, cached = _cache_key_value(grp.getgrgid, gid, _gid_to_grp_cache)
421 if entry and not cached:
422 _name_to_grp_cache[entry.gr_name] = entry
426 def grp_from_name(name):
427 """Return password database entry for name (may be a cached value).
428 Return None if no entry is found.
430 global _gid_to_grp_cache, _name_to_grp_cache
431 entry, cached = _cache_key_value(grp.getgrnam, name, _name_to_grp_cache)
432 if entry and not cached:
433 _gid_to_grp_cache[entry.gr_gid] = entry
439 """Get the user's login name."""
443 _username = pwd_from_uid(uid)[0] or 'user%d' % uid
449 """Get the user's full name."""
451 if not _userfullname:
453 entry = pwd_from_uid(uid)
455 _userfullname = entry[4].split(',')[0] or entry[0]
456 if not _userfullname:
457 _userfullname = 'user%d' % uid
463 """Get the FQDN of this machine."""
466 _hostname = socket.getfqdn()
470 _resource_path = None
471 def resource_path(subdir=''):
472 global _resource_path
473 if not _resource_path:
474 _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
475 return os.path.join(_resource_path, subdir)
477 def format_filesize(size):
482 exponent = int(math.log(size) / math.log(unit))
483 size_prefix = "KMGTPE"[exponent - 1]
484 return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
487 class NotOk(Exception):
492 def __init__(self, outp):
496 while self._read(65536): pass
498 def read(self, size):
499 """Read 'size' bytes from input stream."""
501 return self._read(size)
504 """Read from input stream until a newline is found."""
506 return self._readline()
508 def write(self, data):
509 """Write 'data' to output stream."""
510 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
511 self.outp.write(data)
514 """Return true if input stream is readable."""
515 raise NotImplemented("Subclasses must implement has_input")
518 """Indicate end of output from last sent command."""
522 """Indicate server error to the client."""
523 s = re.sub(r'\s+', ' ', str(s))
524 self.write('\nerror %s\n' % s)
526 def _check_ok(self, onempty):
529 for rl in linereader(self):
530 #log('%d got line: %r\n' % (os.getpid(), rl))
531 if not rl: # empty line
535 elif rl.startswith('error '):
536 #log('client: error: %s\n' % rl[6:])
540 raise Exception('server exited unexpectedly; see errors above')
542 def drain_and_check_ok(self):
543 """Remove all data for the current command from input stream."""
546 return self._check_ok(onempty)
549 """Verify that server action completed successfully."""
551 raise Exception('expected "ok", got %r' % rl)
552 return self._check_ok(onempty)
555 class Conn(BaseConn):
556 def __init__(self, inp, outp):
557 BaseConn.__init__(self, outp)
560 def _read(self, size):
561 return self.inp.read(size)
564 return self.inp.readline()
567 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
569 assert(rl[0] == self.inp.fileno())
575 def checked_reader(fd, n):
577 rl, _, _ = select.select([fd], [], [])
580 if not buf: raise Exception("Unexpected EOF reading %d more bytes" % n)
585 MAX_PACKET = 128 * 1024
586 def mux(p, outfd, outr, errr):
589 while p.poll() is None:
590 rl, _, _ = select.select(fds, [], [])
593 buf = os.read(outr, MAX_PACKET)
595 os.write(outfd, struct.pack('!IB', len(buf), 1) + buf)
597 buf = os.read(errr, 1024)
599 os.write(outfd, struct.pack('!IB', len(buf), 2) + buf)
601 os.write(outfd, struct.pack('!IB', 0, 3))
604 class DemuxConn(BaseConn):
605 """A helper class for bup's client-server protocol."""
606 def __init__(self, infd, outp):
607 BaseConn.__init__(self, outp)
608 # Anything that comes through before the sync string was not
609 # multiplexed and can be assumed to be debug/log before mux init.
611 while tail != 'BUPMUX':
612 b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
614 raise IOError('demux: unexpected EOF during initialization')
616 sys.stderr.write(tail[:-6]) # pre-mux log messages
623 def write(self, data):
625 BaseConn.write(self, data)
627 def _next_packet(self, timeout):
628 if self.closed: return False
629 rl, wl, xl = select.select([self.infd], [], [], timeout)
630 if not rl: return False
631 assert(rl[0] == self.infd)
632 ns = ''.join(checked_reader(self.infd, 5))
633 n, fdw = struct.unpack('!IB', ns)
634 assert(n <= MAX_PACKET)
636 self.reader = checked_reader(self.infd, n)
638 for buf in checked_reader(self.infd, n):
639 sys.stderr.write(buf)
642 debug2("DemuxConn: marked closed\n")
645 def _load_buf(self, timeout):
646 if self.buf is not None:
648 while not self.closed:
649 while not self.reader:
650 if not self._next_packet(timeout):
653 self.buf = next(self.reader)
655 except StopIteration:
659 def _read_parts(self, ix_fn):
660 while self._load_buf(None):
661 assert(self.buf is not None)
663 if i is None or i == len(self.buf):
668 self.buf = self.buf[i:]
676 return buf.index('\n')+1
679 return ''.join(self._read_parts(find_eol))
681 def _read(self, size):
683 def until_size(buf): # Closes on csize
684 if len(buf) < csize[0]:
689 return ''.join(self._read_parts(until_size))
692 return self._load_buf(0)
696 """Generate a list of input lines from 'f' without terminating newlines."""
704 def chunkyreader(f, count = None):
705 """Generate a list of chunks of data read from 'f'.
707 If count is None, read until EOF is reached.
709 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
710 reached while reading, raise IOError.
714 b = f.read(min(count, 65536))
716 raise IOError('EOF with %d bytes remaining' % count)
727 def atomically_replaced_file(name, mode='w', buffering=-1):
728 """Yield a file that will be atomically renamed name when leaving the block.
730 This contextmanager yields an open file object that is backed by a
731 temporary file which will be renamed (atomically) to the target
732 name if everything succeeds.
734 The mode and buffering arguments are handled exactly as with open,
735 and the yielded file will have very restrictive permissions, as
740 with atomically_replaced_file('foo.txt', 'w') as f:
741 f.write('hello jack.')
745 (ffd, tempname) = tempfile.mkstemp(dir=os.path.dirname(name),
746 text=('b' not in mode))
749 f = os.fdopen(ffd, mode, buffering)
757 os.rename(tempname, name)
759 unlink(tempname) # nonexistant file is ignored
763 """Append "/" to 's' if it doesn't aleady end in "/"."""
764 if s and not s.endswith('/'):
770 def _mmap_do(f, sz, flags, prot, close):
772 st = os.fstat(f.fileno())
775 # trying to open a zero-length map gives an error, but an empty
776 # string has all the same behaviour of a zero-length map, ie. it has
779 map = mmap.mmap(f.fileno(), sz, flags, prot)
781 f.close() # map will persist beyond file close
785 def mmap_read(f, sz = 0, close=True):
786 """Create a read-only memory mapped region on file 'f'.
787 If sz is 0, the region will cover the entire file.
789 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close)
792 def mmap_readwrite(f, sz = 0, close=True):
793 """Create a read-write memory mapped region on file 'f'.
794 If sz is 0, the region will cover the entire file.
796 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE,
800 def mmap_readwrite_private(f, sz = 0, close=True):
801 """Create a read-write memory mapped region on file 'f'.
802 If sz is 0, the region will cover the entire file.
803 The map is private, which means the changes are never flushed back to the
806 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
810 _mincore = getattr(_helpers, 'mincore', None)
812 # ./configure ensures that we're on Linux if MINCORE_INCORE isn't defined.
813 MINCORE_INCORE = getattr(_helpers, 'MINCORE_INCORE', 1)
815 _fmincore_chunk_size = None
816 def _set_fmincore_chunk_size():
817 global _fmincore_chunk_size
818 pref_chunk_size = 64 * 1024 * 1024
819 chunk_size = sc_page_size
820 if (sc_page_size < pref_chunk_size):
821 chunk_size = sc_page_size * (pref_chunk_size / sc_page_size)
822 _fmincore_chunk_size = chunk_size
825 """Return the mincore() data for fd as a bytearray whose values can be
826 tested via MINCORE_INCORE, or None if fd does not fully
827 support the operation."""
829 if (st.st_size == 0):
831 if not _fmincore_chunk_size:
832 _set_fmincore_chunk_size()
833 pages_per_chunk = _fmincore_chunk_size / sc_page_size;
834 page_count = (st.st_size + sc_page_size - 1) / sc_page_size;
835 chunk_count = page_count / _fmincore_chunk_size
838 result = bytearray(page_count)
839 for ci in xrange(chunk_count):
840 pos = _fmincore_chunk_size * ci;
841 msize = min(_fmincore_chunk_size, st.st_size - pos)
843 m = mmap.mmap(fd, msize, mmap.MAP_PRIVATE, 0, 0, pos)
844 except mmap.error as ex:
845 if ex.errno == errno.EINVAL or ex.errno == errno.ENODEV:
846 # Perhaps the file was a pipe, i.e. "... | bup split ..."
850 _mincore(m, msize, 0, result, ci * pages_per_chunk)
851 except OSError as ex:
852 if ex.errno == errno.ENOSYS:
858 def parse_timestamp(epoch_str):
859 """Return the number of nanoseconds since the epoch that are described
860 by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
861 throw a ValueError that may contain additional information."""
862 ns_per = {'s' : 1000000000,
866 match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
868 if re.match(r'^([-+]?[0-9]+)$', epoch_str):
869 raise ValueError('must include units, i.e. 100ns, 100ms, ...')
871 (n, units) = match.group(1, 2)
875 return n * ns_per[units]
879 """Parse data size information into a float number.
881 Here are some examples of conversions:
882 199.2k means 203981 bytes
883 1GB means 1073741824 bytes
884 2.1 tb means 2199023255552 bytes
886 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
888 raise ValueError("can't parse %r as a number" % s)
889 (val, unit) = g.groups()
892 if unit in ['t', 'tb']:
893 mult = 1024*1024*1024*1024
894 elif unit in ['g', 'gb']:
895 mult = 1024*1024*1024
896 elif unit in ['m', 'mb']:
898 elif unit in ['k', 'kb']:
900 elif unit in ['', 'b']:
903 raise ValueError("invalid unit %r in number %r" % (unit, s))
908 """Count the number of elements in an iterator. (consumes the iterator)"""
909 return reduce(lambda x,y: x+1, l)
914 """Append an error message to the list of saved errors.
916 Once processing is able to stop and output the errors, the saved errors are
917 accessible in the module variable helpers.saved_errors.
919 saved_errors.append(e)
928 def die_if_errors(msg=None, status=1):
932 msg = 'warning: %d errors encountered\n' % len(saved_errors)
938 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
940 The new exception handler will make sure that bup will exit without an ugly
941 stacktrace when Ctrl-C is hit.
943 oldhook = sys.excepthook
944 def newhook(exctype, value, traceback):
945 if exctype == KeyboardInterrupt:
946 log('\nInterrupted.\n')
948 return oldhook(exctype, value, traceback)
949 sys.excepthook = newhook
952 def columnate(l, prefix):
953 """Format elements of 'l' in columns with 'prefix' leading each line.
955 The number of columns is determined automatically based on the string
961 clen = max(len(s) for s in l)
962 ncols = (tty_width() - len(prefix)) // (clen + 2)
967 while len(l) % ncols:
969 rows = len(l) // ncols
970 for s in range(0, len(l), rows):
971 cols.append(l[s:s+rows])
973 for row in zip(*cols):
974 out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
978 def parse_date_or_fatal(str, fatal):
979 """Parses the given date or calls Option.fatal().
980 For now we expect a string that contains a float."""
983 except ValueError as e:
984 raise fatal('invalid date format (should be a float): %r' % e)
989 def parse_excludes(options, fatal):
990 """Traverse the options and extract all excludes, or call Option.fatal()."""
994 (option, parameter) = flag
995 if option == '--exclude':
996 excluded_paths.append(resolve_parent(parameter))
997 elif option == '--exclude-from':
999 f = open(resolve_parent(parameter))
1000 except IOError as e:
1001 raise fatal("couldn't read %s" % parameter)
1002 for exclude_path in f.readlines():
1003 # FIXME: perhaps this should be rstrip('\n')
1004 exclude_path = resolve_parent(exclude_path.strip())
1006 excluded_paths.append(exclude_path)
1007 return sorted(frozenset(excluded_paths))
1010 def parse_rx_excludes(options, fatal):
1011 """Traverse the options and extract all rx excludes, or call
1013 excluded_patterns = []
1015 for flag in options:
1016 (option, parameter) = flag
1017 if option == '--exclude-rx':
1019 excluded_patterns.append(re.compile(parameter))
1020 except re.error as ex:
1021 fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex))
1022 elif option == '--exclude-rx-from':
1024 f = open(resolve_parent(parameter))
1025 except IOError as e:
1026 raise fatal("couldn't read %s" % parameter)
1027 for pattern in f.readlines():
1028 spattern = pattern.rstrip('\n')
1032 excluded_patterns.append(re.compile(spattern))
1033 except re.error as ex:
1034 fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex))
1035 return excluded_patterns
1038 def should_rx_exclude_path(path, exclude_rxs):
1039 """Return True if path matches a regular expression in exclude_rxs."""
1040 for rx in exclude_rxs:
1042 debug1('Skipping %r: excluded by rx pattern %r.\n'
1043 % (path, rx.pattern))
1048 # FIXME: Carefully consider the use of functions (os.path.*, etc.)
1049 # that resolve against the current filesystem in the strip/graft
1050 # functions for example, but elsewhere as well. I suspect bup's not
1051 # always being careful about that. For some cases, the contents of
1052 # the current filesystem should be irrelevant, and consulting it might
1053 # produce the wrong result, perhaps via unintended symlink resolution,
1056 def path_components(path):
1057 """Break path into a list of pairs of the form (name,
1058 full_path_to_name). Path must start with '/'.
1060 '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
1061 if not path.startswith('/'):
1062 raise Exception('path must start with "/": %s' % path)
1063 # Since we assume path startswith('/'), we can skip the first element.
1064 result = [('', '/')]
1065 norm_path = os.path.abspath(path)
1066 if norm_path == '/':
1069 for p in norm_path.split('/')[1:]:
1070 full_path += '/' + p
1071 result.append((p, full_path))
1075 def stripped_path_components(path, strip_prefixes):
1076 """Strip any prefix in strip_prefixes from path and return a list
1077 of path components where each component is (name,
1078 none_or_full_fs_path_to_name). Assume path startswith('/').
1079 See thelpers.py for examples."""
1080 normalized_path = os.path.abspath(path)
1081 sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
1082 for bp in sorted_strip_prefixes:
1083 normalized_bp = os.path.abspath(bp)
1084 if normalized_bp == '/':
1086 if normalized_path.startswith(normalized_bp):
1087 prefix = normalized_path[:len(normalized_bp)]
1089 for p in normalized_path[len(normalized_bp):].split('/'):
1093 result.append((p, prefix))
1096 return path_components(path)
1099 def grafted_path_components(graft_points, path):
1100 # Create a result that consists of some number of faked graft
1101 # directories before the graft point, followed by all of the real
1102 # directories from path that are after the graft point. Arrange
1103 # for the directory at the graft point in the result to correspond
1104 # to the "orig" directory in --graft orig=new. See t/thelpers.py
1105 # for some examples.
1107 # Note that given --graft orig=new, orig and new have *nothing* to
1108 # do with each other, even if some of their component names
1109 # match. i.e. --graft /foo/bar/baz=/foo/bar/bax is semantically
1110 # equivalent to --graft /foo/bar/baz=/x/y/z, or even
1113 # FIXME: This can't be the best solution...
1114 clean_path = os.path.abspath(path)
1115 for graft_point in graft_points:
1116 old_prefix, new_prefix = graft_point
1117 # Expand prefixes iff not absolute paths.
1118 old_prefix = os.path.normpath(old_prefix)
1119 new_prefix = os.path.normpath(new_prefix)
1120 if clean_path.startswith(old_prefix):
1121 escaped_prefix = re.escape(old_prefix)
1122 grafted_path = re.sub(r'^' + escaped_prefix, new_prefix, clean_path)
1123 # Handle /foo=/ (at least) -- which produces //whatever.
1124 grafted_path = '/' + grafted_path.lstrip('/')
1125 clean_path_components = path_components(clean_path)
1126 # Count the components that were stripped.
1127 strip_count = 0 if old_prefix == '/' else old_prefix.count('/')
1128 new_prefix_parts = new_prefix.split('/')
1129 result_prefix = grafted_path.split('/')[:new_prefix.count('/')]
1130 result = [(p, None) for p in result_prefix] \
1131 + clean_path_components[strip_count:]
1132 # Now set the graft point name to match the end of new_prefix.
1133 graft_point = len(result_prefix)
1134 result[graft_point] = \
1135 (new_prefix_parts[-1], clean_path_components[strip_count][1])
1136 if new_prefix == '/': # --graft ...=/ is a special case.
1139 return path_components(clean_path)
1145 _localtime = getattr(_helpers, 'localtime', None)
1148 bup_time = namedtuple('bup_time', ['tm_year', 'tm_mon', 'tm_mday',
1149 'tm_hour', 'tm_min', 'tm_sec',
1150 'tm_wday', 'tm_yday',
1151 'tm_isdst', 'tm_gmtoff', 'tm_zone'])
1153 # Define a localtime() that returns bup_time when possible. Note:
1154 # this means that any helpers.localtime() results may need to be
1155 # passed through to_py_time() before being passed to python's time
1156 # module, which doesn't appear willing to ignore the extra items.
1158 def localtime(time):
1159 return bup_time(*_helpers.localtime(time))
1160 def utc_offset_str(t):
1161 """Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
1162 If the current UTC offset does not represent an integer number
1163 of minutes, the fractional component will be truncated."""
1164 off = localtime(t).tm_gmtoff
1165 # Note: // doesn't truncate like C for negative values, it rounds down.
1166 offmin = abs(off) // 60
1168 h = (offmin - m) // 60
1169 return "%+03d%02d" % (-h if off < 0 else h, m)
1171 if isinstance(x, time.struct_time):
1173 return time.struct_time(x[:9])
1175 localtime = time.localtime
1176 def utc_offset_str(t):
1177 return time.strftime('%z', localtime(t))
1182 _some_invalid_save_parts_rx = re.compile(r'[[ ~^:?*\\]|\.\.|//|@{')
1184 def valid_save_name(name):
1185 # Enforce a superset of the restrictions in git-check-ref-format(1)
1187 or name.startswith('/') or name.endswith('/') \
1188 or name.endswith('.'):
1190 if _some_invalid_save_parts_rx.search(name):
1193 if ord(c) < 0x20 or ord(c) == 0x7f:
1195 for part in name.split('/'):
1196 if part.startswith('.') or part.endswith('.lock'):
1201 _period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
1203 def period_as_secs(s):
1206 match = _period_rx.match(s)
1209 mag = int(match.group(1))
1210 scale = match.group(2)
1211 return mag * {'s': 1,
1215 'w': 60 * 60 * 24 * 7,
1216 'm': 60 * 60 * 24 * 31,
1217 'y': 60 * 60 * 24 * 366}[scale]