"""Helper functions and classes for bup."""
+from ctypes import sizeof, c_void_p
+from os import environ
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
-import hashlib, heapq, operator, time, platform, grp
+import config, hashlib, heapq, operator, time, grp
+
from bup import _version, _helpers
import bup._helpers as _helpers
+import math
# This function should really be in helpers, not in bup.options. But we
# want options.py to be standalone so people can include it in other projects.
buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
+# If the platform doesn't have fdatasync (OS X), fall back to fsync.
+try:
+ fdatasync = os.fdatasync
+except AttributeError:
+ fdatasync = os.fsync
+
+
# Write (blockingly) to sockets that may or may not be in blocking mode.
# We need this because our stderr is sometimes eaten by subprocesses
# (probably ssh) that sometimes make it nonblocking, if only temporarily,
raise
-def next(it):
- """Get the next item from an iterator, None if we reached the end."""
- try:
+_unspecified_next_default = object()
+
+def _fallback_next(it, default=_unspecified_next_default):
+ """Retrieve the next item from the iterator by calling its
+ next() method. If default is given, it is returned if the
+ iterator is exhausted, otherwise StopIteration is raised."""
+
+ if default is _unspecified_next_default:
return it.next()
- except StopIteration:
- return None
+ else:
+ try:
+ return it.next()
+ except StopIteration:
+ return default
+
+if sys.version_info < (2, 6):
+ next = _fallback_next
def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
count = 0
total = sum(len(it) for it in iters)
iters = (iter(it) for it in iters)
- heap = ((next(it),it) for it in iters)
+ heap = ((next(it, None),it) for it in iters)
heap = [(e,it) for e,it in heap if e]
heapq.heapify(heap)
pass # it doesn't exist, that's what you asked for
-def readpipe(argv):
+def readpipe(argv, preexec_fn=None):
"""Run a subprocess and return its output."""
- p = subprocess.Popen(argv, stdout=subprocess.PIPE)
- r = p.stdout.read()
- p.wait()
- return r
+ p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn)
+ out, err = p.communicate()
+ if p.returncode != 0:
+ raise Exception('subprocess %r failed with status %d'
+ % (' '.join(argv), p.returncode))
+ return out
+
+
+def _argmax_base(command):
+ base_size = 2048
+ for c in command:
+ base_size += len(command) + 1
+ for k, v in environ.iteritems():
+ base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
+ return base_size
+
+
+def _argmax_args_size(args):
+ return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
+
+
+def batchpipe(command, args, preexec_fn=None):
+ """If args is not empty, yield the output produced by calling the
+command list with args as a sequence of strings (It may be necessary
+to return multiple strings in order to respect ARG_MAX)."""
+ base_size = _argmax_base(command)
+ while args:
+ room = config.arg_max - base_size
+ i = 0
+ while i < len(args):
+ next_size = _argmax_args_size(args[i:i+1])
+ if room - next_size < 0:
+ break
+ room -= next_size
+ i += 1
+ sub_args = args[:i]
+ args = args[i:]
+ assert(len(sub_args))
+ yield readpipe(command + sub_args, preexec_fn=preexec_fn)
def realpath(p):
def is_superuser():
- if platform.system().startswith('CYGWIN'):
+ if sys.platform.startswith('cygwin'):
import ctypes
return ctypes.cdll.shell32.IsUserAnAdmin()
else:
_resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
return os.path.join(_resource_path, subdir)
+def format_filesize(size):
+ unit = 1024.0
+ size = float(size)
+ if size < unit:
+ return "%d" % (size)
+ exponent = int(math.log(size) / math.log(unit))
+ size_prefix = "KMGTPE"[exponent - 1]
+ return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
+
class NotOk(Exception):
pass
close)
+def parse_timestamp(epoch_str):
+ """Return the number of nanoseconds since the epoch that are described
+by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed,
+throw a ValueError that may contain additional information."""
+ ns_per = {'s' : 1000000000,
+ 'ms' : 1000000,
+ 'us' : 1000,
+ 'ns' : 1}
+ match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str)
+ if not match:
+ if re.match(r'^([-+]?[0-9]+)$', epoch_str):
+ raise ValueError('must include units, i.e. 100ns, 100ms, ...')
+ raise ValueError()
+ (n, units) = match.group(1, 2)
+ if not n:
+ n = 1
+ n = int(n)
+ return n * ns_per[units]
+
+
def parse_num(s):
"""Parse data size information into a float number.
oldhook = sys.excepthook
def newhook(exctype, value, traceback):
if exctype == KeyboardInterrupt:
- log('Interrupted.\n')
+ log('\nInterrupted.\n')
else:
return oldhook(exctype, value, traceback)
sys.excepthook = newhook
raise fatal("couldn't read %s" % parameter)
for exclude_path in f.readlines():
excluded_paths.append(realpath(exclude_path.strip()))
- return excluded_paths
+ return sorted(frozenset(excluded_paths))
+
+
+def parse_rx_excludes(options, fatal):
+ """Traverse the options and extract all rx excludes, or call
+ Option.fatal()."""
+ excluded_patterns = []
+
+ for flag in options:
+ (option, parameter) = flag
+ if option == '--exclude-rx':
+ try:
+ excluded_patterns.append(re.compile(parameter))
+ except re.error, ex:
+ fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex))
+ elif option == '--exclude-rx-from':
+ try:
+ f = open(realpath(parameter))
+ except IOError, e:
+ raise fatal("couldn't read %s" % parameter)
+ for pattern in f.readlines():
+ spattern = pattern.rstrip('\n')
+ try:
+ excluded_patterns.append(re.compile(spattern))
+ except re.error, ex:
+ fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex))
+ return excluded_patterns
+
+
+def should_rx_exclude_path(path, exclude_rxs):
+ """Return True if path matches a regular expression in exclude_rxs."""
+ for rx in exclude_rxs:
+ if rx.search(path):
+ debug1('Skipping %r: excluded by rx pattern %r.\n'
+ % (path, rx.pattern))
+ return True
+ return False
# FIXME: Carefully consider the use of functions (os.path.*, etc.)
full_path_to_name). Path must start with '/'.
Example:
'/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
- assert(path.startswith('/'))
+ if not path.startswith('/'):
+ raise Exception, 'path must start with "/": %s' % path
# Since we assume path startswith('/'), we can skip the first element.
result = [('', '/')]
norm_path = os.path.abspath(path)
projects / bup.git / blobdiff