"""Helper functions and classes for bup."""
from collections import namedtuple
+from contextlib import contextmanager
from ctypes import sizeof, c_void_p
from os import environ
-from contextlib import contextmanager
+from pipes import quote
+from subprocess import PIPE, Popen
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
import hashlib, heapq, math, operator, time, grp, tempfile
from bup import _helpers
+from bup import compat
+# This function should really be in helpers, not in bup.options. But we
+# want options.py to be standalone so people can include it in other projects.
+from bup.options import _tty_width as tty_width
+
+
+class Nonlocal:
+ """Helper to deal with Python scoping issues"""
+ pass
+
sc_page_size = os.sysconf('SC_PAGE_SIZE')
assert(sc_page_size > 0)
if sc_arg_max == -1: # "no definite limit" - let's choose 2M
sc_arg_max = 2 * 1024 * 1024
-# This function should really be in helpers, not in bup.options. But we
-# want options.py to be standalone so people can include it in other projects.
-from bup.options import _tty_width
-tty_width = _tty_width
+
+def last(iterable):
+ result = None
+ for result in iterable:
+ pass
+ return result
def atoi(s):
fdatasync = _fdatasync
+def partition(predicate, stream):
+ """Returns (leading_matches_it, rest_it), where leading_matches_it
+ must be completely exhausted before traversing rest_it.
+
+ """
+ stream = iter(stream)
+ ns = Nonlocal()
+ ns.first_nonmatch = None
+ def leading_matches():
+ for x in stream:
+ if predicate(x):
+ yield x
+ else:
+ ns.first_nonmatch = (x,)
+ break
+ def rest():
+ if ns.first_nonmatch:
+ yield ns.first_nonmatch[0]
+ for x in stream:
+ yield x
+ return (leading_matches(), rest())
+
+
+def lines_until_sentinel(f, sentinel, ex_type):
+ # sentinel must end with \n and must contain only one \n
+ while True:
+ line = f.readline()
+ if not (line and line.endswith('\n')):
+ raise ex_type('Hit EOF while reading line')
+ if line == sentinel:
+ return
+ yield line
+
+
+def stat_if_exists(path):
+ try:
+ return os.stat(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ return None
+
+
# Write (blockingly) to sockets that may or may not be in blocking mode.
# We need this because our stderr is sometimes eaten by subprocesses
# (probably ssh) that sometimes make it nonblocking, if only temporarily,
raise
-_unspecified_next_default = object()
-
-def _fallback_next(it, default=_unspecified_next_default):
- """Retrieve the next item from the iterator by calling its
- next() method. If default is given, it is returned if the
- iterator is exhausted, otherwise StopIteration is raised."""
-
- if default is _unspecified_next_default:
- return it.next()
- else:
- try:
- return it.next()
- except StopIteration:
- return default
-
-if sys.version_info < (2, 6):
- next = _fallback_next
-
-
def merge_iter(iters, pfreq, pfunc, pfinal, key=None):
if key:
samekey = lambda e, pe: getattr(e, key) == getattr(pe, key, None)
yield e
count += 1
try:
- e = it.next() # Don't use next() function, it's too expensive
+ e = next(it)
except StopIteration:
heapq.heappop(heap) # remove current
else:
raise
+def shstr(cmd):
+ if isinstance(cmd, compat.str_type):
+ return cmd
+ else:
+ return ' '.join(map(quote, cmd))
+
+exc = subprocess.check_call
+
+def exo(cmd,
+ input=None,
+ stdin=None,
+ stderr=None,
+ shell=False,
+ check=True,
+ preexec_fn=None):
+ if input:
+ assert stdin in (None, PIPE)
+ stdin = PIPE
+ p = Popen(cmd,
+ stdin=stdin, stdout=PIPE, stderr=stderr,
+ shell=shell,
+ preexec_fn=preexec_fn)
+ out, err = p.communicate(input)
+ if check and p.returncode != 0:
+ raise Exception('subprocess %r failed with status %d, stderr: %r'
+ % (' '.join(map(quote, cmd)), p.returncode, err))
+ return out, err, p
+
def readpipe(argv, preexec_fn=None, shell=False):
"""Run a subprocess and return its output."""
p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
if not self._next_packet(timeout):
return False
try:
- self.buf = self.reader.next()
+ self.buf = next(self.reader)
return True
except StopIteration:
self.reader = None
# Perhaps the file was a pipe, i.e. "... | bup split ..."
return None
raise ex
- _mincore(m, msize, 0, result, ci * pages_per_chunk);
+ try:
+ _mincore(m, msize, 0, result, ci * pages_per_chunk)
+ except OSError as ex:
+ if ex.errno == errno.ENOSYS:
+ return None
+ raise
return result
Example:
'/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
if not path.startswith('/'):
- raise Exception, 'path must start with "/": %s' % path
+ raise Exception('path must start with "/": %s' % path)
# Since we assume path startswith('/'), we can skip the first element.
result = [('', '/')]
norm_path = os.path.abspath(path)
if part.startswith('.') or part.endswith('.lock'):
return False
return True
+
+
+_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+
+def period_as_secs(s):
+ if s == 'forever':
+ return float('inf')
+ match = _period_rx.match(s)
+ if not match:
+ return None
+ mag = int(match.group(1))
+ scale = match.group(2)
+ return mag * {'s': 1,
+ 'min': 60,
+ 'h': 60 * 60,
+ 'd': 60 * 60 * 24,
+ 'w': 60 * 60 * 24 * 7,
+ 'm': 60 * 60 * 24 * 31,
+ 'y': 60 * 60 * 24 * 366}[scale]
projects / bup.git / blobdiff