from ctypes import sizeof, c_void_p
from math import floor
from os import environ
-from pipes import quote
from subprocess import PIPE, Popen
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
import hashlib, heapq, math, operator, time, grp, tempfile
from bup import _helpers
from bup import compat
-from bup.compat import byte_int
-from bup.io import path_msg
+from bup.compat import argv_bytes, byte_int
+from bup.io import byte_stream, path_msg
# This function should really be in helpers, not in bup.options. But we
# want options.py to be standalone so people can include it in other projects.
from bup.options import _tty_width as tty_width
+buglvl = int(os.environ.get('BUP_DEBUG', 0))
+
+
class Nonlocal:
"""Helper to deal with Python scoping issues"""
pass
pass
return result
-
-def atoi(s):
- """Convert s (ascii bytes) to an integer. Return 0 if s is not a number."""
- try:
- return int(s or b'0')
- except ValueError:
- return 0
-
-
-def atof(s):
- """Convert s (ascii bytes) to a float. Return 0 if s is not a number."""
- try:
- return float(s or b'0')
- except ValueError:
- return 0
-
-
-buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
-
-
try:
_fdatasync = os.fdatasync
except AttributeError:
# sentinel must end with \n and must contain only one \n
while True:
line = f.readline()
- if not (line and line.endswith('\n')):
+ if not (line and line.endswith(b'\n')):
raise ex_type('Hit EOF while reading line')
if line == sentinel:
return
log(s)
-istty1 = os.isatty(1) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 1)
-istty2 = os.isatty(2) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 2)
+istty1 = os.isatty(1) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 1)
+istty2 = os.isatty(2) or (int(os.environ.get('BUP_FORCE_TTY', 0)) & 2)
_last_progress = ''
def progress(s):
"""Calls log() if stderr is a TTY. Does nothing otherwise."""
raise
+_bq_simple_id_rx = re.compile(br'^[-_./a-zA-Z0-9]+$')
+_sq_simple_id_rx = re.compile(r'^[-_./a-zA-Z0-9]+$')
+
+def bquote(x):
+ if x == b'':
+ return b"''"
+ if _bq_simple_id_rx.match(x):
+ return x
+ return b"'%s'" % x.replace(b"'", b"'\"'\"'")
+
+def squote(x):
+ if x == '':
+ return "''"
+ if _sq_simple_id_rx.match(x):
+ return x
+ return "'%s'" % x.replace("'", "'\"'\"'")
+
+def quote(x):
+ if isinstance(x, bytes):
+ return bquote(x)
+ if isinstance(x, compat.str_type):
+ return squote(x)
+ assert False
+
def shstr(cmd):
- if isinstance(cmd, compat.str_type):
+ """Return a shell quoted string for cmd if it's a sequence, else cmd.
+
+ cmd must be a string, bytes, or a sequence of one or the other,
+ and the assumption is that if cmd is a string or bytes, then it's
+ already quoted (because it's what's actually being passed to
+ call() and friends. e.g. log(shstr(cmd)); call(cmd)
+
+ """
+ if isinstance(cmd, (bytes, compat.str_type)):
return cmd
- else:
- return ' '.join(map(quote, cmd))
+ elif all(isinstance(x, bytes) for x in cmd):
+ return b' '.join(map(bquote, cmd))
+ elif all(isinstance(x, compat.str_type) for x in cmd):
+ return ' '.join(map(squote, cmd))
+ raise TypeError('unsupported shstr argument: ' + repr(cmd))
+
exc = subprocess.check_call
stderr=None,
shell=False,
check=True,
- preexec_fn=None):
+ preexec_fn=None,
+ close_fds=True):
if input:
assert stdin in (None, PIPE)
stdin = PIPE
p = Popen(cmd,
stdin=stdin, stdout=PIPE, stderr=stderr,
shell=shell,
- preexec_fn=preexec_fn)
+ preexec_fn=preexec_fn,
+ close_fds=close_fds)
out, err = p.communicate(input)
if check and p.returncode != 0:
- raise Exception('subprocess %r failed with status %d, stderr: %r'
- % (b' '.join(map(quote, cmd)), p.returncode, err))
+ raise Exception('subprocess %r failed with status %d%s'
+ % (b' '.join(map(quote, cmd)), p.returncode,
+ ', stderr: %r' % err if err else ''))
return out, err, p
def readpipe(argv, preexec_fn=None, shell=False):
"""Run a subprocess and return its output."""
- p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
- shell=shell)
- out, err = p.communicate()
- if p.returncode != 0:
- raise Exception('subprocess %r failed with status %d'
- % (b' '.join(argv), p.returncode))
- return out
+ return exo(argv, preexec_fn=preexec_fn, shell=shell)[0]
def _argmax_base(command):
"""Get the FQDN of this machine."""
global _hostname
if not _hostname:
- _hostname = socket.getfqdn().encode('iso-8859-1')
+ _hostname = _helpers.gethostname()
return _hostname
return "%d" % (size)
exponent = int(math.log(size) // math.log(unit))
size_prefix = "KMGTPE"[exponent - 1]
- return "%.1f%s" % (size // math.pow(unit, exponent), size_prefix)
+ return "%.1f%s" % (size / math.pow(unit, exponent), size_prefix)
class NotOk(Exception):
def close(self):
while self._read(65536): pass
+ def _read(self, size):
+ raise NotImplementedError("Subclasses must implement _read")
+
def read(self, size):
"""Read 'size' bytes from input stream."""
self.outp.flush()
return self._read(size)
+ def _readline(self, size):
+ raise NotImplementedError("Subclasses must implement _readline")
+
def readline(self):
"""Read from input stream until a newline is found."""
self.outp.flush()
def has_input(self):
"""Return true if input stream is readable."""
- raise NotImplemented("Subclasses must implement has_input")
+ raise NotImplementedError("Subclasses must implement has_input")
def ok(self):
"""Indicate end of output from last sent command."""
- self.write('\nok\n')
+ self.write(b'\nok\n')
def error(self, s):
"""Indicate server error to the client."""
- s = re.sub(r'\s+', ' ', str(s))
- self.write('\nerror %s\n' % s)
+ s = re.sub(br'\s+', b' ', s)
+ self.write(b'\nerror %s\n' % s)
def _check_ok(self, onempty):
self.outp.flush()
- rl = ''
+ rl = b''
for rl in linereader(self):
#log('%d got line: %r\n' % (os.getpid(), rl))
if not rl: # empty line
continue
- elif rl == 'ok':
+ elif rl == b'ok':
return None
- elif rl.startswith('error '):
+ elif rl.startswith(b'error '):
#log('client: error: %s\n' % rl[6:])
return NotOk(rl[6:])
else:
BaseConn.__init__(self, outp)
# Anything that comes through before the sync string was not
# multiplexed and can be assumed to be debug/log before mux init.
- tail = ''
- while tail != 'BUPMUX':
+ tail = b''
+ while tail != b'BUPMUX':
b = os.read(infd, (len(tail) < 6) and (6-len(tail)) or 1)
if not b:
raise IOError('demux: unexpected EOF during initialization')
tail += b
- sys.stderr.write(tail[:-6]) # pre-mux log messages
+ byte_stream(sys.stderr).write(tail[:-6]) # pre-mux log messages
tail = tail[-6:]
self.infd = infd
self.reader = None
rl, wl, xl = select.select([self.infd], [], [], timeout)
if not rl: return False
assert(rl[0] == self.infd)
- ns = ''.join(checked_reader(self.infd, 5))
+ ns = b''.join(checked_reader(self.infd, 5))
n, fdw = struct.unpack('!IB', ns)
assert(n <= MAX_PACKET)
if fdw == 1:
self.reader = checked_reader(self.infd, n)
elif fdw == 2:
for buf in checked_reader(self.infd, n):
- sys.stderr.write(buf)
+ byte_stream(sys.stderr).write(buf)
elif fdw == 3:
self.closed = True
debug2("DemuxConn: marked closed\n")
def _readline(self):
def find_eol(buf):
try:
- return buf.index('\n')+1
+ return buf.index(b'\n')+1
except ValueError:
return None
- return ''.join(self._read_parts(find_eol))
+ return b''.join(self._read_parts(find_eol))
def _read(self, size):
csize = [size]
return None
else:
return csize[0]
- return ''.join(self._read_parts(until_size))
+ return b''.join(self._read_parts(until_size))
def has_input(self):
return self._load_buf(0)
_set_fmincore_chunk_size()
pages_per_chunk = _fmincore_chunk_size // sc_page_size;
page_count = (st.st_size + sc_page_size - 1) // sc_page_size;
- chunk_count = page_count // _fmincore_chunk_size
- if chunk_count < 1:
- chunk_count = 1
+ chunk_count = (st.st_size + _fmincore_chunk_size - 1) // _fmincore_chunk_size
result = bytearray(page_count)
for ci in compat.range(chunk_count):
pos = _fmincore_chunk_size * ci;
return int(num*mult)
-def count(l):
- """Count the number of elements in an iterator. (consumes the iterator)"""
- return reduce(lambda x,y: x+1, l)
-
-
saved_errors = []
def add_error(e):
"""Append an error message to the list of saved errors.
The number of columns is determined automatically based on the string
lengths.
"""
+ binary = isinstance(prefix, bytes)
+ nothing = b'' if binary else ''
+ nl = b'\n' if binary else '\n'
if not l:
- return ""
+ return nothing
l = l[:]
clen = max(len(s) for s in l)
ncols = (tty_width() - len(prefix)) // (clen + 2)
clen = 0
cols = []
while len(l) % ncols:
- l.append('')
+ l.append(nothing)
rows = len(l) // ncols
for s in compat.range(0, len(l), rows):
cols.append(l[s:s+rows])
- out = ''
+ out = nothing
+ fmt = b'%-*s' if binary else '%-*s'
for row in zip(*cols):
- out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
+ out += prefix + nothing.join((fmt % (clen+2, s)) for s in row) + nl
return out
for flag in options:
(option, parameter) = flag
if option == '--exclude':
- excluded_paths.append(resolve_parent(parameter))
+ excluded_paths.append(resolve_parent(argv_bytes(parameter)))
elif option == '--exclude-from':
try:
- f = open(resolve_parent(parameter))
+ f = open(resolve_parent(argv_bytes(parameter)), 'rb')
except IOError as e:
- raise fatal("couldn't read %s" % parameter)
+ raise fatal("couldn't read %r" % parameter)
for exclude_path in f.readlines():
# FIXME: perhaps this should be rstrip('\n')
exclude_path = resolve_parent(exclude_path.strip())
(option, parameter) = flag
if option == '--exclude-rx':
try:
- excluded_patterns.append(re.compile(parameter))
+ excluded_patterns.append(re.compile(argv_bytes(parameter)))
except re.error as ex:
- fatal('invalid --exclude-rx pattern (%s): %s' % (parameter, ex))
+ fatal('invalid --exclude-rx pattern (%r): %s' % (parameter, ex))
elif option == '--exclude-rx-from':
try:
- f = open(resolve_parent(parameter))
+ f = open(resolve_parent(parameter), 'rb')
except IOError as e:
- raise fatal("couldn't read %s" % parameter)
+ raise fatal("couldn't read %r" % parameter)
for pattern in f.readlines():
- spattern = pattern.rstrip('\n')
+ spattern = pattern.rstrip(b'\n')
if not spattern:
continue
try:
excluded_patterns.append(re.compile(spattern))
except re.error as ex:
- fatal('invalid --exclude-rx pattern (%s): %s' % (spattern, ex))
+ fatal('invalid --exclude-rx pattern (%r): %s' % (spattern, ex))
return excluded_patterns
# module, which doesn't appear willing to ignore the extra items.
if _localtime:
def localtime(time):
- return bup_time(*_helpers.localtime(floor(time)))
+ return bup_time(*_helpers.localtime(int(floor(time))))
def utc_offset_str(t):
"""Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
If the current UTC offset does not represent an integer number
return True
-_period_rx = re.compile(r'^([0-9]+)(s|min|h|d|w|m|y)$')
+_period_rx = re.compile(br'^([0-9]+)(s|min|h|d|w|m|y)$')
def period_as_secs(s):
- if s == 'forever':
+ if s == b'forever':
return float('inf')
match = _period_rx.match(s)
if not match:
return None
mag = int(match.group(1))
scale = match.group(2)
- return mag * {'s': 1,
- 'min': 60,
- 'h': 60 * 60,
- 'd': 60 * 60 * 24,
- 'w': 60 * 60 * 24 * 7,
- 'm': 60 * 60 * 24 * 31,
- 'y': 60 * 60 * 24 * 366}[scale]
+ return mag * {b's': 1,
+ b'min': 60,
+ b'h': 60 * 60,
+ b'd': 60 * 60 * 24,
+ b'w': 60 * 60 * 24 * 7,
+ b'm': 60 * 60 * 24 * 31,
+ b'y': 60 * 60 * 24 * 366}[scale]
projects / bup.git / blobdiff