from collections import namedtuple
from contextlib import contextmanager
from ctypes import sizeof, c_void_p
+from math import floor
from os import environ
from pipes import quote
from subprocess import PIPE, Popen
from bup import _helpers
from bup import compat
+from bup.compat import byte_int
+from bup.io import path_msg
# This function should really be in helpers, not in bup.options. But we
# want options.py to be standalone so people can include it in other projects.
from bup.options import _tty_width as tty_width
def atoi(s):
- """Convert the string 's' to an integer. Return 0 if s is not a number."""
+ """Convert s (ascii bytes) to an integer. Return 0 if s is not a number."""
try:
- return int(s or '0')
+ return int(s or b'0')
except ValueError:
return 0
def atof(s):
- """Convert the string 's' to a float. Return 0 if s is not a number."""
+ """Convert s (ascii bytes) to a float. Return 0 if s is not a number."""
try:
- return float(s or '0')
+ return float(s or b'0')
except ValueError:
return 0
out, err = p.communicate(input)
if check and p.returncode != 0:
raise Exception('subprocess %r failed with status %d, stderr: %r'
- % (' '.join(map(quote, cmd)), p.returncode, err))
+ % (b' '.join(map(quote, cmd)), p.returncode, err))
return out, err, p
def readpipe(argv, preexec_fn=None, shell=False):
out, err = p.communicate()
if p.returncode != 0:
raise Exception('subprocess %r failed with status %d'
- % (' '.join(argv), p.returncode))
+ % (b' '.join(argv), p.returncode))
return out
"""Get the FQDN of this machine."""
global _hostname
if not _hostname:
- _hostname = socket.getfqdn()
+ _hostname = socket.getfqdn().encode('iso-8859-1')
return _hostname
def slashappend(s):
"""Append "/" to 's' if it doesn't aleady end in "/"."""
- if s and not s.endswith('/'):
- return s + '/'
+ assert isinstance(s, bytes)
+ if s and not s.endswith(b'/'):
+ return s + b'/'
else:
return s
def parse_num(s):
- """Parse data size information into a float number.
+ """Parse string or bytes as a possibly unit suffixed number.
- Here are some examples of conversions:
+ For example:
199.2k means 203981 bytes
1GB means 1073741824 bytes
2.1 tb means 2199023255552 bytes
"""
+ if isinstance(s, bytes):
+ # FIXME: should this raise a ValueError for UnicodeDecodeError
+ # (perhaps with the latter as the context).
+ s = s.decode('ascii')
g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
if not g:
raise ValueError("can't parse %r as a number" % s)
full_path_to_name). Path must start with '/'.
Example:
'/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
- if not path.startswith('/'):
- raise Exception('path must start with "/": %s' % path)
+ if not path.startswith(b'/'):
+ raise Exception('path must start with "/": %s' % path_msg(path))
# Since we assume path startswith('/'), we can skip the first element.
- result = [('', '/')]
+ result = [(b'', b'/')]
norm_path = os.path.abspath(path)
- if norm_path == '/':
+ if norm_path == b'/':
return result
- full_path = ''
- for p in norm_path.split('/')[1:]:
- full_path += '/' + p
+ full_path = b''
+ for p in norm_path.split(b'/')[1:]:
+ full_path += b'/' + p
result.append((p, full_path))
return result
sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
for bp in sorted_strip_prefixes:
normalized_bp = os.path.abspath(bp)
- if normalized_bp == '/':
+ if normalized_bp == b'/':
continue
if normalized_path.startswith(normalized_bp):
prefix = normalized_path[:len(normalized_bp)]
result = []
- for p in normalized_path[len(normalized_bp):].split('/'):
+ for p in normalized_path[len(normalized_bp):].split(b'/'):
if p: # not root
- prefix += '/'
+ prefix += b'/'
prefix += p
result.append((p, prefix))
return result
new_prefix = os.path.normpath(new_prefix)
if clean_path.startswith(old_prefix):
escaped_prefix = re.escape(old_prefix)
- grafted_path = re.sub(r'^' + escaped_prefix, new_prefix, clean_path)
+ grafted_path = re.sub(br'^' + escaped_prefix, new_prefix, clean_path)
# Handle /foo=/ (at least) -- which produces //whatever.
- grafted_path = '/' + grafted_path.lstrip('/')
+ grafted_path = b'/' + grafted_path.lstrip(b'/')
clean_path_components = path_components(clean_path)
# Count the components that were stripped.
- strip_count = 0 if old_prefix == '/' else old_prefix.count('/')
- new_prefix_parts = new_prefix.split('/')
- result_prefix = grafted_path.split('/')[:new_prefix.count('/')]
+ strip_count = 0 if old_prefix == b'/' else old_prefix.count(b'/')
+ new_prefix_parts = new_prefix.split(b'/')
+ result_prefix = grafted_path.split(b'/')[:new_prefix.count(b'/')]
result = [(p, None) for p in result_prefix] \
+ clean_path_components[strip_count:]
# Now set the graft point name to match the end of new_prefix.
graft_point = len(result_prefix)
result[graft_point] = \
(new_prefix_parts[-1], clean_path_components[strip_count][1])
- if new_prefix == '/': # --graft ...=/ is a special case.
+ if new_prefix == b'/': # --graft ...=/ is a special case.
return result[1:]
return result
return path_components(clean_path)
# module, which doesn't appear willing to ignore the extra items.
if _localtime:
def localtime(time):
- return bup_time(*_helpers.localtime(time))
+ return bup_time(*_helpers.localtime(floor(time)))
def utc_offset_str(t):
"""Return the local offset from UTC as "+hhmm" or "-hhmm" for time t.
If the current UTC offset does not represent an integer number
offmin = abs(off) // 60
m = offmin % 60
h = (offmin - m) // 60
- return "%+03d%02d" % (-h if off < 0 else h, m)
+ return b'%+03d%02d' % (-h if off < 0 else h, m)
def to_py_time(x):
if isinstance(x, time.struct_time):
return x
else:
localtime = time.localtime
def utc_offset_str(t):
- return time.strftime('%z', localtime(t))
+ return time.strftime(b'%z', localtime(t))
def to_py_time(x):
return x
-_some_invalid_save_parts_rx = re.compile(r'[\[ ~^:?*\\]|\.\.|//|@{')
+_some_invalid_save_parts_rx = re.compile(br'[\[ ~^:?*\\]|\.\.|//|@{')
def valid_save_name(name):
# Enforce a superset of the restrictions in git-check-ref-format(1)
- if name == '@' \
- or name.startswith('/') or name.endswith('/') \
- or name.endswith('.'):
+ if name == b'@' \
+ or name.startswith(b'/') or name.endswith(b'/') \
+ or name.endswith(b'.'):
return False
if _some_invalid_save_parts_rx.search(name):
return False
for c in name:
- if ord(c) < 0x20 or ord(c) == 0x7f:
+ if byte_int(c) < 0x20 or byte_int(c) == 0x7f:
return False
- for part in name.split('/'):
- if part.startswith('.') or part.endswith('.lock'):
+ for part in name.split(b'/'):
+ if part.startswith(b'.') or part.endswith(b'.lock'):
return False
return True
from __future__ import absolute_import
from time import tzset
-import helpers, math, os, os.path, stat, subprocess
+import helpers, math, os, os.path, re, subprocess
from wvtest import *
-from bup.compat import environ
+from bup.compat import bytes_from_byte, bytes_from_uint, environ
from bup.helpers import (atomically_replaced_file, batchpipe, detect_fakeroot,
grafted_path_components, mkdirp, parse_num,
path_components, readpipe, stripped_path_components,
import bup._helpers as _helpers
-bup_tmp = os.path.realpath('../../../t/tmp')
+bup_tmp = os.path.realpath(b'../../../t/tmp')
mkdirp(bup_tmp)
def test_parse_num():
with no_lingering_errors():
pn = parse_num
+ WVPASSEQ(pn(b'1'), 1)
WVPASSEQ(pn('1'), 1)
WVPASSEQ(pn('0'), 0)
WVPASSEQ(pn('1.5k'), 1536)
@wvtest
def test_detect_fakeroot():
with no_lingering_errors():
- if os.getenv('FAKEROOTKEY'):
+ if b'FAKEROOTKEY' in environ:
WVPASS(detect_fakeroot())
else:
WVPASS(not detect_fakeroot())
@wvtest
def test_path_components():
with no_lingering_errors():
- WVPASSEQ(path_components('/'), [('', '/')])
- WVPASSEQ(path_components('/foo'), [('', '/'), ('foo', '/foo')])
- WVPASSEQ(path_components('/foo/'), [('', '/'), ('foo', '/foo')])
- WVPASSEQ(path_components('/foo/bar'),
- [('', '/'), ('foo', '/foo'), ('bar', '/foo/bar')])
- WVEXCEPT(Exception, path_components, 'foo')
+ WVPASSEQ(path_components(b'/'), [(b'', b'/')])
+ WVPASSEQ(path_components(b'/foo'), [(b'', b'/'), (b'foo', b'/foo')])
+ WVPASSEQ(path_components(b'/foo/'), [(b'', b'/'), (b'foo', b'/foo')])
+ WVPASSEQ(path_components(b'/foo/bar'),
+ [(b'', b'/'), (b'foo', b'/foo'), (b'bar', b'/foo/bar')])
+ WVEXCEPT(Exception, path_components, b'foo')
@wvtest
def test_stripped_path_components():
with no_lingering_errors():
- WVPASSEQ(stripped_path_components('/', []), [('', '/')])
- WVPASSEQ(stripped_path_components('/', ['']), [('', '/')])
- WVPASSEQ(stripped_path_components('/', ['/']), [('', '/')])
- WVPASSEQ(stripped_path_components('/foo', ['/']),
- [('', '/'), ('foo', '/foo')])
- WVPASSEQ(stripped_path_components('/', ['/foo']), [('', '/')])
- WVPASSEQ(stripped_path_components('/foo', ['/bar']),
- [('', '/'), ('foo', '/foo')])
- WVPASSEQ(stripped_path_components('/foo', ['/foo']), [('', '/foo')])
- WVPASSEQ(stripped_path_components('/foo/bar', ['/foo']),
- [('', '/foo'), ('bar', '/foo/bar')])
- WVPASSEQ(stripped_path_components('/foo/bar', ['/bar', '/foo', '/baz']),
- [('', '/foo'), ('bar', '/foo/bar')])
- WVPASSEQ(stripped_path_components('/foo/bar/baz', ['/foo/bar/baz']),
- [('', '/foo/bar/baz')])
- WVEXCEPT(Exception, stripped_path_components, 'foo', [])
+ WVPASSEQ(stripped_path_components(b'/', []), [(b'', b'/')])
+ WVPASSEQ(stripped_path_components(b'/', [b'']), [(b'', b'/')])
+ WVPASSEQ(stripped_path_components(b'/', [b'/']), [(b'', b'/')])
+ WVPASSEQ(stripped_path_components(b'/foo', [b'/']),
+ [(b'', b'/'), (b'foo', b'/foo')])
+ WVPASSEQ(stripped_path_components(b'/', [b'/foo']), [(b'', b'/')])
+ WVPASSEQ(stripped_path_components(b'/foo', [b'/bar']),
+ [(b'', b'/'), (b'foo', b'/foo')])
+ WVPASSEQ(stripped_path_components(b'/foo', [b'/foo']), [(b'', b'/foo')])
+ WVPASSEQ(stripped_path_components(b'/foo/bar', [b'/foo']),
+ [(b'', b'/foo'), (b'bar', b'/foo/bar')])
+ WVPASSEQ(stripped_path_components(b'/foo/bar', [b'/bar', b'/foo', b'/baz']),
+ [(b'', b'/foo'), (b'bar', b'/foo/bar')])
+ WVPASSEQ(stripped_path_components(b'/foo/bar/baz', [b'/foo/bar/baz']),
+ [(b'', b'/foo/bar/baz')])
+ WVEXCEPT(Exception, stripped_path_components, b'foo', [])
@wvtest
def test_grafted_path_components():
with no_lingering_errors():
- WVPASSEQ(grafted_path_components([('/chroot', '/')], '/foo'),
- [('', '/'), ('foo', '/foo')])
- WVPASSEQ(grafted_path_components([('/foo/bar', '/')],
- '/foo/bar/baz/bax'),
- [('', '/foo/bar'),
- ('baz', '/foo/bar/baz'),
- ('bax', '/foo/bar/baz/bax')])
- WVPASSEQ(grafted_path_components([('/foo/bar/baz', '/bax')],
- '/foo/bar/baz/1/2'),
- [('', None),
- ('bax', '/foo/bar/baz'),
- ('1', '/foo/bar/baz/1'),
- ('2', '/foo/bar/baz/1/2')])
- WVPASSEQ(grafted_path_components([('/foo', '/bar/baz/bax')],
- '/foo/bar'),
- [('', None),
- ('bar', None),
- ('baz', None),
- ('bax', '/foo'),
- ('bar', '/foo/bar')])
- WVPASSEQ(grafted_path_components([('/foo/bar/baz', '/a/b/c')],
- '/foo/bar/baz'),
- [('', None), ('a', None), ('b', None), ('c', '/foo/bar/baz')])
- WVPASSEQ(grafted_path_components([('/', '/a/b/c/')], '/foo/bar'),
- [('', None), ('a', None), ('b', None), ('c', '/'),
- ('foo', '/foo'), ('bar', '/foo/bar')])
- WVEXCEPT(Exception, grafted_path_components, 'foo', [])
+ WVPASSEQ(grafted_path_components([(b'/chroot', b'/')], b'/foo'),
+ [(b'', b'/'), (b'foo', b'/foo')])
+ WVPASSEQ(grafted_path_components([(b'/foo/bar', b'/')],
+ b'/foo/bar/baz/bax'),
+ [(b'', b'/foo/bar'),
+ (b'baz', b'/foo/bar/baz'),
+ (b'bax', b'/foo/bar/baz/bax')])
+ WVPASSEQ(grafted_path_components([(b'/foo/bar/baz', b'/bax')],
+ b'/foo/bar/baz/1/2'),
+ [(b'', None),
+ (b'bax', b'/foo/bar/baz'),
+ (b'1', b'/foo/bar/baz/1'),
+ (b'2', b'/foo/bar/baz/1/2')])
+ WVPASSEQ(grafted_path_components([(b'/foo', b'/bar/baz/bax')],
+ b'/foo/bar'),
+ [(b'', None),
+ (b'bar', None),
+ (b'baz', None),
+ (b'bax', b'/foo'),
+ (b'bar', b'/foo/bar')])
+ WVPASSEQ(grafted_path_components([(b'/foo/bar/baz', b'/a/b/c')],
+ b'/foo/bar/baz'),
+ [(b'', None), (b'a', None), (b'b', None), (b'c', b'/foo/bar/baz')])
+ WVPASSEQ(grafted_path_components([(b'/', b'/a/b/c/')], b'/foo/bar'),
+ [(b'', None), (b'a', None), (b'b', None), (b'c', b'/'),
+ (b'foo', b'/foo'), (b'bar', b'/foo/bar')])
+ WVEXCEPT(Exception, grafted_path_components, b'foo', [])
@wvtest
def test_readpipe():
with no_lingering_errors():
- x = readpipe(['echo', '42'])
+ x = readpipe([b'echo', b'42'])
WVPASSEQ(x, b'42\n')
try:
- readpipe(['bash', '-c', 'exit 42'])
+ readpipe([b'bash', b'-c', b'exit 42'])
except Exception as ex:
- WVPASSEQ(str(ex),
- "subprocess 'bash -c exit 42' failed with status 42")
+ if not re.match("^subprocess b?'bash -c exit 42' failed with status 42$",
+ str(ex)):
+ WVPASSEQ(str(ex),
+ "^subprocess b?'bash -c exit 42' failed with status 42$")
+
@wvtest
def test_batchpipe():
with no_lingering_errors():
- for chunk in batchpipe(['echo'], []):
+ for chunk in batchpipe([b'echo'], []):
WVPASS(False)
out = b''
- for chunk in batchpipe(['echo'], ['42']):
+ for chunk in batchpipe([b'echo'], [b'42']):
out += chunk
WVPASSEQ(out, b'42\n')
try:
- batchpipe(['bash', '-c'], ['exit 42'])
+ batchpipe([b'bash', b'-c'], [b'exit 42'])
except Exception as ex:
WVPASSEQ(str(ex),
"subprocess 'bash -c exit 42' failed with status 42")
# Force batchpipe to break the args into batches of 3. This
# approach assumes all args are the same length.
arg_max = \
- helpers._argmax_base(['echo']) + helpers._argmax_args_size(args[:3])
+ helpers._argmax_base([b'echo']) + helpers._argmax_args_size(args[:3])
batches = batchpipe(['echo'], args, arg_max=arg_max)
WVPASSEQ(next(batches), b'0 1 2\n')
WVPASSEQ(next(batches), b'3 4 5\n')
WVPASSEQ(next(batches, None), None)
- batches = batchpipe(['echo'], [str(x) for x in range(5)], arg_max=arg_max)
+ batches = batchpipe([b'echo'], [str(x) for x in range(5)], arg_max=arg_max)
WVPASSEQ(next(batches), b'0 1 2\n')
WVPASSEQ(next(batches), b'3 4\n')
WVPASSEQ(next(batches, None), None)
def test_valid_save_name():
with no_lingering_errors():
valid = helpers.valid_save_name
- WVPASS(valid('x'))
- WVPASS(valid('x@'))
- WVFAIL(valid('@'))
- WVFAIL(valid('/'))
- WVFAIL(valid('/foo'))
- WVFAIL(valid('foo/'))
- WVFAIL(valid('/foo/'))
- WVFAIL(valid('foo//bar'))
- WVFAIL(valid('.'))
- WVFAIL(valid('bar.'))
- WVFAIL(valid('foo@{'))
- for x in ' ~^:?*[\\':
- WVFAIL(valid('foo' + x))
+ WVPASS(valid(b'x'))
+ WVPASS(valid(b'x@'))
+ WVFAIL(valid(b'@'))
+ WVFAIL(valid(b'/'))
+ WVFAIL(valid(b'/foo'))
+ WVFAIL(valid(b'foo/'))
+ WVFAIL(valid(b'/foo/'))
+ WVFAIL(valid(b'foo//bar'))
+ WVFAIL(valid(b'.'))
+ WVFAIL(valid(b'bar.'))
+ WVFAIL(valid(b'foo@{'))
+ for x in b' ~^:?*[\\':
+ WVFAIL(valid(b'foo' + bytes_from_byte(x)))
for i in range(20):
- WVFAIL(valid('foo' + chr(i)))
- WVFAIL(valid('foo' + chr(0x7f)))
- WVFAIL(valid('foo..bar'))
- WVFAIL(valid('bar.lock/baz'))
- WVFAIL(valid('foo/bar.lock/baz'))
- WVFAIL(valid('.bar/baz'))
- WVFAIL(valid('foo/.bar/baz'))
+ WVFAIL(valid(b'foo' + bytes_from_uint(i)))
+ WVFAIL(valid(b'foo' + bytes_from_uint(0x7f)))
+ WVFAIL(valid(b'foo..bar'))
+ WVFAIL(valid(b'bar.lock/baz'))
+ WVFAIL(valid(b'foo/bar.lock/baz'))
+ WVFAIL(valid(b'.bar/baz'))
+ WVFAIL(valid(b'foo/.bar/baz'))
projects / bup.git / commitdiff