#
# This code is covered under the terms of the GNU Library General
# Public License as described in the bup LICENSE file.
-import errno, os, sys, stat, time, pwd, grp, socket
-from cStringIO import StringIO
+
+from __future__ import absolute_import, print_function
+from copy import deepcopy
+from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
+from io import BytesIO
+from time import gmtime, strftime
+import errno, os, sys, stat, time, pwd, grp, socket, struct
+
from bup import vint, xstat
from bup.drecurse import recursive_dirlist
-from bup.helpers import add_error, mkdirp, log, is_superuser
+from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize
from bup.helpers import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name
from bup.xstat import utime, lutime
xattr = None
posix1e = None
-if not (sys.platform.startswith('cygwin') or sys.platform.startswith('darwin')):
+if not (sys.platform.startswith('cygwin') \
+ or sys.platform.startswith('darwin') \
+ or sys.platform.startswith('netbsd')):
try:
import posix1e
except ImportError:
# not on Linux, in which case files don't have any linux attrs anyway, so
# lacking the functions isn't a problem.
get_linux_file_attr = set_linux_file_attr = None
-
+
+
+# See the bup_get_linux_file_attr() comments.
+_suppress_linux_file_attr = \
+ sys.byteorder == 'big' and struct.calcsize('@l') > struct.calcsize('@i')
+
+def check_linux_file_attr_api():
+ global get_linux_file_attr, set_linux_file_attr
+ if not (get_linux_file_attr or set_linux_file_attr):
+ return
+ if _suppress_linux_file_attr:
+ log('Warning: Linux attr support disabled (see "bup help index").\n')
+ get_linux_file_attr = set_linux_file_attr = None
+
# WARNING: the metadata encoding is *not* stable yet. Caveat emptor!
# FIXME: Add nfsv4 acl handling - see nfs4-acl-tools.
# FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.).
# FIXME: Consider pack('vvvvsss', ...) optimization.
-# FIXME: Consider caching users/groups.
## FS notes:
#
_rec_tag_hardlink_target = 8 # hard link target path
_rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current)
+_warned_about_attr_einval = None
+
class ApplyError(Exception):
# Thrown when unable to apply any given bit of metadata to a path.
# must be non-negative and < 10**9.
def _add_common(self, path, st):
+ assert(st.st_uid >= 0)
+ assert(st.st_gid >= 0)
self.uid = st.st_uid
self.gid = st.st_gid
self.atime = st.st_atime
st = None
try:
st = xstat.lstat(path)
- except OSError, e:
+ except OSError as e:
if e.errno != errno.ENOENT:
raise
if st:
if stat.S_ISDIR(st.st_mode):
try:
os.rmdir(path)
- except OSError, e:
+ except OSError as e:
if e.errno in (errno.ENOTEMPTY, errno.EEXIST):
msg = 'refusing to overwrite non-empty dir ' + path
raise Exception(msg)
if stat.S_ISREG(self.mode):
assert(self._recognized_file_type())
- fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0600)
+ fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0o600)
os.close(fd)
elif stat.S_ISDIR(self.mode):
assert(self._recognized_file_type())
- os.mkdir(path, 0700)
+ os.mkdir(path, 0o700)
elif stat.S_ISCHR(self.mode):
assert(self._recognized_file_type())
- os.mknod(path, 0600 | stat.S_IFCHR, self.rdev)
+ os.mknod(path, 0o600 | stat.S_IFCHR, self.rdev)
elif stat.S_ISBLK(self.mode):
assert(self._recognized_file_type())
- os.mknod(path, 0600 | stat.S_IFBLK, self.rdev)
+ os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev)
elif stat.S_ISFIFO(self.mode):
assert(self._recognized_file_type())
- os.mknod(path, 0600 | stat.S_IFIFO)
+ os.mknod(path, 0o600 | stat.S_IFIFO)
elif stat.S_ISSOCK(self.mode):
try:
- os.mknod(path, 0600 | stat.S_IFSOCK)
- except OSError, e:
+ os.mknod(path, 0o600 | stat.S_IFSOCK)
+ except OSError as e:
if e.errno in (errno.EINVAL, errno.EPERM):
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.bind(path)
# on MacOS, symlink() permissions depend on umask, and there's
# no way to chown a symlink after creating it, so we have to
# be careful here!
- oldumask = os.umask((self.mode & 0777) ^ 0777)
+ oldumask = os.umask((self.mode & 0o777) ^ 0o777)
try:
os.symlink(self.symlink_target, path)
finally:
if lutime and stat.S_ISLNK(self.mode):
try:
lutime(path, (self.atime, self.mtime))
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EACCES:
raise ApplyError('lutime: %s' % e)
else:
else:
try:
utime(path, (self.atime, self.mtime))
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EACCES:
raise ApplyError('utime: %s' % e)
else:
raise
- # Implement tar/rsync-like semantics; see bup-restore(1).
- # FIXME: should we consider caching user/group name <-> id
- # mappings, getgroups(), etc.?
uid = gid = -1 # By default, do nothing.
if is_superuser():
uid = self.uid
if uid != -1 or gid != -1:
try:
os.lchown(path, uid, gid)
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EPERM:
add_error('lchown: %s' % e)
elif sys.platform.startswith('cygwin') \
raise
if _have_lchmod:
- os.lchmod(path, stat.S_IMODE(self.mode))
+ try:
+ os.lchmod(path, stat.S_IMODE(self.mode))
+ except errno.ENOSYS: # Function not implemented
+ pass
elif not stat.S_ISLNK(self.mode):
os.chmod(path, stat.S_IMODE(self.mode))
try:
if stat.S_ISLNK(st.st_mode):
self.symlink_target = os.readlink(path)
- except OSError, e:
- add_error('readlink: %s', e)
+ except OSError as e:
+ add_error('readlink: %s' % e)
def _encode_symlink_target(self):
return self.symlink_target
def _load_symlink_target_rec(self, port):
- self.symlink_target = vint.read_bvec(port)
+ target = vint.read_bvec(port)
+ self.symlink_target = target
+ self.size = len(target)
## Hardlink targets
# The numeric/text distinction only matters when reading/restoring
# a stored record.
def _add_posix1e_acl(self, path, st):
- if not posix1e: return
+ if not posix1e or not posix1e.HAS_EXTENDED_CHECK:
+ return
if not stat.S_ISLNK(st.st_mode):
acls = None
def_acls = None
if stat.S_ISDIR(st.st_mode):
def_acl = posix1e.ACL(filedef=path)
def_acls = [def_acl, def_acl]
- except EnvironmentError, e:
+ except EnvironmentError as e:
if e.errno not in (errno.EOPNOTSUPP, errno.ENOSYS):
raise
if acls:
def apply_acl(acl_rep, kind):
try:
acl = posix1e.ACL(text = acl_rep)
- except IOError, e:
+ except IOError as e:
if e.errno == 0:
# pylibacl appears to return an IOError with errno
# set to 0 if a group referred to by the ACL rep
raise
try:
acl.applyto(path, kind)
- except IOError, e:
+ except IOError as e:
if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP:
raise ApplyError('POSIX1e ACL applyto: %s' % e)
else:
## Linux attributes (lsattr(1), chattr(1))
def _add_linux_attr(self, path, st):
+ check_linux_file_attr_api()
if not get_linux_file_attr: return
if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
try:
attr = get_linux_file_attr(path)
if attr != 0:
self.linux_attr = attr
- except OSError, e:
+ except OSError as e:
if e.errno == errno.EACCES:
add_error('read Linux attr: %s' % e)
- elif e.errno in (errno.ENOTTY, errno.ENOSYS, errno.EOPNOTSUPP):
+ elif e.errno in (ENOTTY, ENOSYS, EOPNOTSUPP):
# Assume filesystem doesn't support attrs.
return
+ elif e.errno == EINVAL:
+ global _warned_about_attr_einval
+ if not _warned_about_attr_einval:
+ log("Ignoring attr EINVAL;"
+ + " if you're not using ntfs-3g, please report: "
+ + repr(path) + '\n')
+ _warned_about_attr_einval = True
+ return
else:
raise
def _apply_linux_attr_rec(self, path, restore_numeric_ids=False):
if self.linux_attr:
+ check_linux_file_attr_api()
if not set_linux_file_attr:
add_error("%s: can't restore linuxattrs: "
"linuxattr support missing.\n" % path)
return
try:
set_linux_file_attr(path, self.linux_attr)
- except OSError, e:
- if e.errno in (errno.ENOTTY, errno.EOPNOTSUPP, errno.ENOSYS,
- errno.EACCES):
+ except OSError as e:
+ if e.errno in (EACCES, ENOTTY, EOPNOTSUPP, ENOSYS):
raise ApplyError('Linux chattr: %s (0x%s)'
% (e, hex(self.linux_attr)))
+ elif e.errno == EINVAL:
+ msg = "if you're not using ntfs-3g, please report"
+ raise ApplyError('Linux chattr: %s (0x%s) (%s)'
+ % (e, hex(self.linux_attr), msg))
else:
raise
if not xattr: return
try:
self.linux_xattr = xattr.get_all(path, nofollow=True)
- except EnvironmentError, e:
+ except EnvironmentError as e:
if e.errno != errno.EOPNOTSUPP:
raise
def _load_linux_xattr_rec(self, file):
data = vint.read_bvec(file)
- memfile = StringIO(data)
+ memfile = BytesIO(data)
result = []
for i in range(vint.read_vuint(memfile)):
key = vint.read_bvec(memfile)
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
- except IOError, e:
+ except IOError as e:
if e.errno == errno.EACCES:
- raise ApplyError('xattr.set: %s' % e)
+ raise ApplyError('xattr.set %r: %s' % (path, e))
else:
raise
for k, v in self.linux_xattr:
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
- except IOError, e:
+ except IOError as e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
- raise ApplyError('xattr.set: %s' % e)
+ raise ApplyError('xattr.set %r: %s' % (path, e))
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
- except IOError, e:
- if e.errno == errno.EPERM:
- raise ApplyError('xattr.remove: %s' % e)
+ except IOError as e:
+ if e.errno in (errno.EPERM, errno.EACCES):
+ raise ApplyError('xattr.remove %r: %s' % (path, e))
else:
raise
def __init__(self):
- self.mode = None
+ self.mode = self.uid = self.gid = self.user = self.group = None
+ self.atime = self.mtime = self.ctime = None
# optional members
self.path = None
self.size = None
self.linux_xattr = None
self.posix1e_acl = None
+ def __eq__(self, other):
+ if not isinstance(other, Metadata): return False
+ if self.mode != other.mode: return False
+ if self.mtime != other.mtime: return False
+ if self.ctime != other.ctime: return False
+ if self.atime != other.atime: return False
+ if self.path != other.path: return False
+ if self.uid != other.uid: return False
+ if self.gid != other.gid: return False
+ if self.size != other.size: return False
+ if self.user != other.user: return False
+ if self.group != other.group: return False
+ if self.symlink_target != other.symlink_target: return False
+ if self.hardlink_target != other.hardlink_target: return False
+ if self.linux_attr != other.linux_attr: return False
+ if self.posix1e_acl != other.posix1e_acl: return False
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __hash__(self):
+ return hash((self.mode,
+ self.mtime,
+ self.ctime,
+ self.atime,
+ self.path,
+ self.uid,
+ self.gid,
+ self.size,
+ self.user,
+ self.group,
+ self.symlink_target,
+ self.hardlink_target,
+ self.linux_attr,
+ self.posix1e_acl))
+
+ def __repr__(self):
+ result = ['<%s instance at %s' % (self.__class__, hex(id(self)))]
+ if self.path is not None:
+ result += ' path:' + repr(self.path)
+ if self.mode is not None:
+ result += ' mode:' + repr(xstat.mode_str(self.mode)
+ + '(%s)' % oct(self.mode))
+ if self.uid is not None:
+ result += ' uid:' + str(self.uid)
+ if self.gid is not None:
+ result += ' gid:' + str(self.gid)
+ if self.user is not None:
+ result += ' user:' + repr(self.user)
+ if self.group is not None:
+ result += ' group:' + repr(self.group)
+ if self.size is not None:
+ result += ' size:' + repr(self.size)
+ for name, val in (('atime', self.atime),
+ ('mtime', self.mtime),
+ ('ctime', self.ctime)):
+ if val is not None:
+ result += ' %s:%r (%d)' \
+ % (name,
+ strftime('%Y-%m-%d %H:%M %z',
+ gmtime(xstat.fstime_floor_secs(val))),
+ val)
+ result += '>'
+ return ''.join(result)
+
def write(self, port, include_path=True):
records = include_path and [(_rec_tag_path, self._encode_path())] or []
records.extend([(_rec_tag_common_v2, self._encode_common()),
vint.write_vuint(port, _rec_tag_end)
def encode(self, include_path=True):
- port = StringIO()
+ port = BytesIO()
self.write(port, include_path)
return port.getvalue()
+ def copy(self):
+ return deepcopy(self)
+
@staticmethod
def read(port):
# This method should either return a valid Metadata object,
self._apply_linux_xattr_rec):
try:
apply_metadata(path, restore_numeric_ids=num_ids)
- except ApplyError, e:
+ except ApplyError as e:
add_error(e)
def same_file(self, other):
m = from_path(p, statinfo=st, archive_path=safe_path,
save_symlinks=save_symlinks)
if verbose:
- print >> sys.stderr, m.path
+ print(m.path, file=sys.stderr)
m.write(output_file, include_path=write_paths)
else:
start_dir = os.getcwd()
m = from_path(p, statinfo=st, archive_path=safe_path,
save_symlinks=save_symlinks)
if verbose:
- print >> sys.stderr, m.path
+ print(m.path, file=sys.stderr)
m.write(output_file, include_path=write_paths)
os.chdir(dirlist_dir)
finally:
'posix1e-acl'])
-def summary_str(meta):
- mode_val = xstat.mode_str(meta.mode)
- user_val = meta.user
- if not user_val:
- user_val = str(meta.uid)
- group_val = meta.group
- if not group_val:
- group_val = str(meta.gid)
- size_or_dev_val = '-'
- if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode):
- size_or_dev_val = '%d,%d' % (os.major(meta.rdev), os.minor(meta.rdev))
- elif meta.size:
- size_or_dev_val = meta.size
- mtime_secs = xstat.fstime_floor_secs(meta.mtime)
- time_val = time.strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs))
- path_val = meta.path or ''
- if stat.S_ISLNK(meta.mode):
- path_val += ' -> ' + meta.symlink_target
- return '%-10s %-11s %11s %16s %s' % (mode_val,
- user_val + "/" + group_val,
- size_or_dev_val,
- time_val,
- path_val)
+def summary_str(meta, numeric_ids = False, classification = None,
+ human_readable = False):
+
+ """Return a string containing the "ls -l" style listing for meta.
+ Classification may be "all", "type", or None."""
+ user_str = group_str = size_or_dev_str = '?'
+ symlink_target = None
+ if meta:
+ name = meta.path
+ mode_str = xstat.mode_str(meta.mode)
+ symlink_target = meta.symlink_target
+ mtime_secs = xstat.fstime_floor_secs(meta.mtime)
+ mtime_str = strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs))
+ if meta.user and not numeric_ids:
+ user_str = meta.user
+ elif meta.uid != None:
+ user_str = str(meta.uid)
+ if meta.group and not numeric_ids:
+ group_str = meta.group
+ elif meta.gid != None:
+ group_str = str(meta.gid)
+ if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode):
+ if meta.rdev:
+ size_or_dev_str = '%d,%d' % (os.major(meta.rdev),
+ os.minor(meta.rdev))
+ elif meta.size != None:
+ if human_readable:
+ size_or_dev_str = format_filesize(meta.size)
+ else:
+ size_or_dev_str = str(meta.size)
+ else:
+ size_or_dev_str = '-'
+ if classification:
+ classification_str = \
+ xstat.classification_str(meta.mode, classification == 'all')
+ else:
+ mode_str = '?' * 10
+ mtime_str = '????-??-?? ??:??'
+ classification_str = '?'
+
+ name = name or ''
+ if classification:
+ name += classification_str
+ if symlink_target:
+ name += ' -> ' + meta.symlink_target
+
+ return '%-10s %-11s %11s %16s %s' % (mode_str,
+ user_str + "/" + group_str,
+ size_or_dev_str,
+ mtime_str,
+ name)
def detailed_str(meta, fields = None):
first_item = True
for meta in _ArchiveIterator(file):
if not first_item:
- print
- print detailed_str(meta)
+ print()
+ print(detailed_str(meta))
first_item = False
elif verbose > 0:
for meta in _ArchiveIterator(file):
- print summary_str(meta)
+ print(summary_str(meta))
elif verbose == 0:
for meta in _ArchiveIterator(file):
if not meta.path:
- print >> sys.stderr, \
- 'bup: no metadata path, but asked to only display path', \
- '(increase verbosity?)'
+ print('bup: no metadata path, but asked to only display path'
+ '(increase verbosity?)')
sys.exit(1)
- print meta.path
+ print(meta.path)
def start_extract(file, create_symlinks=True):
if not meta: # Hit end record.
break
if verbose:
- print >> sys.stderr, meta.path
+ print(meta.path, file=sys.stderr)
xpath = _clean_up_extract_path(meta.path)
if not xpath:
add_error(Exception('skipping risky path "%s"' % meta.path))
all_dirs.append(meta)
else:
if verbose:
- print >> sys.stderr, meta.path
+ print(meta.path, file=sys.stderr)
meta.apply_to_path(path=xpath,
restore_numeric_ids=restore_numeric_ids)
all_dirs.sort(key = lambda x : len(x.path), reverse=True)
# Don't need to check xpath -- won't be in all_dirs if not OK.
xpath = _clean_up_extract_path(dir.path)
if verbose:
- print >> sys.stderr, dir.path
+ print(dir.path, file=sys.stderr)
dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
else:
meta.path = xpath
if verbose:
- print >> sys.stderr, '+', meta.path
+ print('+', meta.path, file=sys.stderr)
_set_up_path(meta, create_symlinks=create_symlinks)
if os.path.isdir(meta.path):
all_dirs.append(meta)
else:
if verbose:
- print >> sys.stderr, '=', meta.path
+ print('=', meta.path, file=sys.stderr)
meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
all_dirs.sort(key = lambda x : len(x.path), reverse=True)
for dir in all_dirs:
# Don't need to check xpath -- won't be in all_dirs if not OK.
xpath = _clean_up_extract_path(dir.path)
if verbose:
- print >> sys.stderr, '=', xpath
+ print('=', xpath, file=sys.stderr)
# Shouldn't have to check for risky paths here (omitted above).
dir.apply_to_path(path=dir.path,
restore_numeric_ids=restore_numeric_ids)
projects / bup.git / blobdiff