X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=lib%2Fbup%2Fmetadata.py;h=e5337c96278563663a1d0c61303df8cbe8d5e162;hb=c40b3dd5fd74e72024fbaad3daf5a958aefa1c54;hp=0622d2b98652fb374f61cfba16acb00cdaa81c00;hpb=5dfba1fda942796b66905230c132d1af4d28e9ac;p=bup.git diff --git a/lib/bup/metadata.py b/lib/bup/metadata.py index 0622d2b..e5337c9 100644 --- a/lib/bup/metadata.py +++ b/lib/bup/metadata.py @@ -4,8 +4,14 @@ # # This code is covered under the terms of the GNU Library General # Public License as described in the bup LICENSE file. -import errno, os, sys, stat, time, pwd, grp, socket -from cStringIO import StringIO + +from __future__ import absolute_import +from copy import deepcopy +from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP +from io import BytesIO +from time import gmtime, strftime +import errno, os, sys, stat, time, pwd, grp, socket, struct + from bup import vint, xstat from bup.drecurse import recursive_dirlist from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize @@ -42,7 +48,20 @@ except ImportError: # not on Linux, in which case files don't have any linux attrs anyway, so # lacking the functions isn't a problem. get_linux_file_attr = set_linux_file_attr = None - + + +# See the bup_get_linux_file_attr() comments. +_suppress_linux_file_attr = \ + sys.byteorder == 'big' and struct.calcsize('@l') > struct.calcsize('@i') + +def check_linux_file_attr_api(): + global get_linux_file_attr, set_linux_file_attr + if not (get_linux_file_attr or set_linux_file_attr): + return + if _suppress_linux_file_attr: + log('Warning: Linux attr support disabled (see "bup help index").\n') + get_linux_file_attr = set_linux_file_attr = None + # WARNING: the metadata encoding is *not* stable yet. Caveat emptor! @@ -173,6 +192,8 @@ _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1) _rec_tag_hardlink_target = 8 # hard link target path _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current) +_warned_about_attr_einval = None + class ApplyError(Exception): # Thrown when unable to apply any given bit of metadata to a path. @@ -199,6 +220,8 @@ class Metadata: # must be non-negative and < 10**9. def _add_common(self, path, st): + assert(st.st_uid >= 0) + assert(st.st_gid >= 0) self.uid = st.st_uid self.gid = st.st_gid self.atime = st.st_atime @@ -291,14 +314,14 @@ class Metadata: st = None try: st = xstat.lstat(path) - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise if st: if stat.S_ISDIR(st.st_mode): try: os.rmdir(path) - except OSError, e: + except OSError as e: if e.errno in (errno.ENOTEMPTY, errno.EEXIST): msg = 'refusing to overwrite non-empty dir ' + path raise Exception(msg) @@ -308,24 +331,24 @@ class Metadata: if stat.S_ISREG(self.mode): assert(self._recognized_file_type()) - fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0600) + fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0o600) os.close(fd) elif stat.S_ISDIR(self.mode): assert(self._recognized_file_type()) - os.mkdir(path, 0700) + os.mkdir(path, 0o700) elif stat.S_ISCHR(self.mode): assert(self._recognized_file_type()) - os.mknod(path, 0600 | stat.S_IFCHR, self.rdev) + os.mknod(path, 0o600 | stat.S_IFCHR, self.rdev) elif stat.S_ISBLK(self.mode): assert(self._recognized_file_type()) - os.mknod(path, 0600 | stat.S_IFBLK, self.rdev) + os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev) elif stat.S_ISFIFO(self.mode): assert(self._recognized_file_type()) - os.mknod(path, 0600 | stat.S_IFIFO) + os.mknod(path, 0o600 | stat.S_IFIFO) elif stat.S_ISSOCK(self.mode): try: - os.mknod(path, 0600 | stat.S_IFSOCK) - except OSError, e: + os.mknod(path, 0o600 | stat.S_IFSOCK) + except OSError as e: if e.errno in (errno.EINVAL, errno.EPERM): s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.bind(path) @@ -337,7 +360,7 @@ class Metadata: # on MacOS, symlink() permissions depend on umask, and there's # no way to chown a symlink after creating it, so we have to # be careful here! - oldumask = os.umask((self.mode & 0777) ^ 0777) + oldumask = os.umask((self.mode & 0o777) ^ 0o777) try: os.symlink(self.symlink_target, path) finally: @@ -357,7 +380,7 @@ class Metadata: if lutime and stat.S_ISLNK(self.mode): try: lutime(path, (self.atime, self.mtime)) - except OSError, e: + except OSError as e: if e.errno == errno.EACCES: raise ApplyError('lutime: %s' % e) else: @@ -365,7 +388,7 @@ class Metadata: else: try: utime(path, (self.atime, self.mtime)) - except OSError, e: + except OSError as e: if e.errno == errno.EACCES: raise ApplyError('utime: %s' % e) else: @@ -399,7 +422,7 @@ class Metadata: if uid != -1 or gid != -1: try: os.lchown(path, uid, gid) - except OSError, e: + except OSError as e: if e.errno == errno.EPERM: add_error('lchown: %s' % e) elif sys.platform.startswith('cygwin') \ @@ -410,7 +433,10 @@ class Metadata: raise if _have_lchmod: - os.lchmod(path, stat.S_IMODE(self.mode)) + try: + os.lchmod(path, stat.S_IMODE(self.mode)) + except errno.ENOSYS: # Function not implemented + pass elif not stat.S_ISLNK(self.mode): os.chmod(path, stat.S_IMODE(self.mode)) @@ -433,14 +459,16 @@ class Metadata: try: if stat.S_ISLNK(st.st_mode): self.symlink_target = os.readlink(path) - except OSError, e: - add_error('readlink: %s', e) + except OSError as e: + add_error('readlink: %s' % e) def _encode_symlink_target(self): return self.symlink_target def _load_symlink_target_rec(self, port): - self.symlink_target = vint.read_bvec(port) + target = vint.read_bvec(port) + self.symlink_target = target + self.size = len(target) ## Hardlink targets @@ -468,7 +496,8 @@ class Metadata: # The numeric/text distinction only matters when reading/restoring # a stored record. def _add_posix1e_acl(self, path, st): - if not posix1e: return + if not posix1e or not posix1e.HAS_EXTENDED_CHECK: + return if not stat.S_ISLNK(st.st_mode): acls = None def_acls = None @@ -479,7 +508,7 @@ class Metadata: if stat.S_ISDIR(st.st_mode): def_acl = posix1e.ACL(filedef=path) def_acls = [def_acl, def_acl] - except EnvironmentError, e: + except EnvironmentError as e: if e.errno not in (errno.EOPNOTSUPP, errno.ENOSYS): raise if acls: @@ -516,7 +545,7 @@ class Metadata: def apply_acl(acl_rep, kind): try: acl = posix1e.ACL(text = acl_rep) - except IOError, e: + except IOError as e: if e.errno == 0: # pylibacl appears to return an IOError with errno # set to 0 if a group referred to by the ACL rep @@ -527,7 +556,7 @@ class Metadata: raise try: acl.applyto(path, kind) - except IOError, e: + except IOError as e: if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP: raise ApplyError('POSIX1e ACL applyto: %s' % e) else: @@ -554,18 +583,27 @@ class Metadata: ## Linux attributes (lsattr(1), chattr(1)) def _add_linux_attr(self, path, st): + check_linux_file_attr_api() if not get_linux_file_attr: return if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode): try: attr = get_linux_file_attr(path) if attr != 0: self.linux_attr = attr - except OSError, e: + except OSError as e: if e.errno == errno.EACCES: add_error('read Linux attr: %s' % e) - elif e.errno in (errno.ENOTTY, errno.ENOSYS, errno.EOPNOTSUPP): + elif e.errno in (ENOTTY, ENOSYS, EOPNOTSUPP): # Assume filesystem doesn't support attrs. return + elif e.errno == EINVAL: + global _warned_about_attr_einval + if not _warned_about_attr_einval: + log("Ignoring attr EINVAL;" + + " if you're not using ntfs-3g, please report: " + + repr(path) + '\n') + _warned_about_attr_einval = True + return else: raise @@ -585,17 +623,21 @@ class Metadata: def _apply_linux_attr_rec(self, path, restore_numeric_ids=False): if self.linux_attr: + check_linux_file_attr_api() if not set_linux_file_attr: add_error("%s: can't restore linuxattrs: " "linuxattr support missing.\n" % path) return try: set_linux_file_attr(path, self.linux_attr) - except OSError, e: - if e.errno in (errno.ENOTTY, errno.EOPNOTSUPP, errno.ENOSYS, - errno.EACCES): + except OSError as e: + if e.errno in (EACCES, ENOTTY, EOPNOTSUPP, ENOSYS): raise ApplyError('Linux chattr: %s (0x%s)' % (e, hex(self.linux_attr))) + elif e.errno == EINVAL: + msg = "if you're not using ntfs-3g, please report" + raise ApplyError('Linux chattr: %s (0x%s) (%s)' + % (e, hex(self.linux_attr), msg)) else: raise @@ -606,7 +648,7 @@ class Metadata: if not xattr: return try: self.linux_xattr = xattr.get_all(path, nofollow=True) - except EnvironmentError, e: + except EnvironmentError as e: if e.errno != errno.EOPNOTSUPP: raise @@ -625,7 +667,7 @@ class Metadata: def _load_linux_xattr_rec(self, file): data = vint.read_bvec(file) - memfile = StringIO(data) + memfile = BytesIO(data) result = [] for i in range(vint.read_vuint(memfile)): key = vint.read_bvec(memfile) @@ -643,9 +685,9 @@ class Metadata: return try: existing_xattrs = set(xattr.list(path, nofollow=True)) - except IOError, e: + except IOError as e: if e.errno == errno.EACCES: - raise ApplyError('xattr.set: %s' % e) + raise ApplyError('xattr.set %r: %s' % (path, e)) else: raise for k, v in self.linux_xattr: @@ -653,19 +695,19 @@ class Metadata: or v != xattr.get(path, k, nofollow=True): try: xattr.set(path, k, v, nofollow=True) - except IOError, e: + except IOError as e: if e.errno == errno.EPERM \ or e.errno == errno.EOPNOTSUPP: - raise ApplyError('xattr.set: %s' % e) + raise ApplyError('xattr.set %r: %s' % (path, e)) else: raise existing_xattrs -= frozenset([k]) for k in existing_xattrs: try: xattr.remove(path, k, nofollow=True) - except IOError, e: - if e.errno == errno.EPERM: - raise ApplyError('xattr.remove: %s' % e) + except IOError as e: + if e.errno in (errno.EPERM, errno.EACCES): + raise ApplyError('xattr.remove %r: %s' % (path, e)) else: raise @@ -681,30 +723,69 @@ class Metadata: self.linux_xattr = None self.posix1e_acl = None + def __eq__(self, other): + if not isinstance(other, Metadata): return False + if self.mode != other.mode: return False + if self.mtime != other.mtime: return False + if self.ctime != other.ctime: return False + if self.atime != other.atime: return False + if self.path != other.path: return False + if self.uid != other.uid: return False + if self.gid != other.gid: return False + if self.size != other.size: return False + if self.user != other.user: return False + if self.group != other.group: return False + if self.symlink_target != other.symlink_target: return False + if self.hardlink_target != other.hardlink_target: return False + if self.linux_attr != other.linux_attr: return False + if self.posix1e_acl != other.posix1e_acl: return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.mode, + self.mtime, + self.ctime, + self.atime, + self.path, + self.uid, + self.gid, + self.size, + self.user, + self.group, + self.symlink_target, + self.hardlink_target, + self.linux_attr, + self.posix1e_acl)) + def __repr__(self): result = ['<%s instance at %s' % (self.__class__, hex(id(self)))] - if self.path: + if self.path is not None: result += ' path:' + repr(self.path) - if self.mode: + if self.mode is not None: result += ' mode:' + repr(xstat.mode_str(self.mode) - + '(%s)' % hex(self.mode)) - if self.uid: + + '(%s)' % oct(self.mode)) + if self.uid is not None: result += ' uid:' + str(self.uid) - if self.gid: + if self.gid is not None: result += ' gid:' + str(self.gid) - if self.user: + if self.user is not None: result += ' user:' + repr(self.user) - if self.group: + if self.group is not None: result += ' group:' + repr(self.group) - if self.size: + if self.size is not None: result += ' size:' + repr(self.size) for name, val in (('atime', self.atime), ('mtime', self.mtime), ('ctime', self.ctime)): - result += ' %s:%r' \ - % (name, - time.strftime('%Y-%m-%d %H:%M %z', - time.gmtime(xstat.fstime_floor_secs(val)))) + if val is not None: + result += ' %s:%r (%d)' \ + % (name, + strftime('%Y-%m-%d %H:%M %z', + gmtime(xstat.fstime_floor_secs(val))), + val) result += '>' return ''.join(result) @@ -725,10 +806,13 @@ class Metadata: vint.write_vuint(port, _rec_tag_end) def encode(self, include_path=True): - port = StringIO() + port = BytesIO() self.write(port, include_path) return port.getvalue() + def copy(self): + return deepcopy(self) + @staticmethod def read(port): # This method should either return a valid Metadata object, @@ -789,7 +873,7 @@ class Metadata: self._apply_linux_xattr_rec): try: apply_metadata(path, restore_numeric_ids=num_ids) - except ApplyError, e: + except ApplyError as e: add_error(e) def same_file(self, other): @@ -889,31 +973,57 @@ all_fields = frozenset(['path', 'posix1e-acl']) -def summary_str(meta, numeric_ids = False, human_readable = False): - mode_val = xstat.mode_str(meta.mode) - user_val = meta.user - if numeric_ids or not user_val: - user_val = str(meta.uid) - group_val = meta.group - if numeric_ids or not group_val: - group_val = str(meta.gid) - size_or_dev_val = '-' - if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode): - size_or_dev_val = '%d,%d' % (os.major(meta.rdev), os.minor(meta.rdev)) - elif meta.size != None: - size_or_dev_val = meta.size - if human_readable: - size_or_dev_val = format_filesize(meta.size) - mtime_secs = xstat.fstime_floor_secs(meta.mtime) - time_val = time.strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs)) - path_val = meta.path or '' - if stat.S_ISLNK(meta.mode): - path_val += ' -> ' + meta.symlink_target - return '%-10s %-11s %11s %16s %s' % (mode_val, - user_val + "/" + group_val, - size_or_dev_val, - time_val, - path_val) +def summary_str(meta, numeric_ids = False, classification = None, + human_readable = False): + + """Return a string containing the "ls -l" style listing for meta. + Classification may be "all", "type", or None.""" + user_str = group_str = size_or_dev_str = '?' + symlink_target = None + if meta: + name = meta.path + mode_str = xstat.mode_str(meta.mode) + symlink_target = meta.symlink_target + mtime_secs = xstat.fstime_floor_secs(meta.mtime) + mtime_str = strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs)) + if meta.user and not numeric_ids: + user_str = meta.user + elif meta.uid != None: + user_str = str(meta.uid) + if meta.group and not numeric_ids: + group_str = meta.group + elif meta.gid != None: + group_str = str(meta.gid) + if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode): + if meta.rdev: + size_or_dev_str = '%d,%d' % (os.major(meta.rdev), + os.minor(meta.rdev)) + elif meta.size != None: + if human_readable: + size_or_dev_str = format_filesize(meta.size) + else: + size_or_dev_str = str(meta.size) + else: + size_or_dev_str = '-' + if classification: + classification_str = \ + xstat.classification_str(meta.mode, classification == 'all') + else: + mode_str = '?' * 10 + mtime_str = '????-??-?? ??:??' + classification_str = '?' + + name = name or '' + if classification: + name += classification_str + if symlink_target: + name += ' -> ' + meta.symlink_target + + return '%-10s %-11s %11s %16s %s' % (mode_str, + user_str + "/" + group_str, + size_or_dev_str, + mtime_str, + name) def detailed_str(meta, fields = None):