X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=lib%2Fbup%2Fmetadata.py;h=bef806677220a899b7d5328fc44a2555f9d51767;hb=6e2080a451a070bf34bd6b86071f6014ba584d15;hp=042eb4d81279867abeb139ecac8c6b3540732f49;hpb=70d8047ce35e0dfc92405ec4ce31ea40758f4a4a;p=bup.git diff --git a/lib/bup/metadata.py b/lib/bup/metadata.py index 042eb4d..bef8066 100644 --- a/lib/bup/metadata.py +++ b/lib/bup/metadata.py @@ -4,11 +4,11 @@ # # This code is covered under the terms of the GNU Library General # Public License as described in the bup LICENSE file. -import errno, os, sys, stat, time, pwd, grp, socket +import errno, os, sys, stat, time, pwd, grp, socket, struct from cStringIO import StringIO from bup import vint, xstat from bup.drecurse import recursive_dirlist -from bup.helpers import add_error, mkdirp, log, is_superuser +from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize from bup.helpers import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name from bup.xstat import utime, lutime @@ -27,7 +27,9 @@ if sys.platform.startswith('linux'): xattr = None posix1e = None -if not (sys.platform.startswith('cygwin') or sys.platform.startswith('darwin')): +if not (sys.platform.startswith('cygwin') \ + or sys.platform.startswith('darwin') \ + or sys.platform.startswith('netbsd')): try: import posix1e except ImportError: @@ -40,7 +42,20 @@ except ImportError: # not on Linux, in which case files don't have any linux attrs anyway, so # lacking the functions isn't a problem. get_linux_file_attr = set_linux_file_attr = None - + + +# See the bup_get_linux_file_attr() comments. +_suppress_linux_file_attr = \ + sys.byteorder == 'big' and struct.calcsize('@l') > struct.calcsize('@i') + +def check_linux_file_attr_api(): + global get_linux_file_attr, set_linux_file_attr + if not (get_linux_file_attr or set_linux_file_attr): + return + if _suppress_linux_file_attr: + log('Warning: Linux attr support disabled (see "bup help index").\n') + get_linux_file_attr = set_linux_file_attr = None + # WARNING: the metadata encoding is *not* stable yet. Caveat emptor! @@ -56,7 +71,6 @@ except ImportError: # FIXME: Add nfsv4 acl handling - see nfs4-acl-tools. # FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.). # FIXME: Consider pack('vvvvsss', ...) optimization. -# FIXME: Consider caching users/groups. ## FS notes: # @@ -163,13 +177,14 @@ def _clean_up_extract_path(p): # must be unique, and must *never* be changed. _rec_tag_end = 0 _rec_tag_path = 1 -_rec_tag_common = 2 # times, user, group, type, perms, etc. +_rec_tag_common = 2 # times, user, group, type, perms, etc. (legacy/broken) _rec_tag_symlink_target = 3 _rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc. _rec_tag_nfsv4_acl = 5 # intended to supplant posix1e? (unimplemented) _rec_tag_linux_attr = 6 # lsattr(1) chattr(1) _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1) _rec_tag_hardlink_target = 8 # hard link target path +_rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current) class ApplyError(Exception): @@ -224,7 +239,6 @@ class Metadata: return self.uid == other.uid \ and self.gid == other.gid \ and self.rdev == other.rdev \ - and self.atime == other.atime \ and self.mtime == other.mtime \ and self.ctime == other.ctime \ and self.user == other.user \ @@ -236,7 +250,7 @@ class Metadata: atime = xstat.nsecs_to_timespec(self.atime) mtime = xstat.nsecs_to_timespec(self.mtime) ctime = xstat.nsecs_to_timespec(self.ctime) - result = vint.pack('VVsVsVvVvVvV', + result = vint.pack('vvsvsvvVvVvV', self.mode, self.uid, self.user, @@ -251,7 +265,10 @@ class Metadata: ctime[1]) return result - def _load_common_rec(self, port): + def _load_common_rec(self, port, legacy_format=False): + unpack_fmt = 'vvsvsvvVvVvV' + if legacy_format: + unpack_fmt = 'VVsVsVvVvVvV' data = vint.read_bvec(port) (self.mode, self.uid, @@ -264,7 +281,7 @@ class Metadata: self.mtime, mtime_ns, self.ctime, - ctime_ns) = vint.unpack('VVsVsVvVvVvV', data) + ctime_ns) = vint.unpack(unpack_fmt, data) self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns)) self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns)) self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns)) @@ -367,9 +384,6 @@ class Metadata: else: raise - # Implement tar/rsync-like semantics; see bup-restore(1). - # FIXME: should we consider caching user/group name <-> id - # mappings, getgroups(), etc.? uid = gid = -1 # By default, do nothing. if is_superuser(): uid = self.uid @@ -409,7 +423,10 @@ class Metadata: raise if _have_lchmod: - os.lchmod(path, stat.S_IMODE(self.mode)) + try: + os.lchmod(path, stat.S_IMODE(self.mode)) + except errno.ENOSYS: # Function not implemented + pass elif not stat.S_ISLNK(self.mode): os.chmod(path, stat.S_IMODE(self.mode)) @@ -433,7 +450,7 @@ class Metadata: if stat.S_ISLNK(st.st_mode): self.symlink_target = os.readlink(path) except OSError, e: - add_error('readlink: %s', e) + add_error('readlink: %s' % e) def _encode_symlink_target(self): return self.symlink_target @@ -476,8 +493,8 @@ class Metadata: acl = posix1e.ACL(file=path) acls = [acl, acl] # txt and num are the same if stat.S_ISDIR(st.st_mode): - acl = posix1e.ACL(filedef=path) - def_acls = [acl_def, acl_def] + def_acl = posix1e.ACL(filedef=path) + def_acls = [def_acl, def_acl] except EnvironmentError, e: if e.errno not in (errno.EOPNOTSUPP, errno.ENOSYS): raise @@ -485,10 +502,10 @@ class Metadata: txt_flags = posix1e.TEXT_ABBREVIATE num_flags = posix1e.TEXT_ABBREVIATE | posix1e.TEXT_NUMERIC_IDS acl_rep = [acls[0].to_any_text('', '\n', txt_flags), - acls[1].to_any_text('', '\n', num_flags)] + acls[1].to_any_text('', '\n', num_flags)] if def_acls: - acl_rep.append(def_acls[2].to_any_text('', '\n', txt_flags)) - acl_rep.append(def_acls[3].to_any_text('', '\n', num_flags)) + acl_rep.append(def_acls[0].to_any_text('', '\n', txt_flags)) + acl_rep.append(def_acls[1].to_any_text('', '\n', num_flags)) self.posix1e_acl = acl_rep def _same_posix1e_acl(self, other): @@ -553,6 +570,7 @@ class Metadata: ## Linux attributes (lsattr(1), chattr(1)) def _add_linux_attr(self, path, st): + check_linux_file_attr_api() if not get_linux_file_attr: return if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode): try: @@ -584,6 +602,7 @@ class Metadata: def _apply_linux_attr_rec(self, path, restore_numeric_ids=False): if self.linux_attr: + check_linux_file_attr_api() if not set_linux_file_attr: add_error("%s: can't restore linuxattrs: " "linuxattr support missing.\n" % path) @@ -591,8 +610,10 @@ class Metadata: try: set_linux_file_attr(path, self.linux_attr) except OSError, e: - if e.errno in (errno.ENOTTY, errno.EOPNOTSUPP, errno.ENOSYS): - raise ApplyError('Linux chattr: %s' % e) + if e.errno in (errno.ENOTTY, errno.EOPNOTSUPP, errno.ENOSYS, + errno.EACCES): + raise ApplyError('Linux chattr: %s (0x%s)' + % (e, hex(self.linux_attr))) else: raise @@ -636,31 +657,39 @@ class Metadata: add_error("%s: can't restore xattr; xattr support missing.\n" % path) return - existing_xattrs = set(xattr.list(path, nofollow=True)) - if self.linux_xattr: - for k, v in self.linux_xattr: - if k not in existing_xattrs \ - or v != xattr.get(path, k, nofollow=True): - try: - xattr.set(path, k, v, nofollow=True) - except IOError, e: - if e.errno == errno.EPERM \ - or e.errno == errno.EOPNOTSUPP: - raise ApplyError('xattr.set: %s' % e) - else: - raise - existing_xattrs -= frozenset([k]) - for k in existing_xattrs: + if not self.linux_xattr: + return + try: + existing_xattrs = set(xattr.list(path, nofollow=True)) + except IOError, e: + if e.errno == errno.EACCES: + raise ApplyError('xattr.set %r: %s' % (path, e)) + else: + raise + for k, v in self.linux_xattr: + if k not in existing_xattrs \ + or v != xattr.get(path, k, nofollow=True): try: - xattr.remove(path, k, nofollow=True) + xattr.set(path, k, v, nofollow=True) except IOError, e: - if e.errno == errno.EPERM: - raise ApplyError('xattr.remove: %s' % e) + if e.errno == errno.EPERM \ + or e.errno == errno.EOPNOTSUPP: + raise ApplyError('xattr.set %r: %s' % (path, e)) else: raise + existing_xattrs -= frozenset([k]) + for k in existing_xattrs: + try: + xattr.remove(path, k, nofollow=True) + except IOError, e: + if e.errno == errno.EPERM: + raise ApplyError('xattr.remove %r: %s' % (path, e)) + else: + raise def __init__(self): - self.mode = None + self.mode = self.uid = self.gid = self.user = self.group = None + self.atime = self.mtime = self.ctime = None # optional members self.path = None self.size = None @@ -670,9 +699,36 @@ class Metadata: self.linux_xattr = None self.posix1e_acl = None + def __repr__(self): + result = ['<%s instance at %s' % (self.__class__, hex(id(self)))] + if self.path: + result += ' path:' + repr(self.path) + if self.mode: + result += ' mode:' + repr(xstat.mode_str(self.mode) + + '(%s)' % hex(self.mode)) + if self.uid: + result += ' uid:' + str(self.uid) + if self.gid: + result += ' gid:' + str(self.gid) + if self.user: + result += ' user:' + repr(self.user) + if self.group: + result += ' group:' + repr(self.group) + if self.size: + result += ' size:' + repr(self.size) + for name, val in (('atime', self.atime), + ('mtime', self.mtime), + ('ctime', self.ctime)): + result += ' %s:%r' \ + % (name, + time.strftime('%Y-%m-%d %H:%M %z', + time.gmtime(xstat.fstime_floor_secs(val)))) + result += '>' + return ''.join(result) + def write(self, port, include_path=True): records = include_path and [(_rec_tag_path, self._encode_path())] or [] - records.extend([(_rec_tag_common, self._encode_common()), + records.extend([(_rec_tag_common_v2, self._encode_common()), (_rec_tag_symlink_target, self._encode_symlink_target()), (_rec_tag_hardlink_target, @@ -706,7 +762,7 @@ class Metadata: while True: # only exit is error (exception) or _rec_tag_end if tag == _rec_tag_path: result._load_path_rec(port) - elif tag == _rec_tag_common: + elif tag == _rec_tag_common_v2: result._load_common_rec(port) elif tag == _rec_tag_symlink_target: result._load_symlink_target_rec(port) @@ -720,6 +776,8 @@ class Metadata: result._load_linux_xattr_rec(port) elif tag == _rec_tag_end: return result + elif tag == _rec_tag_common: # Should be very rare. + result._load_common_rec(port, legacy_format = True) else: # unknown record vint.skip_bvec(port) tag = vint.read_vuint(port) @@ -743,13 +801,14 @@ class Metadata: + ' with unrecognized mode "0x%x"\n' % self.mode) return num_ids = restore_numeric_ids - try: - self._apply_common_rec(path, restore_numeric_ids=num_ids) - self._apply_posix1e_acl_rec(path, restore_numeric_ids=num_ids) - self._apply_linux_attr_rec(path, restore_numeric_ids=num_ids) - self._apply_linux_xattr_rec(path, restore_numeric_ids=num_ids) - except ApplyError, e: - add_error(e) + for apply_metadata in (self._apply_common_rec, + self._apply_posix1e_acl_rec, + self._apply_linux_attr_rec, + self._apply_linux_xattr_rec): + try: + apply_metadata(path, restore_numeric_ids=num_ids) + except ApplyError, e: + add_error(e) def same_file(self, other): """Compare this to other for equivalency. Return true if @@ -848,29 +907,57 @@ all_fields = frozenset(['path', 'posix1e-acl']) -def summary_str(meta): - mode_val = xstat.mode_str(meta.mode) - user_val = meta.user - if not user_val: - user_val = str(meta.uid) - group_val = meta.group - if not group_val: - group_val = str(meta.gid) - size_or_dev_val = '-' - if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode): - size_or_dev_val = '%d,%d' % (os.major(meta.rdev), os.minor(meta.rdev)) - elif meta.size: - size_or_dev_val = meta.size - mtime_secs = xstat.fstime_floor_secs(meta.mtime) - time_val = time.strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs)) - path_val = meta.path or '' - if stat.S_ISLNK(meta.mode): - path_val += ' -> ' + meta.symlink_target - return '%-10s %-11s %11s %16s %s' % (mode_val, - user_val + "/" + group_val, - size_or_dev_val, - time_val, - path_val) +def summary_str(meta, numeric_ids = False, classification = None, + human_readable = False): + + """Return a string containing the "ls -l" style listing for meta. + Classification may be "all", "type", or None.""" + user_str = group_str = size_or_dev_str = '?' + symlink_target = None + if meta: + name = meta.path + mode_str = xstat.mode_str(meta.mode) + symlink_target = meta.symlink_target + mtime_secs = xstat.fstime_floor_secs(meta.mtime) + mtime_str = time.strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs)) + if meta.user and not numeric_ids: + user_str = meta.user + elif meta.uid != None: + user_str = str(meta.uid) + if meta.group and not numeric_ids: + group_str = meta.group + elif meta.gid != None: + group_str = str(meta.gid) + if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode): + if meta.rdev: + size_or_dev_str = '%d,%d' % (os.major(meta.rdev), + os.minor(meta.rdev)) + elif meta.size != None: + if human_readable: + size_or_dev_str = format_filesize(meta.size) + else: + size_or_dev_str = str(meta.size) + else: + size_or_dev_str = '-' + if classification: + classification_str = \ + xstat.classification_str(meta.mode, classification == 'all') + else: + mode_str = '?' * 10 + mtime_str = '????-??-?? ??:??' + classification_str = '?' + + name = name or '' + if classification: + name += classification_str + if symlink_target: + name += ' -> ' + meta.symlink_target + + return '%-10s %-11s %11s %16s %s' % (mode_str, + user_str + "/" + group_str, + size_or_dev_str, + mtime_str, + name) def detailed_str(meta, fields = None): @@ -926,16 +1013,12 @@ def detailed_str(meta, fields = None): if 'linux-xattr' in fields and meta.linux_xattr: for name, value in meta.linux_xattr: result.append('linux-xattr: %s -> %s' % (name, repr(value))) - if 'posix1e-acl' in fields and meta.posix1e_acl and posix1e: - flags = posix1e.TEXT_ABBREVIATE + if 'posix1e-acl' in fields and meta.posix1e_acl: + acl = meta.posix1e_acl[0] + result.append('posix1e-acl: ' + acl + '\n') if stat.S_ISDIR(meta.mode): - acl = meta.posix1e_acl[0] - default_acl = meta.posix1e_acl[2] - result.append(acl.to_any_text('posix1e-acl: ', '\n', flags)) - result.append(acl.to_any_text('posix1e-acl-default: ', '\n', flags)) - else: - acl = meta.posix1e_acl[0] - result.append(acl.to_any_text('posix1e-acl: ', '\n', flags)) + def_acl = meta.posix1e_acl[2] + result.append('posix1e-acl-default: ' + def_acl + '\n') return '\n'.join(result)