X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=lib%2Fbup%2Fmetadata.py;h=83f04f771475457641c96e369d500b45be946a0b;hb=8512701b4a41b4a7472975cc4c0a906d01b1a514;hp=e5337c96278563663a1d0c61303df8cbe8d5e162;hpb=c40b3dd5fd74e72024fbaad3daf5a958aefa1c54;p=bup.git diff --git a/lib/bup/metadata.py b/lib/bup/metadata.py index e5337c9..83f04f7 100644 --- a/lib/bup/metadata.py +++ b/lib/bup/metadata.py @@ -5,41 +5,43 @@ # This code is covered under the terms of the GNU Library General # Public License as described in the bup LICENSE file. -from __future__ import absolute_import +from __future__ import absolute_import, print_function from copy import deepcopy from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP from io import BytesIO from time import gmtime, strftime -import errno, os, sys, stat, time, pwd, grp, socket, struct +import errno, os, sys, stat, time, socket, struct from bup import vint, xstat from bup.drecurse import recursive_dirlist from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize -from bup.helpers import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name +from bup.io import path_msg +from bup.pwdgrp import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name from bup.xstat import utime, lutime xattr = None if sys.platform.startswith('linux'): + # prefer python-pyxattr (it's a lot faster), but fall back to python-xattr + # as the two are incompatible and only one can be installed on a system try: import xattr except ImportError: log('Warning: Linux xattr support missing; install python-pyxattr.\n') - if xattr: + if xattr and getattr(xattr, 'get_all', None) is None: try: - xattr.get_all - except AttributeError: - log('Warning: python-xattr module is too old; ' - 'install python-pyxattr instead.\n') + from xattr import pyxattr_compat as xattr + if not isinstance(xattr.NS_USER, bytes): + xattr = None + except ImportError: xattr = None + if xattr is None: + log('Warning: python-xattr module is too old; ' + 'upgrade or install python-pyxattr instead.\n') -posix1e = None -if not (sys.platform.startswith('cygwin') \ - or sys.platform.startswith('darwin') \ - or sys.platform.startswith('netbsd')): - try: - import posix1e - except ImportError: - log('Warning: POSIX ACL support missing; install python-pylibacl.\n') +try: + from bup._helpers import read_acl, apply_acl +except ImportError: + read_acl = apply_acl = None try: from bup._helpers import get_linux_file_attr, set_linux_file_attr @@ -120,59 +122,59 @@ def _clean_up_path_for_archive(p): result = p # Take everything after any '/../'. - pos = result.rfind('/../') + pos = result.rfind(b'/../') if pos != -1: - result = result[result.rfind('/../') + 4:] + result = result[result.rfind(b'/../') + 4:] # Take everything after any remaining '../'. - if result.startswith("../"): + if result.startswith(b"../"): result = result[3:] # Remove any '/./' sequences. - pos = result.find('/./') + pos = result.find(b'/./') while pos != -1: - result = result[0:pos] + '/' + result[pos + 3:] - pos = result.find('/./') + result = result[0:pos] + b'/' + result[pos + 3:] + pos = result.find(b'/./') # Remove any leading '/'s. - result = result.lstrip('/') + result = result.lstrip(b'/') # Replace '//' with '/' everywhere. - pos = result.find('//') + pos = result.find(b'//') while pos != -1: - result = result[0:pos] + '/' + result[pos + 2:] - pos = result.find('//') + result = result[0:pos] + b'/' + result[pos + 2:] + pos = result.find(b'//') # Take everything after any remaining './'. - if result.startswith('./'): + if result.startswith(b'./'): result = result[2:] # Take everything before any remaining '/.'. - if result.endswith('/.'): + if result.endswith(b'/.'): result = result[:-2] - if result == '' or result.endswith('/..'): - result = '.' + if result == b'' or result.endswith(b'/..'): + result = b'.' return result def _risky_path(p): - if p.startswith('/'): + if p.startswith(b'/'): return True - if p.find('/../') != -1: + if p.find(b'/../') != -1: return True - if p.startswith('../'): + if p.startswith(b'../'): return True - if p.endswith('/..'): + if p.endswith(b'/..'): return True return False def _clean_up_extract_path(p): - result = p.lstrip('/') - if result == '': - return '.' + result = p.lstrip(b'/') + if result == b'': + return b'.' elif _risky_path(result): return None else: @@ -183,7 +185,7 @@ def _clean_up_extract_path(p): # must be unique, and must *never* be changed. _rec_tag_end = 0 _rec_tag_path = 1 -_rec_tag_common = 2 # times, user, group, type, perms, etc. (legacy/broken) +_rec_tag_common_v1 = 2 # times, user, group, type, perms, etc. (legacy/broken) _rec_tag_symlink_target = 3 _rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc. _rec_tag_nfsv4_acl = 5 # intended to supplant posix1e? (unimplemented) @@ -191,6 +193,7 @@ _rec_tag_linux_attr = 6 # lsattr(1) chattr(1) _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1) _rec_tag_hardlink_target = 8 # hard link target path _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current) +_rec_tag_common_v3 = 10 # adds optional size to v2 _warned_about_attr_einval = None @@ -222,12 +225,13 @@ class Metadata: def _add_common(self, path, st): assert(st.st_uid >= 0) assert(st.st_gid >= 0) + self.size = st.st_size self.uid = st.st_uid self.gid = st.st_gid self.atime = st.st_atime self.mtime = st.st_mtime self.ctime = st.st_ctime - self.user = self.group = '' + self.user = self.group = b'' entry = pwd_from_uid(st.st_uid) if entry: self.user = entry.pw_name @@ -252,7 +256,8 @@ class Metadata: and self.mtime == other.mtime \ and self.ctime == other.ctime \ and self.user == other.user \ - and self.group == other.group + and self.group == other.group \ + and self.size == other.size def _encode_common(self): if not self.mode: @@ -260,7 +265,7 @@ class Metadata: atime = xstat.nsecs_to_timespec(self.atime) mtime = xstat.nsecs_to_timespec(self.mtime) ctime = xstat.nsecs_to_timespec(self.ctime) - result = vint.pack('vvsvsvvVvVvV', + result = vint.pack('vvsvsvvVvVvVv', self.mode, self.uid, self.user, @@ -272,26 +277,36 @@ class Metadata: mtime[0], mtime[1], ctime[0], - ctime[1]) + ctime[1], + self.size if self.size is not None else -1) return result - def _load_common_rec(self, port, legacy_format=False): - unpack_fmt = 'vvsvsvvVvVvV' - if legacy_format: + def _load_common_rec(self, port, version=3): + if version == 3: + # Added trailing size to v2, negative when None. + unpack_fmt = 'vvsvsvvVvVvVv' + elif version == 2: + unpack_fmt = 'vvsvsvvVvVvV' + elif version == 1: unpack_fmt = 'VVsVsVvVvVvV' + else: + raise Exception('unexpected common_rec version %d' % version) data = vint.read_bvec(port) - (self.mode, - self.uid, - self.user, - self.gid, - self.group, - self.rdev, - self.atime, - atime_ns, - self.mtime, - mtime_ns, - self.ctime, - ctime_ns) = vint.unpack(unpack_fmt, data) + values = vint.unpack(unpack_fmt, data) + if version == 3: + (self.mode, self.uid, self.user, self.gid, self.group, + self.rdev, + self.atime, atime_ns, + self.mtime, mtime_ns, + self.ctime, ctime_ns, size) = values + if size >= 0: + self.size = size + else: + (self.mode, self.uid, self.user, self.gid, self.group, + self.rdev, + self.atime, atime_ns, + self.mtime, mtime_ns, + self.ctime, ctime_ns) = values self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns)) self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns)) self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns)) @@ -307,7 +322,8 @@ class Metadata: def _create_via_common_rec(self, path, create_symlinks=True): if not self.mode: - raise ApplyError('no metadata - cannot create path ' + path) + raise ApplyError('no metadata - cannot create path ' + + path_msg(path)) # If the path already exists and is a dir, try rmdir. # If the path already exists and is anything else, try unlink. @@ -323,8 +339,8 @@ class Metadata: os.rmdir(path) except OSError as e: if e.errno in (errno.ENOTEMPTY, errno.EEXIST): - msg = 'refusing to overwrite non-empty dir ' + path - raise Exception(msg) + raise Exception('refusing to overwrite non-empty dir ' + + path_msg(path)) raise else: os.unlink(path) @@ -344,7 +360,7 @@ class Metadata: os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev) elif stat.S_ISFIFO(self.mode): assert(self._recognized_file_type()) - os.mknod(path, 0o600 | stat.S_IFIFO) + os.mkfifo(path, 0o600 | stat.S_IFIFO) elif stat.S_ISSOCK(self.mode): try: os.mknod(path, 0o600 | stat.S_IFSOCK) @@ -369,11 +385,11 @@ class Metadata: else: assert(not self._recognized_file_type()) add_error('not creating "%s" with unrecognized mode "0x%x"\n' - % (path, self.mode)) + % (path_msg(path), self.mode)) def _apply_common_rec(self, path, restore_numeric_ids=False): if not self.mode: - raise ApplyError('no metadata - cannot apply to ' + path) + raise ApplyError('no metadata - cannot apply to ' + path_msg(path)) # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2). # EACCES errors at this stage are fatal for the current path. @@ -396,8 +412,10 @@ class Metadata: uid = gid = -1 # By default, do nothing. if is_superuser(): - uid = self.uid - gid = self.gid + if self.uid is not None: + uid = self.uid + if self.gid is not None: + gid = self.gid if not restore_numeric_ids: if self.uid != 0 and self.user: entry = pwd_from_name(self.user) @@ -428,15 +446,20 @@ class Metadata: elif sys.platform.startswith('cygwin') \ and e.errno == errno.EINVAL: add_error('lchown: unknown uid/gid (%d/%d) for %s' - % (uid, gid, path)) + % (uid, gid, path_msg(path))) else: raise if _have_lchmod: try: os.lchmod(path, stat.S_IMODE(self.mode)) - except errno.ENOSYS: # Function not implemented - pass + except OSError as e: + # - "Function not implemented" + # - "Operation not supported" might be generated by glibc + if e.errno in (errno.ENOSYS, errno.EOPNOTSUPP): + pass + else: + raise elif not stat.S_ISLNK(self.mode): os.chmod(path, stat.S_IMODE(self.mode)) @@ -459,6 +482,9 @@ class Metadata: try: if stat.S_ISLNK(st.st_mode): self.symlink_target = os.readlink(path) + # might have read a different link than the + # one that was in place when we did stat() + self.size = len(self.symlink_target) except OSError as e: add_error('readlink: %s' % e) @@ -468,7 +494,10 @@ class Metadata: def _load_symlink_target_rec(self, port): target = vint.read_bvec(port) self.symlink_target = target - self.size = len(target) + if self.size is None: + self.size = len(target) + else: + assert(self.size == len(target)) ## Hardlink targets @@ -496,30 +525,11 @@ class Metadata: # The numeric/text distinction only matters when reading/restoring # a stored record. def _add_posix1e_acl(self, path, st): - if not posix1e or not posix1e.HAS_EXTENDED_CHECK: + if not read_acl: return if not stat.S_ISLNK(st.st_mode): - acls = None - def_acls = None - try: - if posix1e.has_extended(path): - acl = posix1e.ACL(file=path) - acls = [acl, acl] # txt and num are the same - if stat.S_ISDIR(st.st_mode): - def_acl = posix1e.ACL(filedef=path) - def_acls = [def_acl, def_acl] - except EnvironmentError as e: - if e.errno not in (errno.EOPNOTSUPP, errno.ENOSYS): - raise - if acls: - txt_flags = posix1e.TEXT_ABBREVIATE - num_flags = posix1e.TEXT_ABBREVIATE | posix1e.TEXT_NUMERIC_IDS - acl_rep = [acls[0].to_any_text('', '\n', txt_flags), - acls[1].to_any_text('', '\n', num_flags)] - if def_acls: - acl_rep.append(def_acls[0].to_any_text('', '\n', txt_flags)) - acl_rep.append(def_acls[1].to_any_text('', '\n', num_flags)) - self.posix1e_acl = acl_rep + isdir = 1 if stat.S_ISDIR(st.st_mode) else 0 + self.posix1e_acl = read_acl(path, isdir) def _same_posix1e_acl(self, other): """Return true or false to indicate similarity in the hardlink sense.""" @@ -530,54 +540,43 @@ class Metadata: if self.posix1e_acl: acls = self.posix1e_acl if len(acls) == 2: - acls.extend(['', '']) + return vint.pack('ssss', acls[0], acls[1], b'', b'') return vint.pack('ssss', acls[0], acls[1], acls[2], acls[3]) else: return None def _load_posix1e_acl_rec(self, port): acl_rep = vint.unpack('ssss', vint.read_bvec(port)) - if acl_rep[2] == '': + if acl_rep[2] == b'': acl_rep = acl_rep[:2] self.posix1e_acl = acl_rep def _apply_posix1e_acl_rec(self, path, restore_numeric_ids=False): - def apply_acl(acl_rep, kind): - try: - acl = posix1e.ACL(text = acl_rep) - except IOError as e: - if e.errno == 0: - # pylibacl appears to return an IOError with errno - # set to 0 if a group referred to by the ACL rep - # doesn't exist on the current system. - raise ApplyError("POSIX1e ACL: can't create %r for %r" - % (acl_rep, path)) - else: - raise - try: - acl.applyto(path, kind) - except IOError as e: - if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP: - raise ApplyError('POSIX1e ACL applyto: %s' % e) - else: - raise + if not self.posix1e_acl: + return - if not posix1e: - if self.posix1e_acl: - add_error("%s: can't restore ACLs; posix1e support missing.\n" - % path) + if not apply_acl: + add_error("%s: can't restore ACLs; posix1e support missing.\n" + % path_msg(path)) return - if self.posix1e_acl: + + try: acls = self.posix1e_acl + offs = 1 if restore_numeric_ids else 0 if len(acls) > 2: - if restore_numeric_ids: - apply_acl(acls[3], posix1e.ACL_TYPE_DEFAULT) - else: - apply_acl(acls[2], posix1e.ACL_TYPE_DEFAULT) - if restore_numeric_ids: - apply_acl(acls[1], posix1e.ACL_TYPE_ACCESS) + apply_acl(path, acls[offs], acls[offs + 2]) else: - apply_acl(acls[0], posix1e.ACL_TYPE_ACCESS) + apply_acl(path, acls[offs]) + except IOError as e: + if e.errno == errno.EINVAL: + # libacl returns with errno set to EINVAL if a user + # (or group) doesn't exist + raise ApplyError("POSIX1e ACL: can't create %r for %r" + % (acls, path_msg(path))) + elif e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP: + raise ApplyError('POSIX1e ACL applyto: %s' % e) + else: + raise ## Linux attributes (lsattr(1), chattr(1)) @@ -601,7 +600,7 @@ class Metadata: if not _warned_about_attr_einval: log("Ignoring attr EINVAL;" + " if you're not using ntfs-3g, please report: " - + repr(path) + '\n') + + path_msg(path) + '\n') _warned_about_attr_einval = True return else: @@ -626,7 +625,7 @@ class Metadata: check_linux_file_attr_api() if not set_linux_file_attr: add_error("%s: can't restore linuxattrs: " - "linuxattr support missing.\n" % path) + "linuxattr support missing.\n" % path_msg(path)) return try: set_linux_file_attr(path, self.linux_attr) @@ -679,7 +678,7 @@ class Metadata: if not xattr: if self.linux_xattr: add_error("%s: can't restore xattr; xattr support missing.\n" - % path) + % path_msg(path)) return if not self.linux_xattr: return @@ -687,7 +686,7 @@ class Metadata: existing_xattrs = set(xattr.list(path, nofollow=True)) except IOError as e: if e.errno == errno.EACCES: - raise ApplyError('xattr.set %r: %s' % (path, e)) + raise ApplyError('xattr.set %r: %s' % (path_msg(path), e)) else: raise for k, v in self.linux_xattr: @@ -698,7 +697,7 @@ class Metadata: except IOError as e: if e.errno == errno.EPERM \ or e.errno == errno.EOPNOTSUPP: - raise ApplyError('xattr.set %r: %s' % (path, e)) + raise ApplyError('xattr.set %r: %s' % (path_msg(path), e)) else: raise existing_xattrs -= frozenset([k]) @@ -707,7 +706,7 @@ class Metadata: xattr.remove(path, k, nofollow=True) except IOError as e: if e.errno in (errno.EPERM, errno.EACCES): - raise ApplyError('xattr.remove %r: %s' % (path, e)) + raise ApplyError('xattr.remove %r: %s' % (path_msg(path), e)) else: raise @@ -765,8 +764,7 @@ class Metadata: if self.path is not None: result += ' path:' + repr(self.path) if self.mode is not None: - result += ' mode:' + repr(xstat.mode_str(self.mode) - + '(%s)' % oct(self.mode)) + result += ' mode: %o (%s)' % (self.mode, xstat.mode_str(self.mode)) if self.uid is not None: result += ' uid:' + str(self.uid) if self.gid is not None: @@ -790,8 +788,12 @@ class Metadata: return ''.join(result) def write(self, port, include_path=True): + port.write(self.encode(include_path=include_path)) + + def encode(self, include_path=True): + ret = [] records = include_path and [(_rec_tag_path, self._encode_path())] or [] - records.extend([(_rec_tag_common_v2, self._encode_common()), + records.extend([(_rec_tag_common_v3, self._encode_common()), (_rec_tag_symlink_target, self._encode_symlink_target()), (_rec_tag_hardlink_target, @@ -801,14 +803,10 @@ class Metadata: (_rec_tag_linux_xattr, self._encode_linux_xattr())]) for tag, data in records: if data: - vint.write_vuint(port, tag) - vint.write_bvec(port, data) - vint.write_vuint(port, _rec_tag_end) - - def encode(self, include_path=True): - port = BytesIO() - self.write(port, include_path) - return port.getvalue() + ret.extend((vint.encode_vuint(tag), + vint.encode_bvec(data))) + ret.append(vint.encode_vuint(_rec_tag_end)) + return b''.join(ret) def copy(self): return deepcopy(self) @@ -828,8 +826,10 @@ class Metadata: while True: # only exit is error (exception) or _rec_tag_end if tag == _rec_tag_path: result._load_path_rec(port) + elif tag == _rec_tag_common_v3: + result._load_common_rec(port, version=3) elif tag == _rec_tag_common_v2: - result._load_common_rec(port) + result._load_common_rec(port, version=2) elif tag == _rec_tag_symlink_target: result._load_symlink_target_rec(port) elif tag == _rec_tag_hardlink_target: @@ -842,8 +842,8 @@ class Metadata: result._load_linux_xattr_rec(port) elif tag == _rec_tag_end: return result - elif tag == _rec_tag_common: # Should be very rare. - result._load_common_rec(port, legacy_format = True) + elif tag == _rec_tag_common_v1: # Should be very rare. + result._load_common_rec(port, version=1) else: # unknown record vint.skip_bvec(port) tag = vint.read_vuint(port) @@ -863,7 +863,7 @@ class Metadata: if not path: raise Exception('Metadata.apply_to_path() called with no path') if not self._recognized_file_type(): - add_error('not applying metadata to "%s"' % path + add_error('not applying metadata to "%s"' % path_msg(path) + ' with unrecognized mode "0x%x"\n' % self.mode) return num_ids = restore_numeric_ids @@ -889,11 +889,17 @@ class Metadata: def from_path(path, statinfo=None, archive_path=None, - save_symlinks=True, hardlink_target=None): + save_symlinks=True, hardlink_target=None, + normalized=False, after_stat=None): + # This function is also a test hook; see test-save-errors + """Return the metadata associated with the path. When normalized is + true, return the metadata appropriate for a typical save, which + may or may not be all of it.""" result = Metadata() result.path = archive_path st = statinfo or xstat.lstat(path) - result.size = st.st_size + if after_stat: + after_stat(path) result._add_common(path, st) if save_symlinks: result._add_symlink_target(path, st) @@ -901,6 +907,10 @@ def from_path(path, statinfo=None, archive_path=None, result._add_posix1e_acl(path, st) result._add_linux_attr(path, st) result._add_linux_xattr(path, st) + if normalized: + # Only store sizes for regular files and symlinks for now. + if not (stat.S_ISREG(result.mode) or stat.S_ISLNK(result.mode)): + result.size = None return result @@ -914,18 +924,19 @@ def save_tree(output_file, paths, for path in paths: safe_path = _clean_up_path_for_archive(path) if safe_path != path: - log('archiving "%s" as "%s"\n' % (path, safe_path)) + log('archiving "%s" as "%s"\n' + % (path_msg(path), path_msg(safe_path))) if not recurse: for p in paths: safe_path = _clean_up_path_for_archive(p) st = xstat.lstat(p) if stat.S_ISDIR(st.st_mode): - safe_path += '/' + safe_path += b'/' m = from_path(p, statinfo=st, archive_path=safe_path, save_symlinks=save_symlinks) if verbose: - print >> sys.stderr, m.path + print(m.path, file=sys.stderr) m.write(output_file, include_path=write_paths) else: start_dir = os.getcwd() @@ -937,7 +948,7 @@ def save_tree(output_file, paths, m = from_path(p, statinfo=st, archive_path=safe_path, save_symlinks=save_symlinks) if verbose: - print >> sys.stderr, m.path + print(m.path, file=sys.stderr) m.write(output_file, include_path=write_paths) os.chdir(dirlist_dir) finally: @@ -973,60 +984,61 @@ all_fields = frozenset(['path', 'posix1e-acl']) -def summary_str(meta, numeric_ids = False, classification = None, - human_readable = False): - - """Return a string containing the "ls -l" style listing for meta. +def summary_bytes(meta, numeric_ids = False, classification = None, + human_readable = False): + """Return bytes containing the "ls -l" style listing for meta. Classification may be "all", "type", or None.""" - user_str = group_str = size_or_dev_str = '?' + user_str = group_str = size_or_dev_str = b'?' symlink_target = None if meta: name = meta.path - mode_str = xstat.mode_str(meta.mode) + mode_str = xstat.mode_str(meta.mode).encode('ascii') symlink_target = meta.symlink_target mtime_secs = xstat.fstime_floor_secs(meta.mtime) - mtime_str = strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs)) + mtime_str = strftime('%Y-%m-%d %H:%M', + time.localtime(mtime_secs)).encode('ascii') if meta.user and not numeric_ids: user_str = meta.user elif meta.uid != None: - user_str = str(meta.uid) + user_str = str(meta.uid).encode() if meta.group and not numeric_ids: group_str = meta.group elif meta.gid != None: - group_str = str(meta.gid) + group_str = str(meta.gid).encode() if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode): if meta.rdev: - size_or_dev_str = '%d,%d' % (os.major(meta.rdev), - os.minor(meta.rdev)) + size_or_dev_str = ('%d,%d' % (os.major(meta.rdev), + os.minor(meta.rdev))).encode() elif meta.size != None: if human_readable: - size_or_dev_str = format_filesize(meta.size) + size_or_dev_str = format_filesize(meta.size).encode() else: - size_or_dev_str = str(meta.size) + size_or_dev_str = str(meta.size).encode() else: - size_or_dev_str = '-' + size_or_dev_str = b'-' if classification: classification_str = \ - xstat.classification_str(meta.mode, classification == 'all') + xstat.classification_str(meta.mode, + classification == 'all').encode() else: - mode_str = '?' * 10 - mtime_str = '????-??-?? ??:??' - classification_str = '?' + mode_str = b'?' * 10 + mtime_str = b'????-??-?? ??:??' + classification_str = b'?' - name = name or '' + name = name or b'' if classification: name += classification_str if symlink_target: - name += ' -> ' + meta.symlink_target + name += b' -> ' + meta.symlink_target - return '%-10s %-11s %11s %16s %s' % (mode_str, - user_str + "/" + group_str, - size_or_dev_str, - mtime_str, - name) + return b'%-10s %-11s %11s %16s %s' % (mode_str, + user_str + b'/' + group_str, + size_or_dev_str, + mtime_str, + name) -def detailed_str(meta, fields = None): +def detailed_bytes(meta, fields = None): # FIXME: should optional fields be omitted, or empty i.e. "rdev: # 0", "link-target:", etc. if not fields: @@ -1034,29 +1046,29 @@ def detailed_str(meta, fields = None): result = [] if 'path' in fields: - path = meta.path or '' - result.append('path: ' + path) + path = meta.path or b'' + result.append(b'path: ' + path) if 'mode' in fields: - result.append('mode: %s (%s)' % (oct(meta.mode), - xstat.mode_str(meta.mode))) + result.append(b'mode: %o (%s)' + % (meta.mode, xstat.mode_str(meta.mode).encode('ascii'))) if 'link-target' in fields and stat.S_ISLNK(meta.mode): - result.append('link-target: ' + meta.symlink_target) + result.append(b'link-target: ' + meta.symlink_target) if 'rdev' in fields: if meta.rdev: - result.append('rdev: %d,%d' % (os.major(meta.rdev), - os.minor(meta.rdev))) + result.append(b'rdev: %d,%d' % (os.major(meta.rdev), + os.minor(meta.rdev))) else: - result.append('rdev: 0') - if 'size' in fields and meta.size: - result.append('size: ' + str(meta.size)) + result.append(b'rdev: 0') + if 'size' in fields and meta.size is not None: + result.append(b'size: %d' % meta.size) if 'uid' in fields: - result.append('uid: ' + str(meta.uid)) + result.append(b'uid: %d' % meta.uid) if 'gid' in fields: - result.append('gid: ' + str(meta.gid)) + result.append(b'gid: %d' % meta.gid) if 'user' in fields: - result.append('user: ' + meta.user) + result.append(b'user: ' + meta.user) if 'group' in fields: - result.append('group: ' + meta.group) + result.append(b'group: ' + meta.group) if 'atime' in fields: # If we don't have xstat.lutime, that means we have to use # utime(), and utime() has no way to set the mtime/atime of a @@ -1064,37 +1076,39 @@ def detailed_str(meta, fields = None): # so let's not report it. (That way scripts comparing # before/after won't trigger.) if xstat.lutime or not stat.S_ISLNK(meta.mode): - result.append('atime: ' + xstat.fstime_to_sec_str(meta.atime)) + result.append(b'atime: ' + xstat.fstime_to_sec_bytes(meta.atime)) else: - result.append('atime: 0') + result.append(b'atime: 0') if 'mtime' in fields: if xstat.lutime or not stat.S_ISLNK(meta.mode): - result.append('mtime: ' + xstat.fstime_to_sec_str(meta.mtime)) + result.append(b'mtime: ' + xstat.fstime_to_sec_bytes(meta.mtime)) else: - result.append('mtime: 0') + result.append(b'mtime: 0') if 'ctime' in fields: - result.append('ctime: ' + xstat.fstime_to_sec_str(meta.ctime)) + result.append(b'ctime: ' + xstat.fstime_to_sec_bytes(meta.ctime)) if 'linux-attr' in fields and meta.linux_attr: - result.append('linux-attr: ' + hex(meta.linux_attr)) + result.append(b'linux-attr: %x' % meta.linux_attr) if 'linux-xattr' in fields and meta.linux_xattr: for name, value in meta.linux_xattr: - result.append('linux-xattr: %s -> %s' % (name, repr(value))) + result.append(b'linux-xattr: %s -> %s' % (name, value)) if 'posix1e-acl' in fields and meta.posix1e_acl: acl = meta.posix1e_acl[0] - result.append('posix1e-acl: ' + acl + '\n') + result.append(b'posix1e-acl: ' + acl + b'\n') if stat.S_ISDIR(meta.mode): def_acl = meta.posix1e_acl[2] - result.append('posix1e-acl-default: ' + def_acl + '\n') - return '\n'.join(result) + result.append(b'posix1e-acl-default: ' + def_acl + b'\n') + return b'\n'.join(result) class _ArchiveIterator: - def next(self): + def __next__(self): try: return Metadata.read(self._file) except EOFError: raise StopIteration() + next = __next__ + def __iter__(self): return self @@ -1102,25 +1116,27 @@ class _ArchiveIterator: self._file = file -def display_archive(file): +def display_archive(file, out): if verbose > 1: first_item = True for meta in _ArchiveIterator(file): if not first_item: - print - print detailed_str(meta) + out.write(b'\n') + out.write(detailed_bytes(meta)) + out.write(b'\n') first_item = False elif verbose > 0: for meta in _ArchiveIterator(file): - print summary_str(meta) + out.write(summary_bytes(meta)) + out.write(b'\n') elif verbose == 0: for meta in _ArchiveIterator(file): if not meta.path: - print >> sys.stderr, \ - 'bup: no metadata path, but asked to only display path', \ - '(increase verbosity?)' + log('bup: no metadata path, but asked to only display path' + ' (increase verbosity?)') sys.exit(1) - print meta.path + out.write(meta.path) + out.write(b'\n') def start_extract(file, create_symlinks=True): @@ -1128,10 +1144,11 @@ def start_extract(file, create_symlinks=True): if not meta: # Hit end record. break if verbose: - print >> sys.stderr, meta.path + print(path_msg(meta.path), file=sys.stderr) xpath = _clean_up_extract_path(meta.path) if not xpath: - add_error(Exception('skipping risky path "%s"' % meta.path)) + add_error(Exception('skipping risky path "%s"' + % path_msg(meta.path))) else: meta.path = xpath _set_up_path(meta, create_symlinks=create_symlinks) @@ -1144,13 +1161,14 @@ def finish_extract(file, restore_numeric_ids=False): break xpath = _clean_up_extract_path(meta.path) if not xpath: - add_error(Exception('skipping risky path "%s"' % dir.path)) + add_error(Exception('skipping risky path "%s"' + % path_msg(meta.path))) else: if os.path.isdir(meta.path): all_dirs.append(meta) else: if verbose: - print >> sys.stderr, meta.path + print(path_msg(meta.path), file=sys.stderr) meta.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids) all_dirs.sort(key = lambda x : len(x.path), reverse=True) @@ -1158,7 +1176,7 @@ def finish_extract(file, restore_numeric_ids=False): # Don't need to check xpath -- won't be in all_dirs if not OK. xpath = _clean_up_extract_path(dir.path) if verbose: - print >> sys.stderr, dir.path + print(path_msg(dir.path), file=sys.stderr) dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids) @@ -1171,24 +1189,25 @@ def extract(file, restore_numeric_ids=False, create_symlinks=True): break xpath = _clean_up_extract_path(meta.path) if not xpath: - add_error(Exception('skipping risky path "%s"' % meta.path)) + add_error(Exception('skipping risky path "%s"' + % path_msg(meta.path))) else: meta.path = xpath if verbose: - print >> sys.stderr, '+', meta.path + print('+', path_msg(meta.path), file=sys.stderr) _set_up_path(meta, create_symlinks=create_symlinks) if os.path.isdir(meta.path): all_dirs.append(meta) else: if verbose: - print >> sys.stderr, '=', meta.path + print('=', path_msg(meta.path), file=sys.stderr) meta.apply_to_path(restore_numeric_ids=restore_numeric_ids) all_dirs.sort(key = lambda x : len(x.path), reverse=True) for dir in all_dirs: # Don't need to check xpath -- won't be in all_dirs if not OK. xpath = _clean_up_extract_path(dir.path) if verbose: - print >> sys.stderr, '=', xpath + print('=', path_msg(xpath), file=sys.stderr) # Shouldn't have to check for risky paths here (omitted above). dir.apply_to_path(path=dir.path, restore_numeric_ids=restore_numeric_ids)