metadata: use python-xattr pyxattr compat code

[bup.git] / lib / bup / metadata.py

diff --git a/lib/bup/metadata.py b/lib/bup/metadata.py
index e5337c96278563663a1d0c61303df8cbe8d5e162..fe1d5f378e5750da4d52c707241c0d7787f4cc19 100644 (file)
--- a/lib/bup/metadata.py
+++ b/lib/bup/metadata.py
@@ -5,7 +5,7 @@
 # This code is covered under the terms of the GNU Library General
 # Public License as described in the bup LICENSE file.
 
 # This code is covered under the terms of the GNU Library General
 # Public License as described in the bup LICENSE file.
 

-from __future__ import absolute_import
+from __future__ import absolute_import, print_function

 from copy import deepcopy
 from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
 from io import BytesIO
 from copy import deepcopy
 from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
 from io import BytesIO


@@ -15,21 +15,23 @@ import errno, os, sys, stat, time, pwd, grp, socket, struct
 from bup import vint, xstat
 from bup.drecurse import recursive_dirlist
 from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize
 from bup import vint, xstat
 from bup.drecurse import recursive_dirlist
 from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize

-from bup.helpers import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name
+from bup.pwdgrp import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name

 from bup.xstat import utime, lutime
 
 xattr = None
 if sys.platform.startswith('linux'):
 from bup.xstat import utime, lutime
 
 xattr = None
 if sys.platform.startswith('linux'):

+    # prefer python-pyxattr (it's a lot faster), but fall back to python-xattr
+    # as the two are incompatible and only one can be installed on a system

     try:
         import xattr
     except ImportError:
         log('Warning: Linux xattr support missing; install python-pyxattr.\n')
     try:
         import xattr
     except ImportError:
         log('Warning: Linux xattr support missing; install python-pyxattr.\n')

-    if xattr:
+    if xattr and getattr(xattr, 'get_all', None) is None:

         try:
         try:

-            xattr.get_all
-        except AttributeError:
+            from xattr import pyxattr_compat as xattr
+        except ImportError:

             log('Warning: python-xattr module is too old; '
             log('Warning: python-xattr module is too old; '

-                'install python-pyxattr instead.\n')
+                'upgrade or install python-pyxattr instead.\n')

             xattr = None
 
 posix1e = None
             xattr = None
 
 posix1e = None


@@ -183,7 +185,7 @@ def _clean_up_extract_path(p):
 # must be unique, and must *never* be changed.
 _rec_tag_end = 0
 _rec_tag_path = 1
 # must be unique, and must *never* be changed.
 _rec_tag_end = 0
 _rec_tag_path = 1

-_rec_tag_common = 2 # times, user, group, type, perms, etc. (legacy/broken)
+_rec_tag_common_v1 = 2 # times, user, group, type, perms, etc. (legacy/broken)

 _rec_tag_symlink_target = 3
 _rec_tag_posix1e_acl = 4      # getfacl(1), setfacl(1), etc.
 _rec_tag_nfsv4_acl = 5        # intended to supplant posix1e? (unimplemented)
 _rec_tag_symlink_target = 3
 _rec_tag_posix1e_acl = 4      # getfacl(1), setfacl(1), etc.
 _rec_tag_nfsv4_acl = 5        # intended to supplant posix1e? (unimplemented)


@@ -191,6 +193,7 @@ _rec_tag_linux_attr = 6       # lsattr(1) chattr(1)
 _rec_tag_linux_xattr = 7      # getfattr(1) setfattr(1)
 _rec_tag_hardlink_target = 8 # hard link target path
 _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current)
 _rec_tag_linux_xattr = 7      # getfattr(1) setfattr(1)
 _rec_tag_hardlink_target = 8 # hard link target path
 _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current)

+_rec_tag_common_v3 = 10  # adds optional size to v2

 
 _warned_about_attr_einval = None
 
 
 _warned_about_attr_einval = None
 


@@ -222,6 +225,7 @@ class Metadata:
     def _add_common(self, path, st):
         assert(st.st_uid >= 0)
         assert(st.st_gid >= 0)
     def _add_common(self, path, st):
         assert(st.st_uid >= 0)
         assert(st.st_gid >= 0)

+        self.size = st.st_size

         self.uid = st.st_uid
         self.gid = st.st_gid
         self.atime = st.st_atime
         self.uid = st.st_uid
         self.gid = st.st_gid
         self.atime = st.st_atime


@@ -252,7 +256,8 @@ class Metadata:
             and self.mtime == other.mtime \
             and self.ctime == other.ctime \
             and self.user == other.user \
             and self.mtime == other.mtime \
             and self.ctime == other.ctime \
             and self.user == other.user \

-            and self.group == other.group
+            and self.group == other.group \
+            and self.size == other.size

 
     def _encode_common(self):
         if not self.mode:
 
     def _encode_common(self):
         if not self.mode:


@@ -260,7 +265,7 @@ class Metadata:
         atime = xstat.nsecs_to_timespec(self.atime)
         mtime = xstat.nsecs_to_timespec(self.mtime)
         ctime = xstat.nsecs_to_timespec(self.ctime)
         atime = xstat.nsecs_to_timespec(self.atime)
         mtime = xstat.nsecs_to_timespec(self.mtime)
         ctime = xstat.nsecs_to_timespec(self.ctime)

-        result = vint.pack('vvsvsvvVvVvV',
+        result = vint.pack('vvsvsvvVvVvVv',

                            self.mode,
                            self.uid,
                            self.user,
                            self.mode,
                            self.uid,
                            self.user,


@@ -272,26 +277,36 @@ class Metadata:
                            mtime[0],
                            mtime[1],
                            ctime[0],
                            mtime[0],
                            mtime[1],
                            ctime[0],

-                           ctime[1])
+                           ctime[1],
+                           self.size if self.size is not None else -1)

         return result
 
         return result
 

-    def _load_common_rec(self, port, legacy_format=False):
-        unpack_fmt = 'vvsvsvvVvVvV'
-        if legacy_format:
+    def _load_common_rec(self, port, version=3):
+        if version == 3:
+            # Added trailing size to v2, negative when None.
+            unpack_fmt = 'vvsvsvvVvVvVv'
+        elif version == 2:
+            unpack_fmt = 'vvsvsvvVvVvV'
+        elif version == 1:

             unpack_fmt = 'VVsVsVvVvVvV'
             unpack_fmt = 'VVsVsVvVvVvV'

+        else:
+            raise Exception('unexpected common_rec version %d' % version)

         data = vint.read_bvec(port)
         data = vint.read_bvec(port)

-        (self.mode,
-         self.uid,
-         self.user,
-         self.gid,
-         self.group,
-         self.rdev,
-         self.atime,
-         atime_ns,
-         self.mtime,
-         mtime_ns,
-         self.ctime,
-         ctime_ns) = vint.unpack(unpack_fmt, data)
+        values = vint.unpack(unpack_fmt, data)
+        if version == 3:
+            (self.mode, self.uid, self.user, self.gid, self.group,
+             self.rdev,
+             self.atime, atime_ns,
+             self.mtime, mtime_ns,
+             self.ctime, ctime_ns, size) = values
+            if size >= 0:
+                self.size = size
+        else:
+            (self.mode, self.uid, self.user, self.gid, self.group,
+             self.rdev,
+             self.atime, atime_ns,
+             self.mtime, mtime_ns,
+             self.ctime, ctime_ns) = values

         self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
         self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
         self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
         self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
         self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
         self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))


@@ -344,7 +359,7 @@ class Metadata:
             os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev)
         elif stat.S_ISFIFO(self.mode):
             assert(self._recognized_file_type())
             os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev)
         elif stat.S_ISFIFO(self.mode):
             assert(self._recognized_file_type())

-            os.mknod(path, 0o600 | stat.S_IFIFO)
+            os.mkfifo(path, 0o600 | stat.S_IFIFO)

         elif stat.S_ISSOCK(self.mode):
             try:
                 os.mknod(path, 0o600 | stat.S_IFSOCK)
         elif stat.S_ISSOCK(self.mode):
             try:
                 os.mknod(path, 0o600 | stat.S_IFSOCK)


@@ -468,7 +483,10 @@ class Metadata:
     def _load_symlink_target_rec(self, port):
         target = vint.read_bvec(port)
         self.symlink_target = target
     def _load_symlink_target_rec(self, port):
         target = vint.read_bvec(port)
         self.symlink_target = target

-        self.size = len(target)
+        if self.size is None:
+            self.size = len(target)
+        else:
+            assert(self.size == len(target))

 
 
     ## Hardlink targets
 
 
     ## Hardlink targets


@@ -791,7 +809,7 @@ class Metadata:
 
     def write(self, port, include_path=True):
         records = include_path and [(_rec_tag_path, self._encode_path())] or []
 
     def write(self, port, include_path=True):
         records = include_path and [(_rec_tag_path, self._encode_path())] or []

-        records.extend([(_rec_tag_common_v2, self._encode_common()),
+        records.extend([(_rec_tag_common_v3, self._encode_common()),

                         (_rec_tag_symlink_target,
                          self._encode_symlink_target()),
                         (_rec_tag_hardlink_target,
                         (_rec_tag_symlink_target,
                          self._encode_symlink_target()),
                         (_rec_tag_hardlink_target,


@@ -828,8 +846,10 @@ class Metadata:
             while True: # only exit is error (exception) or _rec_tag_end
                 if tag == _rec_tag_path:
                     result._load_path_rec(port)
             while True: # only exit is error (exception) or _rec_tag_end
                 if tag == _rec_tag_path:
                     result._load_path_rec(port)

+                elif tag == _rec_tag_common_v3:
+                    result._load_common_rec(port, version=3)

                 elif tag == _rec_tag_common_v2:
                 elif tag == _rec_tag_common_v2:

-                    result._load_common_rec(port)
+                    result._load_common_rec(port, version=2)

                 elif tag == _rec_tag_symlink_target:
                     result._load_symlink_target_rec(port)
                 elif tag == _rec_tag_hardlink_target:
                 elif tag == _rec_tag_symlink_target:
                     result._load_symlink_target_rec(port)
                 elif tag == _rec_tag_hardlink_target:


@@ -842,8 +862,8 @@ class Metadata:
                     result._load_linux_xattr_rec(port)
                 elif tag == _rec_tag_end:
                     return result
                     result._load_linux_xattr_rec(port)
                 elif tag == _rec_tag_end:
                     return result

-                elif tag == _rec_tag_common: # Should be very rare.
-                    result._load_common_rec(port, legacy_format = True)
+                elif tag == _rec_tag_common_v1: # Should be very rare.
+                    result._load_common_rec(port, version=1)

                 else: # unknown record
                     vint.skip_bvec(port)
                 tag = vint.read_vuint(port)
                 else: # unknown record
                     vint.skip_bvec(port)
                 tag = vint.read_vuint(port)


@@ -889,11 +909,14 @@ class Metadata:
 
 
 def from_path(path, statinfo=None, archive_path=None,
 
 
 def from_path(path, statinfo=None, archive_path=None,

-              save_symlinks=True, hardlink_target=None):
+              save_symlinks=True, hardlink_target=None,
+              normalized=False):
+    """Return the metadata associated with the path.  When normalized is
+    true, return the metadata appropriate for a typical save, which
+    may or may not be all of it."""

     result = Metadata()
     result.path = archive_path
     st = statinfo or xstat.lstat(path)
     result = Metadata()
     result.path = archive_path
     st = statinfo or xstat.lstat(path)

-    result.size = st.st_size

     result._add_common(path, st)
     if save_symlinks:
         result._add_symlink_target(path, st)
     result._add_common(path, st)
     if save_symlinks:
         result._add_symlink_target(path, st)


@@ -901,6 +924,10 @@ def from_path(path, statinfo=None, archive_path=None,
     result._add_posix1e_acl(path, st)
     result._add_linux_attr(path, st)
     result._add_linux_xattr(path, st)
     result._add_posix1e_acl(path, st)
     result._add_linux_attr(path, st)
     result._add_linux_xattr(path, st)

+    if normalized:
+        # Only store sizes for regular files and symlinks for now.
+        if not (stat.S_ISREG(result.mode) or stat.S_ISLNK(result.mode)):
+            result.size = None

     return result
 
 
     return result
 
 


@@ -925,7 +952,7 @@ def save_tree(output_file, paths,
             m = from_path(p, statinfo=st, archive_path=safe_path,
                           save_symlinks=save_symlinks)
             if verbose:
             m = from_path(p, statinfo=st, archive_path=safe_path,
                           save_symlinks=save_symlinks)
             if verbose:

-                print >> sys.stderr, m.path
+                print(m.path, file=sys.stderr)

             m.write(output_file, include_path=write_paths)
     else:
         start_dir = os.getcwd()
             m.write(output_file, include_path=write_paths)
     else:
         start_dir = os.getcwd()


@@ -937,7 +964,7 @@ def save_tree(output_file, paths,
                 m = from_path(p, statinfo=st, archive_path=safe_path,
                               save_symlinks=save_symlinks)
                 if verbose:
                 m = from_path(p, statinfo=st, archive_path=safe_path,
                               save_symlinks=save_symlinks)
                 if verbose:

-                    print >> sys.stderr, m.path
+                    print(m.path, file=sys.stderr)

                 m.write(output_file, include_path=write_paths)
                 os.chdir(dirlist_dir)
         finally:
                 m.write(output_file, include_path=write_paths)
                 os.chdir(dirlist_dir)
         finally:


@@ -1047,7 +1074,7 @@ def detailed_str(meta, fields = None):
                                            os.minor(meta.rdev)))
         else:
             result.append('rdev: 0')
                                            os.minor(meta.rdev)))
         else:
             result.append('rdev: 0')

-    if 'size' in fields and meta.size:
+    if 'size' in fields and meta.size is not None:

         result.append('size: ' + str(meta.size))
     if 'uid' in fields:
         result.append('uid: ' + str(meta.uid))
         result.append('size: ' + str(meta.size))
     if 'uid' in fields:
         result.append('uid: ' + str(meta.uid))


@@ -1107,20 +1134,19 @@ def display_archive(file):
         first_item = True
         for meta in _ArchiveIterator(file):
             if not first_item:
         first_item = True
         for meta in _ArchiveIterator(file):
             if not first_item:

-                print
-            print detailed_str(meta)
+                print()
+            print(detailed_str(meta))

             first_item = False
     elif verbose > 0:
         for meta in _ArchiveIterator(file):
             first_item = False
     elif verbose > 0:
         for meta in _ArchiveIterator(file):

-            print summary_str(meta)
+            print(summary_str(meta))

     elif verbose == 0:
         for meta in _ArchiveIterator(file):
             if not meta.path:
     elif verbose == 0:
         for meta in _ArchiveIterator(file):
             if not meta.path:

-                print >> sys.stderr, \
-                    'bup: no metadata path, but asked to only display path', \
-                    '(increase verbosity?)'
+                print('bup: no metadata path, but asked to only display path'
+                     '(increase verbosity?)')

                 sys.exit(1)
                 sys.exit(1)

-            print meta.path
+            print(meta.path)

 
 
 def start_extract(file, create_symlinks=True):
 
 
 def start_extract(file, create_symlinks=True):


@@ -1128,7 +1154,7 @@ def start_extract(file, create_symlinks=True):
         if not meta: # Hit end record.
             break
         if verbose:
         if not meta: # Hit end record.
             break
         if verbose:

-            print >> sys.stderr, meta.path
+            print(meta.path, file=sys.stderr)

         xpath = _clean_up_extract_path(meta.path)
         if not xpath:
             add_error(Exception('skipping risky path "%s"' % meta.path))
         xpath = _clean_up_extract_path(meta.path)
         if not xpath:
             add_error(Exception('skipping risky path "%s"' % meta.path))


@@ -1150,7 +1176,7 @@ def finish_extract(file, restore_numeric_ids=False):
                 all_dirs.append(meta)
             else:
                 if verbose:
                 all_dirs.append(meta)
             else:
                 if verbose:

-                    print >> sys.stderr, meta.path
+                    print(meta.path, file=sys.stderr)

                 meta.apply_to_path(path=xpath,
                                    restore_numeric_ids=restore_numeric_ids)
     all_dirs.sort(key = lambda x : len(x.path), reverse=True)
                 meta.apply_to_path(path=xpath,
                                    restore_numeric_ids=restore_numeric_ids)
     all_dirs.sort(key = lambda x : len(x.path), reverse=True)


@@ -1158,7 +1184,7 @@ def finish_extract(file, restore_numeric_ids=False):
         # Don't need to check xpath -- won't be in all_dirs if not OK.
         xpath = _clean_up_extract_path(dir.path)
         if verbose:
         # Don't need to check xpath -- won't be in all_dirs if not OK.
         xpath = _clean_up_extract_path(dir.path)
         if verbose:

-            print >> sys.stderr, dir.path
+            print(dir.path, file=sys.stderr)

         dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
 
 
         dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
 
 


@@ -1175,20 +1201,20 @@ def extract(file, restore_numeric_ids=False, create_symlinks=True):
         else:
             meta.path = xpath
             if verbose:
         else:
             meta.path = xpath
             if verbose:

-                print >> sys.stderr, '+', meta.path
+                print('+', meta.path, file=sys.stderr)

             _set_up_path(meta, create_symlinks=create_symlinks)
             if os.path.isdir(meta.path):
                 all_dirs.append(meta)
             else:
                 if verbose:
             _set_up_path(meta, create_symlinks=create_symlinks)
             if os.path.isdir(meta.path):
                 all_dirs.append(meta)
             else:
                 if verbose:

-                    print >> sys.stderr, '=', meta.path
+                    print('=', meta.path, file=sys.stderr)

                 meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
     all_dirs.sort(key = lambda x : len(x.path), reverse=True)
     for dir in all_dirs:
         # Don't need to check xpath -- won't be in all_dirs if not OK.
         xpath = _clean_up_extract_path(dir.path)
         if verbose:
                 meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
     all_dirs.sort(key = lambda x : len(x.path), reverse=True)
     for dir in all_dirs:
         # Don't need to check xpath -- won't be in all_dirs if not OK.
         xpath = _clean_up_extract_path(dir.path)
         if verbose:

-            print >> sys.stderr, '=', xpath
+            print('=', xpath, file=sys.stderr)

         # Shouldn't have to check for risky paths here (omitted above).
         dir.apply_to_path(path=dir.path,
                           restore_numeric_ids=restore_numeric_ids)
         # Shouldn't have to check for risky paths here (omitted above).
         dir.apply_to_path(path=dir.path,
                           restore_numeric_ids=restore_numeric_ids)