1 """Metadata read/write support for bup."""
3 # Copyright (C) 2010 Rob Browning
5 # This code is covered under the terms of the GNU Library General
6 # Public License as described in the bup LICENSE file.
8 from __future__ import absolute_import, print_function
9 from copy import deepcopy
10 from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
11 from io import BytesIO
12 from time import gmtime, strftime
13 import errno, os, sys, stat, time, socket, struct
15 from bup import vint, xstat
16 from bup.drecurse import recursive_dirlist
17 from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize
18 from bup.io import path_msg
19 from bup.pwdgrp import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name
20 from bup.xstat import utime, lutime
23 if sys.platform.startswith('linux'):
24 # prefer python-pyxattr (it's a lot faster), but fall back to python-xattr
25 # as the two are incompatible and only one can be installed on a system
29 log('Warning: Linux xattr support missing; install python-pyxattr.\n')
30 if xattr and getattr(xattr, 'get_all', None) is None:
32 from xattr import pyxattr_compat as xattr
33 if not isinstance(xattr.NS_USER, bytes):
38 log('Warning: python-xattr module is too old; '
39 'upgrade or install python-pyxattr instead.\n')
42 from bup._helpers import read_acl, apply_acl
44 read_acl = apply_acl = None
47 from bup._helpers import get_linux_file_attr, set_linux_file_attr
49 # No need for a warning here; the only reason they won't exist is that we're
50 # not on Linux, in which case files don't have any linux attrs anyway, so
51 # lacking the functions isn't a problem.
52 get_linux_file_attr = set_linux_file_attr = None
55 # See the bup_get_linux_file_attr() comments.
56 _suppress_linux_file_attr = \
57 sys.byteorder == 'big' and struct.calcsize('@l') > struct.calcsize('@i')
59 def check_linux_file_attr_api():
60 global get_linux_file_attr, set_linux_file_attr
61 if not (get_linux_file_attr or set_linux_file_attr):
63 if _suppress_linux_file_attr:
64 log('Warning: Linux attr support disabled (see "bup help index").\n')
65 get_linux_file_attr = set_linux_file_attr = None
68 # WARNING: the metadata encoding is *not* stable yet. Caveat emptor!
70 # Q: Consider hardlink support?
71 # Q: Is it OK to store raw linux attr (chattr) flags?
72 # Q: Can anything other than S_ISREG(x) or S_ISDIR(x) support posix1e ACLs?
73 # Q: Is the application of posix1e has_extended() correct?
74 # Q: Is one global --numeric-ids argument sufficient?
75 # Q: Do nfsv4 acls trump posix1e acls? (seems likely)
76 # Q: Add support for crtime -- ntfs, and (only internally?) ext*?
78 # FIXME: Fix relative/abs path detection/stripping wrt other platforms.
79 # FIXME: Add nfsv4 acl handling - see nfs4-acl-tools.
80 # FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.).
81 # FIXME: Consider pack('vvvvsss', ...) optimization.
85 # osx (varies between hfs and hfs+):
86 # type - regular dir char block fifo socket ...
87 # perms - rwxrwxrwxsgt
88 # times - ctime atime mtime
91 # hard-link-info (hfs+ only)
94 # attributes-osx see chflags
100 # type - regular dir ...
101 # times - creation, modification, posix change, access
104 # attributes - see attrib
106 # forks (alternate data streams)
110 # type - regular dir ...
111 # perms - rwxrwxrwx (maybe - see wikipedia)
112 # times - creation, modification, access
113 # attributes - see attrib
117 _have_lchmod = hasattr(os, 'lchmod')
120 def _clean_up_path_for_archive(p):
121 # Not the most efficient approach.
124 # Take everything after any '/../'.
125 pos = result.rfind(b'/../')
127 result = result[result.rfind(b'/../') + 4:]
129 # Take everything after any remaining '../'.
130 if result.startswith(b"../"):
133 # Remove any '/./' sequences.
134 pos = result.find(b'/./')
136 result = result[0:pos] + b'/' + result[pos + 3:]
137 pos = result.find(b'/./')
139 # Remove any leading '/'s.
140 result = result.lstrip(b'/')
142 # Replace '//' with '/' everywhere.
143 pos = result.find(b'//')
145 result = result[0:pos] + b'/' + result[pos + 2:]
146 pos = result.find(b'//')
148 # Take everything after any remaining './'.
149 if result.startswith(b'./'):
152 # Take everything before any remaining '/.'.
153 if result.endswith(b'/.'):
156 if result == b'' or result.endswith(b'/..'):
163 if p.startswith(b'/'):
165 if p.find(b'/../') != -1:
167 if p.startswith(b'../'):
169 if p.endswith(b'/..'):
174 def _clean_up_extract_path(p):
175 result = p.lstrip(b'/')
178 elif _risky_path(result):
184 # These tags are currently conceptually private to Metadata, and they
185 # must be unique, and must *never* be changed.
188 _rec_tag_common_v1 = 2 # times, user, group, type, perms, etc. (legacy/broken)
189 _rec_tag_symlink_target = 3
190 _rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc.
191 _rec_tag_nfsv4_acl = 5 # intended to supplant posix1e? (unimplemented)
192 _rec_tag_linux_attr = 6 # lsattr(1) chattr(1)
193 _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1)
194 _rec_tag_hardlink_target = 8 # hard link target path
195 _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current)
196 _rec_tag_common_v3 = 10 # adds optional size to v2
198 _warned_about_attr_einval = None
201 class ApplyError(Exception):
202 # Thrown when unable to apply any given bit of metadata to a path.
207 # Metadata is stored as a sequence of tagged binary records. Each
208 # record will have some subset of add, encode, load, create, and
209 # apply methods, i.e. _add_foo...
211 # We do allow an "empty" object as a special case, i.e. no
212 # records. One can be created by trying to write Metadata(), and
213 # for such an object, read() will return None. This is used by
214 # "bup save", for example, as a placeholder in cases where
217 # NOTE: if any relevant fields are added or removed, be sure to
218 # update same_file() below.
222 # Timestamps are (sec, ns), relative to 1970-01-01 00:00:00, ns
223 # must be non-negative and < 10**9.
225 def _add_common(self, path, st):
226 assert(st.st_uid >= 0)
227 assert(st.st_gid >= 0)
228 self.size = st.st_size
231 self.atime = st.st_atime
232 self.mtime = st.st_mtime
233 self.ctime = st.st_ctime
234 self.user = self.group = b''
235 entry = pwd_from_uid(st.st_uid)
237 self.user = entry.pw_name
238 entry = grp_from_gid(st.st_gid)
240 self.group = entry.gr_name
241 self.mode = st.st_mode
242 # Only collect st_rdev if we might need it for a mknod()
243 # during restore. On some platforms (i.e. kFreeBSD), it isn't
244 # stable for other file types. For example "cp -a" will
245 # change it for a plain file.
246 if stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
247 self.rdev = st.st_rdev
251 def _same_common(self, other):
252 """Return true or false to indicate similarity in the hardlink sense."""
253 return self.uid == other.uid \
254 and self.gid == other.gid \
255 and self.rdev == other.rdev \
256 and self.mtime == other.mtime \
257 and self.ctime == other.ctime \
258 and self.user == other.user \
259 and self.group == other.group \
260 and self.size == other.size
262 def _encode_common(self):
265 atime = xstat.nsecs_to_timespec(self.atime)
266 mtime = xstat.nsecs_to_timespec(self.mtime)
267 ctime = xstat.nsecs_to_timespec(self.ctime)
268 result = vint.pack('vvsvsvvVvVvVv',
281 self.size if self.size is not None else -1)
284 def _load_common_rec(self, port, version=3):
286 # Added trailing size to v2, negative when None.
287 unpack_fmt = 'vvsvsvvVvVvVv'
289 unpack_fmt = 'vvsvsvvVvVvV'
291 unpack_fmt = 'VVsVsVvVvVvV'
293 raise Exception('unexpected common_rec version %d' % version)
294 data = vint.read_bvec(port)
295 values = vint.unpack(unpack_fmt, data)
297 (self.mode, self.uid, self.user, self.gid, self.group,
299 self.atime, atime_ns,
300 self.mtime, mtime_ns,
301 self.ctime, ctime_ns, size) = values
305 (self.mode, self.uid, self.user, self.gid, self.group,
307 self.atime, atime_ns,
308 self.mtime, mtime_ns,
309 self.ctime, ctime_ns) = values
310 self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
311 self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
312 self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
314 def _recognized_file_type(self):
315 return stat.S_ISREG(self.mode) \
316 or stat.S_ISDIR(self.mode) \
317 or stat.S_ISCHR(self.mode) \
318 or stat.S_ISBLK(self.mode) \
319 or stat.S_ISFIFO(self.mode) \
320 or stat.S_ISSOCK(self.mode) \
321 or stat.S_ISLNK(self.mode)
323 def _create_via_common_rec(self, path, create_symlinks=True):
325 raise ApplyError('no metadata - cannot create path '
328 # If the path already exists and is a dir, try rmdir.
329 # If the path already exists and is anything else, try unlink.
332 st = xstat.lstat(path)
334 if e.errno != errno.ENOENT:
337 if stat.S_ISDIR(st.st_mode):
341 if e.errno in (errno.ENOTEMPTY, errno.EEXIST):
342 raise Exception('refusing to overwrite non-empty dir '
348 if stat.S_ISREG(self.mode):
349 assert(self._recognized_file_type())
350 fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0o600)
352 elif stat.S_ISDIR(self.mode):
353 assert(self._recognized_file_type())
354 os.mkdir(path, 0o700)
355 elif stat.S_ISCHR(self.mode):
356 assert(self._recognized_file_type())
357 os.mknod(path, 0o600 | stat.S_IFCHR, self.rdev)
358 elif stat.S_ISBLK(self.mode):
359 assert(self._recognized_file_type())
360 os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev)
361 elif stat.S_ISFIFO(self.mode):
362 assert(self._recognized_file_type())
363 os.mkfifo(path, 0o600 | stat.S_IFIFO)
364 elif stat.S_ISSOCK(self.mode):
366 os.mknod(path, 0o600 | stat.S_IFSOCK)
368 if e.errno in (errno.EINVAL, errno.EPERM):
369 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
373 elif stat.S_ISLNK(self.mode):
374 assert(self._recognized_file_type())
375 if self.symlink_target and create_symlinks:
376 # on MacOS, symlink() permissions depend on umask, and there's
377 # no way to chown a symlink after creating it, so we have to
379 oldumask = os.umask((self.mode & 0o777) ^ 0o777)
381 os.symlink(self.symlink_target, path)
384 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
386 assert(not self._recognized_file_type())
387 add_error('not creating "%s" with unrecognized mode "0x%x"\n'
388 % (path_msg(path), self.mode))
390 def _apply_common_rec(self, path, restore_numeric_ids=False):
392 raise ApplyError('no metadata - cannot apply to ' + path_msg(path))
394 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
395 # EACCES errors at this stage are fatal for the current path.
396 if lutime and stat.S_ISLNK(self.mode):
398 lutime(path, (self.atime, self.mtime))
400 if e.errno == errno.EACCES:
401 raise ApplyError('lutime: %s' % e)
406 utime(path, (self.atime, self.mtime))
408 if e.errno == errno.EACCES:
409 raise ApplyError('utime: %s' % e)
413 uid = gid = -1 # By default, do nothing.
415 if self.uid is not None:
417 if self.gid is not None:
419 if not restore_numeric_ids:
420 if self.uid != 0 and self.user:
421 entry = pwd_from_name(self.user)
424 if self.gid != 0 and self.group:
425 entry = grp_from_name(self.group)
428 else: # not superuser - only consider changing the group/gid
429 user_gids = os.getgroups()
430 if self.gid in user_gids:
432 if not restore_numeric_ids and self.gid != 0:
433 # The grp might not exist on the local system.
434 grps = filter(None, [grp_from_gid(x) for x in user_gids])
435 if self.group in [x.gr_name for x in grps]:
436 g = grp_from_name(self.group)
440 if uid != -1 or gid != -1:
442 os.lchown(path, uid, gid)
444 if e.errno == errno.EPERM:
445 add_error('lchown: %s' % e)
446 elif sys.platform.startswith('cygwin') \
447 and e.errno == errno.EINVAL:
448 add_error('lchown: unknown uid/gid (%d/%d) for %s'
449 % (uid, gid, path_msg(path)))
455 os.lchmod(path, stat.S_IMODE(self.mode))
457 # - "Function not implemented"
458 # - "Operation not supported" might be generated by glibc
459 if e.errno in (errno.ENOSYS, errno.EOPNOTSUPP):
463 elif not stat.S_ISLNK(self.mode):
464 os.chmod(path, stat.S_IMODE(self.mode))
469 def _encode_path(self):
471 return vint.pack('s', self.path)
475 def _load_path_rec(self, port):
476 self.path = vint.unpack('s', vint.read_bvec(port))[0]
481 def _add_symlink_target(self, path, st):
483 if stat.S_ISLNK(st.st_mode):
484 self.symlink_target = os.readlink(path)
485 # might have read a different link than the
486 # one that was in place when we did stat()
487 self.size = len(self.symlink_target)
489 add_error('readlink: %s' % e)
491 def _encode_symlink_target(self):
492 return self.symlink_target
494 def _load_symlink_target_rec(self, port):
495 target = vint.read_bvec(port)
496 self.symlink_target = target
497 if self.size is None:
498 self.size = len(target)
500 assert(self.size == len(target))
505 def _add_hardlink_target(self, target):
506 self.hardlink_target = target
508 def _same_hardlink_target(self, other):
509 """Return true or false to indicate similarity in the hardlink sense."""
510 return self.hardlink_target == other.hardlink_target
512 def _encode_hardlink_target(self):
513 return self.hardlink_target
515 def _load_hardlink_target_rec(self, port):
516 self.hardlink_target = vint.read_bvec(port)
519 ## POSIX1e ACL records
521 # Recorded as a list:
522 # [txt_id_acl, num_id_acl]
523 # or, if a directory:
524 # [txt_id_acl, num_id_acl, txt_id_default_acl, num_id_default_acl]
525 # The numeric/text distinction only matters when reading/restoring
527 def _add_posix1e_acl(self, path, st):
530 if not stat.S_ISLNK(st.st_mode):
531 isdir = 1 if stat.S_ISDIR(st.st_mode) else 0
532 self.posix1e_acl = read_acl(path, isdir)
534 def _same_posix1e_acl(self, other):
535 """Return true or false to indicate similarity in the hardlink sense."""
536 return self.posix1e_acl == other.posix1e_acl
538 def _encode_posix1e_acl(self):
539 # Encode as two strings (w/default ACL string possibly empty).
541 acls = self.posix1e_acl
543 return vint.pack('ssss', acls[0], acls[1], b'', b'')
544 return vint.pack('ssss', acls[0], acls[1], acls[2], acls[3])
548 def _load_posix1e_acl_rec(self, port):
549 acl_rep = vint.unpack('ssss', vint.read_bvec(port))
550 if acl_rep[2] == b'':
551 acl_rep = acl_rep[:2]
552 self.posix1e_acl = acl_rep
554 def _apply_posix1e_acl_rec(self, path, restore_numeric_ids=False):
555 if not self.posix1e_acl:
559 add_error("%s: can't restore ACLs; posix1e support missing.\n"
564 acls = self.posix1e_acl
565 offs = 1 if restore_numeric_ids else 0
567 apply_acl(path, acls[offs], acls[offs + 2])
569 apply_acl(path, acls[offs])
571 if e.errno == errno.EINVAL:
572 # libacl returns with errno set to EINVAL if a user
573 # (or group) doesn't exist
574 raise ApplyError("POSIX1e ACL: can't create %r for %r"
575 % (acls, path_msg(path)))
576 elif e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP:
577 raise ApplyError('POSIX1e ACL applyto: %s' % e)
582 ## Linux attributes (lsattr(1), chattr(1))
584 def _add_linux_attr(self, path, st):
585 check_linux_file_attr_api()
586 if not get_linux_file_attr: return
587 if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
589 attr = get_linux_file_attr(path)
591 self.linux_attr = attr
593 if e.errno == errno.EACCES:
594 add_error('read Linux attr: %s' % e)
595 elif e.errno in (ENOTTY, ENOSYS, EOPNOTSUPP):
596 # Assume filesystem doesn't support attrs.
598 elif e.errno == EINVAL:
599 global _warned_about_attr_einval
600 if not _warned_about_attr_einval:
601 log("Ignoring attr EINVAL;"
602 + " if you're not using ntfs-3g, please report: "
603 + path_msg(path) + '\n')
604 _warned_about_attr_einval = True
609 def _same_linux_attr(self, other):
610 """Return true or false to indicate similarity in the hardlink sense."""
611 return self.linux_attr == other.linux_attr
613 def _encode_linux_attr(self):
615 return vint.pack('V', self.linux_attr)
619 def _load_linux_attr_rec(self, port):
620 data = vint.read_bvec(port)
621 self.linux_attr = vint.unpack('V', data)[0]
623 def _apply_linux_attr_rec(self, path, restore_numeric_ids=False):
625 check_linux_file_attr_api()
626 if not set_linux_file_attr:
627 add_error("%s: can't restore linuxattrs: "
628 "linuxattr support missing.\n" % path_msg(path))
631 set_linux_file_attr(path, self.linux_attr)
633 if e.errno in (EACCES, ENOTTY, EOPNOTSUPP, ENOSYS):
634 raise ApplyError('Linux chattr: %s (0x%s)'
635 % (e, hex(self.linux_attr)))
636 elif e.errno == EINVAL:
637 msg = "if you're not using ntfs-3g, please report"
638 raise ApplyError('Linux chattr: %s (0x%s) (%s)'
639 % (e, hex(self.linux_attr), msg))
644 ## Linux extended attributes (getfattr(1), setfattr(1))
646 def _add_linux_xattr(self, path, st):
649 self.linux_xattr = xattr.get_all(path, nofollow=True)
650 except EnvironmentError as e:
651 if e.errno != errno.EOPNOTSUPP:
654 def _same_linux_xattr(self, other):
655 """Return true or false to indicate similarity in the hardlink sense."""
656 return self.linux_xattr == other.linux_xattr
658 def _encode_linux_xattr(self):
660 result = vint.pack('V', len(self.linux_xattr))
661 for name, value in self.linux_xattr:
662 result += vint.pack('ss', name, value)
667 def _load_linux_xattr_rec(self, file):
668 data = vint.read_bvec(file)
669 memfile = BytesIO(data)
671 for i in range(vint.read_vuint(memfile)):
672 key = vint.read_bvec(memfile)
673 value = vint.read_bvec(memfile)
674 result.append((key, value))
675 self.linux_xattr = result
677 def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
680 add_error("%s: can't restore xattr; xattr support missing.\n"
683 if not self.linux_xattr:
686 existing_xattrs = set(xattr.list(path, nofollow=True))
688 if e.errno == errno.EACCES:
689 raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
692 for k, v in self.linux_xattr:
693 if k not in existing_xattrs \
694 or v != xattr.get(path, k, nofollow=True):
696 xattr.set(path, k, v, nofollow=True)
698 if e.errno == errno.EPERM \
699 or e.errno == errno.EOPNOTSUPP:
700 raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
703 existing_xattrs -= frozenset([k])
704 for k in existing_xattrs:
706 xattr.remove(path, k, nofollow=True)
708 if e.errno in (errno.EPERM, errno.EACCES):
709 raise ApplyError('xattr.remove %r: %s' % (path_msg(path), e))
714 __slots__ = ('mode', 'uid', 'atime', 'mtime', 'ctime',
715 'path', 'size', 'symlink_target', 'hardlink_target',
716 'linux_attr', 'linux_xattr', 'posix1e_acl')
717 self.mode = self.uid = self.gid = self.user = self.group = None
718 self.atime = self.mtime = self.ctime = None
722 self.symlink_target = None
723 self.hardlink_target = None
724 self.linux_attr = None
725 self.linux_xattr = None
726 self.posix1e_acl = None
728 def __eq__(self, other):
729 if not isinstance(other, Metadata): return False
730 if self.mode != other.mode: return False
731 if self.mtime != other.mtime: return False
732 if self.ctime != other.ctime: return False
733 if self.atime != other.atime: return False
734 if self.path != other.path: return False
735 if self.uid != other.uid: return False
736 if self.gid != other.gid: return False
737 if self.size != other.size: return False
738 if self.user != other.user: return False
739 if self.group != other.group: return False
740 if self.symlink_target != other.symlink_target: return False
741 if self.hardlink_target != other.hardlink_target: return False
742 if self.linux_attr != other.linux_attr: return False
743 if self.posix1e_acl != other.posix1e_acl: return False
746 def __ne__(self, other):
747 return not self.__eq__(other)
750 return hash((self.mode,
761 self.hardlink_target,
766 result = ['<%s instance at %s' % (self.__class__, hex(id(self)))]
767 if self.path is not None:
768 result += ' path:' + repr(self.path)
769 if self.mode is not None:
770 result += ' mode: %o (%s)' % (self.mode, xstat.mode_str(self.mode))
771 if self.uid is not None:
772 result += ' uid:' + str(self.uid)
773 if self.gid is not None:
774 result += ' gid:' + str(self.gid)
775 if self.user is not None:
776 result += ' user:' + repr(self.user)
777 if self.group is not None:
778 result += ' group:' + repr(self.group)
779 if self.size is not None:
780 result += ' size:' + repr(self.size)
781 for name, val in (('atime', self.atime),
782 ('mtime', self.mtime),
783 ('ctime', self.ctime)):
785 result += ' %s:%r (%d)' \
787 strftime('%Y-%m-%d %H:%M %z',
788 gmtime(xstat.fstime_floor_secs(val))),
791 return ''.join(result)
793 def write(self, port, include_path=True):
794 port.write(self.encode(include_path=include_path))
796 def encode(self, include_path=True):
798 records = include_path and [(_rec_tag_path, self._encode_path())] or []
799 records.extend([(_rec_tag_common_v3, self._encode_common()),
800 (_rec_tag_symlink_target,
801 self._encode_symlink_target()),
802 (_rec_tag_hardlink_target,
803 self._encode_hardlink_target()),
804 (_rec_tag_posix1e_acl, self._encode_posix1e_acl()),
805 (_rec_tag_linux_attr, self._encode_linux_attr()),
806 (_rec_tag_linux_xattr, self._encode_linux_xattr())])
807 for tag, data in records:
809 ret.extend((vint.encode_vuint(tag),
810 vint.encode_bvec(data)))
811 ret.append(vint.encode_vuint(_rec_tag_end))
815 return deepcopy(self)
819 # This method should either return a valid Metadata object,
820 # return None if there was no information at all (just a
821 # _rec_tag_end), throw EOFError if there was nothing at all to
822 # read, or throw an Exception if a valid object could not be
824 tag = vint.read_vuint(port)
825 if tag == _rec_tag_end:
827 try: # From here on, EOF is an error.
829 while True: # only exit is error (exception) or _rec_tag_end
830 if tag == _rec_tag_path:
831 result._load_path_rec(port)
832 elif tag == _rec_tag_common_v3:
833 result._load_common_rec(port, version=3)
834 elif tag == _rec_tag_common_v2:
835 result._load_common_rec(port, version=2)
836 elif tag == _rec_tag_symlink_target:
837 result._load_symlink_target_rec(port)
838 elif tag == _rec_tag_hardlink_target:
839 result._load_hardlink_target_rec(port)
840 elif tag == _rec_tag_posix1e_acl:
841 result._load_posix1e_acl_rec(port)
842 elif tag == _rec_tag_linux_attr:
843 result._load_linux_attr_rec(port)
844 elif tag == _rec_tag_linux_xattr:
845 result._load_linux_xattr_rec(port)
846 elif tag == _rec_tag_end:
848 elif tag == _rec_tag_common_v1: # Should be very rare.
849 result._load_common_rec(port, version=1)
850 else: # unknown record
852 tag = vint.read_vuint(port)
854 raise Exception("EOF while reading Metadata")
857 return stat.S_ISDIR(self.mode)
859 def create_path(self, path, create_symlinks=True):
860 self._create_via_common_rec(path, create_symlinks=create_symlinks)
862 def apply_to_path(self, path=None, restore_numeric_ids=False):
863 # apply metadata to path -- file must exist
867 raise Exception('Metadata.apply_to_path() called with no path')
868 if not self._recognized_file_type():
869 add_error('not applying metadata to "%s"' % path_msg(path)
870 + ' with unrecognized mode "0x%x"\n' % self.mode)
872 num_ids = restore_numeric_ids
873 for apply_metadata in (self._apply_common_rec,
874 self._apply_posix1e_acl_rec,
875 self._apply_linux_attr_rec,
876 self._apply_linux_xattr_rec):
878 apply_metadata(path, restore_numeric_ids=num_ids)
879 except ApplyError as e:
882 def same_file(self, other):
883 """Compare this to other for equivalency. Return true if
884 their information implies they could represent the same file
885 on disk, in the hardlink sense. Assume they're both regular
887 return self._same_common(other) \
888 and self._same_hardlink_target(other) \
889 and self._same_posix1e_acl(other) \
890 and self._same_linux_attr(other) \
891 and self._same_linux_xattr(other)
894 def from_path(path, statinfo=None, archive_path=None,
895 save_symlinks=True, hardlink_target=None,
896 normalized=False, after_stat=None):
897 # This function is also a test hook; see test-save-errors
898 """Return the metadata associated with the path. When normalized is
899 true, return the metadata appropriate for a typical save, which
900 may or may not be all of it."""
902 result.path = archive_path
903 st = statinfo or xstat.lstat(path)
906 result._add_common(path, st)
908 result._add_symlink_target(path, st)
909 result._add_hardlink_target(hardlink_target)
910 result._add_posix1e_acl(path, st)
911 result._add_linux_attr(path, st)
912 result._add_linux_xattr(path, st)
914 # Only store sizes for regular files and symlinks for now.
915 if not (stat.S_ISREG(result.mode) or stat.S_ISLNK(result.mode)):
920 def save_tree(output_file, paths,
926 # Issue top-level rewrite warnings.
928 safe_path = _clean_up_path_for_archive(path)
929 if safe_path != path:
930 log('archiving "%s" as "%s"\n'
931 % (path_msg(path), path_msg(safe_path)))
935 safe_path = _clean_up_path_for_archive(p)
937 if stat.S_ISDIR(st.st_mode):
939 m = from_path(p, statinfo=st, archive_path=safe_path,
940 save_symlinks=save_symlinks)
942 print(m.path, file=sys.stderr)
943 m.write(output_file, include_path=write_paths)
945 start_dir = os.getcwd()
947 for (p, st) in recursive_dirlist(paths, xdev=xdev):
948 dirlist_dir = os.getcwd()
950 safe_path = _clean_up_path_for_archive(p)
951 m = from_path(p, statinfo=st, archive_path=safe_path,
952 save_symlinks=save_symlinks)
954 print(m.path, file=sys.stderr)
955 m.write(output_file, include_path=write_paths)
956 os.chdir(dirlist_dir)
961 def _set_up_path(meta, create_symlinks=True):
962 # Allow directories to exist as a special case -- might have
963 # been created by an earlier longer path.
967 parent = os.path.dirname(meta.path)
970 meta.create_path(meta.path, create_symlinks=create_symlinks)
973 all_fields = frozenset(['path',
990 def summary_bytes(meta, numeric_ids = False, classification = None,
991 human_readable = False):
992 """Return bytes containing the "ls -l" style listing for meta.
993 Classification may be "all", "type", or None."""
994 user_str = group_str = size_or_dev_str = b'?'
995 symlink_target = None
998 mode_str = xstat.mode_str(meta.mode).encode('ascii')
999 symlink_target = meta.symlink_target
1000 mtime_secs = xstat.fstime_floor_secs(meta.mtime)
1001 mtime_str = strftime('%Y-%m-%d %H:%M',
1002 time.localtime(mtime_secs)).encode('ascii')
1003 if meta.user and not numeric_ids:
1004 user_str = meta.user
1005 elif meta.uid != None:
1006 user_str = str(meta.uid).encode()
1007 if meta.group and not numeric_ids:
1008 group_str = meta.group
1009 elif meta.gid != None:
1010 group_str = str(meta.gid).encode()
1011 if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode):
1013 size_or_dev_str = ('%d,%d' % (os.major(meta.rdev),
1014 os.minor(meta.rdev))).encode()
1015 elif meta.size != None:
1017 size_or_dev_str = format_filesize(meta.size).encode()
1019 size_or_dev_str = str(meta.size).encode()
1021 size_or_dev_str = b'-'
1023 classification_str = \
1024 xstat.classification_str(meta.mode,
1025 classification == 'all').encode()
1027 mode_str = b'?' * 10
1028 mtime_str = b'????-??-?? ??:??'
1029 classification_str = b'?'
1033 name += classification_str
1035 name += b' -> ' + meta.symlink_target
1037 return b'%-10s %-11s %11s %16s %s' % (mode_str,
1038 user_str + b'/' + group_str,
1044 def detailed_bytes(meta, fields = None):
1045 # FIXME: should optional fields be omitted, or empty i.e. "rdev:
1046 # 0", "link-target:", etc.
1051 if 'path' in fields:
1052 path = meta.path or b''
1053 result.append(b'path: ' + path)
1054 if 'mode' in fields:
1055 result.append(b'mode: %o (%s)'
1056 % (meta.mode, xstat.mode_str(meta.mode).encode('ascii')))
1057 if 'link-target' in fields and stat.S_ISLNK(meta.mode):
1058 result.append(b'link-target: ' + meta.symlink_target)
1059 if 'rdev' in fields:
1061 result.append(b'rdev: %d,%d' % (os.major(meta.rdev),
1062 os.minor(meta.rdev)))
1064 result.append(b'rdev: 0')
1065 if 'size' in fields and meta.size is not None:
1066 result.append(b'size: %d' % meta.size)
1068 result.append(b'uid: %d' % meta.uid)
1070 result.append(b'gid: %d' % meta.gid)
1071 if 'user' in fields:
1072 result.append(b'user: ' + meta.user)
1073 if 'group' in fields:
1074 result.append(b'group: ' + meta.group)
1075 if 'atime' in fields:
1076 # If we don't have xstat.lutime, that means we have to use
1077 # utime(), and utime() has no way to set the mtime/atime of a
1078 # symlink. Thus, the mtime/atime of a symlink is meaningless,
1079 # so let's not report it. (That way scripts comparing
1080 # before/after won't trigger.)
1081 if xstat.lutime or not stat.S_ISLNK(meta.mode):
1082 result.append(b'atime: ' + xstat.fstime_to_sec_bytes(meta.atime))
1084 result.append(b'atime: 0')
1085 if 'mtime' in fields:
1086 if xstat.lutime or not stat.S_ISLNK(meta.mode):
1087 result.append(b'mtime: ' + xstat.fstime_to_sec_bytes(meta.mtime))
1089 result.append(b'mtime: 0')
1090 if 'ctime' in fields:
1091 result.append(b'ctime: ' + xstat.fstime_to_sec_bytes(meta.ctime))
1092 if 'linux-attr' in fields and meta.linux_attr:
1093 result.append(b'linux-attr: %x' % meta.linux_attr)
1094 if 'linux-xattr' in fields and meta.linux_xattr:
1095 for name, value in meta.linux_xattr:
1096 result.append(b'linux-xattr: %s -> %s' % (name, value))
1097 if 'posix1e-acl' in fields and meta.posix1e_acl:
1098 acl = meta.posix1e_acl[0]
1099 result.append(b'posix1e-acl: ' + acl + b'\n')
1100 if stat.S_ISDIR(meta.mode):
1101 def_acl = meta.posix1e_acl[2]
1102 result.append(b'posix1e-acl-default: ' + def_acl + b'\n')
1103 return b'\n'.join(result)
1106 class _ArchiveIterator:
1109 return Metadata.read(self._file)
1111 raise StopIteration()
1118 def __init__(self, file):
1122 def display_archive(file, out):
1125 for meta in _ArchiveIterator(file):
1128 out.write(detailed_bytes(meta))
1132 for meta in _ArchiveIterator(file):
1133 out.write(summary_bytes(meta))
1136 for meta in _ArchiveIterator(file):
1138 log('bup: no metadata path, but asked to only display path'
1139 ' (increase verbosity?)')
1141 out.write(meta.path)
1145 def start_extract(file, create_symlinks=True):
1146 for meta in _ArchiveIterator(file):
1147 if not meta: # Hit end record.
1150 print(path_msg(meta.path), file=sys.stderr)
1151 xpath = _clean_up_extract_path(meta.path)
1153 add_error(Exception('skipping risky path "%s"'
1154 % path_msg(meta.path)))
1157 _set_up_path(meta, create_symlinks=create_symlinks)
1160 def finish_extract(file, restore_numeric_ids=False):
1162 for meta in _ArchiveIterator(file):
1163 if not meta: # Hit end record.
1165 xpath = _clean_up_extract_path(meta.path)
1167 add_error(Exception('skipping risky path "%s"'
1168 % path_msg(meta.path)))
1170 if os.path.isdir(meta.path):
1171 all_dirs.append(meta)
1174 print(path_msg(meta.path), file=sys.stderr)
1175 meta.apply_to_path(path=xpath,
1176 restore_numeric_ids=restore_numeric_ids)
1177 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
1178 for dir in all_dirs:
1179 # Don't need to check xpath -- won't be in all_dirs if not OK.
1180 xpath = _clean_up_extract_path(dir.path)
1182 print(path_msg(dir.path), file=sys.stderr)
1183 dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
1186 def extract(file, restore_numeric_ids=False, create_symlinks=True):
1187 # For now, just store all the directories and handle them last,
1190 for meta in _ArchiveIterator(file):
1191 if not meta: # Hit end record.
1193 xpath = _clean_up_extract_path(meta.path)
1195 add_error(Exception('skipping risky path "%s"'
1196 % path_msg(meta.path)))
1200 print('+', path_msg(meta.path), file=sys.stderr)
1201 _set_up_path(meta, create_symlinks=create_symlinks)
1202 if os.path.isdir(meta.path):
1203 all_dirs.append(meta)
1206 print('=', path_msg(meta.path), file=sys.stderr)
1207 meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
1208 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
1209 for dir in all_dirs:
1210 # Don't need to check xpath -- won't be in all_dirs if not OK.
1211 xpath = _clean_up_extract_path(dir.path)
1213 print('=', path_msg(xpath), file=sys.stderr)
1214 # Shouldn't have to check for risky paths here (omitted above).
1215 dir.apply_to_path(path=dir.path,
1216 restore_numeric_ids=restore_numeric_ids)