1 """Metadata read/write support for bup."""
3 # Copyright (C) 2010 Rob Browning
5 # This code is covered under the terms of the GNU Library General
6 # Public License as described in the bup LICENSE file.
8 from __future__ import absolute_import, print_function
9 from copy import deepcopy
10 from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
11 from io import BytesIO
12 from time import gmtime, strftime
13 import errno, os, sys, stat, time, pwd, grp, socket, struct
15 from bup import vint, xstat
16 from bup.drecurse import recursive_dirlist
17 from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize
18 from bup.pwdgrp import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name
19 from bup.xstat import utime, lutime
22 if sys.platform.startswith('linux'):
26 log('Warning: Linux xattr support missing; install python-pyxattr.\n')
30 except AttributeError:
31 log('Warning: python-xattr module is too old; '
32 'install python-pyxattr instead.\n')
36 if not (sys.platform.startswith('cygwin') \
37 or sys.platform.startswith('darwin') \
38 or sys.platform.startswith('netbsd')):
42 log('Warning: POSIX ACL support missing; install python-pylibacl.\n')
45 from bup._helpers import get_linux_file_attr, set_linux_file_attr
47 # No need for a warning here; the only reason they won't exist is that we're
48 # not on Linux, in which case files don't have any linux attrs anyway, so
49 # lacking the functions isn't a problem.
50 get_linux_file_attr = set_linux_file_attr = None
53 # See the bup_get_linux_file_attr() comments.
54 _suppress_linux_file_attr = \
55 sys.byteorder == 'big' and struct.calcsize('@l') > struct.calcsize('@i')
57 def check_linux_file_attr_api():
58 global get_linux_file_attr, set_linux_file_attr
59 if not (get_linux_file_attr or set_linux_file_attr):
61 if _suppress_linux_file_attr:
62 log('Warning: Linux attr support disabled (see "bup help index").\n')
63 get_linux_file_attr = set_linux_file_attr = None
66 # WARNING: the metadata encoding is *not* stable yet. Caveat emptor!
68 # Q: Consider hardlink support?
69 # Q: Is it OK to store raw linux attr (chattr) flags?
70 # Q: Can anything other than S_ISREG(x) or S_ISDIR(x) support posix1e ACLs?
71 # Q: Is the application of posix1e has_extended() correct?
72 # Q: Is one global --numeric-ids argument sufficient?
73 # Q: Do nfsv4 acls trump posix1e acls? (seems likely)
74 # Q: Add support for crtime -- ntfs, and (only internally?) ext*?
76 # FIXME: Fix relative/abs path detection/stripping wrt other platforms.
77 # FIXME: Add nfsv4 acl handling - see nfs4-acl-tools.
78 # FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.).
79 # FIXME: Consider pack('vvvvsss', ...) optimization.
83 # osx (varies between hfs and hfs+):
84 # type - regular dir char block fifo socket ...
85 # perms - rwxrwxrwxsgt
86 # times - ctime atime mtime
89 # hard-link-info (hfs+ only)
92 # attributes-osx see chflags
98 # type - regular dir ...
99 # times - creation, modification, posix change, access
102 # attributes - see attrib
104 # forks (alternate data streams)
108 # type - regular dir ...
109 # perms - rwxrwxrwx (maybe - see wikipedia)
110 # times - creation, modification, access
111 # attributes - see attrib
115 _have_lchmod = hasattr(os, 'lchmod')
118 def _clean_up_path_for_archive(p):
119 # Not the most efficient approach.
122 # Take everything after any '/../'.
123 pos = result.rfind('/../')
125 result = result[result.rfind('/../') + 4:]
127 # Take everything after any remaining '../'.
128 if result.startswith("../"):
131 # Remove any '/./' sequences.
132 pos = result.find('/./')
134 result = result[0:pos] + '/' + result[pos + 3:]
135 pos = result.find('/./')
137 # Remove any leading '/'s.
138 result = result.lstrip('/')
140 # Replace '//' with '/' everywhere.
141 pos = result.find('//')
143 result = result[0:pos] + '/' + result[pos + 2:]
144 pos = result.find('//')
146 # Take everything after any remaining './'.
147 if result.startswith('./'):
150 # Take everything before any remaining '/.'.
151 if result.endswith('/.'):
154 if result == '' or result.endswith('/..'):
161 if p.startswith('/'):
163 if p.find('/../') != -1:
165 if p.startswith('../'):
167 if p.endswith('/..'):
172 def _clean_up_extract_path(p):
173 result = p.lstrip('/')
176 elif _risky_path(result):
182 # These tags are currently conceptually private to Metadata, and they
183 # must be unique, and must *never* be changed.
186 _rec_tag_common_v1 = 2 # times, user, group, type, perms, etc. (legacy/broken)
187 _rec_tag_symlink_target = 3
188 _rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc.
189 _rec_tag_nfsv4_acl = 5 # intended to supplant posix1e? (unimplemented)
190 _rec_tag_linux_attr = 6 # lsattr(1) chattr(1)
191 _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1)
192 _rec_tag_hardlink_target = 8 # hard link target path
193 _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current)
194 _rec_tag_common_v3 = 10 # adds optional size to v2
196 _warned_about_attr_einval = None
199 class ApplyError(Exception):
200 # Thrown when unable to apply any given bit of metadata to a path.
205 # Metadata is stored as a sequence of tagged binary records. Each
206 # record will have some subset of add, encode, load, create, and
207 # apply methods, i.e. _add_foo...
209 # We do allow an "empty" object as a special case, i.e. no
210 # records. One can be created by trying to write Metadata(), and
211 # for such an object, read() will return None. This is used by
212 # "bup save", for example, as a placeholder in cases where
215 # NOTE: if any relevant fields are added or removed, be sure to
216 # update same_file() below.
220 # Timestamps are (sec, ns), relative to 1970-01-01 00:00:00, ns
221 # must be non-negative and < 10**9.
223 def _add_common(self, path, st):
224 assert(st.st_uid >= 0)
225 assert(st.st_gid >= 0)
226 self.size = st.st_size
229 self.atime = st.st_atime
230 self.mtime = st.st_mtime
231 self.ctime = st.st_ctime
232 self.user = self.group = ''
233 entry = pwd_from_uid(st.st_uid)
235 self.user = entry.pw_name
236 entry = grp_from_gid(st.st_gid)
238 self.group = entry.gr_name
239 self.mode = st.st_mode
240 # Only collect st_rdev if we might need it for a mknod()
241 # during restore. On some platforms (i.e. kFreeBSD), it isn't
242 # stable for other file types. For example "cp -a" will
243 # change it for a plain file.
244 if stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
245 self.rdev = st.st_rdev
249 def _same_common(self, other):
250 """Return true or false to indicate similarity in the hardlink sense."""
251 return self.uid == other.uid \
252 and self.gid == other.gid \
253 and self.rdev == other.rdev \
254 and self.mtime == other.mtime \
255 and self.ctime == other.ctime \
256 and self.user == other.user \
257 and self.group == other.group \
258 and self.size == other.size
260 def _encode_common(self):
263 atime = xstat.nsecs_to_timespec(self.atime)
264 mtime = xstat.nsecs_to_timespec(self.mtime)
265 ctime = xstat.nsecs_to_timespec(self.ctime)
266 result = vint.pack('vvsvsvvVvVvVv',
279 self.size if self.size is not None else -1)
282 def _load_common_rec(self, port, version=3):
284 # Added trailing size to v2, negative when None.
285 unpack_fmt = 'vvsvsvvVvVvVv'
287 unpack_fmt = 'vvsvsvvVvVvV'
289 unpack_fmt = 'VVsVsVvVvVvV'
291 raise Exception('unexpected common_rec version %d' % version)
292 data = vint.read_bvec(port)
293 values = vint.unpack(unpack_fmt, data)
295 (self.mode, self.uid, self.user, self.gid, self.group,
297 self.atime, atime_ns,
298 self.mtime, mtime_ns,
299 self.ctime, ctime_ns, size) = values
303 (self.mode, self.uid, self.user, self.gid, self.group,
305 self.atime, atime_ns,
306 self.mtime, mtime_ns,
307 self.ctime, ctime_ns) = values
308 self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
309 self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
310 self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
312 def _recognized_file_type(self):
313 return stat.S_ISREG(self.mode) \
314 or stat.S_ISDIR(self.mode) \
315 or stat.S_ISCHR(self.mode) \
316 or stat.S_ISBLK(self.mode) \
317 or stat.S_ISFIFO(self.mode) \
318 or stat.S_ISSOCK(self.mode) \
319 or stat.S_ISLNK(self.mode)
321 def _create_via_common_rec(self, path, create_symlinks=True):
323 raise ApplyError('no metadata - cannot create path ' + path)
325 # If the path already exists and is a dir, try rmdir.
326 # If the path already exists and is anything else, try unlink.
329 st = xstat.lstat(path)
331 if e.errno != errno.ENOENT:
334 if stat.S_ISDIR(st.st_mode):
338 if e.errno in (errno.ENOTEMPTY, errno.EEXIST):
339 msg = 'refusing to overwrite non-empty dir ' + path
345 if stat.S_ISREG(self.mode):
346 assert(self._recognized_file_type())
347 fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0o600)
349 elif stat.S_ISDIR(self.mode):
350 assert(self._recognized_file_type())
351 os.mkdir(path, 0o700)
352 elif stat.S_ISCHR(self.mode):
353 assert(self._recognized_file_type())
354 os.mknod(path, 0o600 | stat.S_IFCHR, self.rdev)
355 elif stat.S_ISBLK(self.mode):
356 assert(self._recognized_file_type())
357 os.mknod(path, 0o600 | stat.S_IFBLK, self.rdev)
358 elif stat.S_ISFIFO(self.mode):
359 assert(self._recognized_file_type())
360 os.mkfifo(path, 0o600 | stat.S_IFIFO)
361 elif stat.S_ISSOCK(self.mode):
363 os.mknod(path, 0o600 | stat.S_IFSOCK)
365 if e.errno in (errno.EINVAL, errno.EPERM):
366 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
370 elif stat.S_ISLNK(self.mode):
371 assert(self._recognized_file_type())
372 if self.symlink_target and create_symlinks:
373 # on MacOS, symlink() permissions depend on umask, and there's
374 # no way to chown a symlink after creating it, so we have to
376 oldumask = os.umask((self.mode & 0o777) ^ 0o777)
378 os.symlink(self.symlink_target, path)
381 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
383 assert(not self._recognized_file_type())
384 add_error('not creating "%s" with unrecognized mode "0x%x"\n'
387 def _apply_common_rec(self, path, restore_numeric_ids=False):
389 raise ApplyError('no metadata - cannot apply to ' + path)
391 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
392 # EACCES errors at this stage are fatal for the current path.
393 if lutime and stat.S_ISLNK(self.mode):
395 lutime(path, (self.atime, self.mtime))
397 if e.errno == errno.EACCES:
398 raise ApplyError('lutime: %s' % e)
403 utime(path, (self.atime, self.mtime))
405 if e.errno == errno.EACCES:
406 raise ApplyError('utime: %s' % e)
410 uid = gid = -1 # By default, do nothing.
414 if not restore_numeric_ids:
415 if self.uid != 0 and self.user:
416 entry = pwd_from_name(self.user)
419 if self.gid != 0 and self.group:
420 entry = grp_from_name(self.group)
423 else: # not superuser - only consider changing the group/gid
424 user_gids = os.getgroups()
425 if self.gid in user_gids:
427 if not restore_numeric_ids and self.gid != 0:
428 # The grp might not exist on the local system.
429 grps = filter(None, [grp_from_gid(x) for x in user_gids])
430 if self.group in [x.gr_name for x in grps]:
431 g = grp_from_name(self.group)
435 if uid != -1 or gid != -1:
437 os.lchown(path, uid, gid)
439 if e.errno == errno.EPERM:
440 add_error('lchown: %s' % e)
441 elif sys.platform.startswith('cygwin') \
442 and e.errno == errno.EINVAL:
443 add_error('lchown: unknown uid/gid (%d/%d) for %s'
450 os.lchmod(path, stat.S_IMODE(self.mode))
451 except errno.ENOSYS: # Function not implemented
453 elif not stat.S_ISLNK(self.mode):
454 os.chmod(path, stat.S_IMODE(self.mode))
459 def _encode_path(self):
461 return vint.pack('s', self.path)
465 def _load_path_rec(self, port):
466 self.path = vint.unpack('s', vint.read_bvec(port))[0]
471 def _add_symlink_target(self, path, st):
473 if stat.S_ISLNK(st.st_mode):
474 self.symlink_target = os.readlink(path)
476 add_error('readlink: %s' % e)
478 def _encode_symlink_target(self):
479 return self.symlink_target
481 def _load_symlink_target_rec(self, port):
482 target = vint.read_bvec(port)
483 self.symlink_target = target
484 if self.size is None:
485 self.size = len(target)
487 assert(self.size == len(target))
492 def _add_hardlink_target(self, target):
493 self.hardlink_target = target
495 def _same_hardlink_target(self, other):
496 """Return true or false to indicate similarity in the hardlink sense."""
497 return self.hardlink_target == other.hardlink_target
499 def _encode_hardlink_target(self):
500 return self.hardlink_target
502 def _load_hardlink_target_rec(self, port):
503 self.hardlink_target = vint.read_bvec(port)
506 ## POSIX1e ACL records
508 # Recorded as a list:
509 # [txt_id_acl, num_id_acl]
510 # or, if a directory:
511 # [txt_id_acl, num_id_acl, txt_id_default_acl, num_id_default_acl]
512 # The numeric/text distinction only matters when reading/restoring
514 def _add_posix1e_acl(self, path, st):
515 if not posix1e or not posix1e.HAS_EXTENDED_CHECK:
517 if not stat.S_ISLNK(st.st_mode):
521 if posix1e.has_extended(path):
522 acl = posix1e.ACL(file=path)
523 acls = [acl, acl] # txt and num are the same
524 if stat.S_ISDIR(st.st_mode):
525 def_acl = posix1e.ACL(filedef=path)
526 def_acls = [def_acl, def_acl]
527 except EnvironmentError as e:
528 if e.errno not in (errno.EOPNOTSUPP, errno.ENOSYS):
531 txt_flags = posix1e.TEXT_ABBREVIATE
532 num_flags = posix1e.TEXT_ABBREVIATE | posix1e.TEXT_NUMERIC_IDS
533 acl_rep = [acls[0].to_any_text('', '\n', txt_flags),
534 acls[1].to_any_text('', '\n', num_flags)]
536 acl_rep.append(def_acls[0].to_any_text('', '\n', txt_flags))
537 acl_rep.append(def_acls[1].to_any_text('', '\n', num_flags))
538 self.posix1e_acl = acl_rep
540 def _same_posix1e_acl(self, other):
541 """Return true or false to indicate similarity in the hardlink sense."""
542 return self.posix1e_acl == other.posix1e_acl
544 def _encode_posix1e_acl(self):
545 # Encode as two strings (w/default ACL string possibly empty).
547 acls = self.posix1e_acl
549 acls.extend(['', ''])
550 return vint.pack('ssss', acls[0], acls[1], acls[2], acls[3])
554 def _load_posix1e_acl_rec(self, port):
555 acl_rep = vint.unpack('ssss', vint.read_bvec(port))
557 acl_rep = acl_rep[:2]
558 self.posix1e_acl = acl_rep
560 def _apply_posix1e_acl_rec(self, path, restore_numeric_ids=False):
561 def apply_acl(acl_rep, kind):
563 acl = posix1e.ACL(text = acl_rep)
566 # pylibacl appears to return an IOError with errno
567 # set to 0 if a group referred to by the ACL rep
568 # doesn't exist on the current system.
569 raise ApplyError("POSIX1e ACL: can't create %r for %r"
574 acl.applyto(path, kind)
576 if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP:
577 raise ApplyError('POSIX1e ACL applyto: %s' % e)
583 add_error("%s: can't restore ACLs; posix1e support missing.\n"
587 acls = self.posix1e_acl
589 if restore_numeric_ids:
590 apply_acl(acls[3], posix1e.ACL_TYPE_DEFAULT)
592 apply_acl(acls[2], posix1e.ACL_TYPE_DEFAULT)
593 if restore_numeric_ids:
594 apply_acl(acls[1], posix1e.ACL_TYPE_ACCESS)
596 apply_acl(acls[0], posix1e.ACL_TYPE_ACCESS)
599 ## Linux attributes (lsattr(1), chattr(1))
601 def _add_linux_attr(self, path, st):
602 check_linux_file_attr_api()
603 if not get_linux_file_attr: return
604 if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
606 attr = get_linux_file_attr(path)
608 self.linux_attr = attr
610 if e.errno == errno.EACCES:
611 add_error('read Linux attr: %s' % e)
612 elif e.errno in (ENOTTY, ENOSYS, EOPNOTSUPP):
613 # Assume filesystem doesn't support attrs.
615 elif e.errno == EINVAL:
616 global _warned_about_attr_einval
617 if not _warned_about_attr_einval:
618 log("Ignoring attr EINVAL;"
619 + " if you're not using ntfs-3g, please report: "
621 _warned_about_attr_einval = True
626 def _same_linux_attr(self, other):
627 """Return true or false to indicate similarity in the hardlink sense."""
628 return self.linux_attr == other.linux_attr
630 def _encode_linux_attr(self):
632 return vint.pack('V', self.linux_attr)
636 def _load_linux_attr_rec(self, port):
637 data = vint.read_bvec(port)
638 self.linux_attr = vint.unpack('V', data)[0]
640 def _apply_linux_attr_rec(self, path, restore_numeric_ids=False):
642 check_linux_file_attr_api()
643 if not set_linux_file_attr:
644 add_error("%s: can't restore linuxattrs: "
645 "linuxattr support missing.\n" % path)
648 set_linux_file_attr(path, self.linux_attr)
650 if e.errno in (EACCES, ENOTTY, EOPNOTSUPP, ENOSYS):
651 raise ApplyError('Linux chattr: %s (0x%s)'
652 % (e, hex(self.linux_attr)))
653 elif e.errno == EINVAL:
654 msg = "if you're not using ntfs-3g, please report"
655 raise ApplyError('Linux chattr: %s (0x%s) (%s)'
656 % (e, hex(self.linux_attr), msg))
661 ## Linux extended attributes (getfattr(1), setfattr(1))
663 def _add_linux_xattr(self, path, st):
666 self.linux_xattr = xattr.get_all(path, nofollow=True)
667 except EnvironmentError as e:
668 if e.errno != errno.EOPNOTSUPP:
671 def _same_linux_xattr(self, other):
672 """Return true or false to indicate similarity in the hardlink sense."""
673 return self.linux_xattr == other.linux_xattr
675 def _encode_linux_xattr(self):
677 result = vint.pack('V', len(self.linux_xattr))
678 for name, value in self.linux_xattr:
679 result += vint.pack('ss', name, value)
684 def _load_linux_xattr_rec(self, file):
685 data = vint.read_bvec(file)
686 memfile = BytesIO(data)
688 for i in range(vint.read_vuint(memfile)):
689 key = vint.read_bvec(memfile)
690 value = vint.read_bvec(memfile)
691 result.append((key, value))
692 self.linux_xattr = result
694 def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
697 add_error("%s: can't restore xattr; xattr support missing.\n"
700 if not self.linux_xattr:
703 existing_xattrs = set(xattr.list(path, nofollow=True))
705 if e.errno == errno.EACCES:
706 raise ApplyError('xattr.set %r: %s' % (path, e))
709 for k, v in self.linux_xattr:
710 if k not in existing_xattrs \
711 or v != xattr.get(path, k, nofollow=True):
713 xattr.set(path, k, v, nofollow=True)
715 if e.errno == errno.EPERM \
716 or e.errno == errno.EOPNOTSUPP:
717 raise ApplyError('xattr.set %r: %s' % (path, e))
720 existing_xattrs -= frozenset([k])
721 for k in existing_xattrs:
723 xattr.remove(path, k, nofollow=True)
725 if e.errno in (errno.EPERM, errno.EACCES):
726 raise ApplyError('xattr.remove %r: %s' % (path, e))
731 self.mode = self.uid = self.gid = self.user = self.group = None
732 self.atime = self.mtime = self.ctime = None
736 self.symlink_target = None
737 self.hardlink_target = None
738 self.linux_attr = None
739 self.linux_xattr = None
740 self.posix1e_acl = None
742 def __eq__(self, other):
743 if not isinstance(other, Metadata): return False
744 if self.mode != other.mode: return False
745 if self.mtime != other.mtime: return False
746 if self.ctime != other.ctime: return False
747 if self.atime != other.atime: return False
748 if self.path != other.path: return False
749 if self.uid != other.uid: return False
750 if self.gid != other.gid: return False
751 if self.size != other.size: return False
752 if self.user != other.user: return False
753 if self.group != other.group: return False
754 if self.symlink_target != other.symlink_target: return False
755 if self.hardlink_target != other.hardlink_target: return False
756 if self.linux_attr != other.linux_attr: return False
757 if self.posix1e_acl != other.posix1e_acl: return False
760 def __ne__(self, other):
761 return not self.__eq__(other)
764 return hash((self.mode,
775 self.hardlink_target,
780 result = ['<%s instance at %s' % (self.__class__, hex(id(self)))]
781 if self.path is not None:
782 result += ' path:' + repr(self.path)
783 if self.mode is not None:
784 result += ' mode:' + repr(xstat.mode_str(self.mode)
785 + '(%s)' % oct(self.mode))
786 if self.uid is not None:
787 result += ' uid:' + str(self.uid)
788 if self.gid is not None:
789 result += ' gid:' + str(self.gid)
790 if self.user is not None:
791 result += ' user:' + repr(self.user)
792 if self.group is not None:
793 result += ' group:' + repr(self.group)
794 if self.size is not None:
795 result += ' size:' + repr(self.size)
796 for name, val in (('atime', self.atime),
797 ('mtime', self.mtime),
798 ('ctime', self.ctime)):
800 result += ' %s:%r (%d)' \
802 strftime('%Y-%m-%d %H:%M %z',
803 gmtime(xstat.fstime_floor_secs(val))),
806 return ''.join(result)
808 def write(self, port, include_path=True):
809 records = include_path and [(_rec_tag_path, self._encode_path())] or []
810 records.extend([(_rec_tag_common_v3, self._encode_common()),
811 (_rec_tag_symlink_target,
812 self._encode_symlink_target()),
813 (_rec_tag_hardlink_target,
814 self._encode_hardlink_target()),
815 (_rec_tag_posix1e_acl, self._encode_posix1e_acl()),
816 (_rec_tag_linux_attr, self._encode_linux_attr()),
817 (_rec_tag_linux_xattr, self._encode_linux_xattr())])
818 for tag, data in records:
820 vint.write_vuint(port, tag)
821 vint.write_bvec(port, data)
822 vint.write_vuint(port, _rec_tag_end)
824 def encode(self, include_path=True):
826 self.write(port, include_path)
827 return port.getvalue()
830 return deepcopy(self)
834 # This method should either return a valid Metadata object,
835 # return None if there was no information at all (just a
836 # _rec_tag_end), throw EOFError if there was nothing at all to
837 # read, or throw an Exception if a valid object could not be
839 tag = vint.read_vuint(port)
840 if tag == _rec_tag_end:
842 try: # From here on, EOF is an error.
844 while True: # only exit is error (exception) or _rec_tag_end
845 if tag == _rec_tag_path:
846 result._load_path_rec(port)
847 elif tag == _rec_tag_common_v3:
848 result._load_common_rec(port, version=3)
849 elif tag == _rec_tag_common_v2:
850 result._load_common_rec(port, version=2)
851 elif tag == _rec_tag_symlink_target:
852 result._load_symlink_target_rec(port)
853 elif tag == _rec_tag_hardlink_target:
854 result._load_hardlink_target_rec(port)
855 elif tag == _rec_tag_posix1e_acl:
856 result._load_posix1e_acl_rec(port)
857 elif tag == _rec_tag_linux_attr:
858 result._load_linux_attr_rec(port)
859 elif tag == _rec_tag_linux_xattr:
860 result._load_linux_xattr_rec(port)
861 elif tag == _rec_tag_end:
863 elif tag == _rec_tag_common_v1: # Should be very rare.
864 result._load_common_rec(port, version=1)
865 else: # unknown record
867 tag = vint.read_vuint(port)
869 raise Exception("EOF while reading Metadata")
872 return stat.S_ISDIR(self.mode)
874 def create_path(self, path, create_symlinks=True):
875 self._create_via_common_rec(path, create_symlinks=create_symlinks)
877 def apply_to_path(self, path=None, restore_numeric_ids=False):
878 # apply metadata to path -- file must exist
882 raise Exception('Metadata.apply_to_path() called with no path')
883 if not self._recognized_file_type():
884 add_error('not applying metadata to "%s"' % path
885 + ' with unrecognized mode "0x%x"\n' % self.mode)
887 num_ids = restore_numeric_ids
888 for apply_metadata in (self._apply_common_rec,
889 self._apply_posix1e_acl_rec,
890 self._apply_linux_attr_rec,
891 self._apply_linux_xattr_rec):
893 apply_metadata(path, restore_numeric_ids=num_ids)
894 except ApplyError as e:
897 def same_file(self, other):
898 """Compare this to other for equivalency. Return true if
899 their information implies they could represent the same file
900 on disk, in the hardlink sense. Assume they're both regular
902 return self._same_common(other) \
903 and self._same_hardlink_target(other) \
904 and self._same_posix1e_acl(other) \
905 and self._same_linux_attr(other) \
906 and self._same_linux_xattr(other)
909 def from_path(path, statinfo=None, archive_path=None,
910 save_symlinks=True, hardlink_target=None,
912 """Return the metadata associated with the path. When normalized is
913 true, return the metadata appropriate for a typical save, which
914 may or may not be all of it."""
916 result.path = archive_path
917 st = statinfo or xstat.lstat(path)
918 result._add_common(path, st)
920 result._add_symlink_target(path, st)
921 result._add_hardlink_target(hardlink_target)
922 result._add_posix1e_acl(path, st)
923 result._add_linux_attr(path, st)
924 result._add_linux_xattr(path, st)
926 # Only store sizes for regular files and symlinks for now.
927 if not (stat.S_ISREG(result.mode) or stat.S_ISLNK(result.mode)):
932 def save_tree(output_file, paths,
938 # Issue top-level rewrite warnings.
940 safe_path = _clean_up_path_for_archive(path)
941 if safe_path != path:
942 log('archiving "%s" as "%s"\n' % (path, safe_path))
946 safe_path = _clean_up_path_for_archive(p)
948 if stat.S_ISDIR(st.st_mode):
950 m = from_path(p, statinfo=st, archive_path=safe_path,
951 save_symlinks=save_symlinks)
953 print(m.path, file=sys.stderr)
954 m.write(output_file, include_path=write_paths)
956 start_dir = os.getcwd()
958 for (p, st) in recursive_dirlist(paths, xdev=xdev):
959 dirlist_dir = os.getcwd()
961 safe_path = _clean_up_path_for_archive(p)
962 m = from_path(p, statinfo=st, archive_path=safe_path,
963 save_symlinks=save_symlinks)
965 print(m.path, file=sys.stderr)
966 m.write(output_file, include_path=write_paths)
967 os.chdir(dirlist_dir)
972 def _set_up_path(meta, create_symlinks=True):
973 # Allow directories to exist as a special case -- might have
974 # been created by an earlier longer path.
978 parent = os.path.dirname(meta.path)
981 meta.create_path(meta.path, create_symlinks=create_symlinks)
984 all_fields = frozenset(['path',
1001 def summary_str(meta, numeric_ids = False, classification = None,
1002 human_readable = False):
1004 """Return a string containing the "ls -l" style listing for meta.
1005 Classification may be "all", "type", or None."""
1006 user_str = group_str = size_or_dev_str = '?'
1007 symlink_target = None
1010 mode_str = xstat.mode_str(meta.mode)
1011 symlink_target = meta.symlink_target
1012 mtime_secs = xstat.fstime_floor_secs(meta.mtime)
1013 mtime_str = strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs))
1014 if meta.user and not numeric_ids:
1015 user_str = meta.user
1016 elif meta.uid != None:
1017 user_str = str(meta.uid)
1018 if meta.group and not numeric_ids:
1019 group_str = meta.group
1020 elif meta.gid != None:
1021 group_str = str(meta.gid)
1022 if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode):
1024 size_or_dev_str = '%d,%d' % (os.major(meta.rdev),
1025 os.minor(meta.rdev))
1026 elif meta.size != None:
1028 size_or_dev_str = format_filesize(meta.size)
1030 size_or_dev_str = str(meta.size)
1032 size_or_dev_str = '-'
1034 classification_str = \
1035 xstat.classification_str(meta.mode, classification == 'all')
1038 mtime_str = '????-??-?? ??:??'
1039 classification_str = '?'
1043 name += classification_str
1045 name += ' -> ' + meta.symlink_target
1047 return '%-10s %-11s %11s %16s %s' % (mode_str,
1048 user_str + "/" + group_str,
1054 def detailed_str(meta, fields = None):
1055 # FIXME: should optional fields be omitted, or empty i.e. "rdev:
1056 # 0", "link-target:", etc.
1061 if 'path' in fields:
1062 path = meta.path or ''
1063 result.append('path: ' + path)
1064 if 'mode' in fields:
1065 result.append('mode: %s (%s)' % (oct(meta.mode),
1066 xstat.mode_str(meta.mode)))
1067 if 'link-target' in fields and stat.S_ISLNK(meta.mode):
1068 result.append('link-target: ' + meta.symlink_target)
1069 if 'rdev' in fields:
1071 result.append('rdev: %d,%d' % (os.major(meta.rdev),
1072 os.minor(meta.rdev)))
1074 result.append('rdev: 0')
1075 if 'size' in fields and meta.size is not None:
1076 result.append('size: ' + str(meta.size))
1078 result.append('uid: ' + str(meta.uid))
1080 result.append('gid: ' + str(meta.gid))
1081 if 'user' in fields:
1082 result.append('user: ' + meta.user)
1083 if 'group' in fields:
1084 result.append('group: ' + meta.group)
1085 if 'atime' in fields:
1086 # If we don't have xstat.lutime, that means we have to use
1087 # utime(), and utime() has no way to set the mtime/atime of a
1088 # symlink. Thus, the mtime/atime of a symlink is meaningless,
1089 # so let's not report it. (That way scripts comparing
1090 # before/after won't trigger.)
1091 if xstat.lutime or not stat.S_ISLNK(meta.mode):
1092 result.append('atime: ' + xstat.fstime_to_sec_str(meta.atime))
1094 result.append('atime: 0')
1095 if 'mtime' in fields:
1096 if xstat.lutime or not stat.S_ISLNK(meta.mode):
1097 result.append('mtime: ' + xstat.fstime_to_sec_str(meta.mtime))
1099 result.append('mtime: 0')
1100 if 'ctime' in fields:
1101 result.append('ctime: ' + xstat.fstime_to_sec_str(meta.ctime))
1102 if 'linux-attr' in fields and meta.linux_attr:
1103 result.append('linux-attr: ' + hex(meta.linux_attr))
1104 if 'linux-xattr' in fields and meta.linux_xattr:
1105 for name, value in meta.linux_xattr:
1106 result.append('linux-xattr: %s -> %s' % (name, repr(value)))
1107 if 'posix1e-acl' in fields and meta.posix1e_acl:
1108 acl = meta.posix1e_acl[0]
1109 result.append('posix1e-acl: ' + acl + '\n')
1110 if stat.S_ISDIR(meta.mode):
1111 def_acl = meta.posix1e_acl[2]
1112 result.append('posix1e-acl-default: ' + def_acl + '\n')
1113 return '\n'.join(result)
1116 class _ArchiveIterator:
1119 return Metadata.read(self._file)
1121 raise StopIteration()
1126 def __init__(self, file):
1130 def display_archive(file):
1133 for meta in _ArchiveIterator(file):
1136 print(detailed_str(meta))
1139 for meta in _ArchiveIterator(file):
1140 print(summary_str(meta))
1142 for meta in _ArchiveIterator(file):
1144 print('bup: no metadata path, but asked to only display path'
1145 '(increase verbosity?)')
1150 def start_extract(file, create_symlinks=True):
1151 for meta in _ArchiveIterator(file):
1152 if not meta: # Hit end record.
1155 print(meta.path, file=sys.stderr)
1156 xpath = _clean_up_extract_path(meta.path)
1158 add_error(Exception('skipping risky path "%s"' % meta.path))
1161 _set_up_path(meta, create_symlinks=create_symlinks)
1164 def finish_extract(file, restore_numeric_ids=False):
1166 for meta in _ArchiveIterator(file):
1167 if not meta: # Hit end record.
1169 xpath = _clean_up_extract_path(meta.path)
1171 add_error(Exception('skipping risky path "%s"' % dir.path))
1173 if os.path.isdir(meta.path):
1174 all_dirs.append(meta)
1177 print(meta.path, file=sys.stderr)
1178 meta.apply_to_path(path=xpath,
1179 restore_numeric_ids=restore_numeric_ids)
1180 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
1181 for dir in all_dirs:
1182 # Don't need to check xpath -- won't be in all_dirs if not OK.
1183 xpath = _clean_up_extract_path(dir.path)
1185 print(dir.path, file=sys.stderr)
1186 dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
1189 def extract(file, restore_numeric_ids=False, create_symlinks=True):
1190 # For now, just store all the directories and handle them last,
1193 for meta in _ArchiveIterator(file):
1194 if not meta: # Hit end record.
1196 xpath = _clean_up_extract_path(meta.path)
1198 add_error(Exception('skipping risky path "%s"' % meta.path))
1202 print('+', meta.path, file=sys.stderr)
1203 _set_up_path(meta, create_symlinks=create_symlinks)
1204 if os.path.isdir(meta.path):
1205 all_dirs.append(meta)
1208 print('=', meta.path, file=sys.stderr)
1209 meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
1210 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
1211 for dir in all_dirs:
1212 # Don't need to check xpath -- won't be in all_dirs if not OK.
1213 xpath = _clean_up_extract_path(dir.path)
1215 print('=', xpath, file=sys.stderr)
1216 # Shouldn't have to check for risky paths here (omitted above).
1217 dir.apply_to_path(path=dir.path,
1218 restore_numeric_ids=restore_numeric_ids)