1 """Metadata read/write support for bup."""
3 # Copyright (C) 2010 Rob Browning
5 # This code is covered under the terms of the GNU Library General
6 # Public License as described in the bup LICENSE file.
7 import errno, os, sys, stat, time, pwd, grp, socket, struct
8 from cStringIO import StringIO
9 from bup import vint, xstat
10 from bup.drecurse import recursive_dirlist
11 from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize
12 from bup.helpers import pwd_from_uid, pwd_from_name, grp_from_gid, grp_from_name
13 from bup.xstat import utime, lutime
16 if sys.platform.startswith('linux'):
20 log('Warning: Linux xattr support missing; install python-pyxattr.\n')
24 except AttributeError:
25 log('Warning: python-xattr module is too old; '
26 'install python-pyxattr instead.\n')
30 if not (sys.platform.startswith('cygwin') \
31 or sys.platform.startswith('darwin') \
32 or sys.platform.startswith('netbsd')):
36 log('Warning: POSIX ACL support missing; install python-pylibacl.\n')
39 from bup._helpers import get_linux_file_attr, set_linux_file_attr
41 # No need for a warning here; the only reason they won't exist is that we're
42 # not on Linux, in which case files don't have any linux attrs anyway, so
43 # lacking the functions isn't a problem.
44 get_linux_file_attr = set_linux_file_attr = None
47 # See the bup_get_linux_file_attr() comments.
48 _suppress_linux_file_attr = \
49 sys.byteorder == 'big' and struct.calcsize('@l') > struct.calcsize('@i')
51 def check_linux_file_attr_api():
52 global get_linux_file_attr, set_linux_file_attr
53 if not (get_linux_file_attr or set_linux_file_attr):
55 if _suppress_linux_file_attr:
56 log('Warning: Linux attr support disabled (see "bup help index").\n')
57 get_linux_file_attr = set_linux_file_attr = None
60 # WARNING: the metadata encoding is *not* stable yet. Caveat emptor!
62 # Q: Consider hardlink support?
63 # Q: Is it OK to store raw linux attr (chattr) flags?
64 # Q: Can anything other than S_ISREG(x) or S_ISDIR(x) support posix1e ACLs?
65 # Q: Is the application of posix1e has_extended() correct?
66 # Q: Is one global --numeric-ids argument sufficient?
67 # Q: Do nfsv4 acls trump posix1e acls? (seems likely)
68 # Q: Add support for crtime -- ntfs, and (only internally?) ext*?
70 # FIXME: Fix relative/abs path detection/stripping wrt other platforms.
71 # FIXME: Add nfsv4 acl handling - see nfs4-acl-tools.
72 # FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.).
73 # FIXME: Consider pack('vvvvsss', ...) optimization.
77 # osx (varies between hfs and hfs+):
78 # type - regular dir char block fifo socket ...
79 # perms - rwxrwxrwxsgt
80 # times - ctime atime mtime
83 # hard-link-info (hfs+ only)
86 # attributes-osx see chflags
92 # type - regular dir ...
93 # times - creation, modification, posix change, access
96 # attributes - see attrib
98 # forks (alternate data streams)
102 # type - regular dir ...
103 # perms - rwxrwxrwx (maybe - see wikipedia)
104 # times - creation, modification, access
105 # attributes - see attrib
109 _have_lchmod = hasattr(os, 'lchmod')
112 def _clean_up_path_for_archive(p):
113 # Not the most efficient approach.
116 # Take everything after any '/../'.
117 pos = result.rfind('/../')
119 result = result[result.rfind('/../') + 4:]
121 # Take everything after any remaining '../'.
122 if result.startswith("../"):
125 # Remove any '/./' sequences.
126 pos = result.find('/./')
128 result = result[0:pos] + '/' + result[pos + 3:]
129 pos = result.find('/./')
131 # Remove any leading '/'s.
132 result = result.lstrip('/')
134 # Replace '//' with '/' everywhere.
135 pos = result.find('//')
137 result = result[0:pos] + '/' + result[pos + 2:]
138 pos = result.find('//')
140 # Take everything after any remaining './'.
141 if result.startswith('./'):
144 # Take everything before any remaining '/.'.
145 if result.endswith('/.'):
148 if result == '' or result.endswith('/..'):
155 if p.startswith('/'):
157 if p.find('/../') != -1:
159 if p.startswith('../'):
161 if p.endswith('/..'):
166 def _clean_up_extract_path(p):
167 result = p.lstrip('/')
170 elif _risky_path(result):
176 # These tags are currently conceptually private to Metadata, and they
177 # must be unique, and must *never* be changed.
180 _rec_tag_common = 2 # times, user, group, type, perms, etc. (legacy/broken)
181 _rec_tag_symlink_target = 3
182 _rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc.
183 _rec_tag_nfsv4_acl = 5 # intended to supplant posix1e? (unimplemented)
184 _rec_tag_linux_attr = 6 # lsattr(1) chattr(1)
185 _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1)
186 _rec_tag_hardlink_target = 8 # hard link target path
187 _rec_tag_common_v2 = 9 # times, user, group, type, perms, etc. (current)
190 class ApplyError(Exception):
191 # Thrown when unable to apply any given bit of metadata to a path.
196 # Metadata is stored as a sequence of tagged binary records. Each
197 # record will have some subset of add, encode, load, create, and
198 # apply methods, i.e. _add_foo...
200 # We do allow an "empty" object as a special case, i.e. no
201 # records. One can be created by trying to write Metadata(), and
202 # for such an object, read() will return None. This is used by
203 # "bup save", for example, as a placeholder in cases where
206 # NOTE: if any relevant fields are added or removed, be sure to
207 # update same_file() below.
211 # Timestamps are (sec, ns), relative to 1970-01-01 00:00:00, ns
212 # must be non-negative and < 10**9.
214 def _add_common(self, path, st):
217 self.atime = st.st_atime
218 self.mtime = st.st_mtime
219 self.ctime = st.st_ctime
220 self.user = self.group = ''
221 entry = pwd_from_uid(st.st_uid)
223 self.user = entry.pw_name
224 entry = grp_from_gid(st.st_gid)
226 self.group = entry.gr_name
227 self.mode = st.st_mode
228 # Only collect st_rdev if we might need it for a mknod()
229 # during restore. On some platforms (i.e. kFreeBSD), it isn't
230 # stable for other file types. For example "cp -a" will
231 # change it for a plain file.
232 if stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
233 self.rdev = st.st_rdev
237 def _same_common(self, other):
238 """Return true or false to indicate similarity in the hardlink sense."""
239 return self.uid == other.uid \
240 and self.gid == other.gid \
241 and self.rdev == other.rdev \
242 and self.mtime == other.mtime \
243 and self.ctime == other.ctime \
244 and self.user == other.user \
245 and self.group == other.group
247 def _encode_common(self):
250 atime = xstat.nsecs_to_timespec(self.atime)
251 mtime = xstat.nsecs_to_timespec(self.mtime)
252 ctime = xstat.nsecs_to_timespec(self.ctime)
253 result = vint.pack('vvsvsvvVvVvV',
268 def _load_common_rec(self, port, legacy_format=False):
269 unpack_fmt = 'vvsvsvvVvVvV'
271 unpack_fmt = 'VVsVsVvVvVvV'
272 data = vint.read_bvec(port)
284 ctime_ns) = vint.unpack(unpack_fmt, data)
285 self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
286 self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
287 self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
289 def _recognized_file_type(self):
290 return stat.S_ISREG(self.mode) \
291 or stat.S_ISDIR(self.mode) \
292 or stat.S_ISCHR(self.mode) \
293 or stat.S_ISBLK(self.mode) \
294 or stat.S_ISFIFO(self.mode) \
295 or stat.S_ISSOCK(self.mode) \
296 or stat.S_ISLNK(self.mode)
298 def _create_via_common_rec(self, path, create_symlinks=True):
300 raise ApplyError('no metadata - cannot create path ' + path)
302 # If the path already exists and is a dir, try rmdir.
303 # If the path already exists and is anything else, try unlink.
306 st = xstat.lstat(path)
308 if e.errno != errno.ENOENT:
311 if stat.S_ISDIR(st.st_mode):
315 if e.errno in (errno.ENOTEMPTY, errno.EEXIST):
316 msg = 'refusing to overwrite non-empty dir ' + path
322 if stat.S_ISREG(self.mode):
323 assert(self._recognized_file_type())
324 fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0600)
326 elif stat.S_ISDIR(self.mode):
327 assert(self._recognized_file_type())
329 elif stat.S_ISCHR(self.mode):
330 assert(self._recognized_file_type())
331 os.mknod(path, 0600 | stat.S_IFCHR, self.rdev)
332 elif stat.S_ISBLK(self.mode):
333 assert(self._recognized_file_type())
334 os.mknod(path, 0600 | stat.S_IFBLK, self.rdev)
335 elif stat.S_ISFIFO(self.mode):
336 assert(self._recognized_file_type())
337 os.mknod(path, 0600 | stat.S_IFIFO)
338 elif stat.S_ISSOCK(self.mode):
340 os.mknod(path, 0600 | stat.S_IFSOCK)
342 if e.errno in (errno.EINVAL, errno.EPERM):
343 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
347 elif stat.S_ISLNK(self.mode):
348 assert(self._recognized_file_type())
349 if self.symlink_target and create_symlinks:
350 # on MacOS, symlink() permissions depend on umask, and there's
351 # no way to chown a symlink after creating it, so we have to
353 oldumask = os.umask((self.mode & 0777) ^ 0777)
355 os.symlink(self.symlink_target, path)
358 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
360 assert(not self._recognized_file_type())
361 add_error('not creating "%s" with unrecognized mode "0x%x"\n'
364 def _apply_common_rec(self, path, restore_numeric_ids=False):
366 raise ApplyError('no metadata - cannot apply to ' + path)
368 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
369 # EACCES errors at this stage are fatal for the current path.
370 if lutime and stat.S_ISLNK(self.mode):
372 lutime(path, (self.atime, self.mtime))
374 if e.errno == errno.EACCES:
375 raise ApplyError('lutime: %s' % e)
380 utime(path, (self.atime, self.mtime))
382 if e.errno == errno.EACCES:
383 raise ApplyError('utime: %s' % e)
387 uid = gid = -1 # By default, do nothing.
391 if not restore_numeric_ids:
392 if self.uid != 0 and self.user:
393 entry = pwd_from_name(self.user)
396 if self.gid != 0 and self.group:
397 entry = grp_from_name(self.group)
400 else: # not superuser - only consider changing the group/gid
401 user_gids = os.getgroups()
402 if self.gid in user_gids:
404 if not restore_numeric_ids and self.gid != 0:
405 # The grp might not exist on the local system.
406 grps = filter(None, [grp_from_gid(x) for x in user_gids])
407 if self.group in [x.gr_name for x in grps]:
408 g = grp_from_name(self.group)
412 if uid != -1 or gid != -1:
414 os.lchown(path, uid, gid)
416 if e.errno == errno.EPERM:
417 add_error('lchown: %s' % e)
418 elif sys.platform.startswith('cygwin') \
419 and e.errno == errno.EINVAL:
420 add_error('lchown: unknown uid/gid (%d/%d) for %s'
427 os.lchmod(path, stat.S_IMODE(self.mode))
428 except errno.ENOSYS: # Function not implemented
430 elif not stat.S_ISLNK(self.mode):
431 os.chmod(path, stat.S_IMODE(self.mode))
436 def _encode_path(self):
438 return vint.pack('s', self.path)
442 def _load_path_rec(self, port):
443 self.path = vint.unpack('s', vint.read_bvec(port))[0]
448 def _add_symlink_target(self, path, st):
450 if stat.S_ISLNK(st.st_mode):
451 self.symlink_target = os.readlink(path)
453 add_error('readlink: %s' % e)
455 def _encode_symlink_target(self):
456 return self.symlink_target
458 def _load_symlink_target_rec(self, port):
459 self.symlink_target = vint.read_bvec(port)
464 def _add_hardlink_target(self, target):
465 self.hardlink_target = target
467 def _same_hardlink_target(self, other):
468 """Return true or false to indicate similarity in the hardlink sense."""
469 return self.hardlink_target == other.hardlink_target
471 def _encode_hardlink_target(self):
472 return self.hardlink_target
474 def _load_hardlink_target_rec(self, port):
475 self.hardlink_target = vint.read_bvec(port)
478 ## POSIX1e ACL records
480 # Recorded as a list:
481 # [txt_id_acl, num_id_acl]
482 # or, if a directory:
483 # [txt_id_acl, num_id_acl, txt_id_default_acl, num_id_default_acl]
484 # The numeric/text distinction only matters when reading/restoring
486 def _add_posix1e_acl(self, path, st):
487 if not posix1e: return
488 if not stat.S_ISLNK(st.st_mode):
492 if posix1e.has_extended(path):
493 acl = posix1e.ACL(file=path)
494 acls = [acl, acl] # txt and num are the same
495 if stat.S_ISDIR(st.st_mode):
496 def_acl = posix1e.ACL(filedef=path)
497 def_acls = [def_acl, def_acl]
498 except EnvironmentError, e:
499 if e.errno not in (errno.EOPNOTSUPP, errno.ENOSYS):
502 txt_flags = posix1e.TEXT_ABBREVIATE
503 num_flags = posix1e.TEXT_ABBREVIATE | posix1e.TEXT_NUMERIC_IDS
504 acl_rep = [acls[0].to_any_text('', '\n', txt_flags),
505 acls[1].to_any_text('', '\n', num_flags)]
507 acl_rep.append(def_acls[0].to_any_text('', '\n', txt_flags))
508 acl_rep.append(def_acls[1].to_any_text('', '\n', num_flags))
509 self.posix1e_acl = acl_rep
511 def _same_posix1e_acl(self, other):
512 """Return true or false to indicate similarity in the hardlink sense."""
513 return self.posix1e_acl == other.posix1e_acl
515 def _encode_posix1e_acl(self):
516 # Encode as two strings (w/default ACL string possibly empty).
518 acls = self.posix1e_acl
520 acls.extend(['', ''])
521 return vint.pack('ssss', acls[0], acls[1], acls[2], acls[3])
525 def _load_posix1e_acl_rec(self, port):
526 acl_rep = vint.unpack('ssss', vint.read_bvec(port))
528 acl_rep = acl_rep[:2]
529 self.posix1e_acl = acl_rep
531 def _apply_posix1e_acl_rec(self, path, restore_numeric_ids=False):
532 def apply_acl(acl_rep, kind):
534 acl = posix1e.ACL(text = acl_rep)
537 # pylibacl appears to return an IOError with errno
538 # set to 0 if a group referred to by the ACL rep
539 # doesn't exist on the current system.
540 raise ApplyError("POSIX1e ACL: can't create %r for %r"
545 acl.applyto(path, kind)
547 if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP:
548 raise ApplyError('POSIX1e ACL applyto: %s' % e)
554 add_error("%s: can't restore ACLs; posix1e support missing.\n"
558 acls = self.posix1e_acl
560 if restore_numeric_ids:
561 apply_acl(acls[3], posix1e.ACL_TYPE_DEFAULT)
563 apply_acl(acls[2], posix1e.ACL_TYPE_DEFAULT)
564 if restore_numeric_ids:
565 apply_acl(acls[1], posix1e.ACL_TYPE_ACCESS)
567 apply_acl(acls[0], posix1e.ACL_TYPE_ACCESS)
570 ## Linux attributes (lsattr(1), chattr(1))
572 def _add_linux_attr(self, path, st):
573 check_linux_file_attr_api()
574 if not get_linux_file_attr: return
575 if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
577 attr = get_linux_file_attr(path)
579 self.linux_attr = attr
581 if e.errno == errno.EACCES:
582 add_error('read Linux attr: %s' % e)
583 elif e.errno in (errno.ENOTTY, errno.ENOSYS, errno.EOPNOTSUPP):
584 # Assume filesystem doesn't support attrs.
589 def _same_linux_attr(self, other):
590 """Return true or false to indicate similarity in the hardlink sense."""
591 return self.linux_attr == other.linux_attr
593 def _encode_linux_attr(self):
595 return vint.pack('V', self.linux_attr)
599 def _load_linux_attr_rec(self, port):
600 data = vint.read_bvec(port)
601 self.linux_attr = vint.unpack('V', data)[0]
603 def _apply_linux_attr_rec(self, path, restore_numeric_ids=False):
605 check_linux_file_attr_api()
606 if not set_linux_file_attr:
607 add_error("%s: can't restore linuxattrs: "
608 "linuxattr support missing.\n" % path)
611 set_linux_file_attr(path, self.linux_attr)
613 if e.errno in (errno.ENOTTY, errno.EOPNOTSUPP, errno.ENOSYS,
615 raise ApplyError('Linux chattr: %s (0x%s)'
616 % (e, hex(self.linux_attr)))
621 ## Linux extended attributes (getfattr(1), setfattr(1))
623 def _add_linux_xattr(self, path, st):
626 self.linux_xattr = xattr.get_all(path, nofollow=True)
627 except EnvironmentError, e:
628 if e.errno != errno.EOPNOTSUPP:
631 def _same_linux_xattr(self, other):
632 """Return true or false to indicate similarity in the hardlink sense."""
633 return self.linux_xattr == other.linux_xattr
635 def _encode_linux_xattr(self):
637 result = vint.pack('V', len(self.linux_xattr))
638 for name, value in self.linux_xattr:
639 result += vint.pack('ss', name, value)
644 def _load_linux_xattr_rec(self, file):
645 data = vint.read_bvec(file)
646 memfile = StringIO(data)
648 for i in range(vint.read_vuint(memfile)):
649 key = vint.read_bvec(memfile)
650 value = vint.read_bvec(memfile)
651 result.append((key, value))
652 self.linux_xattr = result
654 def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
657 add_error("%s: can't restore xattr; xattr support missing.\n"
660 if not self.linux_xattr:
663 existing_xattrs = set(xattr.list(path, nofollow=True))
665 if e.errno == errno.EACCES:
666 raise ApplyError('xattr.set %r: %s' % (path, e))
669 for k, v in self.linux_xattr:
670 if k not in existing_xattrs \
671 or v != xattr.get(path, k, nofollow=True):
673 xattr.set(path, k, v, nofollow=True)
675 if e.errno == errno.EPERM \
676 or e.errno == errno.EOPNOTSUPP:
677 raise ApplyError('xattr.set %r: %s' % (path, e))
680 existing_xattrs -= frozenset([k])
681 for k in existing_xattrs:
683 xattr.remove(path, k, nofollow=True)
685 if e.errno == errno.EPERM:
686 raise ApplyError('xattr.remove %r: %s' % (path, e))
691 self.mode = self.uid = self.gid = self.user = self.group = None
692 self.atime = self.mtime = self.ctime = None
696 self.symlink_target = None
697 self.hardlink_target = None
698 self.linux_attr = None
699 self.linux_xattr = None
700 self.posix1e_acl = None
703 result = ['<%s instance at %s' % (self.__class__, hex(id(self)))]
705 result += ' path:' + repr(self.path)
707 result += ' mode:' + repr(xstat.mode_str(self.mode)
708 + '(%s)' % hex(self.mode))
710 result += ' uid:' + str(self.uid)
712 result += ' gid:' + str(self.gid)
714 result += ' user:' + repr(self.user)
716 result += ' group:' + repr(self.group)
718 result += ' size:' + repr(self.size)
719 for name, val in (('atime', self.atime),
720 ('mtime', self.mtime),
721 ('ctime', self.ctime)):
724 time.strftime('%Y-%m-%d %H:%M %z',
725 time.gmtime(xstat.fstime_floor_secs(val))))
727 return ''.join(result)
729 def write(self, port, include_path=True):
730 records = include_path and [(_rec_tag_path, self._encode_path())] or []
731 records.extend([(_rec_tag_common_v2, self._encode_common()),
732 (_rec_tag_symlink_target,
733 self._encode_symlink_target()),
734 (_rec_tag_hardlink_target,
735 self._encode_hardlink_target()),
736 (_rec_tag_posix1e_acl, self._encode_posix1e_acl()),
737 (_rec_tag_linux_attr, self._encode_linux_attr()),
738 (_rec_tag_linux_xattr, self._encode_linux_xattr())])
739 for tag, data in records:
741 vint.write_vuint(port, tag)
742 vint.write_bvec(port, data)
743 vint.write_vuint(port, _rec_tag_end)
745 def encode(self, include_path=True):
747 self.write(port, include_path)
748 return port.getvalue()
752 # This method should either return a valid Metadata object,
753 # return None if there was no information at all (just a
754 # _rec_tag_end), throw EOFError if there was nothing at all to
755 # read, or throw an Exception if a valid object could not be
757 tag = vint.read_vuint(port)
758 if tag == _rec_tag_end:
760 try: # From here on, EOF is an error.
762 while True: # only exit is error (exception) or _rec_tag_end
763 if tag == _rec_tag_path:
764 result._load_path_rec(port)
765 elif tag == _rec_tag_common_v2:
766 result._load_common_rec(port)
767 elif tag == _rec_tag_symlink_target:
768 result._load_symlink_target_rec(port)
769 elif tag == _rec_tag_hardlink_target:
770 result._load_hardlink_target_rec(port)
771 elif tag == _rec_tag_posix1e_acl:
772 result._load_posix1e_acl_rec(port)
773 elif tag == _rec_tag_linux_attr:
774 result._load_linux_attr_rec(port)
775 elif tag == _rec_tag_linux_xattr:
776 result._load_linux_xattr_rec(port)
777 elif tag == _rec_tag_end:
779 elif tag == _rec_tag_common: # Should be very rare.
780 result._load_common_rec(port, legacy_format = True)
781 else: # unknown record
783 tag = vint.read_vuint(port)
785 raise Exception("EOF while reading Metadata")
788 return stat.S_ISDIR(self.mode)
790 def create_path(self, path, create_symlinks=True):
791 self._create_via_common_rec(path, create_symlinks=create_symlinks)
793 def apply_to_path(self, path=None, restore_numeric_ids=False):
794 # apply metadata to path -- file must exist
798 raise Exception('Metadata.apply_to_path() called with no path')
799 if not self._recognized_file_type():
800 add_error('not applying metadata to "%s"' % path
801 + ' with unrecognized mode "0x%x"\n' % self.mode)
803 num_ids = restore_numeric_ids
804 for apply_metadata in (self._apply_common_rec,
805 self._apply_posix1e_acl_rec,
806 self._apply_linux_attr_rec,
807 self._apply_linux_xattr_rec):
809 apply_metadata(path, restore_numeric_ids=num_ids)
810 except ApplyError, e:
813 def same_file(self, other):
814 """Compare this to other for equivalency. Return true if
815 their information implies they could represent the same file
816 on disk, in the hardlink sense. Assume they're both regular
818 return self._same_common(other) \
819 and self._same_hardlink_target(other) \
820 and self._same_posix1e_acl(other) \
821 and self._same_linux_attr(other) \
822 and self._same_linux_xattr(other)
825 def from_path(path, statinfo=None, archive_path=None,
826 save_symlinks=True, hardlink_target=None):
828 result.path = archive_path
829 st = statinfo or xstat.lstat(path)
830 result.size = st.st_size
831 result._add_common(path, st)
833 result._add_symlink_target(path, st)
834 result._add_hardlink_target(hardlink_target)
835 result._add_posix1e_acl(path, st)
836 result._add_linux_attr(path, st)
837 result._add_linux_xattr(path, st)
841 def save_tree(output_file, paths,
847 # Issue top-level rewrite warnings.
849 safe_path = _clean_up_path_for_archive(path)
850 if safe_path != path:
851 log('archiving "%s" as "%s"\n' % (path, safe_path))
855 safe_path = _clean_up_path_for_archive(p)
857 if stat.S_ISDIR(st.st_mode):
859 m = from_path(p, statinfo=st, archive_path=safe_path,
860 save_symlinks=save_symlinks)
862 print >> sys.stderr, m.path
863 m.write(output_file, include_path=write_paths)
865 start_dir = os.getcwd()
867 for (p, st) in recursive_dirlist(paths, xdev=xdev):
868 dirlist_dir = os.getcwd()
870 safe_path = _clean_up_path_for_archive(p)
871 m = from_path(p, statinfo=st, archive_path=safe_path,
872 save_symlinks=save_symlinks)
874 print >> sys.stderr, m.path
875 m.write(output_file, include_path=write_paths)
876 os.chdir(dirlist_dir)
881 def _set_up_path(meta, create_symlinks=True):
882 # Allow directories to exist as a special case -- might have
883 # been created by an earlier longer path.
887 parent = os.path.dirname(meta.path)
890 meta.create_path(meta.path, create_symlinks=create_symlinks)
893 all_fields = frozenset(['path',
910 def summary_str(meta, numeric_ids = False, classification = None,
911 human_readable = False):
913 """Return a string containing the "ls -l" style listing for meta.
914 Classification may be "all", "type", or None."""
915 user_str = group_str = size_or_dev_str = '?'
916 symlink_target = None
919 mode_str = xstat.mode_str(meta.mode)
920 symlink_target = meta.symlink_target
921 mtime_secs = xstat.fstime_floor_secs(meta.mtime)
922 mtime_str = time.strftime('%Y-%m-%d %H:%M', time.localtime(mtime_secs))
923 if meta.user and not numeric_ids:
925 elif meta.uid != None:
926 user_str = str(meta.uid)
927 if meta.group and not numeric_ids:
928 group_str = meta.group
929 elif meta.gid != None:
930 group_str = str(meta.gid)
931 if stat.S_ISCHR(meta.mode) or stat.S_ISBLK(meta.mode):
933 size_or_dev_str = '%d,%d' % (os.major(meta.rdev),
935 elif meta.size != None:
937 size_or_dev_str = format_filesize(meta.size)
939 size_or_dev_str = str(meta.size)
941 size_or_dev_str = '-'
943 classification_str = \
944 xstat.classification_str(meta.mode, classification == 'all')
947 mtime_str = '????-??-?? ??:??'
948 classification_str = '?'
952 name += classification_str
954 name += ' -> ' + meta.symlink_target
956 return '%-10s %-11s %11s %16s %s' % (mode_str,
957 user_str + "/" + group_str,
963 def detailed_str(meta, fields = None):
964 # FIXME: should optional fields be omitted, or empty i.e. "rdev:
965 # 0", "link-target:", etc.
971 path = meta.path or ''
972 result.append('path: ' + path)
974 result.append('mode: %s (%s)' % (oct(meta.mode),
975 xstat.mode_str(meta.mode)))
976 if 'link-target' in fields and stat.S_ISLNK(meta.mode):
977 result.append('link-target: ' + meta.symlink_target)
980 result.append('rdev: %d,%d' % (os.major(meta.rdev),
981 os.minor(meta.rdev)))
983 result.append('rdev: 0')
984 if 'size' in fields and meta.size:
985 result.append('size: ' + str(meta.size))
987 result.append('uid: ' + str(meta.uid))
989 result.append('gid: ' + str(meta.gid))
991 result.append('user: ' + meta.user)
992 if 'group' in fields:
993 result.append('group: ' + meta.group)
994 if 'atime' in fields:
995 # If we don't have xstat.lutime, that means we have to use
996 # utime(), and utime() has no way to set the mtime/atime of a
997 # symlink. Thus, the mtime/atime of a symlink is meaningless,
998 # so let's not report it. (That way scripts comparing
999 # before/after won't trigger.)
1000 if xstat.lutime or not stat.S_ISLNK(meta.mode):
1001 result.append('atime: ' + xstat.fstime_to_sec_str(meta.atime))
1003 result.append('atime: 0')
1004 if 'mtime' in fields:
1005 if xstat.lutime or not stat.S_ISLNK(meta.mode):
1006 result.append('mtime: ' + xstat.fstime_to_sec_str(meta.mtime))
1008 result.append('mtime: 0')
1009 if 'ctime' in fields:
1010 result.append('ctime: ' + xstat.fstime_to_sec_str(meta.ctime))
1011 if 'linux-attr' in fields and meta.linux_attr:
1012 result.append('linux-attr: ' + hex(meta.linux_attr))
1013 if 'linux-xattr' in fields and meta.linux_xattr:
1014 for name, value in meta.linux_xattr:
1015 result.append('linux-xattr: %s -> %s' % (name, repr(value)))
1016 if 'posix1e-acl' in fields and meta.posix1e_acl:
1017 acl = meta.posix1e_acl[0]
1018 result.append('posix1e-acl: ' + acl + '\n')
1019 if stat.S_ISDIR(meta.mode):
1020 def_acl = meta.posix1e_acl[2]
1021 result.append('posix1e-acl-default: ' + def_acl + '\n')
1022 return '\n'.join(result)
1025 class _ArchiveIterator:
1028 return Metadata.read(self._file)
1030 raise StopIteration()
1035 def __init__(self, file):
1039 def display_archive(file):
1042 for meta in _ArchiveIterator(file):
1045 print detailed_str(meta)
1048 for meta in _ArchiveIterator(file):
1049 print summary_str(meta)
1051 for meta in _ArchiveIterator(file):
1053 print >> sys.stderr, \
1054 'bup: no metadata path, but asked to only display path', \
1055 '(increase verbosity?)'
1060 def start_extract(file, create_symlinks=True):
1061 for meta in _ArchiveIterator(file):
1062 if not meta: # Hit end record.
1065 print >> sys.stderr, meta.path
1066 xpath = _clean_up_extract_path(meta.path)
1068 add_error(Exception('skipping risky path "%s"' % meta.path))
1071 _set_up_path(meta, create_symlinks=create_symlinks)
1074 def finish_extract(file, restore_numeric_ids=False):
1076 for meta in _ArchiveIterator(file):
1077 if not meta: # Hit end record.
1079 xpath = _clean_up_extract_path(meta.path)
1081 add_error(Exception('skipping risky path "%s"' % dir.path))
1083 if os.path.isdir(meta.path):
1084 all_dirs.append(meta)
1087 print >> sys.stderr, meta.path
1088 meta.apply_to_path(path=xpath,
1089 restore_numeric_ids=restore_numeric_ids)
1090 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
1091 for dir in all_dirs:
1092 # Don't need to check xpath -- won't be in all_dirs if not OK.
1093 xpath = _clean_up_extract_path(dir.path)
1095 print >> sys.stderr, dir.path
1096 dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
1099 def extract(file, restore_numeric_ids=False, create_symlinks=True):
1100 # For now, just store all the directories and handle them last,
1103 for meta in _ArchiveIterator(file):
1104 if not meta: # Hit end record.
1106 xpath = _clean_up_extract_path(meta.path)
1108 add_error(Exception('skipping risky path "%s"' % meta.path))
1112 print >> sys.stderr, '+', meta.path
1113 _set_up_path(meta, create_symlinks=create_symlinks)
1114 if os.path.isdir(meta.path):
1115 all_dirs.append(meta)
1118 print >> sys.stderr, '=', meta.path
1119 meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
1120 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
1121 for dir in all_dirs:
1122 # Don't need to check xpath -- won't be in all_dirs if not OK.
1123 xpath = _clean_up_extract_path(dir.path)
1125 print >> sys.stderr, '=', xpath
1126 # Shouldn't have to check for risky paths here (omitted above).
1127 dir.apply_to_path(path=dir.path,
1128 restore_numeric_ids=restore_numeric_ids)