1 """Metadata read/write support for bup."""
3 # Copyright (C) 2010 Rob Browning
5 # This code is covered under the terms of the GNU Library General
6 # Public License as described in the bup LICENSE file.
7 import errno, os, sys, stat, pwd, grp, struct, re
8 from cStringIO import StringIO
9 from bup import vint, xstat
10 from bup.drecurse import recursive_dirlist
11 from bup.helpers import add_error, mkdirp, log, is_superuser
12 from bup.xstat import utime, lutime, lstat
13 import bup._helpers as _helpers
18 log('Warning: Linux xattr support missing; install python-pyxattr.\n')
23 except AttributeError:
24 log('Warning: python-xattr module is too old; '
25 'install python-pyxattr instead.\n')
30 log('Warning: POSIX ACL support missing; install python-pylibacl.\n')
33 from bup._helpers import get_linux_file_attr, set_linux_file_attr
35 # No need for a warning here; the only reason they won't exist is that we're
36 # not on Linux, in which case files don't have any linux attrs anyway, so
37 # lacking the functions isn't a problem.
38 get_linux_file_attr = set_linux_file_attr = None
41 # WARNING: the metadata encoding is *not* stable yet. Caveat emptor!
43 # Q: Consider hardlink support?
44 # Q: Is it OK to store raw linux attr (chattr) flags?
45 # Q: Can anything other than S_ISREG(x) or S_ISDIR(x) support posix1e ACLs?
46 # Q: Is the application of posix1e has_extended() correct?
47 # Q: Is one global --numeric-ids argument sufficient?
48 # Q: Do nfsv4 acls trump posix1e acls? (seems likely)
49 # Q: Add support for crtime -- ntfs, and (only internally?) ext*?
51 # FIXME: Fix relative/abs path detection/stripping wrt other platforms.
52 # FIXME: Add nfsv4 acl handling - see nfs4-acl-tools.
53 # FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.).
54 # FIXME: Consider pack('vvvvsss', ...) optimization.
55 # FIXME: Consider caching users/groups.
59 # osx (varies between hfs and hfs+):
60 # type - regular dir char block fifo socket ...
61 # perms - rwxrwxrwxsgt
62 # times - ctime atime mtime
65 # hard-link-info (hfs+ only)
68 # attributes-osx see chflags
74 # type - regular dir ...
75 # times - creation, modification, posix change, access
78 # attributes - see attrib
80 # forks (alternate data streams)
84 # type - regular dir ...
85 # perms - rwxrwxrwx (maybe - see wikipedia)
86 # times - creation, modification, access
87 # attributes - see attrib
91 _have_lchmod = hasattr(os, 'lchmod')
94 def _clean_up_path_for_archive(p):
95 # Not the most efficient approach.
98 # Take everything after any '/../'.
99 pos = result.rfind('/../')
101 result = result[result.rfind('/../') + 4:]
103 # Take everything after any remaining '../'.
104 if result.startswith("../"):
107 # Remove any '/./' sequences.
108 pos = result.find('/./')
110 result = result[0:pos] + '/' + result[pos + 3:]
111 pos = result.find('/./')
113 # Remove any leading '/'s.
114 result = result.lstrip('/')
116 # Replace '//' with '/' everywhere.
117 pos = result.find('//')
119 result = result[0:pos] + '/' + result[pos + 2:]
120 pos = result.find('//')
122 # Take everything after any remaining './'.
123 if result.startswith('./'):
126 # Take everything before any remaining '/.'.
127 if result.endswith('/.'):
130 if result == '' or result.endswith('/..'):
137 if p.startswith('/'):
139 if p.find('/../') != -1:
141 if p.startswith('../'):
143 if p.endswith('/..'):
148 def _clean_up_extract_path(p):
149 result = p.lstrip('/')
152 elif _risky_path(result):
158 # These tags are currently conceptually private to Metadata, and they
159 # must be unique, and must *never* be changed.
162 _rec_tag_common = 2 # times, owner, group, type, perms, etc.
163 _rec_tag_symlink_target = 3
164 _rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc.
165 _rec_tag_nfsv4_acl = 5 # intended to supplant posix1e acls?
166 _rec_tag_linux_attr = 6 # lsattr(1) chattr(1)
167 _rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1)
170 class ApplyError(Exception):
171 # Thrown when unable to apply any given bit of metadata to a path.
176 # Metadata is stored as a sequence of tagged binary records. Each
177 # record will have some subset of add, encode, load, create, and
178 # apply methods, i.e. _add_foo...
182 # Timestamps are (sec, ns), relative to 1970-01-01 00:00:00, ns
183 # must be non-negative and < 10**9.
185 def _add_common(self, path, st):
186 self.mode = st.st_mode
189 self.rdev = st.st_rdev
190 self.atime = st.st_atime
191 self.mtime = st.st_mtime
192 self.ctime = st.st_ctime
193 self.owner = self.group = ''
195 self.owner = pwd.getpwuid(st.st_uid)[0]
197 add_error("no user name for id %s '%s'" % (st.st_gid, path))
199 self.group = grp.getgrgid(st.st_gid)[0]
201 add_error("no group name for id %s '%s'" % (st.st_gid, path))
203 def _encode_common(self):
204 atime = xstat.nsecs_to_timespec(self.atime)
205 mtime = xstat.nsecs_to_timespec(self.mtime)
206 ctime = xstat.nsecs_to_timespec(self.ctime)
207 result = vint.pack('VVsVsVvVvVvV',
222 def _load_common_rec(self, port):
223 data = vint.read_bvec(port)
235 ctime_ns) = vint.unpack('VVsVsVvVvVvV', data)
236 self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
237 self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
238 self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
240 def _recognized_file_type(self):
241 return stat.S_ISREG(self.mode) \
242 or stat.S_ISDIR(self.mode) \
243 or stat.S_ISCHR(self.mode) \
244 or stat.S_ISBLK(self.mode) \
245 or stat.S_ISFIFO(self.mode) \
246 or stat.S_ISLNK(self.mode)
248 def _create_via_common_rec(self, path, create_symlinks=True):
249 # If the path already exists and is a dir, try rmdir.
250 # If the path already exists and is anything else, try unlink.
253 st = xstat.lstat(path)
255 if e.errno != errno.ENOENT:
258 if stat.S_ISDIR(st.st_mode):
262 if e.errno == errno.ENOTEMPTY:
263 msg = 'refusing to overwrite non-empty dir' + path
269 if stat.S_ISREG(self.mode):
270 assert(self._recognized_file_type())
271 fd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL, 0600)
273 elif stat.S_ISDIR(self.mode):
274 assert(self._recognized_file_type())
276 elif stat.S_ISCHR(self.mode):
277 assert(self._recognized_file_type())
278 os.mknod(path, 0600 | stat.S_IFCHR, self.rdev)
279 elif stat.S_ISBLK(self.mode):
280 assert(self._recognized_file_type())
281 os.mknod(path, 0600 | stat.S_IFBLK, self.rdev)
282 elif stat.S_ISFIFO(self.mode):
283 assert(self._recognized_file_type())
284 os.mknod(path, 0600 | stat.S_IFIFO)
285 elif stat.S_ISLNK(self.mode):
286 assert(self._recognized_file_type())
287 if self.symlink_target and create_symlinks:
288 # on MacOS, symlink() permissions depend on umask, and there's
289 # no way to chown a symlink after creating it, so we have to
291 oldumask = os.umask((self.mode & 0777) ^ 0777)
293 os.symlink(self.symlink_target, path)
296 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
298 assert(not self._recognized_file_type())
299 add_error('not creating "%s" with unrecognized mode "0x%x"\n'
302 def _apply_common_rec(self, path, restore_numeric_ids=False):
303 # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
304 # EACCES errors at this stage are fatal for the current path.
305 if stat.S_ISLNK(self.mode):
307 lutime(path, (self.atime, self.mtime))
309 if e.errno == errno.EACCES:
310 raise ApplyError('lutime: %s' % e)
315 utime(path, (self.atime, self.mtime))
317 if e.errno == errno.EACCES:
318 raise ApplyError('utime: %s' % e)
322 # Don't try to restore owner unless we're root, and even
323 # if asked, don't try to restore the owner or group if
324 # it doesn't exist in the system db.
327 if not restore_numeric_ids:
330 add_error('ignoring missing owner for "%s"\n' % path)
332 if not is_superuser():
333 uid = -1 # Not root; assume we can't change owner.
336 uid = pwd.getpwnam(self.owner)[2]
339 fmt = 'ignoring unknown owner %s for "%s"\n'
340 add_error(fmt % (self.owner, path))
343 add_error('ignoring missing group for "%s"\n' % path)
346 gid = grp.getgrnam(self.group)[2]
349 add_error('ignoring unknown group %s for "%s"\n'
350 % (self.group, path))
353 os.lchown(path, uid, gid)
355 if e.errno == errno.EPERM:
356 add_error('lchown: %s' % e)
361 os.lchmod(path, stat.S_IMODE(self.mode))
362 elif not stat.S_ISLNK(self.mode):
363 os.chmod(path, stat.S_IMODE(self.mode))
368 def _encode_path(self):
370 return vint.pack('s', self.path)
374 def _load_path_rec(self, port):
375 self.path = vint.unpack('s', vint.read_bvec(port))[0]
380 def _add_symlink_target(self, path, st):
382 if stat.S_ISLNK(st.st_mode):
383 self.symlink_target = os.readlink(path)
385 add_error('readlink: %s', e)
387 def _encode_symlink_target(self):
388 return self.symlink_target
390 def _load_symlink_target_rec(self, port):
391 self.symlink_target = vint.read_bvec(port)
394 ## POSIX1e ACL records
396 # Recorded as a list:
397 # [txt_id_acl, num_id_acl]
398 # or, if a directory:
399 # [txt_id_acl, num_id_acl, txt_id_default_acl, num_id_default_acl]
400 # The numeric/text distinction only matters when reading/restoring
402 def _add_posix1e_acl(self, path, st):
403 if not posix1e: return
404 if not stat.S_ISLNK(st.st_mode):
406 if posix1e.has_extended(path):
407 acl = posix1e.ACL(file=path)
408 self.posix1e_acl = [acl, acl] # txt and num are the same
409 if stat.S_ISDIR(st.st_mode):
410 acl = posix1e.ACL(filedef=path)
411 self.posix1e_acl.extend([acl, acl])
412 except EnvironmentError, e:
413 if e.errno != errno.EOPNOTSUPP:
416 def _encode_posix1e_acl(self):
417 # Encode as two strings (w/default ACL string possibly empty).
419 acls = self.posix1e_acl
420 txt_flags = posix1e.TEXT_ABBREVIATE
421 num_flags = posix1e.TEXT_ABBREVIATE | posix1e.TEXT_NUMERIC_IDS
422 acl_reps = [acls[0].to_any_text('', '\n', txt_flags),
423 acls[1].to_any_text('', '\n', num_flags)]
427 acl_reps.append(acls[2].to_any_text('', '\n', txt_flags))
428 acl_reps.append(acls[3].to_any_text('', '\n', num_flags))
429 return vint.pack('ssss',
430 acl_reps[0], acl_reps[1], acl_reps[2], acl_reps[3])
434 def _load_posix1e_acl_rec(self, port):
435 data = vint.read_bvec(port)
436 acl_reps = vint.unpack('ssss', data)
437 if acl_reps[2] == '':
438 acl_reps = acl_reps[:2]
439 self.posix1e_acl = [posix1e.ACL(text=x) for x in acl_reps]
441 def _apply_posix1e_acl_rec(self, path, restore_numeric_ids=False):
444 add_error("%s: can't restore ACLs; posix1e support missing.\n"
448 acls = self.posix1e_acl
450 if restore_numeric_ids:
451 acls[3].applyto(path, posix1e.ACL_TYPE_DEFAULT)
453 acls[2].applyto(path, posix1e.ACL_TYPE_DEFAULT)
454 if restore_numeric_ids:
455 acls[1].applyto(path, posix1e.ACL_TYPE_ACCESS)
457 acls[0].applyto(path, posix1e.ACL_TYPE_ACCESS)
460 ## Linux attributes (lsattr(1), chattr(1))
462 def _add_linux_attr(self, path, st):
463 if not get_linux_file_attr: return
464 if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
466 attr = get_linux_file_attr(path)
468 self.linux_attr = attr
470 if e.errno == errno.EACCES:
471 add_error('read Linux attr: %s' % e)
472 elif e.errno == errno.ENOTTY: # Inappropriate ioctl for device.
473 add_error('read Linux attr: %s' % e)
477 def _encode_linux_attr(self):
479 return vint.pack('V', self.linux_attr)
483 def _load_linux_attr_rec(self, port):
484 data = vint.read_bvec(port)
485 self.linux_attr = vint.unpack('V', data)[0]
487 def _apply_linux_attr_rec(self, path, restore_numeric_ids=False):
489 if not set_linux_file_attr:
490 add_error("%s: can't restore linuxattrs: "
491 "linuxattr support missing.\n" % path)
493 set_linux_file_attr(path, self.linux_attr)
496 ## Linux extended attributes (getfattr(1), setfattr(1))
498 def _add_linux_xattr(self, path, st):
501 self.linux_xattr = xattr.get_all(path, nofollow=True)
502 except EnvironmentError, e:
503 if e.errno != errno.EOPNOTSUPP:
506 def _encode_linux_xattr(self):
508 result = vint.pack('V', len(self.linux_xattr))
509 for name, value in self.linux_xattr:
510 result += vint.pack('ss', name, value)
515 def _load_linux_xattr_rec(self, file):
516 data = vint.read_bvec(file)
517 memfile = StringIO(data)
519 for i in range(vint.read_vuint(memfile)):
520 key = vint.read_bvec(memfile)
521 value = vint.read_bvec(memfile)
522 result.append((key, value))
523 self.linux_xattr = result
525 def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
528 add_error("%s: can't restore xattr; xattr support missing.\n"
531 existing_xattrs = set(xattr.list(path, nofollow=True))
533 for k, v in self.linux_xattr:
534 if k not in existing_xattrs \
535 or v != xattr.get(path, k, nofollow=True):
537 xattr.set(path, k, v, nofollow=True)
539 if e.errno == errno.EPERM:
540 raise ApplyError('xattr.set: %s' % e)
543 existing_xattrs -= frozenset([k])
544 for k in existing_xattrs:
546 xattr.remove(path, k, nofollow=True)
548 if e.errno == errno.EPERM:
549 raise ApplyError('xattr.remove: %s' % e)
556 self.symlink_target = None
557 self.linux_attr = None
558 self.linux_xattr = None
559 self.posix1e_acl = None
560 self.posix1e_acl_default = None
562 def write(self, port, include_path=True):
563 records = include_path and [(_rec_tag_path, self._encode_path())] or []
564 records.extend([(_rec_tag_common, self._encode_common()),
565 (_rec_tag_symlink_target, self._encode_symlink_target()),
566 (_rec_tag_posix1e_acl, self._encode_posix1e_acl()),
567 (_rec_tag_linux_attr, self._encode_linux_attr()),
568 (_rec_tag_linux_xattr, self._encode_linux_xattr())])
569 for tag, data in records:
571 vint.write_vuint(port, tag)
572 vint.write_bvec(port, data)
573 vint.write_vuint(port, _rec_tag_end)
577 # This method should either: return a valid Metadata object;
578 # throw EOFError if there was nothing at all to read; throw an
579 # Exception if a valid object could not be read completely.
580 tag = vint.read_vuint(port)
581 try: # From here on, EOF is an error.
583 while True: # only exit is error (exception) or _rec_tag_end
584 if tag == _rec_tag_path:
585 result._load_path_rec(port)
586 elif tag == _rec_tag_common:
587 result._load_common_rec(port)
588 elif tag == _rec_tag_symlink_target:
589 result._load_symlink_target_rec(port)
590 elif tag == _rec_tag_posix1e_acl:
591 result._load_posix1e_acl_rec(port)
592 elif tag ==_rec_tag_nfsv4_acl:
593 result._load_nfsv4_acl_rec(port)
594 elif tag == _rec_tag_linux_attr:
595 result._load_linux_attr_rec(port)
596 elif tag == _rec_tag_linux_xattr:
597 result._load_linux_xattr_rec(port)
598 elif tag == _rec_tag_end:
600 else: # unknown record
602 tag = vint.read_vuint(port)
604 raise Exception("EOF while reading Metadata")
607 return stat.S_ISDIR(self.mode)
609 def create_path(self, path, create_symlinks=True):
610 self._create_via_common_rec(path, create_symlinks=create_symlinks)
612 def apply_to_path(self, path=None, restore_numeric_ids=False):
613 # apply metadata to path -- file must exist
617 raise Exception('Metadata.apply_to_path() called with no path');
618 if not self._recognized_file_type():
619 add_error('not applying metadata to "%s"' % path
620 + ' with unrecognized mode "0x%x"\n' % self.mode)
622 num_ids = restore_numeric_ids
624 self._apply_common_rec(path, restore_numeric_ids=num_ids)
625 self._apply_posix1e_acl_rec(path, restore_numeric_ids=num_ids)
626 self._apply_linux_attr_rec(path, restore_numeric_ids=num_ids)
627 self._apply_linux_xattr_rec(path, restore_numeric_ids=num_ids)
628 except ApplyError, e:
632 def from_path(path, statinfo=None, archive_path=None, save_symlinks=True):
634 result.path = archive_path
635 st = statinfo or xstat.lstat(path)
636 result._add_common(path, st)
638 result._add_symlink_target(path, st)
639 result._add_posix1e_acl(path, st)
640 result._add_linux_attr(path, st)
641 result._add_linux_xattr(path, st)
645 def save_tree(output_file, paths,
651 # Issue top-level rewrite warnings.
653 safe_path = _clean_up_path_for_archive(path)
654 if safe_path != path:
655 log('archiving "%s" as "%s"\n' % (path, safe_path))
657 start_dir = os.getcwd()
659 for (p, st) in recursive_dirlist(paths, xdev=xdev):
660 dirlist_dir = os.getcwd()
662 safe_path = _clean_up_path_for_archive(p)
663 m = from_path(p, statinfo=st, archive_path=safe_path,
664 save_symlinks=save_symlinks)
666 print >> sys.stderr, m.path
667 m.write(output_file, include_path=write_paths)
668 os.chdir(dirlist_dir)
673 def _set_up_path(meta, create_symlinks=True):
674 # Allow directories to exist as a special case -- might have
675 # been created by an earlier longer path.
679 parent = os.path.dirname(meta.path)
682 meta.create_path(meta.path, create_symlinks=create_symlinks)
685 class _ArchiveIterator:
688 return Metadata.read(self._file)
690 raise StopIteration()
695 def __init__(self, file):
699 def display_archive(file):
700 for meta in _ArchiveIterator(file):
702 print meta.path # FIXME
707 def start_extract(file, create_symlinks=True):
708 for meta in _ArchiveIterator(file):
710 print >> sys.stderr, meta.path
711 xpath = _clean_up_extract_path(meta.path)
713 add_error(Exception('skipping risky path "%s"' % meta.path))
716 _set_up_path(meta, create_symlinks=create_symlinks)
719 def finish_extract(file, restore_numeric_ids=False):
721 for meta in _ArchiveIterator(file):
722 xpath = _clean_up_extract_path(meta.path)
724 add_error(Exception('skipping risky path "%s"' % dir.path))
726 if os.path.isdir(meta.path):
727 all_dirs.append(meta)
730 print >> sys.stderr, meta.path
731 meta.apply_to_path(path=xpath,
732 restore_numeric_ids=restore_numeric_ids)
733 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
735 # Don't need to check xpath -- won't be in all_dirs if not OK.
736 xpath = _clean_up_extract_path(dir.path)
738 print >> sys.stderr, dir.path
739 dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
742 def extract(file, restore_numeric_ids=False, create_symlinks=True):
743 # For now, just store all the directories and handle them last,
746 for meta in _ArchiveIterator(file):
747 xpath = _clean_up_extract_path(meta.path)
749 add_error(Exception('skipping risky path "%s"' % meta.path))
753 print >> sys.stderr, '+', meta.path
754 _set_up_path(meta, create_symlinks=create_symlinks)
755 if os.path.isdir(meta.path):
756 all_dirs.append(meta)
759 print >> sys.stderr, '=', meta.path
760 meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
761 all_dirs.sort(key = lambda x : len(x.path), reverse=True)
763 # Don't need to check xpath -- won't be in all_dirs if not OK.
764 xpath = _clean_up_extract_path(dir.path)
766 print >> sys.stderr, '=', xpath
767 # Shouldn't have to check for risky paths here (omitted above).
768 dir.apply_to_path(path=dir.path,
769 restore_numeric_ids=restore_numeric_ids)