--- /dev/null
+% bup-meta(1) Bup %BUP_VERSION%
+% Rob Browning <rlb@defaultvalue.org>
+% %BUP_DATE%
+
+# NAME
+
+bup-meta - create or extract a metadata archive
+
+# SYNOPSIS
+
+bup meta \-\-create
+ ~ [-R] [-v] [-q] [\-\-no-symlinks] [\-\-no-paths] [-f *file*] \<*paths*...\>
+
+bup meta \-\-list
+ ~ [-v] [-q] [-f *file*]
+
+bup meta \-\-extract
+ ~ [-v] [-q] [\-\-numeric-ids] [\-\-no-symlinks] [-f *file*]
+
+bup meta \-\-start-extract
+ ~ [-v] [-q] [\-\-numeric-ids] [\-\-no-symlinks] [-f *file*]
+
+bup meta \-\-finish-extract
+ ~ [-v] [-q] [\-\-numeric-ids] [-f *file*]
+
+# DESCRIPTION
+
+`bup meta` either creates or extracts a metadata archive. A metadata
+archive contains the metadata information (timestamps, ownership,
+access permissions, etc.) for a set of filesystem paths.
+
+# OPTIONS
+
+-c, \-\-create
+: Create a metadata archive for the specified *path*s. Write the
+ archive to standard output unless **\-\-file** is specified.
+
+-t, \-\-list
+: Display information about the metadata in an archive. Read the
+ archive from standard output unless **\-\-file** is specified.
+
+-x, \-\-extract
+: Extract a metadata archive. Conceptually, perform **\-\-start-extract**
+ followed by **\-\-finish-extract**. Read the archive from standard input
+ unless **\-\-file** is specified.
+
+\-\-start-extract
+: Build a filesystem tree matching the paths stored in a metadata
+ archive. By itself, this command does not produce a full
+ restoration of the metadata. For a full restoration, this command
+ must be followed by a call to **\-\-finish-extract**. Once this
+ command has finished, all of the normal files described by the
+ metadata will exist and be empty. Restoring the data in those
+ files, and then calling **\-\-finish-extract** should restore the
+ original tree. The archive will be read from standard input
+ unless **\-\-file** is specified.
+
+\-\-finish-extract
+: Finish applying the metadata stored in an archive to the
+ filesystem. Normally, this command should follow a call to
+ **\-\-start-extract**. The archive will be read from standard input
+ unless **\-\-file** is specified.
+
+-f, \-\-file=*filename*
+: Read the metadata archive from *filename* or write it to
+ *filename* as appropriate. If *filename* is "-", then read from
+ standard input or write to standard output.
+
+-R, \-\-recurse
+: Recursively descend into subdirectories during **\-\-create**.
+
+\-\-numeric-ids
+: Apply numeric user and group IDs (rather than text IDs) during
+ **\-\-extract** or **\-\-finish-extract**.
+
+\-\-symlinks
+: Record symbolic link targets when creating an archive, or restore
+ symbolic links when extracting an archive (during **\-\-extract**
+ or **\-\-start-extract**). This option is enabled by default.
+ Specify **\-\-no-symlinks** to disable it.
+
+\-\-paths
+: Record pathnames when creating an archive. This option is enabled
+ by default. Specify **\-\-no-paths** to disable it.
+
+-v, --verbose
+: Be more verbose (can be used more than once).
+
+-q, --quiet
+: Be quiet.
+
+# EXAMPLES
+
+ # Create a metadata archive for /etc.
+ $ bup meta -cRf etc.meta /etc
+ bup: removing leading "/" from "/etc"
+
+ # Extract the etc.meta archive (files will be empty).
+ $ mkdir tmp && cd tmp
+ $ bup meta -xf ../etc.meta
+ $ ls
+ etc
+
+ # Restore /etc completely.
+ $ mkdir tmp && cd tmp
+ $ bup meta \-\-start-extract -f ../etc.meta
+ ...fill in all regular file contents using some other tool...
+ $ bup meta \-\-finish-extract -f ../etc.meta
+
+# BUGS
+
+Hard links are not handled yet.
+
+# BUP
+
+Part of the `bup`(1) suite.
runtests-cmdline: all
t/test.sh
+ t/test-meta.sh
stupid:
PATH=/bin:/usr/bin $(MAKE) test
.*~ *~ */*~ lib/*/*~ lib/*/*/*~ \
*.pyc */*.pyc lib/*/*.pyc lib/*/*/*.pyc \
bup bup-* cmd/bup-* lib/bup/_version.py randomgen memtest \
- out[12] out2[tc] tags[12] tags2[tc]
+ out[12] out2[tc] tags[12] tags2[tc] \
+ testfs.img lib/bup/t/testfs.img
chmod u+rwx lib/bup/t/pybuptest.tmp
rm -rf *.tmp t/*.tmp lib/*/*/*.tmp build lib/bup/build
+ if test -e testfs; then rmdir testfs; fi
+ if test -e lib/bup/t/testfs; then rmdir lib/bup/t/testfs; fi
- Install the needed python libraries (including the development
libraries). On Debian or Ubuntu, this is usually:
apt-get install python2.6-dev python-fuse
+ apt-get install python-pyxattr python-pylibacl
Substitute python2.5-dev or python2.4-dev if you have an older system.
--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (C) 2010 Rob Browning
+#
+# This code is covered under the terms of the GNU Library General
+# Public License as described in the bup LICENSE file.
+
+# TODO: Add tar-like -C option.
+# TODO: Add tar-like -v support to --list.
+
+import sys
+from bup import metadata
+from bup import options
+from bup.helpers import handle_ctrl_c, log, saved_errors
+
+optspec = """
+bup meta --create [OPTION ...] <PATH ...>
+bup meta --extract [OPTION ...]
+bup meta --start-extract [OPTION ...]
+bup meta --finish-extract [OPTION ...]
+--
+c,create write metadata for PATHs to stdout (or --file)
+t,list display metadata
+x,extract perform --start-extract followed by --finish-extract
+start-extract build tree matching metadata provided on standard input (or --file)
+finish-extract finish applying standard input (or --file) metadata to filesystem
+f,file= specify source or destination file
+R,recurse recurse into subdirectories
+xdev,one-file-system don't cross filesystem boundaries
+numeric-ids apply numeric IDs (user, group, etc.), not names, during restore
+symlinks handle symbolic links (default is true)
+paths include paths in metadata (default is true)
+v,verbose increase log output (can be used more than once)
+q,quiet don't show progress meter
+"""
+
+action = None
+target_filename = ''
+should_recurse = False
+restore_numeric_ids = False
+include_paths = True
+handle_symlinks = True
+xdev = False
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, remainder) = o.parse(sys.argv[1:])
+
+for flag, value in flags:
+ if flag == '--create' or flag == '-c':
+ action = 'create'
+ elif flag == '--list' or flag == '-t':
+ action = 'list'
+ elif flag == '--extract' or flag == '-x':
+ action = 'extract'
+ elif flag == '--start-extract':
+ action = 'start-extract'
+ elif flag == '--finish-extract':
+ action = 'finish-extract'
+ elif flag == '--file' or flag == '-f':
+ target_filename = value
+ elif flag == '--recurse' or flag == '-R':
+ should_recurse = True
+ elif flag == '--no-recurse':
+ should_recurse = False
+ elif flag in frozenset(['--xdev', '--one-file-system']):
+ xdev = True
+ elif flag in frozenset(['--no-xdev', '--no-one-file-system']):
+ xdev = False
+ elif flag == '--numeric-ids':
+ restore_numeric_ids = True
+ elif flag == '--no-numeric-ids':
+ restore_numeric_ids = False
+ elif flag == '--paths':
+ include_paths = True
+ elif flag == '--no-paths':
+ include_paths = False
+ elif flag == '--symlinks':
+ handle_symlinks = True
+ elif flag == '--no-symlinks':
+ handle_symlinks = False
+ elif flag == '--verbose' or flag == '-v':
+ metadata.verbose += 1
+ elif flag == '--quiet' or flag == '-q':
+ metadata.verbose = 0
+
+if not action:
+ o.fatal("no action specified")
+
+if action == 'create':
+ if len(remainder) < 1:
+ o.fatal("no paths specified for create")
+ if target_filename != '-':
+ output_file = open(target_filename, 'w')
+ else:
+ output_file = sys.stdout
+ metadata.save_tree(output_file,
+ remainder,
+ recurse=should_recurse,
+ write_paths=include_paths,
+ save_symlinks=handle_symlinks,
+ xdev=xdev)
+
+elif action == 'list':
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --list")
+ if target_filename != '-':
+ src = open(target_filename, 'r')
+ else:
+ src = sys.stdin
+ metadata.display_archive(src)
+
+elif action == 'start-extract':
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --start-extract")
+ if target_filename != '-':
+ src = open(target_filename, 'r')
+ else:
+ src = sys.stdin
+ metadata.start_extract(src, create_symlinks=handle_symlinks)
+
+elif action == 'finish-extract':
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --finish-extract")
+ if target_filename != '-':
+ src = open(target_filename, 'r')
+ else:
+ src = sys.stdin
+ num_ids = restore_numeric_ids
+ metadata.finish_extract(src, restore_numeric_ids=num_ids)
+
+elif action == 'extract':
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --extract")
+ if target_filename != '-':
+ src = open(target_filename, 'r')
+ else:
+ src = sys.stdin
+ metadata.extract(src,
+ restore_numeric_ids=restore_numeric_ids,
+ create_symlinks=handle_symlinks)
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
+else:
+ sys.exit(0)
--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (C) 2010 Rob Browning
+#
+# This code is covered under the terms of the GNU Library General
+# Public License as described in the bup LICENSE file.
+
+import errno
+import posix1e
+import stat
+import sys
+from bup import metadata
+from bup import options
+from bup import xstat
+from bup.helpers import handle_ctrl_c, saved_errors, add_error, log
+
+
+def fstimestr(fstime):
+ (s, ns) = fstime.secs_nsecs()
+ if ns == 0:
+ return '%d' % s
+ else:
+ return '%d.%09d' % (s, ns)
+
+
+optspec = """
+bup pathinfo [OPTION ...] <PATH ...>
+--
+v,verbose increase log output (can be used more than once)
+q,quiet don't show progress meter
+exclude-fields= exclude comma-separated fields
+include-fields= include comma-separated fields (definitive if first)
+"""
+
+target_filename = ''
+all_fields = frozenset(['path',
+ 'mode',
+ 'link-target',
+ 'rdev',
+ 'uid',
+ 'gid',
+ 'owner',
+ 'group',
+ 'atime',
+ 'mtime',
+ 'ctime',
+ 'linux-attr',
+ 'linux-xattr',
+ 'posix1e-acl'])
+active_fields = all_fields
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, remainder) = o.parse(sys.argv[1:])
+
+treat_include_fields_as_definitive = True
+for flag, value in flags:
+ if flag == '--verbose' or flag == '-v':
+ metadata.verbose += 1
+ elif flag == '--quiet' or flag == '-q':
+ metadata.verbose = 0
+ elif flag == '--exclude-fields':
+ exclude_fields = frozenset(value.split(','))
+ for f in exclude_fields:
+ if not f in all_fields:
+ o.fatal(f + ' is not a valid field name')
+ active_fields = active_fields - exclude_fields
+ treat_include_fields_as_definitive = False
+ elif flag == '--include-fields':
+ include_fields = frozenset(value.split(','))
+ for f in include_fields:
+ if not f in all_fields:
+ o.fatal(f + ' is not a valid field name')
+ if treat_include_fields_as_definitive:
+ active_fields = include_fields
+ treat_include_fields_as_definitive = False
+ else:
+ active_fields = active_fields | include_fields
+
+for path in remainder:
+ try:
+ m = metadata.from_path(path, archive_path = path)
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ add_error(e)
+ continue
+ else:
+ raise
+ if 'path' in active_fields:
+ print 'path:', m.path
+ if 'mode' in active_fields:
+ print 'mode:', oct(m.mode)
+ if 'link-target' in active_fields and stat.S_ISLNK(m.mode):
+ print 'link-target:', m.symlink_target
+ if 'rdev' in active_fields:
+ print 'rdev:', m.rdev
+ if 'uid' in active_fields:
+ print 'uid:', m.uid
+ if 'gid' in active_fields:
+ print 'gid:', m.gid
+ if 'owner' in active_fields:
+ print 'owner:', m.owner
+ if 'group' in active_fields:
+ print 'group:', m.group
+ if 'atime' in active_fields:
+ print 'atime: ' + fstimestr(m.atime)
+ if 'mtime' in active_fields:
+ print 'mtime: ' + fstimestr(m.mtime)
+ if 'ctime' in active_fields:
+ print 'ctime: ' + fstimestr(m.ctime)
+ if 'linux-attr' in active_fields and m.linux_attr:
+ print 'linux-attr:', hex(m.linux_attr)
+ if 'linux-xattr' in active_fields and m.linux_xattr:
+ for name, value in m.linux_xattr:
+ print 'linux-xattr: %s -> %s' % (name, repr(value))
+ if 'posix1e-acl' in active_fields and m.posix1e_acl:
+ flags = posix1e.TEXT_ABBREVIATE
+ if stat.S_ISDIR(m.mode):
+ acl = m.posix1e_acl[0]
+ default_acl = m.posix1e_acl[2]
+ print acl.to_any_text('posix1e-acl: ', '\n', flags)
+ print acl.to_any_text('posix1e-acl-default: ', '\n', flags)
+ else:
+ acl = m.posix1e_acl[0]
+ print acl.to_any_text('posix1e-acl: ', '\n', flags)
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
+else:
+ sys.exit(0)
+#define _LARGEFILE64_SOURCE 1
#undef NDEBUG
#include "bupsplit.h"
#include <Python.h>
#include <assert.h>
-#include <stdint.h>
+#include <errno.h>
#include <fcntl.h>
#include <arpa/inet.h>
+#include <stdint.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
+#ifdef linux
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#endif
+
static int istty2 = 0;
// Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV...
}
-static PyMethodDef faster_methods[] = {
+#ifdef linux
+static PyObject *bup_get_linux_file_attr(PyObject *self, PyObject *args)
+{
+ int rc;
+ unsigned long attr;
+ char *path;
+ int fd;
+
+ if (!PyArg_ParseTuple(args, "s", &path))
+ return NULL;
+
+ fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
+ if (fd == -1)
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
+
+ attr = 0;
+ rc = ioctl(fd, FS_IOC_GETFLAGS, &attr);
+ if (rc == -1)
+ {
+ close(fd);
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
+ }
+
+ close(fd);
+ return Py_BuildValue("k", attr);
+}
+
+
+static PyObject *bup_set_linux_file_attr(PyObject *self, PyObject *args)
+{
+ int rc;
+ unsigned long attr;
+ char *path;
+ int fd;
+
+ if (!PyArg_ParseTuple(args, "sk", &path, &attr))
+ return NULL;
+
+ fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
+ if(fd == -1)
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
+
+ rc = ioctl(fd, FS_IOC_SETFLAGS, &attr);
+ if (rc == -1)
+ {
+ close(fd);
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
+ }
+
+ close(fd);
+ Py_RETURN_TRUE;
+}
+#endif /* def linux */
+
+
+#if defined(_ATFILE_SOURCE) \
+ || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L
+#define HAVE_BUP_UTIMENSAT 1
+
+static PyObject *bup_utimensat(PyObject *self, PyObject *args)
+{
+ int rc, dirfd, flags;
+ char *path;
+ long access, access_ns, modification, modification_ns;
+ struct timespec ts[2];
+
+ if (!PyArg_ParseTuple(args, "is((ll)(ll))i",
+ &dirfd,
+ &path,
+ &access, &access_ns,
+ &modification, &modification_ns,
+ &flags))
+ return NULL;
+
+ if (isnan(access))
+ {
+ PyErr_SetString(PyExc_ValueError, "access time is NaN");
+ return NULL;
+ }
+ else if (isinf(access))
+ {
+ PyErr_SetString(PyExc_ValueError, "access time is infinite");
+ return NULL;
+ }
+ else if (isnan(modification))
+ {
+ PyErr_SetString(PyExc_ValueError, "modification time is NaN");
+ return NULL;
+ }
+ else if (isinf(modification))
+ {
+ PyErr_SetString(PyExc_ValueError, "modification time is infinite");
+ return NULL;
+ }
+
+ if (isnan(access_ns))
+ {
+ PyErr_SetString(PyExc_ValueError, "access time ns is NaN");
+ return NULL;
+ }
+ else if (isinf(access_ns))
+ {
+ PyErr_SetString(PyExc_ValueError, "access time ns is infinite");
+ return NULL;
+ }
+ else if (isnan(modification_ns))
+ {
+ PyErr_SetString(PyExc_ValueError, "modification time ns is NaN");
+ return NULL;
+ }
+ else if (isinf(modification_ns))
+ {
+ PyErr_SetString(PyExc_ValueError, "modification time ns is infinite");
+ return NULL;
+ }
+
+ ts[0].tv_sec = access;
+ ts[0].tv_nsec = access_ns;
+ ts[1].tv_sec = modification;
+ ts[1].tv_nsec = modification_ns;
+
+ rc = utimensat(dirfd, path, ts, flags);
+ if (rc != 0)
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
+
+ Py_RETURN_TRUE;
+}
+
+#endif /* defined(_ATFILE_SOURCE)
+ || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */
+
+
+#ifdef linux /* and likely others */
+
+#define HAVE_BUP_STAT 1
+static PyObject *bup_stat(PyObject *self, PyObject *args)
+{
+ int rc;
+ char *filename;
+
+ if (!PyArg_ParseTuple(args, "s", &filename))
+ return NULL;
+
+ struct stat st;
+ rc = stat(filename, &st);
+ if (rc != 0)
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
+
+ return Py_BuildValue("kkkkkkkk"
+ "(ll)"
+ "(ll)"
+ "(ll)",
+ (unsigned long) st.st_mode,
+ (unsigned long) st.st_ino,
+ (unsigned long) st.st_dev,
+ (unsigned long) st.st_nlink,
+ (unsigned long) st.st_uid,
+ (unsigned long) st.st_gid,
+ (unsigned long) st.st_rdev,
+ (unsigned long) st.st_size,
+ (long) st.st_atime,
+ (long) st.st_atim.tv_nsec,
+ (long) st.st_mtime,
+ (long) st.st_mtim.tv_nsec,
+ (long) st.st_ctime,
+ (long) st.st_ctim.tv_nsec);
+}
+
+
+#define HAVE_BUP_LSTAT 1
+static PyObject *bup_lstat(PyObject *self, PyObject *args)
+{
+ int rc;
+ char *filename;
+
+ if (!PyArg_ParseTuple(args, "s", &filename))
+ return NULL;
+
+ struct stat st;
+ rc = lstat(filename, &st);
+ if (rc != 0)
+ return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
+
+ return Py_BuildValue("kkkkkkkk"
+ "(ll)"
+ "(ll)"
+ "(ll)",
+ (unsigned long) st.st_mode,
+ (unsigned long) st.st_ino,
+ (unsigned long) st.st_dev,
+ (unsigned long) st.st_nlink,
+ (unsigned long) st.st_uid,
+ (unsigned long) st.st_gid,
+ (unsigned long) st.st_rdev,
+ (unsigned long) st.st_size,
+ (long) st.st_atime,
+ (long) st.st_atim.tv_nsec,
+ (long) st.st_mtime,
+ (long) st.st_mtim.tv_nsec,
+ (long) st.st_ctime,
+ (long) st.st_ctim.tv_nsec);
+}
+
+
+#define HAVE_BUP_FSTAT 1
+static PyObject *bup_fstat(PyObject *self, PyObject *args)
+{
+ int rc, fd;
+
+ if (!PyArg_ParseTuple(args, "i", &fd))
+ return NULL;
+
+ struct stat st;
+ rc = fstat(fd, &st);
+ if (rc != 0)
+ return PyErr_SetFromErrno(PyExc_IOError);
+
+ return Py_BuildValue("kkkkkkkk"
+ "(ll)"
+ "(ll)"
+ "(ll)",
+ (unsigned long) st.st_mode,
+ (unsigned long) st.st_ino,
+ (unsigned long) st.st_dev,
+ (unsigned long) st.st_nlink,
+ (unsigned long) st.st_uid,
+ (unsigned long) st.st_gid,
+ (unsigned long) st.st_rdev,
+ (unsigned long) st.st_size,
+ (long) st.st_atime,
+ (long) st.st_atim.tv_nsec,
+ (long) st.st_mtime,
+ (long) st.st_mtim.tv_nsec,
+ (long) st.st_ctime,
+ (long) st.st_ctim.tv_nsec);
+}
+
+#endif /* def linux */
+
+
+static PyMethodDef helper_methods[] = {
{ "selftest", selftest, METH_VARARGS,
"Check that the rolling checksum rolls correctly (for unit tests)." },
{ "blobbits", blobbits, METH_VARARGS,
"open() the given filename for read with O_NOATIME if possible" },
{ "fadvise_done", fadvise_done, METH_VARARGS,
"Inform the kernel that we're finished with earlier parts of a file" },
+#ifdef linux
+ { "get_linux_file_attr", bup_get_linux_file_attr, METH_VARARGS,
+ "Return the Linux attributes for the given file." },
+ { "set_linux_file_attr", bup_set_linux_file_attr, METH_VARARGS,
+ "Set the Linux attributes for the given file." },
+#endif
+#ifdef HAVE_BUP_UTIMENSAT
+ { "utimensat", bup_utimensat, METH_VARARGS,
+ "Change file timestamps with nanosecond precision." },
+#endif
+#ifdef HAVE_BUP_STAT
+ { "stat", bup_stat, METH_VARARGS,
+ "Extended version of stat." },
+#endif
+#ifdef HAVE_BUP_LSTAT
+ { "lstat", bup_lstat, METH_VARARGS,
+ "Extended version of lstat." },
+#endif
+#ifdef HAVE_BUP_FSTAT
+ { "fstat", bup_fstat, METH_VARARGS,
+ "Extended version of fstat." },
+#endif
{ NULL, NULL, 0, NULL }, // sentinel
};
+
PyMODINIT_FUNC init_helpers(void)
{
- char *e = getenv("BUP_FORCE_TTY");
- Py_InitModule("_helpers", faster_methods);
+ char *e;
+ PyObject *m = Py_InitModule("_helpers", helper_methods);
+ if (m == NULL)
+ return;
+#ifdef HAVE_BUP_UTIMENSAT
+ PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD));
+ PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW",
+ Py_BuildValue("i", AT_SYMLINK_NOFOLLOW));
+#endif
+#ifdef HAVE_BUP_STAT
+ Py_INCREF(Py_True);
+ PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_True);
+#else
+ Py_INCREF(Py_False);
+ PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_False);
+#endif
+ e = getenv("BUP_FORCE_TTY");
istty2 = isatty(2) || (atoi(e ? e : "0") & 2);
unpythonize_argv();
}
import stat, os
from bup.helpers import *
+import bup.xstat as xstat
try:
O_LARGEFILE = os.O_LARGEFILE
os.fchdir(self.fd)
def stat(self):
- return os.fstat(self.fd)
+ return xstat.fstat(self.fd)
_IFMT = stat.S_IFMT(0xffffffff) # avoid function call in inner loop
l = []
for n in os.listdir('.'):
try:
- st = os.lstat(n)
+ st = xstat.lstat(n)
except OSError, e:
add_error(Exception('%s: %s' % (realpath(n), str(e))))
continue
assert(type(paths) != type(''))
for path in paths:
try:
- pst = os.lstat(path)
+ pst = xstat.lstat(path)
if stat.S_ISLNK(pst.st_mode):
yield (path, pst)
continue
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
import heapq, operator, time
from bup import _version, _helpers
+import bup._helpers as _helpers
# This function should really be in helpers, not in bup.options. But we
# want options.py to be standalone so people can include it in other projects.
return out
+def detect_fakeroot():
+ "Return True if we appear to be running under fakeroot."
+ return os.getenv("FAKEROOTKEY") != None
+
+
_username = None
def username():
"""Get the user's login name."""
log('%-70s\n' % e)
+def clear_errors():
+ global saved_errors
+ saved_errors = []
+
+
def handle_ctrl_c():
"""Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
def from_stat(self, st, tstart):
old = (self.dev, self.ctime, self.mtime,
self.uid, self.gid, self.size, self.flags & IX_EXISTS)
- new = (st.st_dev, int(st.st_ctime), int(st.st_mtime),
+ new = (st.st_dev,
+ int(st.st_ctime.approx_secs()),
+ int(st.st_mtime.approx_secs()),
st.st_uid, st.st_gid, st.st_size, IX_EXISTS)
self.dev = st.st_dev
- self.ctime = int(st.st_ctime)
- self.mtime = int(st.st_mtime)
+ self.ctime = int(st.st_ctime.approx_secs())
+ self.mtime = int(st.st_mtime.approx_secs())
self.uid = st.st_uid
self.gid = st.st_gid
self.size = st.st_size
self.mode = st.st_mode
self.flags |= IX_EXISTS
- if int(st.st_ctime) >= tstart or old != new \
+ if int(st.st_ctime.approx_secs()) >= tstart or old != new \
or self.sha == EMPTY_SHA or not self.gitmode:
self.invalidate()
self._fixup()
if st:
isdir = stat.S_ISDIR(st.st_mode)
assert(isdir == endswith)
- e = NewEntry(basename, name, st.st_dev, int(st.st_ctime),
- int(st.st_mtime), st.st_uid, st.st_gid,
+ e = NewEntry(basename, name, st.st_dev,
+ int(st.st_ctime.approx_secs()),
+ int(st.st_mtime.approx_secs()),
+ st.st_uid, st.st_gid,
st.st_size, st.st_mode, gitmode, sha, flags,
0, 0)
else:
--- /dev/null
+"""Metadata read/write support for bup."""
+
+# Copyright (C) 2010 Rob Browning
+#
+# This code is covered under the terms of the GNU Library General
+# Public License as described in the bup LICENSE file.
+
+import errno, os, sys, stat, pwd, grp, struct, xattr, posix1e, re
+
+from cStringIO import StringIO
+from bup import vint
+from bup.drecurse import recursive_dirlist
+from bup.helpers import add_error, mkdirp, log
+from bup.xstat import utime, lutime, lstat, FSTime
+import bup._helpers as _helpers
+
+if _helpers.get_linux_file_attr:
+ from bup._helpers import get_linux_file_attr, set_linux_file_attr
+
+# WARNING: the metadata encoding is *not* stable yet. Caveat emptor!
+
+# Q: Consider hardlink support?
+# Q: Is it OK to store raw linux attr (chattr) flags?
+# Q: Can anything other than S_ISREG(x) or S_ISDIR(x) support posix1e ACLs?
+# Q: Is the application of posix1e has_extended() correct?
+# Q: Is one global --numeric-ids argument sufficient?
+# Q: Do nfsv4 acls trump posix1e acls? (seems likely)
+# Q: Add support for crtime -- ntfs, and (only internally?) ext*?
+
+# FIXME: Fix relative/abs path detection/stripping wrt other platforms.
+# FIXME: Add nfsv4 acl handling - see nfs4-acl-tools.
+# FIXME: Consider other entries mentioned in stat(2) (S_IFDOOR, etc.).
+# FIXME: Consider pack('vvvvsss', ...) optimization.
+# FIXME: Consider caching users/groups.
+
+## FS notes:
+#
+# osx (varies between hfs and hfs+):
+# type - regular dir char block fifo socket ...
+# perms - rwxrwxrwxsgt
+# times - ctime atime mtime
+# uid
+# gid
+# hard-link-info (hfs+ only)
+# link-target
+# device-major/minor
+# attributes-osx see chflags
+# content-type
+# content-creator
+# forks
+#
+# ntfs
+# type - regular dir ...
+# times - creation, modification, posix change, access
+# hard-link-info
+# link-target
+# attributes - see attrib
+# ACLs
+# forks (alternate data streams)
+# crtime?
+#
+# fat
+# type - regular dir ...
+# perms - rwxrwxrwx (maybe - see wikipedia)
+# times - creation, modification, access
+# attributes - see attrib
+
+verbose = 0
+
+_have_lchmod = hasattr(os, 'lchmod')
+
+
+def _clean_up_path_for_archive(p):
+ # Not the most efficient approach.
+ result = p
+
+ # Take everything after any '/../'.
+ pos = result.rfind('/../')
+ if(pos != -1):
+ result = result[result.rfind('/../') + 4:]
+
+ # Take everything after any remaining '../'.
+ if result.startswith("../"):
+ result = result[3:]
+
+ # Remove any '/./' sequences.
+ pos = result.find('/./')
+ while pos != -1:
+ result = result[0:pos] + '/' + result[pos + 3:]
+ pos = result.find('/./')
+
+ # Remove any leading '/'s.
+ result = result.lstrip('/')
+
+ # Replace '//' with '/' everywhere.
+ pos = result.find('//')
+ while pos != -1:
+ result = result[0:pos] + '/' + result[pos + 2:]
+ pos = result.find('//')
+
+ # Take everything after any remaining './'.
+ if result.startswith('./'):
+ result = result[2:]
+
+ # Take everything before any remaining '/.'.
+ if result.endswith('/.'):
+ result = result[:-2]
+
+ if result == '' or result.endswith('/..'):
+ result = '.'
+
+ return result
+
+
+def _risky_path(p):
+ if p.startswith('/'):
+ return True
+ if p.find('/../') != -1:
+ return True
+ if p.startswith('../'):
+ return True
+ if p.endswith('/..'):
+ return True
+ return False
+
+
+def _clean_up_extract_path(p):
+ result = p.lstrip('/')
+ if result == '':
+ return '.'
+ elif _risky_path(result):
+ return None
+ else:
+ return result
+
+
+# These tags are currently conceptually private to Metadata, and they
+# must be unique, and must *never* be changed.
+_rec_tag_end = 0
+_rec_tag_path = 1
+_rec_tag_common = 2 # times, owner, group, type, perms, etc.
+_rec_tag_symlink_target = 3
+_rec_tag_posix1e_acl = 4 # getfacl(1), setfacl(1), etc.
+_rec_tag_nfsv4_acl = 5 # intended to supplant posix1e acls?
+_rec_tag_linux_attr = 6 # lsattr(1) chattr(1)
+_rec_tag_linux_xattr = 7 # getfattr(1) setfattr(1)
+
+
+class ApplyError(Exception):
+ # Thrown when unable to apply any given bit of metadata to a path.
+ pass
+
+
+class Metadata:
+ # Metadata is stored as a sequence of tagged binary records. Each
+ # record will have some subset of add, encode, load, create, and
+ # apply methods, i.e. _add_foo...
+
+ ## Common records
+
+ # Timestamps are (sec, ns), relative to 1970-01-01 00:00:00, ns
+ # must be non-negative and < 10**9.
+
+ def _add_common(self, path, st):
+ self.mode = st.st_mode
+ self.uid = st.st_uid
+ self.gid = st.st_gid
+ self.rdev = st.st_rdev
+ self.atime = st.st_atime
+ self.mtime = st.st_mtime
+ self.ctime = st.st_ctime
+ self.owner = self.group = ''
+ try:
+ self.owner = pwd.getpwuid(st.st_uid)[0]
+ except KeyError, e:
+ add_error("no user name for id %s '%s'" % (st.st_gid, path))
+ try:
+ self.group = grp.getgrgid(st.st_gid)[0]
+ except KeyError, e:
+ add_error("no group name for id %s '%s'" % (st.st_gid, path))
+
+ def _encode_common(self):
+ atime = self.atime.to_timespec()
+ mtime = self.mtime.to_timespec()
+ ctime = self.ctime.to_timespec()
+ result = vint.pack('VVsVsVvVvVvV',
+ self.mode,
+ self.uid,
+ self.owner,
+ self.gid,
+ self.group,
+ self.rdev,
+ atime[0],
+ atime[1],
+ mtime[0],
+ mtime[1],
+ ctime[0],
+ ctime[1])
+ return result
+
+ def _load_common_rec(self, port):
+ data = vint.read_bvec(port)
+ (self.mode,
+ self.uid,
+ self.owner,
+ self.gid,
+ self.group,
+ self.rdev,
+ self.atime,
+ atime_ns,
+ self.mtime,
+ mtime_ns,
+ self.ctime,
+ ctime_ns) = vint.unpack('VVsVsVvVvVvV', data)
+ self.atime = FSTime.from_timespec((self.atime, atime_ns))
+ self.mtime = FSTime.from_timespec((self.mtime, mtime_ns))
+ self.ctime = FSTime.from_timespec((self.ctime, ctime_ns))
+
+ def _create_via_common_rec(self, path, create_symlinks=True):
+ # If the path already exists and is a dir, try rmdir.
+ # If the path already exists and is anything else, try unlink.
+ st = None
+ try:
+ st = lstat(path)
+ except IOError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ if st:
+ if stat.S_ISDIR(st.st_mode):
+ try:
+ os.rmdir(path)
+ except OSError, e:
+ if e.errno == errno.ENOTEMPTY:
+ msg = 'refusing to overwrite non-empty dir' + path
+ raise Exception(msg)
+ raise
+ else:
+ os.unlink(path)
+
+ if stat.S_ISREG(self.mode):
+ os.mknod(path, 0600 | stat.S_IFREG)
+ elif stat.S_ISDIR(self.mode):
+ os.mkdir(path, 0700)
+ elif stat.S_ISCHR(self.mode):
+ os.mknod(path, 0600 | stat.S_IFCHR, self.rdev)
+ elif stat.S_ISBLK(self.mode):
+ os.mknod(path, 0600 | stat.S_IFBLK, self.rdev)
+ elif stat.S_ISFIFO(self.mode):
+ os.mknod(path, 0600 | stat.S_IFIFO)
+ elif stat.S_ISLNK(self.mode):
+ if(self.symlink_target and create_symlinks):
+ os.symlink(self.symlink_target, path)
+ # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
+ # Otherwise, do nothing.
+
+ def _apply_common_rec(self, path, restore_numeric_ids=False):
+ # FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
+ # EACCES errors at this stage are fatal for the current path.
+ if stat.S_ISLNK(self.mode):
+ try:
+ lutime(path, (self.atime, self.mtime))
+ except IOError, e:
+ if e.errno == errno.EACCES:
+ raise ApplyError('lutime: %s' % e)
+ else:
+ raise
+ else:
+ try:
+ utime(path, (self.atime, self.mtime))
+ except IOError, e:
+ if e.errno == errno.EACCES:
+ raise ApplyError('utime: %s' % e)
+ else:
+ raise
+
+ # Don't try to restore owner unless we're root, and even
+ # if asked, don't try to restore the owner or group if
+ # it doesn't exist in the system db.
+ uid = self.uid
+ gid = self.gid
+ if not restore_numeric_ids:
+ if not self.owner:
+ uid = -1
+ add_error('ignoring missing owner for "%s"\n' % path)
+ else:
+ if os.geteuid() != 0:
+ uid = -1 # Not root; assume we can't change owner.
+ else:
+ try:
+ uid = pwd.getpwnam(self.owner)[2]
+ except KeyError:
+ uid = -1
+ fmt = 'ignoring unknown owner %s for "%s"\n'
+ add_error(fmt % (self.owner, path))
+ if not self.group:
+ gid = -1
+ add_error('ignoring missing group for "%s"\n' % path)
+ else:
+ try:
+ gid = grp.getgrnam(self.group)[2]
+ except KeyError:
+ gid = -1
+ add_error('ignoring unknown group %s for "%s"\n'
+ % (self.group, path))
+
+ try:
+ os.lchown(path, uid, gid)
+ except OSError, e:
+ if e.errno == errno.EPERM:
+ add_error('lchown: %s' % e)
+ else:
+ raise
+
+ if _have_lchmod:
+ os.lchmod(path, stat.S_IMODE(self.mode))
+ elif not stat.S_ISLNK(self.mode):
+ os.chmod(path, stat.S_IMODE(self.mode))
+
+
+ ## Path records
+
+ def _encode_path(self):
+ if self.path:
+ return vint.pack('s', self.path)
+ else:
+ return None
+
+ def _load_path_rec(self, port):
+ self.path = vint.unpack('s', vint.read_bvec(port))[0]
+
+
+ ## Symlink targets
+
+ def _add_symlink_target(self, path, st):
+ try:
+ if(stat.S_ISLNK(st.st_mode)):
+ self.symlink_target = os.readlink(path)
+ except OSError, e:
+ add_error('readlink: %s', e)
+
+ def _encode_symlink_target(self):
+ return self.symlink_target
+
+ def _load_symlink_target_rec(self, port):
+ self.symlink_target = vint.read_bvec(port)
+
+
+ ## POSIX1e ACL records
+
+ # Recorded as a list:
+ # [txt_id_acl, num_id_acl]
+ # or, if a directory:
+ # [txt_id_acl, num_id_acl, txt_id_default_acl, num_id_default_acl]
+ # The numeric/text distinction only matters when reading/restoring
+ # a stored record.
+ def _add_posix1e_acl(self, path, st):
+ if not stat.S_ISLNK(st.st_mode):
+ try:
+ if posix1e.has_extended(path):
+ acl = posix1e.ACL(file=path)
+ self.posix1e_acl = [acl, acl] # txt and num are the same
+ if stat.S_ISDIR(st.st_mode):
+ acl = posix1e.ACL(filedef=path)
+ self.posix1e_acl.extend([acl, acl])
+ except EnvironmentError, e:
+ if e.errno != errno.EOPNOTSUPP:
+ raise
+
+ def _encode_posix1e_acl(self):
+ # Encode as two strings (w/default ACL string possibly empty).
+ if self.posix1e_acl:
+ acls = self.posix1e_acl
+ txt_flags = posix1e.TEXT_ABBREVIATE
+ num_flags = posix1e.TEXT_ABBREVIATE | posix1e.TEXT_NUMERIC_IDS
+ acl_reps = [acls[0].to_any_text('', '\n', txt_flags),
+ acls[1].to_any_text('', '\n', num_flags)]
+ if(len(acls) < 3):
+ acl_reps += ['', '']
+ else:
+ acl_reps.append(acls[2].to_any_text('', '\n', txt_flags))
+ acl_reps.append(acls[3].to_any_text('', '\n', num_flags))
+ return vint.pack('ssss',
+ acl_reps[0], acl_reps[1], acl_reps[2], acl_reps[3])
+ else:
+ return None
+
+ def _load_posix1e_acl_rec(self, port):
+ data = vint.read_bvec(port)
+ acl_reps = vint.unpack('ssss', data)
+ if(acl_reps[2] == ''):
+ acl_reps = acl_reps[:2]
+ self.posix1e_acl = [posix1e.ACL(text=x) for x in acl_reps]
+
+ def _apply_posix1e_acl_rec(self, path, restore_numeric_ids=False):
+ if(self.posix1e_acl):
+ acls = self.posix1e_acl
+ if(len(acls) > 2):
+ if restore_numeric_ids:
+ acls[3].applyto(path, posix1e.ACL_TYPE_DEFAULT)
+ else:
+ acls[2].applyto(path, posix1e.ACL_TYPE_DEFAULT)
+ if restore_numeric_ids:
+ acls[1].applyto(path, posix1e.ACL_TYPE_ACCESS)
+ else:
+ acls[0].applyto(path, posix1e.ACL_TYPE_ACCESS)
+
+
+ ## Linux attributes (lsattr(1), chattr(1))
+
+ def _add_linux_attr(self, path, st):
+ if stat.S_ISREG(st.st_mode) or stat.S_ISDIR(st.st_mode):
+ try:
+ attr = get_linux_file_attr(path)
+ if(attr != 0):
+ self.linux_attr = attr
+ except IOError, e:
+ if e.errno == errno.EACCES:
+ add_error('read Linux attr: %s' % e)
+ elif e.errno == errno.ENOTTY: # Inappropriate ioctl for device.
+ add_error('read Linux attr: %s' % e)
+ else:
+ raise
+
+ def _encode_linux_attr(self):
+ if self.linux_attr:
+ return vint.pack('V', self.linux_attr)
+ else:
+ return None
+
+ def _load_linux_attr_rec(self, port):
+ data = vint.read_bvec(port)
+ self.linux_attr = vint.unpack('V', data)[0]
+
+ def _apply_linux_attr_rec(self, path, restore_numeric_ids=False):
+ if(self.linux_attr):
+ set_linux_file_attr(path, self.linux_attr)
+
+
+ ## Linux extended attributes (getfattr(1), setfattr(1))
+
+ def _add_linux_xattr(self, path, st):
+ try:
+ self.linux_xattr = xattr.get_all(path, nofollow=True)
+ except EnvironmentError, e:
+ if e.errno != errno.EOPNOTSUPP:
+ raise
+
+ def _encode_linux_xattr(self):
+ if self.linux_xattr:
+ result = vint.pack('V', len(self.linux_xattr))
+ for name, value in self.linux_xattr:
+ result += vint.pack('ss', name, value)
+ return result
+ else:
+ return None
+
+ def _load_linux_xattr_rec(self, file):
+ data = vint.read_bvec(file)
+ memfile = StringIO(data)
+ result = []
+ for i in range(vint.read_vuint(memfile)):
+ key = vint.read_bvec(memfile)
+ value = vint.read_bvec(memfile)
+ result.append((key, value))
+ self.linux_xattr = result
+
+ def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
+ existing_xattrs = set(xattr.list(path, nofollow=True))
+ if(self.linux_xattr):
+ for k, v in self.linux_xattr:
+ if k not in existing_xattrs \
+ or v != xattr.get(path, k, nofollow=True):
+ try:
+ xattr.set(path, k, v, nofollow=True)
+ except IOError, e:
+ if e.errno == errno.EPERM:
+ raise ApplyError('xattr.set: %s' % e)
+ else:
+ raise
+ existing_xattrs -= frozenset([k])
+ for k in existing_xattrs:
+ try:
+ xattr.remove(path, k, nofollow=True)
+ except IOError, e:
+ if e.errno == errno.EPERM:
+ raise ApplyError('xattr.remove: %s' % e)
+ else:
+ raise
+
+ def __init__(self):
+ # optional members
+ self.path = None
+ self.symlink_target = None
+ self.linux_attr = None
+ self.linux_xattr = None
+ self.posix1e_acl = None
+ self.posix1e_acl_default = None
+
+ def write(self, port, include_path=True):
+ records = [(_rec_tag_path, self._encode_path())] if include_path else []
+ records.extend([(_rec_tag_common, self._encode_common()),
+ (_rec_tag_symlink_target, self._encode_symlink_target()),
+ (_rec_tag_posix1e_acl, self._encode_posix1e_acl()),
+ (_rec_tag_linux_attr, self._encode_linux_attr()),
+ (_rec_tag_linux_xattr, self._encode_linux_xattr())])
+ for tag, data in records:
+ if data:
+ vint.write_vuint(port, tag)
+ vint.write_bvec(port, data)
+ vint.write_vuint(port, _rec_tag_end)
+
+ @staticmethod
+ def read(port):
+ # This method should either: return a valid Metadata object;
+ # throw EOFError if there was nothing at all to read; throw an
+ # Exception if a valid object could not be read completely.
+ tag = vint.read_vuint(port)
+ try: # From here on, EOF is an error.
+ result = Metadata()
+ while(True): # only exit is error (exception) or _rec_tag_end
+ if tag == _rec_tag_path:
+ result._load_path_rec(port)
+ elif tag == _rec_tag_common:
+ result._load_common_rec(port)
+ elif tag == _rec_tag_symlink_target:
+ result._load_symlink_target_rec(port)
+ elif tag == _rec_tag_posix1e_acl:
+ result._load_posix1e_acl_rec(port)
+ elif tag ==_rec_tag_nfsv4_acl:
+ result._load_nfsv4_acl_rec(port)
+ elif tag == _rec_tag_linux_attr:
+ result._load_linux_attr_rec(port)
+ elif tag == _rec_tag_linux_xattr:
+ result._load_linux_xattr_rec(port)
+ elif tag == _rec_tag_end:
+ return result
+ else: # unknown record
+ vint.skip_bvec(port)
+ tag = vint.read_vuint(port)
+ except EOFError:
+ raise Exception("EOF while reading Metadata")
+
+ def isdir(self):
+ return stat.S_ISDIR(self.mode)
+
+ def create_path(self, path, create_symlinks=True):
+ self._create_via_common_rec(path, create_symlinks=create_symlinks)
+
+ def apply_to_path(self, path=None, restore_numeric_ids=False):
+ # apply metadata to path -- file must exist
+ if not path:
+ path = self.path
+ if not path:
+ raise Exception('Metadata.apply_to_path() called with no path');
+ num_ids = restore_numeric_ids
+ try:
+ self._apply_common_rec(path, restore_numeric_ids=num_ids)
+ self._apply_posix1e_acl_rec(path, restore_numeric_ids=num_ids)
+ self._apply_linux_attr_rec(path, restore_numeric_ids=num_ids)
+ self._apply_linux_xattr_rec(path, restore_numeric_ids=num_ids)
+ except ApplyError, e:
+ add_error(e)
+
+
+def from_path(path, statinfo=None, archive_path=None, save_symlinks=True):
+ result = Metadata()
+ result.path = archive_path
+ st = statinfo if statinfo else lstat(path)
+ result._add_common(path, st)
+ if(save_symlinks):
+ result._add_symlink_target(path, st)
+ result._add_posix1e_acl(path, st)
+ result._add_linux_attr(path, st)
+ result._add_linux_xattr(path, st)
+ return result
+
+
+def save_tree(output_file, paths,
+ recurse=False,
+ write_paths=True,
+ save_symlinks=True,
+ xdev=False):
+
+ # Issue top-level rewrite warnings.
+ for path in paths:
+ safe_path = _clean_up_path_for_archive(path)
+ if(safe_path != path):
+ log('archiving "%s" as "%s"\n' % (path, safe_path))
+
+ start_dir = os.getcwd()
+ try:
+ for (p, st) in recursive_dirlist(paths, xdev=xdev):
+ dirlist_dir = os.getcwd()
+ os.chdir(start_dir)
+ safe_path = _clean_up_path_for_archive(p)
+ m = from_path(p, statinfo=st, archive_path=safe_path,
+ save_symlinks=save_symlinks)
+ if verbose:
+ print >> sys.stderr, m.path
+ m.write(output_file, include_path=write_paths)
+ os.chdir(dirlist_dir)
+ finally:
+ os.chdir(start_dir)
+
+
+def _set_up_path(meta, create_symlinks=True):
+ # Allow directories to exist as a special case -- might have
+ # been created by an earlier longer path.
+ if meta.isdir():
+ mkdirp(meta.path)
+ else:
+ parent = os.path.dirname(meta.path)
+ if parent:
+ mkdirp(parent)
+ meta.create_path(meta.path, create_symlinks=create_symlinks)
+
+
+class _ArchiveIterator:
+ def next(self):
+ try:
+ return Metadata.read(self._file)
+ except EOFError:
+ raise StopIteration()
+
+ def __iter__(self):
+ return self
+
+ def __init__(self, file):
+ self._file = file
+
+
+def display_archive(file):
+ for meta in _ArchiveIterator(file):
+ if verbose:
+ print meta.path # FIXME
+ else:
+ print meta.path
+
+
+def start_extract(file, create_symlinks=True):
+ for meta in _ArchiveIterator(file):
+ if verbose:
+ print >> sys.stderr, meta.path
+ xpath = _clean_up_extract_path(meta.path)
+ if not xpath:
+ add_error(Exception('skipping risky path "%s"' % meta.path))
+ else:
+ meta.path = xpath
+ _set_up_path(meta, create_symlinks=create_symlinks)
+
+
+def finish_extract(file, restore_numeric_ids=False):
+ all_dirs = []
+ for meta in _ArchiveIterator(file):
+ xpath = _clean_up_extract_path(meta.path)
+ if not xpath:
+ add_error(Exception('skipping risky path "%s"' % dir.path))
+ else:
+ if os.path.isdir(meta.path):
+ all_dirs.append(meta)
+ else:
+ if verbose:
+ print >> sys.stderr, meta.path
+ meta.apply_to_path(path=xpath,
+ restore_numeric_ids=restore_numeric_ids)
+ all_dirs.sort(key = lambda x : len(x.path), reverse=True)
+ for dir in all_dirs:
+ # Don't need to check xpath -- won't be in all_dirs if not OK.
+ xpath = _clean_up_extract_path(dir.path)
+ if verbose:
+ print >> sys.stderr, dir.path
+ dir.apply_to_path(path=xpath, restore_numeric_ids=restore_numeric_ids)
+
+
+def extract(file, restore_numeric_ids=False, create_symlinks=True):
+ # For now, just store all the directories and handle them last,
+ # longest first.
+ all_dirs = []
+ for meta in _ArchiveIterator(file):
+ xpath = _clean_up_extract_path(meta.path)
+ if not xpath:
+ add_error(Exception('skipping risky path "%s"' % meta.path))
+ else:
+ meta.path = xpath
+ if verbose:
+ print >> sys.stderr, '+', meta.path
+ _set_up_path(meta, create_symlinks=create_symlinks)
+ if os.path.isdir(meta.path):
+ all_dirs.append(meta)
+ else:
+ if verbose:
+ print >> sys.stderr, '=', meta.path
+ meta.apply_to_path(restore_numeric_ids=restore_numeric_ids)
+ all_dirs.sort(key = lambda x : len(x.path), reverse=True)
+ for dir in all_dirs:
+ # Don't need to check xpath -- won't be in all_dirs if not OK.
+ xpath = _clean_up_extract_path(dir.path)
+ if verbose:
+ print >> sys.stderr, '=', xpath
+ # Shouldn't have to check for risky paths here (omitted above).
+ dir.apply_to_path(path=dir.path,
+ restore_numeric_ids=restore_numeric_ids)
+import math
import os
+import bup._helpers as _helpers
from bup.helpers import *
from wvtest import *
WVPASSEQ(pn('1e+9 k'), 1000000000 * 1024)
WVPASSEQ(pn('-3e-3mb'), int(-0.003 * 1024 * 1024))
+@wvtest
+def test_detect_fakeroot():
+ if os.getenv('FAKEROOTKEY'):
+ WVPASS(detect_fakeroot())
+ else:
+ WVPASS(not detect_fakeroot())
+
@wvtest
def test_strip_path():
prefix = "/var/backup/daily.0/localhost"
import os
from bup import index
from bup.helpers import *
+import bup.xstat as xstat
from wvtest import *
@wvtest
@wvtest
def index_writer():
unlink('index.tmp')
- ds = os.stat('.')
- fs = os.stat('tindex.py')
+ ds = xstat.stat('.')
+ fs = xstat.stat('tindex.py')
w = index.Writer('index.tmp')
w.add('/var/tmp/sporky', fs)
w.add('/etc/passwd', fs)
def index_dirty():
unlink('index.tmp')
unlink('index2.tmp')
- ds = os.stat('.')
- fs = os.stat('tindex.py')
+ ds = xstat.stat('.')
+ fs = xstat.stat('tindex.py')
w1 = index.Writer('index.tmp')
w1.add('/a/b/x', fs)
--- /dev/null
+import grp
+import pwd
+import stat
+import subprocess
+import tempfile
+import xattr
+import bup.helpers as helpers
+from bup import metadata
+from bup.helpers import clear_errors, detect_fakeroot
+from wvtest import *
+
+
+top_dir = os.getcwd()
+
+
+def ex(*cmd):
+ try:
+ cmd_str = ' '.join(cmd)
+ print >> sys.stderr, cmd_str
+ rc = subprocess.call(cmd)
+ if rc < 0:
+ print >> sys.stderr, 'terminated by signal', - rc
+ sys.exit(1)
+ elif rc > 0:
+ print >> sys.stderr, 'returned exit status', rc
+ sys.exit(1)
+ except OSError, e:
+ print >> sys.stderr, 'subprocess call failed:', e
+ sys.exit(1)
+
+
+def setup_testfs():
+ # Set up testfs with user_xattr, etc.
+ subprocess.call(['umount', 'testfs'])
+ ex('dd', 'if=/dev/zero', 'of=testfs.img', 'bs=1M', 'count=32')
+ ex('mke2fs', '-F', '-j', '-m', '0', 'testfs.img')
+ ex('rm', '-rf', 'testfs')
+ os.mkdir('testfs')
+ ex('mount', '-o', 'loop,acl,user_xattr', 'testfs.img', 'testfs')
+ # Hide, so that tests can't create risks.
+ ex('chown', 'root:root', 'testfs')
+ os.chmod('testfs', 0700)
+
+
+def cleanup_testfs():
+ subprocess.call(['umount', 'testfs'])
+ subprocess.call(['rm', '-f', 'testfs.img'])
+
+
+@wvtest
+def test_clean_up_archive_path():
+ cleanup = metadata._clean_up_path_for_archive
+ WVPASSEQ(cleanup('foo'), 'foo')
+ WVPASSEQ(cleanup('/foo'), 'foo')
+ WVPASSEQ(cleanup('///foo'), 'foo')
+ WVPASSEQ(cleanup('/foo/bar'), 'foo/bar')
+ WVPASSEQ(cleanup('foo/./bar'), 'foo/bar')
+ WVPASSEQ(cleanup('/foo/./bar'), 'foo/bar')
+ WVPASSEQ(cleanup('/foo/./bar/././baz'), 'foo/bar/baz')
+ WVPASSEQ(cleanup('/foo/./bar///././baz'), 'foo/bar/baz')
+ WVPASSEQ(cleanup('//./foo/./bar///././baz/.///'), 'foo/bar/baz/')
+ WVPASSEQ(cleanup('./foo/./.bar'), 'foo/.bar')
+ WVPASSEQ(cleanup('./foo/.'), 'foo')
+ WVPASSEQ(cleanup('./foo/..'), '.')
+ WVPASSEQ(cleanup('//./..//.../..//.'), '.')
+ WVPASSEQ(cleanup('//./..//..././/.'), '...')
+ WVPASSEQ(cleanup('/////.'), '.')
+ WVPASSEQ(cleanup('/../'), '.')
+ WVPASSEQ(cleanup(''), '.')
+
+
+@wvtest
+def test_risky_path():
+ risky = metadata._risky_path
+ WVPASS(risky('/foo'))
+ WVPASS(risky('///foo'))
+ WVPASS(risky('/../foo'))
+ WVPASS(risky('../foo'))
+ WVPASS(risky('foo/..'))
+ WVPASS(risky('foo/../'))
+ WVPASS(risky('foo/../bar'))
+ WVFAIL(risky('foo'))
+ WVFAIL(risky('foo/'))
+ WVFAIL(risky('foo///'))
+ WVFAIL(risky('./foo'))
+ WVFAIL(risky('foo/.'))
+ WVFAIL(risky('./foo/.'))
+ WVFAIL(risky('foo/bar'))
+ WVFAIL(risky('foo/./bar'))
+
+
+@wvtest
+def test_clean_up_extract_path():
+ cleanup = metadata._clean_up_extract_path
+ WVPASSEQ(cleanup('/foo'), 'foo')
+ WVPASSEQ(cleanup('///foo'), 'foo')
+ WVFAIL(cleanup('/../foo'))
+ WVFAIL(cleanup('../foo'))
+ WVFAIL(cleanup('foo/..'))
+ WVFAIL(cleanup('foo/../'))
+ WVFAIL(cleanup('foo/../bar'))
+ WVPASSEQ(cleanup('foo'), 'foo')
+ WVPASSEQ(cleanup('foo/'), 'foo/')
+ WVPASSEQ(cleanup('foo///'), 'foo///')
+ WVPASSEQ(cleanup('./foo'), './foo')
+ WVPASSEQ(cleanup('foo/.'), 'foo/.')
+ WVPASSEQ(cleanup('./foo/.'), './foo/.')
+ WVPASSEQ(cleanup('foo/bar'), 'foo/bar')
+ WVPASSEQ(cleanup('foo/./bar'), 'foo/./bar')
+ WVPASSEQ(cleanup('/'), '.')
+ WVPASSEQ(cleanup('./'), './')
+ WVPASSEQ(cleanup('///foo/bar'), 'foo/bar')
+ WVPASSEQ(cleanup('///foo/bar'), 'foo/bar')
+
+
+@wvtest
+def test_from_path_error():
+ if os.geteuid() == 0 or detect_fakeroot():
+ return
+ tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-')
+ try:
+ path = tmpdir + '/foo'
+ subprocess.call(['mkdir', path])
+ m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ WVPASSEQ(m.path, path)
+ subprocess.call(['chmod', '000', path])
+ metadata.from_path(path, archive_path=path, save_symlinks=True)
+ errmsg = helpers.saved_errors[0] if helpers.saved_errors else ''
+ WVPASS(errmsg.startswith('read Linux attr'))
+ clear_errors()
+ finally:
+ subprocess.call(['rm', '-rf', tmpdir])
+
+
+@wvtest
+def test_apply_to_path_restricted_access():
+ if os.geteuid() == 0 or detect_fakeroot():
+ return
+ tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-')
+ try:
+ path = tmpdir + '/foo'
+ subprocess.call(['mkdir', path])
+ clear_errors()
+ m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ WVPASSEQ(m.path, path)
+ subprocess.call(['chmod', '000', tmpdir])
+ m.apply_to_path(path)
+ errmsg = str(helpers.saved_errors[0]) if helpers.saved_errors else ''
+ WVPASS(errmsg.startswith('utime: '))
+ clear_errors()
+ finally:
+ subprocess.call(['rm', '-rf', tmpdir])
+
+
+@wvtest
+def test_restore_restricted_user_group():
+ if os.geteuid() == 0 or detect_fakeroot():
+ return
+ tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-')
+ try:
+ path = tmpdir + '/foo'
+ subprocess.call(['mkdir', path])
+ m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ WVPASSEQ(m.path, path)
+ WVPASSEQ(m.apply_to_path(path), None)
+ orig_uid = m.uid
+ m.uid = 0;
+ m.apply_to_path(path, restore_numeric_ids=True)
+ errmsg = str(helpers.saved_errors[0]) if helpers.saved_errors else ''
+ WVPASS(errmsg.startswith('lchown: '))
+ clear_errors()
+ m.uid = orig_uid
+ m.gid = 0;
+ m.apply_to_path(path, restore_numeric_ids=True)
+ errmsg = str(helpers.saved_errors[0]) if helpers.saved_errors else ''
+ WVPASS(errmsg.startswith('lchown: '))
+ clear_errors()
+ finally:
+ subprocess.call(['rm', '-rf', tmpdir])
+
+
+@wvtest
+def test_restore_nonexistent_user_group():
+ tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-')
+ try:
+ path = tmpdir + '/foo'
+ subprocess.call(['mkdir', path])
+ m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ WVPASSEQ(m.path, path)
+ m.owner = max([x.pw_name for x in pwd.getpwall()], key=len) + 'x'
+ m.group = max([x.gr_name for x in grp.getgrall()], key=len) + 'x'
+ WVPASSEQ(m.apply_to_path(path, restore_numeric_ids=True), None)
+ WVPASSEQ(os.stat(path).st_uid, m.uid)
+ WVPASSEQ(os.stat(path).st_gid, m.gid)
+ WVPASSEQ(m.apply_to_path(path, restore_numeric_ids=False), None)
+ WVPASSEQ(os.stat(path).st_uid, os.geteuid())
+ WVPASSEQ(os.stat(path).st_gid, os.getgid())
+ finally:
+ subprocess.call(['rm', '-rf', tmpdir])
+
+
+@wvtest
+def test_restore_over_existing_target():
+ tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-')
+ try:
+ path = tmpdir + '/foo'
+ os.mkdir(path)
+ dir_m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ os.rmdir(path)
+ open(path, 'w').close()
+ file_m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ # Restore dir over file.
+ WVPASSEQ(dir_m.create_path(path, create_symlinks=True), None)
+ WVPASS(stat.S_ISDIR(os.stat(path).st_mode))
+ # Restore dir over dir.
+ WVPASSEQ(dir_m.create_path(path, create_symlinks=True), None)
+ WVPASS(stat.S_ISDIR(os.stat(path).st_mode))
+ # Restore file over dir.
+ WVPASSEQ(file_m.create_path(path, create_symlinks=True), None)
+ WVPASS(stat.S_ISREG(os.stat(path).st_mode))
+ # Restore file over file.
+ WVPASSEQ(file_m.create_path(path, create_symlinks=True), None)
+ WVPASS(stat.S_ISREG(os.stat(path).st_mode))
+ # Restore file over non-empty dir.
+ os.remove(path)
+ os.mkdir(path)
+ open(path + '/bar', 'w').close()
+ WVEXCEPT(Exception, file_m.create_path, path, create_symlinks=True)
+ # Restore dir over non-empty dir.
+ os.remove(path + '/bar')
+ os.mkdir(path + '/bar')
+ WVEXCEPT(Exception, dir_m.create_path, path, create_symlinks=True)
+ finally:
+ subprocess.call(['rm', '-rf', tmpdir])
+
+
+@wvtest
+def test_handling_of_incorrect_existing_linux_xattrs():
+ if os.geteuid() != 0 or detect_fakeroot():
+ return
+ setup_testfs()
+ subprocess.check_call('rm -rf testfs/*', shell=True)
+ path = 'testfs/foo'
+ open(path, 'w').close()
+ xattr.set(path, 'foo', 'bar', namespace=xattr.NS_USER)
+ m = metadata.from_path(path, archive_path=path, save_symlinks=True)
+ xattr.set(path, 'baz', 'bax', namespace=xattr.NS_USER)
+ m.apply_to_path(path, restore_numeric_ids=False)
+ WVPASSEQ(xattr.list(path), ['user.foo'])
+ WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
+ xattr.set(path, 'foo', 'baz', namespace=xattr.NS_USER)
+ m.apply_to_path(path, restore_numeric_ids=False)
+ WVPASSEQ(xattr.list(path), ['user.foo'])
+ WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
+ xattr.remove(path, 'foo', namespace=xattr.NS_USER)
+ m.apply_to_path(path, restore_numeric_ids=False)
+ WVPASSEQ(xattr.list(path), ['user.foo'])
+ WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
+ os.chdir(top_dir)
+ cleanup_testfs()
--- /dev/null
+from bup import vint
+from wvtest import *
+from cStringIO import StringIO
+
+
+def encode_and_decode_vuint(x):
+ f = StringIO()
+ vint.write_vuint(f, x)
+ return vint.read_vuint(StringIO(f.getvalue()))
+
+
+@wvtest
+def test_vuint():
+ for x in (0, 1, 42, 128, 10**16):
+ WVPASSEQ(encode_and_decode_vuint(x), x)
+ WVEXCEPT(Exception, vint.write_vuint, StringIO(), -1)
+ WVEXCEPT(EOFError, vint.read_vuint, StringIO())
+
+
+def encode_and_decode_vint(x):
+ f = StringIO()
+ vint.write_vint(f, x)
+ return vint.read_vint(StringIO(f.getvalue()))
+
+
+@wvtest
+def test_vint():
+ values = (0, 1, 42, 64, 10**16)
+ for x in values:
+ WVPASSEQ(encode_and_decode_vint(x), x)
+ for x in [-x for x in values]:
+ WVPASSEQ(encode_and_decode_vint(x), x)
+ WVEXCEPT(EOFError, vint.read_vint, StringIO())
+
+
+def encode_and_decode_bvec(x):
+ f = StringIO()
+ vint.write_bvec(f, x)
+ return vint.read_bvec(StringIO(f.getvalue()))
+
+
+@wvtest
+def test_bvec():
+ values = ('', 'x', 'foo', '\0', '\0foo', 'foo\0bar\0')
+ for x in values:
+ WVPASSEQ(encode_and_decode_bvec(x), x)
+ WVEXCEPT(EOFError, vint.read_bvec, StringIO())
+ outf = StringIO()
+ for x in ('foo', 'bar', 'baz', 'bax'):
+ vint.write_bvec(outf, x)
+ inf = StringIO(outf.getvalue())
+ WVPASSEQ(vint.read_bvec(inf), 'foo')
+ WVPASSEQ(vint.read_bvec(inf), 'bar')
+ vint.skip_bvec(inf)
+ WVPASSEQ(vint.read_bvec(inf), 'bax')
+
+
+def pack_and_unpack(types, *values):
+ data = vint.pack(types, *values)
+ return vint.unpack(types, data)
+
+
+@wvtest
+def test_pack_and_unpack():
+ tests = [('', []),
+ ('s', ['foo']),
+ ('ss', ['foo', 'bar']),
+ ('sV', ['foo', 0]),
+ ('sv', ['foo', -1]),
+ ('V', [0]),
+ ('Vs', [0, 'foo']),
+ ('VV', [0, 1]),
+ ('Vv', [0, -1]),
+ ('v', [0]),
+ ('vs', [0, 'foo']),
+ ('vV', [0, 1]),
+ ('vv', [0, -1])]
+ for test in tests:
+ (types, values) = test
+ WVPASSEQ(pack_and_unpack(types, *values), values)
+ WVEXCEPT(Exception, vint.pack, 's')
+ WVEXCEPT(Exception, vint.pack, 's', 'foo', 'bar')
+ WVEXCEPT(Exception, vint.pack, 'x', 1)
+ WVEXCEPT(Exception, vint.unpack, 's', '')
+ WVEXCEPT(Exception, vint.unpack, 'x', '')
--- /dev/null
+import math
+from wvtest import *
+import bup._helpers as _helpers
+from bup.xstat import FSTime
+
+def _test_fstime():
+ def approx_eq(x, y):
+ return math.fabs(x - y) < 1 / 10e8
+ def ts_eq_ish(x, y):
+ return approx_eq(x[0], y[0]) and approx_eq(x[1], y[1])
+ def fst_eq_ish(x, y):
+ return approx_eq(x.approx_secs(), y.approx_secs())
+ def s_ns_eq_ish(fst, s, ns):
+ (fst_s, fst_ns) = fst.secs_nsecs()
+ return approx_eq(fst_s, s) and approx_eq(fst_ns, ns)
+ from_secs = FSTime.from_secs
+ from_ts = FSTime.from_timespec
+ WVPASS(from_secs(0) == from_secs(0))
+ WVPASS(from_secs(0) < from_secs(1))
+ WVPASS(from_secs(-1) < from_secs(1))
+ WVPASS(from_secs(1) > from_secs(0))
+ WVPASS(from_secs(1) > from_secs(-1))
+
+ WVPASS(fst_eq_ish(from_secs(0), from_ts((0, 0))))
+ WVPASS(fst_eq_ish(from_secs(1), from_ts((1, 0))))
+ WVPASS(fst_eq_ish(from_secs(-1), from_ts((-1, 0))))
+ WVPASS(fst_eq_ish(from_secs(-0.5), from_ts((-1, 10**9 / 2))))
+ WVPASS(fst_eq_ish(from_secs(-1.5), from_ts((-2, 10**9 / 2))))
+ WVPASS(fst_eq_ish(from_secs(1 / 10e8), from_ts((0, 1))))
+ WVPASS(fst_eq_ish(from_secs(-1 / 10e8), from_ts((-1, 10**9 - 1))))
+
+ WVPASS(ts_eq_ish(from_secs(0).to_timespec(), (0, 0)))
+ WVPASS(ts_eq_ish(from_secs(1).to_timespec(), (1, 0)))
+ WVPASS(ts_eq_ish(from_secs(-1).to_timespec(), (-1, 0)))
+ WVPASS(ts_eq_ish(from_secs(-0.5).to_timespec(), (-1, 10**9 / 2)))
+ WVPASS(ts_eq_ish(from_secs(-1.5).to_timespec(), (-2, 10**9 / 2)))
+ WVPASS(ts_eq_ish(from_secs(1 / 10e8).to_timespec(), (0, 1)))
+ WVPASS(ts_eq_ish(from_secs(-1 / 10e8).to_timespec(), (-1, 10**9 - 1)))
+
+ WVPASS(s_ns_eq_ish(from_secs(0), 0, 0))
+ WVPASS(s_ns_eq_ish(from_secs(1), 1, 0))
+ WVPASS(s_ns_eq_ish(from_secs(-1), -1, 0))
+ WVPASS(s_ns_eq_ish(from_secs(-0.5), 0, - 10**9 / 2))
+ WVPASS(s_ns_eq_ish(from_secs(-1.5), -1, - 10**9 / 2))
+ WVPASS(s_ns_eq_ish(from_secs(-1 / 10e8), 0, -1))
+
+@wvtest
+def test_fstime():
+ _test_fstime();
+ if _helpers._have_ns_fs_timestamps: # Test native python timestamp rep too.
+ orig = _helpers._have_ns_fs_timestamps
+ try:
+ _helpers._have_ns_fs_timestamps = None
+ _test_fstime();
+ finally:
+ _helpers._have_ns_fs_timestamps = orig
--- /dev/null
+"""Binary encodings for bup."""
+
+# Copyright (C) 2010 Rob Browning
+#
+# This code is covered under the terms of the GNU Library General
+# Public License as described in the bup LICENSE file.
+
+from cStringIO import StringIO
+
+# Variable length integers are encoded as vints -- see jakarta lucene.
+
+def write_vuint(port, x):
+ if x < 0:
+ raise Exception("vuints must not be negative")
+ elif x == 0:
+ port.write('\0')
+ else:
+ while x:
+ seven_bits = x & 0x7f
+ x >>= 7
+ if x:
+ port.write(chr(0x80 | seven_bits))
+ else:
+ port.write(chr(seven_bits))
+
+
+def read_vuint(port):
+ c = port.read(1)
+ if c == '':
+ raise EOFError('encountered EOF while reading vuint');
+ result = 0
+ offset = 0
+ while c:
+ b = ord(c)
+ if b & 0x80:
+ result |= ((b & 0x7f) << offset)
+ offset += 7
+ c = port.read(1)
+ else:
+ result |= (b << offset)
+ break
+ return result
+
+
+def write_vint(port, x):
+ # Sign is handled with the second bit of the first byte. All else
+ # matches vuint.
+ if x == 0:
+ port.write('\0')
+ else:
+ if x < 0:
+ x = -x
+ sign_and_six_bits = (x & 0x3f) | 0x40
+ else:
+ sign_and_six_bits = x & 0x3f
+ x >>= 6
+ if x:
+ port.write(chr(0x80 | sign_and_six_bits))
+ write_vuint(port, x)
+ else:
+ port.write(chr(sign_and_six_bits))
+
+
+def read_vint(port):
+ c = port.read(1)
+ if c == '':
+ raise EOFError('encountered EOF while reading vint');
+ negative = False
+ result = 0
+ offset = 0
+ # Handle first byte with sign bit specially.
+ if c:
+ b = ord(c)
+ if b & 0x40:
+ negative = True
+ result |= (b & 0x3f)
+ if b & 0x80:
+ offset += 6
+ c = port.read(1)
+ else:
+ return -result if negative else result
+ while c:
+ b = ord(c)
+ if b & 0x80:
+ result |= ((b & 0x7f) << offset)
+ offset += 7
+ c = port.read(1)
+ else:
+ result |= (b << offset)
+ break
+ return -result if negative else result
+
+
+def write_bvec(port, x):
+ write_vuint(port, len(x))
+ port.write(x)
+
+
+def read_bvec(port):
+ n = read_vuint(port)
+ return port.read(n)
+
+
+def skip_bvec(port):
+ port.read(read_vuint(port))
+
+
+def pack(types, *args):
+ if len(types) != len(args):
+ raise Exception('number of arguments does not match format string')
+ port = StringIO()
+ for (type, value) in zip(types, args):
+ if type == 'V':
+ write_vuint(port, value)
+ elif type == 'v':
+ write_vint(port, value)
+ elif type == 's':
+ write_bvec(port, value)
+ else:
+ raise Exception('unknown xpack format string item "' + type + '"')
+ return port.getvalue()
+
+
+def unpack(types, data):
+ result = []
+ port = StringIO(data)
+ for type in types:
+ if type == 'V':
+ result.append(read_vuint(port))
+ elif type == 'v':
+ result.append(read_vint(port))
+ elif type == 's':
+ result.append(read_bvec(port))
+ else:
+ raise Exception('unknown xunpack format string item "' + type + '"')
+ return result
--- /dev/null
+"""Enhanced stat operations for bup."""
+import math
+import os
+import bup._helpers as _helpers
+
+
+try:
+ _have_utimensat = _helpers.utimensat
+except AttributeError, e:
+ _have_utimensat = False
+
+
+class FSTime():
+ # Class to represent filesystem timestamps. Use integer
+ # nanoseconds on platforms where we have the higher resolution
+ # lstat. Use the native python stat representation (floating
+ # point seconds) otherwise.
+
+ def __cmp__(self, x):
+ return self._value.__cmp__(x._value)
+
+ def to_timespec(self):
+ """Return (s, ns) where ns is always non-negative
+ and t = s + ns / 10e8""" # metadata record rep (and libc rep)
+ s_ns = self.secs_nsecs()
+ if s_ns[0] > 0 or s_ns[1] >= 0:
+ return s_ns
+ return (s_ns[0] - 1, 10**9 + s_ns[1]) # ns is negative
+
+ @staticmethod
+ def from_secs(secs):
+ ts = FSTime()
+ ts._value = int(secs * 10**9)
+ return ts
+
+ @staticmethod
+ def from_timespec(timespec):
+ ts = FSTime()
+ ts._value = timespec[0] * 10**9 + timespec[1]
+ return ts
+
+ def approx_secs(self):
+ return self._value / 10e8;
+
+ def secs_nsecs(self):
+ "Return a (s, ns) pair: -1.5s -> (-1, -10**9 / 2)."
+ if self._value >= 0:
+ return (self._value / 10**9, self._value % 10**9)
+ abs_val = -self._value
+ return (- (abs_val / 10**9), - (abs_val % 10**9))
+
+ if _helpers._have_ns_fs_timestamps: # Use integer nanoseconds.
+
+ @staticmethod
+ def from_stat_time(stat_time):
+ return FSTime.from_timespec(stat_time)
+
+ else: # Use python default floating-point seconds.
+
+ @staticmethod
+ def from_stat_time(stat_time):
+ ts = FSTime()
+ x = math.modf(stat_time)
+ ts._value = int(x[1]) + int(x[0] * 10**9)
+ return ts
+
+
+if _have_utimensat:
+
+ def lutime(path, times):
+ atime = times[0].to_timespec()
+ mtime = times[1].to_timespec()
+ return _helpers.utimensat(_helpers.AT_FDCWD, path, (atime, mtime),
+ _helpers.AT_SYMLINK_NOFOLLOW)
+ def utime(path, times):
+ atime = times[0].to_timespec()
+ mtime = times[1].to_timespec()
+ return _helpers.utimensat(_helpers.AT_FDCWD, path, (atime, mtime), 0)
+
+else:
+
+ def lutime(path, times):
+ return None
+
+ def utime(path, times):
+ atime = times[0].approx_secs()
+ mtime = times[1].approx_secs()
+ os.utime(path, (atime, mtime))
+
+
+class stat_result():
+
+ @staticmethod
+ def from_stat_rep(st):
+ result = stat_result()
+ if _helpers._have_ns_fs_timestamps:
+ (result.st_mode,
+ result.st_ino,
+ result.st_dev,
+ result.st_nlink,
+ result.st_uid,
+ result.st_gid,
+ result.st_rdev,
+ result.st_size,
+ atime,
+ mtime,
+ ctime) = st
+ else:
+ result.st_mode = st.st_mode
+ result.st_ino = st.st_ino
+ result.st_dev = st.st_dev
+ result.st_nlink = st.st_nlink
+ result.st_uid = st.st_uid
+ result.st_gid = st.st_gid
+ result.st_rdev = st.st_rdev
+ result.st_size = st.st_size
+ atime = FSTime.from_stat_time(st.st_atime)
+ mtime = FSTime.from_stat_time(st.st_mtime)
+ ctime = FSTime.from_stat_time(st.st_ctime)
+ result.st_atime = FSTime.from_stat_time(atime)
+ result.st_mtime = FSTime.from_stat_time(mtime)
+ result.st_ctime = FSTime.from_stat_time(ctime)
+ return result
+
+
+try:
+ _stat = _helpers.stat
+except AttributeError, e:
+ _stat = os.stat
+
+def stat(path):
+ return stat_result.from_stat_rep(_stat(path))
+
+
+try:
+ _fstat = _helpers.fstat
+except AttributeError, e:
+ _fstat = os.fstat
+
+def fstat(path):
+ return stat_result.from_stat_rep(_fstat(path))
+
+
+try:
+ _lstat = _helpers.lstat
+except AttributeError, e:
+ _lstat = os.lstat
+
+def lstat(path):
+ return stat_result.from_stat_rep(_lstat(path))
--- /dev/null
+#!/usr/bin/env bash
+. wvtest.sh
+set -e -o pipefail
+
+TOP="$(pwd)"
+export BUP_DIR="$TOP/buptest.tmp"
+
+bup()
+{
+ "$TOP/bup" "$@"
+}
+
+# Very simple metadata tests -- "make install" to a temp directory,
+# then check that bup meta can reproduce the metadata correctly
+# (according to coreutils stat) via create, extract, start-extract,
+# and finish-extract. The current tests are crude, and this does not
+# test devices, varying users/groups, acls, attrs, etc.
+
+genstat()
+{
+ (
+ export PATH="${TOP}:${PATH}" # pick up bup
+ # Skip atime (test elsewhere) to avoid the observer effect.
+ find . | sort | xargs bup xstat --exclude-fields ctime,atime
+ )
+}
+
+actually-root()
+{
+ test "$(whoami)" == root -a -z "${FAKEROOTKEY}"
+}
+
+force-delete()
+{
+ if ! actually-root
+ then
+ rm -rf "$@"
+ else
+ # Go to greater lengths to deal with any test detritus.
+ for f in "$@"
+ do
+ test -e "$@" || continue
+ chattr -fR = "$@" || true
+ setfacl -Rb "$@"
+ rm -r "$@"
+ done
+ fi
+}
+
+test-src-create-extract()
+{
+ # Test bup meta create/extract for ./src -> ./src-restore.
+ # Also writes to ./src-stat and ./src-restore-stat.
+ (
+ (cd src && WVPASS genstat) > src-stat
+ WVPASS bup meta --create --recurse --file src.meta src
+ # Test extract.
+ force-delete src-restore
+ mkdir src-restore
+ cd src-restore
+ WVPASS bup meta --extract --file ../src.meta
+ WVPASS test -d src
+ (cd src && genstat >../../src-restore-stat) || WVFAIL
+ WVPASS diff -U5 ../src-stat ../src-restore-stat
+ # Test start/finish extract.
+ force-delete src
+ WVPASS bup meta --start-extract --file ../src.meta
+ WVPASS test -d src
+ WVPASS bup meta --finish-extract --file ../src.meta
+ (cd src && genstat >../../src-restore-stat) || WVFAIL
+ WVPASS diff -U5 ../src-stat ../src-restore-stat
+ )
+}
+
+if actually-root
+then
+ umount "${TOP}/bupmeta.tmp/testfs" || true
+fi
+
+force-delete "${BUP_DIR}"
+force-delete "${TOP}/bupmeta.tmp"
+
+# Create a test tree.
+(
+ mkdir -p "${TOP}/bupmeta.tmp"
+ make DESTDIR="${TOP}/bupmeta.tmp/src" install
+ mkdir "${TOP}/bupmeta.tmp/src/misc"
+ cp -a cmd/bup-* "${TOP}/bupmeta.tmp/src/misc/"
+) || WVFAIL
+
+# Use the test tree to check bup meta.
+WVSTART 'meta - general'
+(
+ cd "${TOP}/bupmeta.tmp"
+ test-src-create-extract
+)
+
+# Root-only tests: ACLs, Linux attr, Linux xattr, etc.
+if actually-root
+then
+ (
+ cleanup_at_exit()
+ {
+ cd "${TOP}"
+ umount "${TOP}/bupmeta.tmp/testfs" || true
+ }
+
+ trap cleanup_at_exit EXIT
+
+ WVSTART 'meta - general (as root)'
+ WVPASS cd "${TOP}/bupmeta.tmp"
+ umount testfs || true
+ dd if=/dev/zero of=testfs.img bs=1M count=32
+ mke2fs -F -j -m 0 testfs.img
+ mkdir testfs
+ mount -o loop,acl,user_xattr testfs.img testfs
+ # Hide, so that tests can't create risks.
+ chown root:root testfs
+ chmod 0700 testfs
+
+ cp -a src testfs/src
+ (cd testfs && test-src-create-extract)
+
+ WVSTART 'meta - atime (as root)'
+ force-delete testfs/src
+ mkdir testfs/src
+ (
+ mkdir testfs/src/foo
+ touch testfs/src/bar
+ PYTHONPATH="${TOP}/lib" \
+ python -c "from bup.xstat import lutime, FSTime; \
+ x = FSTime.from_secs(42);\
+ lutime('testfs/src/foo', (x, x));\
+ lutime('testfs/src/bar', (x, x));"
+ cd testfs
+ WVPASS bup meta -v --create --recurse --file src.meta src
+ bup meta -tvf src.meta
+ # Test extract.
+ force-delete src-restore
+ mkdir src-restore
+ cd src-restore
+ WVPASS bup meta --extract --file ../src.meta
+ WVPASSEQ "$(bup xstat --include-fields=atime src/foo)" "atime: 42"
+ WVPASSEQ "$(bup xstat --include-fields=atime src/bar)" "atime: 42"
+ # Test start/finish extract.
+ force-delete src
+ WVPASS bup meta --start-extract --file ../src.meta
+ WVPASS test -d src
+ WVPASS bup meta --finish-extract --file ../src.meta
+ WVPASSEQ "$(bup xstat --include-fields=atime src/foo)" "atime: 42"
+ WVPASSEQ "$(bup xstat --include-fields=atime src/bar)" "atime: 42"
+ )
+
+ WVSTART 'meta - Linux attr (as root)'
+ force-delete testfs/src
+ mkdir testfs/src
+ (
+ touch testfs/src/foo
+ mkdir testfs/src/bar
+ chattr +acdeijstuADST testfs/src/foo
+ chattr +acdeijstuADST testfs/src/bar
+ (cd testfs && test-src-create-extract)
+ )
+
+ WVSTART 'meta - Linux xattr (as root)'
+ force-delete testfs/src
+ mkdir testfs/src
+ (
+ touch testfs/src/foo
+ mkdir testfs/src/bar
+ attr -s foo -V bar testfs/src/foo
+ attr -s foo -V bar testfs/src/bar
+ (cd testfs && test-src-create-extract)
+ )
+
+ WVSTART 'meta - POSIX.1e ACLs (as root)'
+ force-delete testfs/src
+ mkdir testfs/src
+ (
+ touch testfs/src/foo
+ mkdir testfs/src/bar
+ setfacl -m u:root:r testfs/src/foo
+ setfacl -m u:root:r testfs/src/bar
+ (cd testfs && test-src-create-extract)
+ )
+ )
+fi
+
+force-delete "${BUP_DIR}"
+force-delete "$TOP/bupmeta.tmp"
+
+exit 0
projects / bup.git / commitdiff