for separating the two steps are described in the man page
for `bup-index`(1).
+By default, metadata will be saved for every path. However, if
+`--strip`, `--strip-path`, or `--graft` is specified, metadata will
+not be saved for the root directory (*/*).
+
# OPTIONS
-r, \--remote=*host*:*path*
\--strip
: strips the path that is given from all files and directories.
- A directory */root/chroot/etc* saved with
- "bup save -n chroot \--strip /root/chroot" would be saved
- as */etc*.
+ A directory */root/chroot/etc* saved with "bup save -n chroot
+ \--strip /root/chroot" would be saved as */etc*. Note that
+ currently, metadata will not be saved for the root directory (*/*)
+ when this option is specified.
\--strip-path=*path-prefix*
: strips the given path prefix *path-prefix* from all
files and directories.
- A directory */root/chroots/webserver* saved with
- "bup save -n webserver \--strip-path=/root/chroots" would
- be saved as */webserver/etc*
+ A directory */root/chroots/webserver* saved with "bup save -n
+ webserver \--strip-path=/root/chroots" would be saved as
+ */webserver/etc*. Note that currently, metadata will not be saved
+ for the root directory (*/*) when this option is specified.
\--graft=*old_path*=*new_path*
: a graft point *old_path*=*new_path* (can be used more than
once).
- A directory */root/chroot/a/etc* saved with
- "bup save -n chroots \--graft /root/chroot/a/etc=/chroots/a"
- would be saved as */chroots/a/etc*
+ A directory */root/chroot/a/etc* saved with "bup save -n chroots
+ \--graft /root/chroot/a/etc=/chroots/a" would be saved as
+ */chroots/a/etc*. Note that currently, metadata will not be saved
+ for the root directory (*/*) when this option is specified.
-*#*, \--compress=*#*
: set the compression level to # (a value from 0-9, where
#!/usr/bin/env python
import sys, stat, time, math
-from bup import hashsplit, git, options, index, client
+from bup import hashsplit, git, options, index, client, metadata
from bup.helpers import *
from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
return dir
+# Metadata is stored in a file named .bupm in each directory. The
+# first metadata entry will be the metadata for the current directory.
+# The remaining entries will be for each of the other directory
+# elements, in the order they're listed in the index.
+#
+# Since the git tree elements are sorted according to
+# git.shalist_item_sort_key, the metalist items are accumulated as
+# (sort_key, metadata) tuples, and then sorted when the .bupm file is
+# created. The sort_key must be computed using the element's real
+# name and mode rather than the git mode and (possibly mangled) name.
+
parts = ['']
shalists = [[]]
+metalists = [[]]
-def _push(part):
+def _push(part, metadata):
assert(part)
parts.append(part)
shalists.append([])
+ # First entry is dir metadata, which is represented with an empty name.
+ metalists.append([('', metadata)])
def _pop(force_tree):
assert(len(parts) >= 1)
part = parts.pop()
shalist = shalists.pop()
+ metalist = metalists.pop()
+ if metalist:
+ sorted_metalist = sorted(metalist, key = lambda x : x[0])
+ metadata = ''.join([m[1].encode() for m in sorted_metalist])
+ shalist.append((0100644, '.bupm', w.new_blob(metadata)))
tree = force_tree or w.new_tree(shalist)
if shalists:
shalists[-1].append((GIT_MODE_TREE,
git.mangle_name(part,
GIT_MODE_TREE, GIT_MODE_TREE),
tree))
- else: # this was the toplevel, so put it back for sanity
+ else:
+ # This was the toplevel, so put it back for sanity (i.e. cd .. from /).
shalists.append(shalist)
+ metalists.append(metalist)
return tree
lastremain = None
def wantrecurse_during(ent):
return not already_saved(ent) or ent.sha_missing()
+
total = ftotal = 0
if opt.progress:
for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
assert(dir.startswith('/'))
if opt.strip:
- stripped_base_path = strip_base_path(dir, extra)
- dirp = stripped_base_path.split('/')
+ dirp = stripped_path_components(dir, extra)
elif opt.strip_path:
- dirp = strip_path(opt.strip_path, dir).split('/')
+ dirp = stripped_path_components(dir, [opt.strip_path])
elif graft_points:
- grafted = graft_path(graft_points, dir)
- dirp = grafted.split('/')
+ dirp = grafted_path_components(graft_points, dir)
else:
- dirp = dir.split('/')
- while parts > dirp:
+ dirp = path_components(dir)
+
+ while parts > [x[0] for x in dirp]:
_pop(force_tree = None)
+
if dir != '/':
- for part in dirp[len(parts):]:
- _push(part)
+ for path_component in dirp[len(parts):]:
+ dir_name, fs_path = path_component
+ if fs_path:
+ meta = metadata.from_path(fs_path)
+ else:
+ meta = metadata.Metadata()
+ _push(dir_name, meta)
if not file:
# no filename portion means this is a subdir. But
id = None
if hashvalid:
id = ent.sha
- shalists[-1].append((ent.gitmode,
- git.mangle_name(file, ent.mode, ent.gitmode),
- id))
+ git_name = git.mangle_name(file, ent.mode, ent.gitmode)
+ git_info = (ent.gitmode, git_name, id)
+ shalists[-1].append(git_info)
+ sort_key = git.shalist_item_sort_key((ent.mode, file, id))
+ metalists[-1].append((sort_key, metadata.from_path(ent.name)))
else:
if stat.S_ISREG(ent.mode):
try:
else:
(mode, id) = (GIT_MODE_SYMLINK, w.new_blob(rl))
else:
- add_error(Exception('skipping special file "%s"' % ent.name))
- lastskip_name = ent.name
+ # Everything else should be fully described by its
+ # metadata, so just record an empty blob, so the paths
+ # in the tree and .bupm will match up.
+ (mode, id) = (GIT_MODE_FILE, w.new_blob(""))
+
if id:
ent.validate(mode, id)
ent.repack()
- shalists[-1].append((mode,
- git.mangle_name(file, ent.mode, ent.gitmode),
- id))
+ git_name = git.mangle_name(file, ent.mode, ent.gitmode)
+ git_info = (mode, git_name, id)
+ shalists[-1].append(git_info)
+ sort_key = git.shalist_item_sort_key((ent.mode, file, id))
+ metalists[-1].append((sort_key, metadata.from_path(ent.name)))
if exists and wasmissing:
count += oldsize
subcount = 0
progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n'
% (pct, count/1024, total/1024, fcount, ftotal))
-while len(parts) > 1:
+while len(parts) > 1: # _pop() all the parts above the indexed items.
_pop(force_tree = None)
assert(len(shalists) == 1)
+assert(len(metalists) == 1)
+
+if not (opt.strip or opt.strip_path or graft_points):
+ # For now, only save metadata for the root directory when there
+ # isn't any path grafting or stripping that might create multiple
+ # roots.
+ shalist = shalists[-1]
+ metadata = ''.join([metadata.from_path('/').encode()])
+ shalist.append((0100644, '.bupm', w.new_blob(metadata)))
tree = w.new_tree(shalists[-1])
+
if opt.tree:
print tree.encode('hex')
if opt.commit or opt.name:
return sum.digest()
-def _shalist_sort_key(ent):
+def shalist_item_sort_key(ent):
(mode, name, id) = ent
assert(mode+0 == mode)
if stat.S_ISDIR(mode):
def tree_encode(shalist):
"""Generate a git tree object from (mode,name,hash) tuples."""
- shalist = sorted(shalist, key = _shalist_sort_key)
+ shalist = sorted(shalist, key = shalist_item_sort_key)
l = []
for (mode,name,bin) in shalist:
assert(mode)
return date
-def strip_path(prefix, path):
- """Strips a given prefix from a path.
-
- First both paths are normalized.
-
- Raises an Exception if no prefix is given.
- """
- if prefix == None:
- raise Exception('no path given')
-
- normalized_prefix = os.path.realpath(prefix)
- debug2("normalized_prefix: %s\n" % normalized_prefix)
- normalized_path = os.path.realpath(path)
- debug2("normalized_path: %s\n" % normalized_path)
- if normalized_path.startswith(normalized_prefix):
- return normalized_path[len(normalized_prefix):]
- else:
- return path
-
-
-def strip_base_path(path, base_paths):
- """Strips the base path from a given path.
-
-
- Determines the base path for the given string and then strips it
- using strip_path().
- Iterates over all base_paths from long to short, to prevent that
- a too short base_path is removed.
- """
- normalized_path = os.path.realpath(path)
- sorted_base_paths = sorted(base_paths, key=len, reverse=True)
- for bp in sorted_base_paths:
- if normalized_path.startswith(os.path.realpath(bp)):
- return strip_path(bp, normalized_path)
- return path
-
-
-def graft_path(graft_points, path):
- normalized_path = os.path.realpath(path)
+def path_components(path):
+ """Break path into a list of pairs of the form (name,
+ full_path_to_name). Path must start with '/'.
+ Example:
+ '/home/foo' -> [('', '/'), ('home', '/home'), ('foo', '/home/foo')]"""
+ assert(path.startswith('/'))
+ # Since we assume path startswith('/'), we can skip the first element.
+ result = [('', '/')]
+ norm_path = os.path.abspath(path)
+ if norm_path == '/':
+ return result
+ full_path = ''
+ for p in norm_path.split('/')[1:]:
+ full_path += '/' + p
+ result.append((p, full_path))
+ return result
+
+
+def stripped_path_components(path, strip_prefixes):
+ """Strip any prefix in strip_prefixes from path and return a list
+ of path components where each component is (name,
+ none_or_full_fs_path_to_name). Assume path startswith('/').
+ See thelpers.py for examples."""
+ normalized_path = os.path.abspath(path)
+ sorted_strip_prefixes = sorted(strip_prefixes, key=len, reverse=True)
+ for bp in sorted_strip_prefixes:
+ normalized_bp = os.path.abspath(bp)
+ if normalized_path.startswith(normalized_bp):
+ prefix = normalized_path[:len(normalized_bp)]
+ result = []
+ for p in normalized_path[len(normalized_bp):].split('/'):
+ if p: # not root
+ prefix += '/'
+ prefix += p
+ result.append((p, prefix))
+ return result
+ # Nothing to strip.
+ return path_components(path)
+
+
+def grafted_path_components(graft_points, path):
+ # Find the first '/' after the graft prefix, match that to the
+ # original source base dir, then move on.
+ clean_path = os.path.abspath(path)
for graft_point in graft_points:
old_prefix, new_prefix = graft_point
- if normalized_path.startswith(old_prefix):
- return re.sub(r'^' + old_prefix, new_prefix, normalized_path)
- return normalized_path
+ if clean_path.startswith(old_prefix):
+ grafted_path = re.sub(r'^' + old_prefix, new_prefix,
+ clean_path)
+ result = [(p, None) for p in grafted_path.split('/')]
+ result[-1] = (result[-1][0], clean_path)
+ return result
+ return path_components(clean_path)
# hashlib is only available in python 2.5 or higher, but the 'sha' module
# record will have some subset of add, encode, load, create, and
# apply methods, i.e. _add_foo...
+ # We do allow an "empty" object as a special case, i.e. no
+ # records. One can be created by trying to write Metadata(), and
+ # for such an object, read() will return None. This is used by
+ # "bup save", for example, as a placeholder in cases where
+ # from_path() fails.
+
## Common records
# Timestamps are (sec, ns), relative to 1970-01-01 00:00:00, ns
# must be non-negative and < 10**9.
def _add_common(self, path, st):
- self.mode = st.st_mode
self.uid = st.st_uid
self.gid = st.st_gid
self.rdev = st.st_rdev
self.group = grp.getgrgid(st.st_gid)[0]
except KeyError, e:
add_error("no group name for id %s '%s'" % (st.st_gid, path))
+ self.mode = st.st_mode
def _encode_common(self):
+ if not self.mode:
+ return None
atime = xstat.nsecs_to_timespec(self.atime)
mtime = xstat.nsecs_to_timespec(self.mtime)
ctime = xstat.nsecs_to_timespec(self.ctime)
or stat.S_ISLNK(self.mode)
def _create_via_common_rec(self, path, create_symlinks=True):
+ if not self.mode:
+ raise ApplyError('no metadata - cannot create path ' + path)
+
# If the path already exists and is a dir, try rmdir.
# If the path already exists and is anything else, try unlink.
st = None
% (path, self.mode))
def _apply_common_rec(self, path, restore_numeric_ids=False):
+ if not self.mode:
+ raise ApplyError('no metadata - cannot apply to ' + path)
+
# FIXME: S_ISDOOR, S_IFMPB, S_IFCMP, S_IFNWK, ... see stat(2).
# EACCES errors at this stage are fatal for the current path.
if lutime and stat.S_ISLNK(self.mode):
raise
def __init__(self):
+ self.mode = None
# optional members
self.path = None
self.size = None
vint.write_bvec(port, data)
vint.write_vuint(port, _rec_tag_end)
+ def encode(self, include_path=True):
+ port = StringIO()
+ self.write(port, include_path)
+ return port.getvalue()
+
@staticmethod
def read(port):
- # This method should either: return a valid Metadata object;
- # throw EOFError if there was nothing at all to read; throw an
- # Exception if a valid object could not be read completely.
+ # This method should either return a valid Metadata object,
+ # return None if there was no information at all (just a
+ # _rec_tag_end), throw EOFError if there was nothing at all to
+ # read, or throw an Exception if a valid object could not be
+ # read completely.
tag = vint.read_vuint(port)
+ if tag == _rec_tag_end:
+ return None
try: # From here on, EOF is an error.
result = Metadata()
while True: # only exit is error (exception) or _rec_tag_end
result = []
if 'path' in fields:
- result.append('path: ' + meta.path)
+ path = meta.path or ''
+ result.append('path: ' + path)
if 'mode' in fields:
result.append('mode: %s (%s)' % (oct(meta.mode),
xstat.mode_str(meta.mode)))
for meta in _ArchiveIterator(file):
if not meta.path:
print >> sys.stderr, \
- 'bup: cannot list path for metadata without path'
+ 'bup: no metadata path, but asked to only display path (increase verbosity?)'
sys.exit(1)
print meta.path
def start_extract(file, create_symlinks=True):
for meta in _ArchiveIterator(file):
+ if not meta: # Hit end record.
+ break
if verbose:
print >> sys.stderr, meta.path
xpath = _clean_up_extract_path(meta.path)
def finish_extract(file, restore_numeric_ids=False):
all_dirs = []
for meta in _ArchiveIterator(file):
+ if not meta: # Hit end record.
+ break
xpath = _clean_up_extract_path(meta.path)
if not xpath:
add_error(Exception('skipping risky path "%s"' % dir.path))
# longest first.
all_dirs = []
for meta in _ArchiveIterator(file):
+ if not meta: # Hit end record.
+ break
xpath = _clean_up_extract_path(meta.path)
if not xpath:
add_error(Exception('skipping risky path "%s"' % meta.path))
WVPASS(not detect_fakeroot())
@wvtest
-def test_strip_path():
- prefix = "/NOT_EXISTING/var/backup/daily.0/localhost"
- empty_prefix = ""
- non_matching_prefix = "/home"
- path = "/NOT_EXISTING/var/backup/daily.0/localhost/etc/"
+def test_path_components():
+ WVPASSEQ(path_components('/'), [('', '/')])
+ WVPASSEQ(path_components('/foo'), [('', '/'), ('foo', '/foo')])
+ WVPASSEQ(path_components('/foo/'), [('', '/'), ('foo', '/foo')])
+ WVPASSEQ(path_components('/foo/bar'),
+ [('', '/'), ('foo', '/foo'), ('bar', '/foo/bar')])
+ WVEXCEPT(Exception, path_components, 'foo')
- WVPASSEQ(strip_path(prefix, path), '/etc')
- WVPASSEQ(strip_path(empty_prefix, path), path)
- WVPASSEQ(strip_path(non_matching_prefix, path), path)
- WVEXCEPT(Exception, strip_path, None, path)
@wvtest
-def test_strip_base_path():
- path = "/NOT_EXISTING/var/backup/daily.0/localhost/etc/"
- base_paths = ["/NOT_EXISTING/var",
- "/NOT_EXISTING/var/backup",
- "/NOT_EXISTING/var/backup/daily.0/localhost"
- ]
- WVPASSEQ(strip_base_path(path, base_paths), '/etc')
+def test_stripped_path_components():
+ WVPASSEQ(stripped_path_components('/', []), [('', '/')])
+ WVPASSEQ(stripped_path_components('/', ['']), [('', '/')])
+ WVPASSEQ(stripped_path_components('/', ['/']), [('', '/')])
+ WVPASSEQ(stripped_path_components('/', ['/foo']), [('', '/')])
+ WVPASSEQ(stripped_path_components('/foo', ['/bar']),
+ [('', '/'), ('foo', '/foo')])
+ WVPASSEQ(stripped_path_components('/foo', ['/foo']), [('', '/foo')])
+ WVPASSEQ(stripped_path_components('/foo/bar', ['/foo']),
+ [('', '/foo'), ('bar', '/foo/bar')])
+ WVPASSEQ(stripped_path_components('/foo/bar', ['/bar', '/foo', '/baz']),
+ [('', '/foo'), ('bar', '/foo/bar')])
+ WVPASSEQ(stripped_path_components('/foo/bar/baz', ['/foo/bar/baz']),
+ [('', '/foo/bar/baz')])
+ WVEXCEPT(Exception, stripped_path_components, 'foo', [])
@wvtest
-def test_strip_symlinked_base_path():
- tmpdir = os.path.join(os.getcwd(),"test_strip_symlinked_base_path.tmp")
- symlink_src = os.path.join(tmpdir, "private", "var")
- symlink_dst = os.path.join(tmpdir, "var")
- path = os.path.join(symlink_dst, "a")
-
- os.mkdir(tmpdir)
- os.mkdir(os.path.join(tmpdir, "private"))
- os.mkdir(symlink_src)
- os.symlink(symlink_src, symlink_dst)
-
- result = strip_base_path(path, [symlink_dst])
-
- os.remove(symlink_dst)
- os.rmdir(symlink_src)
- os.rmdir(os.path.join(tmpdir, "private"))
- os.rmdir(tmpdir)
-
- WVPASSEQ(result, "/a")
-
-@wvtest
-def test_graft_path():
- middle_matching_old_path = "/NOT_EXISTING/user"
- non_matching_old_path = "/NOT_EXISTING/usr"
- matching_old_path = "/NOT_EXISTING/home"
- matching_full_path = "/NOT_EXISTING/home/user"
- new_path = "/opt"
-
- all_graft_points = [(middle_matching_old_path, new_path),
- (non_matching_old_path, new_path),
- (matching_old_path, new_path)]
-
- path = "/NOT_EXISTING/home/user/"
-
- WVPASSEQ(graft_path([(middle_matching_old_path, new_path)], path),
- "/NOT_EXISTING/home/user")
- WVPASSEQ(graft_path([(non_matching_old_path, new_path)], path),
- "/NOT_EXISTING/home/user")
- WVPASSEQ(graft_path([(matching_old_path, new_path)], path), "/opt/user")
- WVPASSEQ(graft_path(all_graft_points, path), "/opt/user")
- WVPASSEQ(graft_path([(matching_full_path, new_path)], path),
- "/opt")
+def test_grafted_path_components():
+ WVPASSEQ(grafted_path_components([('/chroot', '/')], '/foo'),
+ [('', '/'), ('foo', '/foo')])
+ WVPASSEQ(grafted_path_components([('/foo/bar', '')], '/foo/bar/baz/bax'),
+ [('', None), ('baz', None), ('bax', '/foo/bar/baz/bax')])
+ WVEXCEPT(Exception, grafted_path_components, 'foo', [])
self.ctime = self.mtime = self.atime = 0
self._subs = None
+ def __repr__(self):
+ return "<bup.vfs.Node object at X - name:%r hash:%s parent:%r>" \
+ % (self.name, self.hash.encode('hex'),
+ self.parent.name if self.parent.name else None)
+
def __cmp__(a, b):
if a is b:
return 0
class Dir(Node):
"""A directory stored inside of bup's repository."""
+
+ def __init__(self, *args):
+ Node.__init__(self, *args)
+ self._metadata_sha = None
+
def _mksubs(self):
self._subs = {}
it = cp().get(self.hash.encode('hex'))
type = it.next()
assert(type == 'tree')
for (mode,mangled_name,sha) in git.tree_decode(''.join(it)):
+ if mangled_name == '.bupm':
+ self._metadata_sha = sha
+ continue
name = mangled_name
(name,bupmode) = git.demangle_name(mangled_name)
if bupmode == git.BUP_CHUNKED:
set -e
rm -rf "$TOP/bupmeta.tmp/src"
mkdir -p "$TOP/bupmeta.tmp/src"
- #cp -a Documentation cmd lib t "$TOP/bupmeta.tmp"/src
cp -pPR Documentation cmd lib t "$TOP/bupmeta.tmp"/src
+
+ # Regression test for metadata sort order. Previously, these two
+ # entries would sort in the wrong order because the metadata
+ # entries were being sorted by mangled name, but the index isn't.
+ dd if=/dev/zero of="$TOP/bupmeta.tmp"/src/foo bs=1k count=33
+ touch -d 2011-11-11 "$TOP/bupmeta.tmp"/src/foo
+ touch -d 2011-12-12 "$TOP/bupmeta.tmp"/src/foo-bar
+
t/mksock "$TOP/bupmeta.tmp/src/test-socket" || true
) || WVFAIL
WVFAIL bup save -t $D/d/e/fifotest
mkfifo $D/d/e/fifotest
WVPASS bup index -u $D/d/e/fifotest
-WVFAIL bup save -t $D/d/e/fifotest
-WVFAIL bup save -t $D/d/e
+WVPASS bup save -t $D/d/e/fifotest
+WVPASS bup save -t $D/d/e
rm -f $D/d/e/fifotest
WVPASS bup index -u $D/d/e
WVFAIL bup save -t $D/d/e/fifotest
COMPRESSION_9_SIZE=$(du -s $D | cut -f1)
WVPASS [ "$COMPRESSION_9_SIZE" -lt "$COMPRESSION_0_SIZE" ]
+
+
+WVSTART "save disjoint top-level directories"
+(
+ set -e
+ top_dir="$(echo $(pwd) | awk -F "/" '{print $2}')"
+ if [ "$top_dir" == tmp ]; then
+ echo "(running from within /tmp; skipping test)"
+ exit 0
+ fi
+ D=bupdata.tmp
+ rm -rf $D
+ mkdir -p $D/x
+ date > $D/x/1
+ tmpdir="$(mktemp --tmpdir=/tmp -d bup-test-XXXXXXX)"
+ cleanup() { set -x; rm -rf "${tmpdir}"; set +x; }
+ trap cleanup EXIT
+ date > "$tmpdir/2"
+
+ export BUP_DIR="$TOP/buptest.tmp"
+ rm -rf "$BUP_DIR"
+
+ WVPASS bup init
+ WVPASS bup index -vu $(pwd)/$D/x "$tmpdir"
+ WVPASS bup save -t -n src $(pwd)/$D/x "$tmpdir"
+ WVPASSEQ "$(bup ls src/latest)" \
+"$top_dir/
+tmp/"
+) || WVFAIL
projects / bup.git / commitdiff