"""
# end of bup preamble
+from __future__ import print_function
+from stat import S_ISDIR
import copy, errno, os, sys, stat, re
-from bup import options, git, metadata, vfs
+from bup import options, git, metadata, vfs2
from bup._helpers import write_sparsely
-from bup.helpers import (add_error, chunkyreader, handle_ctrl_c, log, mkdirp,
- parse_rx_excludes, progress, qprogress, saved_errors,
- should_rx_exclude_path, unlink)
+from bup.compat import wrap_main
+from bup.helpers import (add_error, chunkyreader, die_if_errors, handle_ctrl_c,
+ log, mkdirp, parse_rx_excludes, progress, qprogress,
+ saved_errors, should_rx_exclude_path, unlink)
+from bup.repo import LocalRepo
optspec = """
sys.stdout.flush()
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)
-def verbose1(s):
- if opt.verbose >= 1:
- print s
-
-
-def verbose2(s):
- if opt.verbose >= 2:
- print s
-
-
-def plog(s):
- if opt.quiet:
- return
- qprogress(s)
-
-
def valid_restore_path(path):
path = os.path.normpath(path)
if path.startswith('/'):
if '/' in path:
return True
-
-def print_info(n, fullname):
- if stat.S_ISDIR(n.mode):
- verbose1('%s/' % fullname)
- elif stat.S_ISLNK(n.mode):
- verbose2('%s@ -> %s' % (fullname, n.readlink()))
- else:
- verbose2(fullname)
-
-
-def create_path(n, fullname, meta):
- if meta:
- meta.create_path(fullname)
- else:
- # These fallbacks are important -- meta could be null if, for
- # example, save created a "fake" item, i.e. a new strip/graft
- # path element, etc. You can find cases like that by
- # searching for "Metadata()".
- unlink(fullname)
- if stat.S_ISDIR(n.mode):
- mkdirp(fullname)
- elif stat.S_ISLNK(n.mode):
- os.symlink(n.readlink(), fullname)
-
-
def parse_owner_mappings(type, options, fatal):
"""Traverse the options and parse all --map-TYPEs, or call Option.fatal()."""
opt_name = '--map-' + type
owner_map[old_id] = new_id
return owner_map
-
def apply_metadata(meta, name, restore_numeric_ids, owner_map):
m = copy.deepcopy(meta)
m.user = owner_map['user'].get(m.user, m.user)
m.uid = owner_map['uid'].get(m.uid, m.uid)
m.gid = owner_map['gid'].get(m.gid, m.gid)
m.apply_to_path(name, restore_numeric_ids = restore_numeric_ids)
-
-
-# Track a list of (restore_path, vfs_path, meta) triples for each path
-# we've written for a given hardlink_target. This allows us to handle
-# the case where we restore a set of hardlinks out of order (with
-# respect to the original save call(s)) -- i.e. when we don't restore
-# the hardlink_target path first. This data also allows us to attempt
-# to handle other situations like hardlink sets that change on disk
-# during a save, or between index and save.
-targets_written = {}
-
-def hardlink_compatible(target_path, target_vfs_path, target_meta,
- src_node, src_meta):
- global top
- if not os.path.exists(target_path):
+
+def hardlink_compatible(prev_path, prev_item, new_item, top):
+ prev_candidate = top + prev_path
+ if not os.path.exists(prev_candidate):
return False
- target_node = top.lresolve(target_vfs_path)
- if src_node.mode != target_node.mode \
- or src_node.mtime != target_node.mtime \
- or src_node.ctime != target_node.ctime \
- or src_node.hash != target_node.hash:
+ prev_meta, new_meta = prev_item.meta, new_item.meta
+ if new_item.oid != prev_item.oid \
+ or new_meta.mtime != prev_meta.mtime \
+ or new_meta.ctime != prev_meta.ctime \
+ or new_meta.mode != prev_meta.mode:
return False
- if not src_meta.same_file(target_meta):
+ # FIXME: should we be checking the path on disk, or the recorded metadata?
+ # The exists() above might seem to suggest the former.
+ if not new_meta.same_file(prev_meta):
return False
return True
-
-def hardlink_if_possible(fullname, node, meta):
+def hardlink_if_possible(fullname, item, top, hardlinks):
"""Find a suitable hardlink target, link to it, and return true,
otherwise return false."""
- # Expect the caller to handle restoring the metadata if
- # hardlinking isn't possible.
- global targets_written
- target = meta.hardlink_target
- target_versions = targets_written.get(target)
+ # The cwd will be dirname(fullname), and fullname will be
+ # absolute, i.e. /foo/bar, and the caller is expected to handle
+ # restoring the metadata if hardlinking isn't possible.
+
+ # FIXME: we can probably replace the target_vfs_path with the
+ # relevant vfs item
+
+ # hardlinks tracks a list of (restore_path, vfs_path, meta)
+ # triples for each path we've written for a given hardlink_target.
+ # This allows us to handle the case where we restore a set of
+ # hardlinks out of order (with respect to the original save
+ # call(s)) -- i.e. when we don't restore the hardlink_target path
+ # first. This data also allows us to attempt to handle other
+ # situations like hardlink sets that change on disk during a save,
+ # or between index and save.
+
+ target = item.meta.hardlink_target
+ assert(target)
+ assert(fullname.startswith('/'))
+ target_versions = hardlinks.get(target)
if target_versions:
# Check every path in the set that we've written so far for a match.
- for (target_path, target_vfs_path, target_meta) in target_versions:
- if hardlink_compatible(target_path, target_vfs_path, target_meta,
- node, meta):
+ for prev_path, prev_item in target_versions:
+ if hardlink_compatible(prev_path, prev_item, item, top):
try:
- os.link(target_path, fullname)
+ os.link(top + prev_path, top + fullname)
return True
except OSError as e:
if e.errno != errno.EXDEV:
raise
else:
target_versions = []
- targets_written[target] = target_versions
- full_vfs_path = node.fullname()
- target_versions.append((fullname, full_vfs_path, meta))
+ hardlinks[target] = target_versions
+ target_versions.append((fullname, item))
return False
+def write_file_content(repo, dest_path, vfs_file):
+ with vfs2.fopen(repo, vfs_file) as inf:
+ with open(dest_path, 'wb') as outf:
+ for b in chunkyreader(inf):
+ outf.write(b)
+
+def write_file_content_sparsely(repo, dest_path, vfs_file):
+ with vfs2.fopen(repo, vfs_file) as inf:
+ outfd = os.open(dest_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ try:
+ trailing_zeros = 0;
+ for b in chunkyreader(inf):
+ trailing_zeros = write_sparsely(outfd, b, 512, trailing_zeros)
+ pos = os.lseek(outfd, trailing_zeros, os.SEEK_END)
+ os.ftruncate(outfd, pos)
+ finally:
+ os.close(outfd)
+
+def restore(repo, parent_path, name, item, top, sparse, numeric_ids, owner_map,
+ exclude_rxs, verbosity, hardlinks):
+ global total_restored
+ mode = vfs2.item_mode(item)
+ treeish = S_ISDIR(mode)
+ fullname = parent_path + '/' + name
+ # Match behavior of index --exclude-rx with respect to paths.
+ if should_rx_exclude_path(fullname + ('/' if treeish else ''),
+ exclude_rxs):
+ return
-def write_file_content(fullname, n):
- outf = open(fullname, 'wb')
- try:
- for b in chunkyreader(n.open()):
- outf.write(b)
- finally:
- outf.close()
-
-
-def write_file_content_sparsely(fullname, n):
- outfd = os.open(fullname, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
- try:
- trailing_zeros = 0;
- for b in chunkyreader(n.open()):
- trailing_zeros = write_sparsely(outfd, b, 512, trailing_zeros)
- pos = os.lseek(outfd, trailing_zeros, os.SEEK_END)
- os.ftruncate(outfd, pos)
- finally:
- os.close(outfd)
-
-
-def find_dir_item_metadata_by_name(dir, name):
- """Find metadata in dir (a node) for an item with the given name,
- or for the directory itself if the name is ''."""
- meta_stream = None
- try:
- mfile = dir.metadata_file() # VFS file -- cannot close().
- if mfile:
- meta_stream = mfile.open()
- # First entry is for the dir itself.
- meta = metadata.Metadata.read(meta_stream)
- if name == '':
- return meta
- for sub in dir:
- if stat.S_ISDIR(sub.mode):
- meta = find_dir_item_metadata_by_name(sub, '')
- else:
- meta = metadata.Metadata.read(meta_stream)
- if sub.name == name:
- return meta
- finally:
- if meta_stream:
- meta_stream.close()
-
-
-def do_root(n, sparse, owner_map, restore_root_meta = True):
- # Very similar to do_node(), except that this function doesn't
- # create a path for n's destination directory (and so ignores
- # n.fullname). It assumes the destination is '.', and restores
- # n's metadata and content there.
- global total_restored, opt
- meta_stream = None
- try:
- # Directory metadata is the first entry in any .bupm file in
- # the directory. Get it.
- mfile = n.metadata_file() # VFS file -- cannot close().
- root_meta = None
- if mfile:
- meta_stream = mfile.open()
- root_meta = metadata.Metadata.read(meta_stream)
- print_info(n, '.')
- total_restored += 1
- plog('Restoring: %d\r' % total_restored)
- for sub in n:
- m = None
- # Don't get metadata if this is a dir -- handled in sub do_node().
- if meta_stream and not stat.S_ISDIR(sub.mode):
- m = metadata.Metadata.read(meta_stream)
- do_node(n, sub, sparse, owner_map, meta = m)
- if root_meta and restore_root_meta:
- apply_metadata(root_meta, '.', opt.numeric_ids, owner_map)
- finally:
- if meta_stream:
- meta_stream.close()
+ if not treeish:
+ # Do this now so we'll have meta.symlink_target for verbose output
+ item = vfs2.augment_item_meta(repo, item, include_size=True)
+ meta = item.meta
+ assert(meta.mode == mode)
+
+ if stat.S_ISDIR(mode):
+ if verbosity >= 1:
+ print('%s/' % fullname)
+ elif stat.S_ISLNK(mode):
+ assert(meta.symlink_target)
+ if verbosity >= 2:
+ print('%s@ -> %s' % (fullname, meta.symlink_target))
+ else:
+ if verbosity >= 2:
+ print(fullname)
-def do_node(top, n, sparse, owner_map, meta = None):
- # Create n.fullname(), relative to the current directory, and
- # restore all of its metadata, when available. The meta argument
- # will be None for dirs, or when there is no .bupm (i.e. no
- # metadata).
- global total_restored, opt
- meta_stream = None
- write_content = sparse and write_file_content_sparsely or write_file_content
+ orig_cwd = os.getcwd()
try:
- fullname = n.fullname(stop_at=top)
- # Match behavior of index --exclude-rx with respect to paths.
- exclude_candidate = '/' + fullname
- if(stat.S_ISDIR(n.mode)):
- exclude_candidate += '/'
- if should_rx_exclude_path(exclude_candidate, exclude_rxs):
- return
- # If this is a directory, its metadata is the first entry in
- # any .bupm file inside the directory. Get it.
- if(stat.S_ISDIR(n.mode)):
- mfile = n.metadata_file() # VFS file -- cannot close().
- if mfile:
- meta_stream = mfile.open()
- meta = metadata.Metadata.read(meta_stream)
- print_info(n, fullname)
-
- created_hardlink = False
- if meta and meta.hardlink_target:
- created_hardlink = hardlink_if_possible(fullname, n, meta)
-
- if not created_hardlink:
- create_path(n, fullname, meta)
- if meta:
+ if treeish:
+ # Assumes contents() returns '.' with the full metadata first
+ sub_items = vfs2.contents(repo, item, want_meta=True)
+ dot, item = next(sub_items, None)
+ assert(dot == '.')
+ item = vfs2.augment_item_meta(repo, item, include_size=True)
+ meta = item.meta
+ meta.create_path(name)
+ os.chdir(name)
+ total_restored += 1
+ if verbosity >= 0:
+ qprogress('Restoring: %d\r' % total_restored)
+ for sub_name, sub_item in sub_items:
+ restore(repo, fullname, sub_name, sub_item, top, sparse,
+ numeric_ids, owner_map, exclude_rxs, verbosity,
+ hardlinks)
+ os.chdir('..')
+ apply_metadata(meta, name, numeric_ids, owner_map)
+ else:
+ created_hardlink = False
+ if meta.hardlink_target:
+ created_hardlink = hardlink_if_possible(fullname, item, top,
+ hardlinks)
+ if not created_hardlink:
+ meta.create_path(name)
if stat.S_ISREG(meta.mode):
- write_content(fullname, n)
- elif stat.S_ISREG(n.mode):
- write_content(fullname, n)
-
- total_restored += 1
- plog('Restoring: %d\r' % total_restored)
- for sub in n:
- m = None
- # Don't get metadata if this is a dir -- handled in sub do_node().
- if meta_stream and not stat.S_ISDIR(sub.mode):
- m = metadata.Metadata.read(meta_stream)
- do_node(top, sub, sparse, owner_map, meta = m)
- if meta and not created_hardlink:
- apply_metadata(meta, fullname, opt.numeric_ids, owner_map)
+ if sparse:
+ write_file_content_sparsely(repo, name, item)
+ else:
+ write_file_content(repo, name, item)
+ total_restored += 1
+ if verbosity >= 0:
+ qprogress('Restoring: %d\r' % total_restored)
+ if not created_hardlink:
+ apply_metadata(meta, name, numeric_ids, owner_map)
finally:
- if meta_stream:
- meta_stream.close()
- n.release()
-
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-git.check_repo_or_die()
-top = vfs.RefList(None)
+ os.chdir(orig_cwd)
-if not extra:
- o.fatal('must specify at least one filename to restore')
+def main():
+ o = options.Options(optspec)
+ opt, flags, extra = o.parse(sys.argv[1:])
+ verbosity = opt.verbose if not opt.quiet else -1
-exclude_rxs = parse_rx_excludes(flags, o.fatal)
+ git.check_repo_or_die()
-owner_map = {}
-for map_type in ('user', 'group', 'uid', 'gid'):
- owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
+ if not extra:
+ o.fatal('must specify at least one filename to restore')
-if opt.outdir:
- mkdirp(opt.outdir)
- os.chdir(opt.outdir)
+ exclude_rxs = parse_rx_excludes(flags, o.fatal)
-ret = 0
-for d in extra:
- if not valid_restore_path(d):
- add_error("ERROR: path %r doesn't include a branch and revision" % d)
- continue
- path,name = os.path.split(d)
- try:
- n = top.lresolve(d)
- except vfs.NodeError as e:
- add_error(e)
- continue
- isdir = stat.S_ISDIR(n.mode)
- if not name or name == '.':
- # Source is /foo/what/ever/ or /foo/what/ever/. -- extract
- # what/ever/* to the current directory, and if name == '.'
- # (i.e. /foo/what/ever/.), then also restore what/ever's
- # metadata to the current directory.
- if not isdir:
- add_error('%r: not a directory' % d)
+ owner_map = {}
+ for map_type in ('user', 'group', 'uid', 'gid'):
+ owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
+
+ if opt.outdir:
+ mkdirp(opt.outdir)
+ os.chdir(opt.outdir)
+
+ repo = LocalRepo()
+ top = os.getcwd()
+ hardlinks = {}
+ for path in extra:
+ if not valid_restore_path(path):
+ add_error("path %r doesn't include a branch and revision" % path)
+ continue
+ try:
+ resolved = vfs2.lresolve(repo, path, want_meta=True)
+ except vfs2.IOError as e:
+ add_error(e)
+ continue
+ path_parent, path_name = os.path.split(path)
+ leaf_name, leaf_item = resolved[-1]
+ if not leaf_item:
+ add_error('error: cannot access %r in %r'
+ % ('/'.join(name for name, item in resolved),
+ path))
+ continue
+ if not path_name or path_name == '.':
+ # Source is /foo/what/ever/ or /foo/what/ever/. -- extract
+ # what/ever/* to the current directory, and if name == '.'
+ # (i.e. /foo/what/ever/.), then also restore what/ever's
+ # metadata to the current directory.
+ treeish = vfs2.item_mode(leaf_item)
+ if not treeish:
+ add_error('%r cannot be restored as a directory' % path)
+ else:
+ items = vfs2.contents(repo, leaf_item, want_meta=True)
+ dot, leaf_item = next(items, None)
+ assert(dot == '.')
+ for sub_name, sub_item in items:
+ restore(repo, '', sub_name, sub_item, top,
+ opt.sparse, opt.numeric_ids, owner_map,
+ exclude_rxs, verbosity, hardlinks)
+ if path_name == '.':
+ leaf_item = vfs2.augment_item_meta(repo, leaf_item,
+ include_size=True)
+ apply_metadata(leaf_item.meta, '.',
+ opt.numeric_ids, owner_map)
else:
- do_root(n, opt.sparse, owner_map, restore_root_meta = (name == '.'))
- else:
- # Source is /foo/what/ever -- extract ./ever to cwd.
- if isinstance(n, vfs.FakeSymlink):
- # Source is actually /foo/what, i.e. a top-level commit
- # like /foo/latest, which is a symlink to ../.commit/SHA.
- # So dereference it, and restore ../.commit/SHA/. to
- # "./what/.".
- target = n.dereference()
- mkdirp(n.name)
- os.chdir(n.name)
- do_root(target, opt.sparse, owner_map)
- else: # Not a directory or fake symlink.
- meta = find_dir_item_metadata_by_name(n.parent, n.name)
- do_node(n.parent, n, opt.sparse, owner_map, meta = meta)
+ restore(repo, '', leaf_name, leaf_item, top,
+ opt.sparse, opt.numeric_ids, owner_map,
+ exclude_rxs, verbosity, hardlinks)
-if not opt.quiet:
- progress('Restoring: %d, done.\n' % total_restored)
+ if verbosity >= 0:
+ progress('Restoring: %d, done.\n' % total_restored)
+ die_if_errors()
-if saved_errors:
- log('WARNING: %d errors encountered while restoring.\n' % len(saved_errors))
- sys.exit(1)
+wrap_main(main)
"""
# end of bup preamble
-import os, sys, struct
+import os, sys, struct, subprocess
from bup import options, git
-from bup.helpers import Conn, debug1, debug2, linereader, log
+from bup.git import MissingObject
+from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel,
+ log)
suspended_w = None
git.update_ref(refname, newval.decode('hex'), oldval.decode('hex'))
conn.ok()
-
-cat_pipe = None
-def cat(conn, id):
- global cat_pipe
+def join(conn, id):
_init_session()
- if not cat_pipe:
- cat_pipe = git.CatPipe()
try:
- for blob in cat_pipe.join(id):
+ for blob in git.cp().join(id):
conn.write(struct.pack('!I', len(blob)))
conn.write(blob)
except KeyError as e:
conn.write('\0\0\0\0')
conn.ok()
+def cat_batch(conn, dummy):
+ _init_session()
+ cat_pipe = git.cp()
+ # For now, avoid potential deadlock by just reading them all
+ for ref in tuple(lines_until_sentinel(conn, '\n', Exception)):
+ ref = ref[:-1]
+ it = cat_pipe.get(ref)
+ info = next(it)
+ if not info[0]:
+ conn.write('missing\n')
+ continue
+ conn.write('%s %s %d\n' % info)
+ for buf in it:
+ conn.write(buf)
+ conn.ok()
+
+def refs(conn, args):
+ limit_to_heads, limit_to_tags = args.split()
+ assert limit_to_heads in ('0', '1')
+ assert limit_to_tags in ('0', '1')
+ limit_to_heads = int(limit_to_heads)
+ limit_to_tags = int(limit_to_tags)
+ _init_session()
+ patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception))
+ for name, oid in git.list_refs(patterns=patterns,
+ limit_to_heads=limit_to_heads,
+ limit_to_tags=limit_to_tags):
+ assert '\n' not in name
+ conn.write('%s %s\n' % (oid.encode('hex'), name))
+ conn.write('\n')
+ conn.ok()
+
+def rev_list(conn, _):
+ _init_session()
+ count = conn.readline()
+ if not count:
+ raise Exception('Unexpected EOF while reading rev-list count')
+ count = None if count == '\n' else int(count)
+ fmt = conn.readline()
+ if not fmt:
+ raise Exception('Unexpected EOF while reading rev-list format')
+ fmt = None if fmt == '\n' else fmt[:-1]
+ refs = tuple(x[:-1] for x in lines_until_sentinel(conn, '\n', Exception))
+ args = git.rev_list_invocation(refs, count=count, format=fmt)
+ p = subprocess.Popen(git.rev_list_invocation(refs, count=count, format=fmt),
+ preexec_fn=git._gitenv(git.repodir),
+ stdout=subprocess.PIPE)
+ while True:
+ out = p.stdout.read(64 * 1024)
+ if not out:
+ break
+ conn.write(out)
+ rv = p.wait() # not fatal
+ if rv:
+ msg = 'git rev-list returned error %d' % rv
+ conn.error(msg)
+ raise GitError(msg)
+ conn.ok()
+
optspec = """
bup server
'receive-objects-v2': receive_objects_v2,
'read-ref': read_ref,
'update-ref': update_ref,
- 'cat': cat,
+ 'join': join,
+ 'cat': join, # apocryphal alias
+ 'cat-batch' : cat_batch,
+ 'refs': refs,
+ 'rev-list': rev_list
}
# FIXME: this protocol is totally lame and not at all future-proof.
from bup import git, ssh
from bup.helpers import (Conn, atomically_replaced_file, chunkyreader, debug1,
- debug2, linereader, mkdirp, progress, qprogress)
+ debug2, linereader, lines_until_sentinel,
+ mkdirp, progress, qprogress)
bwlimit = None
(oldval or '').encode('hex')))
self.check_ok()
- def cat(self, id):
- self._require_command('cat')
+ def join(self, id):
+ self._require_command('join')
self.check_busy()
- self._busy = 'cat'
+ self._busy = 'join'
+ # Send 'cat' so we'll work fine with older versions
self.conn.write('cat %s\n' % re.sub(r'[\n\r]', '_', id))
while 1:
sz = struct.unpack('!I', self.conn.read(4))[0]
if not sz: break
yield self.conn.read(sz)
+ # FIXME: ok to assume the only NotOk is a KerError? (it is true atm)
e = self.check_ok()
self._not_busy()
if e:
raise KeyError(str(e))
+ def cat_batch(self, refs):
+ self._require_command('cat-batch')
+ self.check_busy()
+ self._busy = 'cat-batch'
+ conn = self.conn
+ conn.write('cat-batch\n')
+ # FIXME: do we want (only) binary protocol?
+ for ref in refs:
+ assert ref
+ assert '\n' not in ref
+ conn.write(ref)
+ conn.write('\n')
+ conn.write('\n')
+ for ref in refs:
+ info = conn.readline()
+ if info == 'missing\n':
+ yield None, None, None, None
+ continue
+ if not (info and info.endswith('\n')):
+ raise ClientError('Hit EOF while looking for object info: %r'
+ % info)
+ oidx, oid_t, size = info.split(' ')
+ size = int(size)
+ cr = chunkyreader(conn, size)
+ yield oidx, oid_t, size, cr
+ detritus = next(cr, None)
+ if detritus:
+ raise ClientError('unexpected leftover data ' + repr(detritus))
+ # FIXME: confusing
+ not_ok = self.check_ok()
+ if not_ok:
+ raise not_ok
+ self._not_busy()
+
+ def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False):
+ patterns = patterns or tuple()
+ self._require_command('refs')
+ self.check_busy()
+ self._busy = 'refs'
+ conn = self.conn
+ conn.write('refs %s %s\n' % (1 if limit_to_heads else 0,
+ 1 if limit_to_tags else 0))
+ for pattern in patterns:
+ assert '\n' not in pattern
+ conn.write(pattern)
+ conn.write('\n')
+ conn.write('\n')
+ for line in lines_until_sentinel(conn, '\n', ClientError):
+ line = line[:-1]
+ oidx, name = line.split(' ')
+ if len(oidx) != 40:
+ raise ClientError('Invalid object fingerprint in %r' % line)
+ if not name:
+ raise ClientError('Invalid reference name in %r' % line)
+ yield name, oidx.decode('hex')
+ # FIXME: confusing
+ not_ok = self.check_ok()
+ if not_ok:
+ raise not_ok
+ self._not_busy()
+
+ def rev_list(self, refs, count=None, parse=None, format=None):
+ self._require_command('rev-list')
+ assert (count is None) or (isinstance(count, Integral))
+ if format:
+ assert '\n' not in format
+ assert parse
+ for ref in refs:
+ assert ref
+ assert '\n' not in ref
+ self.check_busy()
+ self._busy = 'rev-list'
+ conn = self.conn
+ conn.write('rev-list\n')
+ if count is not None:
+ conn.write(str(count))
+ conn.write('\n')
+ if format:
+ conn.write(format)
+ conn.write('\n')
+ for ref in refs:
+ conn.write(ref)
+ conn.write('\n')
+ conn.write('\n')
+ if not format:
+ for _ in xrange(len(refs)):
+ line = conn.readline()
+ if not line:
+ raise ClientError('unexpected EOF')
+ line = line.strip()
+ assert len(line) == 40
+ yield line
+ else:
+ for _ in xrange(len(refs)):
+ line = conn.readline()
+ if not line:
+ raise ClientError('unexpected EOF')
+ if not line.startswith('commit '):
+ raise ClientError('unexpected line ' + repr(line))
+ yield line[7:], parse(conn)
+ # FIXME: confusing
+ not_ok = self.check_ok()
+ if not_ok:
+ raise not_ok
+ self._not_busy()
+
class PackWriter_Remote(git.PackWriter):
def __init__(self, conn, objcache_maker, suggest_packs,
return None
-def rev_list(ref_or_refs, count=None, parse=None, format=None, repo_dir=None):
- """Yield information about commits as per "git rev-list". If a format
- is not provided, yield one hex hash at a time. If a format is
- provided, pass it to rev-list and call parse(git_stdout) for each
- commit with the stream positioned just after the rev-list "commit
- HASH" header line. When a format is provided yield (oidx,
- parse(git_stdout)) for each commit.
-
- """
- assert bool(parse) == bool(format)
+def rev_list_invocation(ref_or_refs, count=None, format=None):
if isinstance(ref_or_refs, compat.str_type):
refs = (ref_or_refs,)
else:
argv = ['git', 'rev-list']
if isinstance(count, Integral):
argv.extend(['-n', str(count)])
- else:
- assert not count
+ elif count:
+ raise ValueError('unexpected count argument %r' % count)
+
if format:
argv.append('--pretty=format:' + format)
for ref in refs:
assert not ref.startswith('-')
argv.append(ref)
argv.append('--')
- p = subprocess.Popen(argv,
+ return argv
+
+
+def rev_list(ref_or_refs, count=None, parse=None, format=None, repo_dir=None):
+ """Yield information about commits as per "git rev-list". If a format
+ is not provided, yield one hex hash at a time. If a format is
+ provided, pass it to rev-list and call parse(git_stdout) for each
+ commit with the stream positioned just after the rev-list "commit
+ HASH" header line. When a format is provided yield (oidx,
+ parse(git_stdout)) for each commit.
+
+ """
+ assert bool(parse) == bool(format)
+ p = subprocess.Popen(rev_list_invocation(ref_or_refs, count=count,
+ format=format),
preexec_fn = _gitenv(repo_dir),
stdout = subprocess.PIPE)
if not format:
"""Helper functions and classes for bup."""
from collections import namedtuple
+from contextlib import contextmanager
from ctypes import sizeof, c_void_p
from os import environ
-from contextlib import contextmanager
+from pipes import quote
+from subprocess import PIPE, Popen
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
import hashlib, heapq, math, operator, time, grp, tempfile
from bup import _helpers
-
+from bup import compat
class Nonlocal:
"""Helper to deal with Python scoping issues"""
tty_width = _tty_width
+def last(iterable):
+ result = None
+ for result in iterable:
+ pass
+ return result
+
+
def atoi(s):
"""Convert the string 's' to an integer. Return 0 if s is not a number."""
try:
return (leading_matches(), rest())
+def lines_until_sentinel(f, sentinel, ex_type):
+ # sentinel must end with \n and must contain only one \n
+ while True:
+ line = f.readline()
+ if not (line and line.endswith('\n')):
+ raise ex_type('Hit EOF while reading line')
+ if line == sentinel:
+ return
+ yield line
+
+
def stat_if_exists(path):
try:
return os.stat(path)
raise
+def shstr(cmd):
+ if isinstance(cmd, compat.str_type):
+ return cmd
+ else:
+ return ' '.join(map(quote, cmd))
+
+exc = subprocess.check_call
+
+def exo(cmd,
+ input=None,
+ stdin=None,
+ stderr=None,
+ shell=False,
+ check=True,
+ preexec_fn=None):
+ if input:
+ assert stdin in (None, PIPE)
+ stdin = PIPE
+ p = Popen(cmd,
+ stdin=stdin, stdout=PIPE, stderr=stderr,
+ shell=shell,
+ preexec_fn=preexec_fn)
+ out, err = p.communicate(input)
+ if check and p.returncode != 0:
+ raise Exception('subprocess %r failed with status %d, stderr: %r'
+ % (' '.join(map(quote, cmd)), p.returncode, err))
+ return out, err, p
+
def readpipe(argv, preexec_fn=None, shell=False):
"""Run a subprocess and return its output."""
p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn,
# This code is covered under the terms of the GNU Library General
# Public License as described in the bup LICENSE file.
+from copy import deepcopy
from errno import EACCES, EINVAL, ENOTTY, ENOSYS, EOPNOTSUPP
from io import BytesIO
from time import gmtime, strftime
self.linux_xattr = None
self.posix1e_acl = None
+ def __eq__(self, other):
+ if not isinstance(other, Metadata): return False
+ if self.mode != other.mode: return False
+ if self.mtime != other.mtime: return False
+ if self.ctime != other.ctime: return False
+ if self.atime != other.atime: return False
+ if self.path != other.path: return False
+ if self.uid != other.uid: return False
+ if self.gid != other.gid: return False
+ if self.size != other.size: return False
+ if self.user != other.user: return False
+ if self.group != other.group: return False
+ if self.symlink_target != other.symlink_target: return False
+ if self.hardlink_target != other.hardlink_target: return False
+ if self.linux_attr != other.linux_attr: return False
+ if self.posix1e_acl != other.posix1e_acl: return False
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __hash__(self):
+ return hash((self.mode,
+ self.mtime,
+ self.ctime,
+ self.atime,
+ self.path,
+ self.uid,
+ self.gid,
+ self.size,
+ self.user,
+ self.group,
+ self.symlink_target,
+ self.hardlink_target,
+ self.linux_attr,
+ self.posix1e_acl))
+
def __repr__(self):
result = ['<%s instance at %s' % (self.__class__, hex(id(self)))]
if self.path is not None:
self.write(port, include_path)
return port.getvalue()
+ def copy(self):
+ return deepcopy(self)
+
@staticmethod
def read(port):
# This method should either return a valid Metadata object,
+from functools import partial
+
from bup import client, git
def __init__(self, repo_dir=None):
self.repo_dir = repo_dir or git.repo()
self._cp = git.cp(repo_dir)
+ self.rev_list = partial(git.rev_list, repo_dir=repo_dir)
+
+ def cat(self, ref):
+ """If ref does not exist, yield (None, None, None). Otherwise yield
+ (oidx, type, size), and then all of the data associated with
+ ref.
+
+ """
+ it = self._cp.get(ref)
+ oidx, typ, size = info = next(it)
+ yield info
+ if oidx:
+ for data in it:
+ yield data
+ assert not next(it, None)
def join(self, ref):
return self._cp.join(ref)
+ def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False):
+ for ref in git.list_refs(patterns=patterns,
+ limit_to_heads=limit_to_heads,
+ limit_to_tags=limit_to_tags,
+ repo_dir=self.repo_dir):
+ yield ref
+
class RemoteRepo:
def __init__(self, address):
self.address = address
self.client = client.Client(address)
+ self.rev_list = self.client.rev_list
+
+ def cat(self, ref):
+ """If ref does not exist, yield (None, None, None). Otherwise yield
+ (oidx, type, size), and then all of the data associated with
+ ref.
+
+ """
+ # Yield all the data here so that we don't finish the
+ # cat_batch iterator (triggering its cleanup) until all of the
+ # data has been read. Otherwise we'd be out of sync with the
+ # server.
+ items = self.client.cat_batch((ref,))
+ oidx, typ, size, it = info = next(items)
+ yield info[:-1]
+ if oidx:
+ for data in it:
+ yield data
+ assert not next(items, None)
def join(self, ref):
- return self.client.cat(ref)
+ return self.client.join(ref)
+
+ def refs(self, patterns=None, limit_to_heads=False, limit_to_tags=False):
+ for ref in self.client.refs(patterns=patterns,
+ limit_to_heads=limit_to_heads,
+ limit_to_tags=limit_to_tags):
+ yield ref
--- /dev/null
+
+from __future__ import print_function
+from collections import namedtuple
+from io import BytesIO
+from os import environ, symlink
+from stat import S_IFDIR, S_IFREG, S_ISDIR, S_ISREG
+from sys import stderr
+from time import localtime, strftime
+
+from wvtest import *
+
+from bup import git, metadata, vfs2 as vfs
+from bup.git import BUP_CHUNKED
+from bup.helpers import exc, exo, shstr
+from bup.metadata import Metadata
+from bup.repo import LocalRepo
+from buptest import no_lingering_errors, test_tempdir
+
+top_dir = '../../..'
+bup_tmp = os.path.realpath('../../../t/tmp')
+bup_path = top_dir + '/bup'
+start_dir = os.getcwd()
+
+def ex(cmd, **kwargs):
+ print(shstr(cmd), file=stderr)
+ return exc(cmd, **kwargs)
+
+TreeDictValue = namedtuple('TreeDictValue', ('name', 'oid', 'meta'))
+
+def tree_items(repo, oid):
+ """Yield (name, entry_oid, meta) for each entry in oid. meta will be
+ a Metadata object for any non-directories and for '.', otherwise
+ None.
+
+ """
+ # This is a simpler approach than the one in the vfs, used to
+ # cross-check its behavior.
+ tree_data, bupm_oid = vfs._tree_data_and_bupm(repo, oid)
+ bupm = vfs._FileReader(repo, bupm_oid) if bupm_oid else None
+ try:
+ maybe_meta = lambda : Metadata.read(bupm) if bupm else None
+ m = maybe_meta()
+ if m:
+ m.size = 0
+ yield TreeDictValue(name='.', oid=oid, meta=m)
+ tree_ents = vfs.ordered_tree_entries(tree_data, bupm=True)
+ for name, mangled_name, kind, gitmode, sub_oid in tree_ents:
+ if mangled_name == '.bupm':
+ continue
+ assert name != '.'
+ if S_ISDIR(gitmode):
+ if kind == BUP_CHUNKED:
+ yield TreeDictValue(name=name, oid=sub_oid,
+ meta=maybe_meta())
+ else:
+ yield TreeDictValue(name=name, oid=sub_oid,
+ meta=vfs.default_dir_mode)
+ else:
+ yield TreeDictValue(name=name, oid=sub_oid, meta=maybe_meta())
+ finally:
+ if bupm:
+ bupm.close()
+
+def tree_dict(repo, oid):
+ return dict((x.name, x) for x in tree_items(repo, oid))
+
+def run_augment_item_meta_tests(repo,
+ file_path, file_size,
+ link_path, link_target):
+ _, file_item = vfs.resolve(repo, file_path)[-1]
+ _, link_item = vfs.lresolve(repo, link_path)[-1]
+ wvpass(isinstance(file_item.meta, Metadata))
+ wvpass(isinstance(link_item.meta, Metadata))
+ # Note: normally, modifying item.meta values is forbidden
+ file_item.meta.size = file_item.meta.size or vfs.item_size(repo, file_item)
+ link_item.meta.size = link_item.meta.size or vfs.item_size(repo, link_item)
+
+ ## Ensure a fully populated item is left alone
+ augmented = vfs.augment_item_meta(repo, file_item)
+ wvpass(augmented is file_item)
+ wvpass(augmented.meta is file_item.meta)
+ augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
+ wvpass(augmented is file_item)
+ wvpass(augmented.meta is file_item.meta)
+
+ ## Ensure a missing size is handled poperly
+ file_item.meta.size = None
+ augmented = vfs.augment_item_meta(repo, file_item)
+ wvpass(augmented is file_item)
+ wvpass(augmented.meta is file_item.meta)
+ augmented = vfs.augment_item_meta(repo, file_item, include_size=True)
+ wvpass(augmented is not file_item)
+ wvpasseq(file_size, augmented.meta.size)
+
+ ## Ensure a meta mode is handled properly
+ mode_item = file_item._replace(meta=vfs.default_file_mode)
+ augmented = vfs.augment_item_meta(repo, mode_item)
+ augmented_w_size = vfs.augment_item_meta(repo, mode_item, include_size=True)
+ for item in (augmented, augmented_w_size):
+ meta = item.meta
+ wvpass(item is not file_item)
+ wvpass(isinstance(meta, Metadata))
+ wvpasseq(vfs.default_file_mode, meta.mode)
+ wvpasseq((0, 0, 0, 0, 0),
+ (meta.uid, meta.gid, meta.atime, meta.mtime, meta.ctime))
+ wvpass(augmented.meta.size is None)
+ wvpasseq(file_size, augmented_w_size.meta.size)
+
+ ## Ensure symlinks are handled properly
+ mode_item = link_item._replace(meta=vfs.default_symlink_mode)
+ augmented = vfs.augment_item_meta(repo, mode_item)
+ wvpass(augmented is not mode_item)
+ wvpass(isinstance(augmented.meta, Metadata))
+ wvpasseq(link_target, augmented.meta.symlink_target)
+ wvpasseq(len(link_target), augmented.meta.size)
+ augmented = vfs.augment_item_meta(repo, mode_item, include_size=True)
+ wvpass(augmented is not mode_item)
+ wvpass(isinstance(augmented.meta, Metadata))
+ wvpasseq(link_target, augmented.meta.symlink_target)
+ wvpasseq(len(link_target), augmented.meta.size)
+
+
+@wvtest
+def test_item_mode():
+ with no_lingering_errors():
+ mode = S_IFDIR | 0o755
+ meta = metadata.from_path('.')
+ oid = '\0' * 20
+ wvpasseq(mode, vfs.item_mode(vfs.Item(oid=oid, meta=mode)))
+ wvpasseq(meta.mode, vfs.item_mode(vfs.Item(oid=oid, meta=meta)))
+
+@wvtest
+def test_misc():
+ with no_lingering_errors():
+ with test_tempdir('bup-tvfs-') as tmpdir:
+ bup_dir = tmpdir + '/bup'
+ environ['GIT_DIR'] = bup_dir
+ environ['BUP_DIR'] = bup_dir
+ git.repodir = bup_dir
+ data_path = tmpdir + '/src'
+ os.mkdir(data_path)
+ with open(data_path + '/file', 'w+') as tmpfile:
+ tmpfile.write(b'canary\n')
+ symlink('file', data_path + '/symlink')
+ ex((bup_path, 'init'))
+ ex((bup_path, 'index', '-v', data_path))
+ ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
+ data_path))
+ repo = LocalRepo()
+
+ wvstart('readlink')
+ ls_tree = exo(('git', 'ls-tree', 'test', 'symlink'))
+ mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
+ assert name == 'symlink'
+ link_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
+ wvpasseq('file', vfs.readlink(repo, link_item))
+
+ ls_tree = exo(('git', 'ls-tree', 'test', 'file'))
+ mode, typ, oidx, name = ls_tree[0].strip().split(None, 3)
+ assert name == 'file'
+ file_item = vfs.Item(oid=oidx.decode('hex'), meta=int(mode, 8))
+ wvexcept(Exception, vfs.readlink, repo, file_item)
+
+ wvstart('item_size')
+ wvpasseq(4, vfs.item_size(repo, link_item))
+ wvpasseq(7, vfs.item_size(repo, file_item))
+ meta = metadata.from_path(__file__)
+ meta.size = 42
+ fake_item = file_item._replace(meta=meta)
+ wvpasseq(42, vfs.item_size(repo, fake_item))
+
+ wvstart('augment_item_meta')
+ run_augment_item_meta_tests(repo,
+ '/test/latest/file', 7,
+ '/test/latest/symlink', 'file')
+
+ wvstart('copy_item')
+ # FIXME: this caused StopIteration
+ #_, file_item = vfs.resolve(repo, '/file')[-1]
+ _, file_item = vfs.resolve(repo, '/test/latest/file')[-1]
+ file_copy = vfs.copy_item(file_item)
+ wvpass(file_copy is not file_item)
+ wvpass(file_copy.meta is not file_item.meta)
+ wvpass(isinstance(file_copy, tuple))
+ wvpass(file_item.meta.user)
+ wvpass(file_copy.meta.user)
+ file_copy.meta.user = None
+ wvpass(file_item.meta.user)
+
+@wvtest
+def test_resolve():
+ with no_lingering_errors():
+ with test_tempdir('bup-tvfs-') as tmpdir:
+ resolve = vfs.resolve
+ lresolve = vfs.lresolve
+ bup_dir = tmpdir + '/bup'
+ environ['GIT_DIR'] = bup_dir
+ environ['BUP_DIR'] = bup_dir
+ git.repodir = bup_dir
+ data_path = tmpdir + '/src'
+ save_time = 100000
+ save_time_str = strftime('%Y-%m-%d-%H%M%S', localtime(save_time))
+ os.mkdir(data_path)
+ with open(data_path + '/file', 'w+') as tmpfile:
+ print('canary', file=tmpfile)
+ symlink('file', data_path + '/symlink')
+ ex((bup_path, 'init'))
+ ex((bup_path, 'index', '-v', data_path))
+ ex((bup_path, 'save', '-d', str(save_time), '-tvvn', 'test',
+ '--strip', data_path))
+ ex((bup_path, 'tag', 'test-tag', 'test'))
+ repo = LocalRepo()
+
+ tip_hash = exo(('git', 'show-ref', 'refs/heads/test'))[0]
+ tip_oidx = tip_hash.strip().split()[0]
+ tip_oid = tip_oidx.decode('hex')
+ tip_meta = Metadata()
+ tip_meta.mode = S_IFDIR | 0o755
+ tip_meta.uid = tip_meta.gid = tip_meta.size = 0
+ tip_meta.atime = tip_meta.mtime = tip_meta.ctime = save_time * 10**9
+ test_revlist = vfs.RevList(meta=tip_meta, oid=tip_oid)
+ tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
+ tip_oidx))[0].strip()
+ tip_tree_oid = tip_tree_oidx.decode('hex')
+ tip_tree = tree_dict(repo, tip_tree_oid)
+
+ wvstart('resolve: /')
+ res = resolve(repo, '/')
+ wvpasseq(1, len(res))
+ wvpasseq((('', vfs._root),), res)
+ ignore, root_item = res[0]
+ root_content = frozenset(vfs.contents(repo, root_item))
+ wvpasseq(frozenset([('.', root_item),
+ ('.tag', vfs._tags),
+ ('test', test_revlist)]),
+ root_content)
+
+ wvstart('resolve: /.tag')
+ res = resolve(repo, '/.tag')
+ wvpasseq(2, len(res))
+ wvpasseq((('', vfs._root), ('.tag', vfs._tags)),
+ res)
+ ignore, tag_item = res[1]
+ tag_content = frozenset(vfs.contents(repo, tag_item))
+ wvpasseq(frozenset([('.', tag_item),
+ ('test-tag', test_revlist)]),
+ tag_content)
+
+ wvstart('resolve: /test')
+ res = resolve(repo, '/test')
+ wvpasseq(2, len(res))
+ wvpasseq((('', vfs._root), ('test', test_revlist)), res)
+ ignore, test_item = res[1]
+ test_content = frozenset(vfs.contents(repo, test_item))
+ expected_latest_item = vfs.Item(meta=S_IFDIR | 0o755,
+ oid=tip_tree_oid)
+ wvpasseq(frozenset([('.', test_revlist),
+ (save_time_str, expected_latest_item),
+ ('latest', expected_latest_item)]),
+ test_content)
+
+ wvstart('resolve: /test/latest')
+ res = resolve(repo, '/test/latest')
+ wvpasseq(3, len(res))
+ expected_latest_item_w_meta = vfs.Item(meta=tip_tree['.'].meta,
+ oid=tip_tree_oid)
+ expected = (('', vfs._root),
+ ('test', test_revlist),
+ ('latest', expected_latest_item_w_meta))
+ wvpasseq(expected, res)
+ ignore, latest_item = res[2]
+ latest_content = frozenset(vfs.contents(repo, latest_item))
+ expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
+ for x in (tip_tree[name]
+ for name in ('.', 'file',
+ 'symlink')))
+ wvpasseq(expected, latest_content)
+
+ wvstart('resolve: /test/latest/foo')
+ res = resolve(repo, '/test/latest/file')
+ wvpasseq(4, len(res))
+ expected_file_item_w_meta = vfs.Item(meta=tip_tree['file'].meta,
+ oid=tip_tree['file'].oid)
+ expected = (('', vfs._root),
+ ('test', test_revlist),
+ ('latest', expected_latest_item_w_meta),
+ ('file', expected_file_item_w_meta))
+ wvpasseq(expected, res)
+
+ wvstart('resolve: /test/latest/symlink')
+ res = resolve(repo, '/test/latest/symlink')
+ wvpasseq(4, len(res))
+ expected = (('', vfs._root),
+ ('test', test_revlist),
+ ('latest', expected_latest_item_w_meta),
+ ('file', expected_file_item_w_meta))
+ wvpasseq(expected, res)
+
+ wvstart('lresolve: /test/latest/symlink')
+ res = lresolve(repo, '/test/latest/symlink')
+ wvpasseq(4, len(res))
+ symlink_value = tip_tree['symlink']
+ expected_symlink_item_w_meta = vfs.Item(meta=symlink_value.meta,
+ oid=symlink_value.oid)
+ expected = (('', vfs._root),
+ ('test', test_revlist),
+ ('latest', expected_latest_item_w_meta),
+ ('symlink', expected_symlink_item_w_meta))
+ wvpasseq(expected, res)
+
+ wvstart('resolve: /test/latest/missing')
+ res = resolve(repo, '/test/latest/missing')
+ wvpasseq(4, len(res))
+ name, item = res[-1]
+ wvpasseq('missing', name)
+ wvpass(item is None)
+
+@wvtest
+def test_resolve_loop():
+ with no_lingering_errors():
+ with test_tempdir('bup-tvfs-resloop-') as tmpdir:
+ resolve = vfs.resolve
+ lresolve = vfs.lresolve
+ bup_dir = tmpdir + '/bup'
+ environ['GIT_DIR'] = bup_dir
+ environ['BUP_DIR'] = bup_dir
+ git.repodir = bup_dir
+ repo = LocalRepo()
+ data_path = tmpdir + '/src'
+ os.mkdir(data_path)
+ symlink('loop', data_path + '/loop')
+ ex((bup_path, 'init'))
+ ex((bup_path, 'index', '-v', data_path))
+ ex((bup_path, 'save', '-d', '100000', '-tvvn', 'test', '--strip',
+ data_path))
+ wvexcept(vfs.Loop, resolve, repo, '/test/latest/loop')
+
+@wvtest
+def test_contents_with_mismatched_bupm_git_ordering():
+ with no_lingering_errors():
+ with test_tempdir('bup-tvfs-') as tmpdir:
+ bup_dir = tmpdir + '/bup'
+ environ['GIT_DIR'] = bup_dir
+ environ['BUP_DIR'] = bup_dir
+ git.repodir = bup_dir
+ data_path = tmpdir + '/src'
+ os.mkdir(data_path)
+ os.mkdir(data_path + '/foo')
+ with open(data_path + '/foo.', 'w+') as tmpfile:
+ tmpfile.write(b'canary\n')
+ ex((bup_path, 'init'))
+ ex((bup_path, 'index', '-v', data_path))
+ ex((bup_path, 'save', '-tvvn', 'test', '--strip',
+ data_path))
+ repo = LocalRepo()
+ tip_sref = exo(('git', 'show-ref', 'refs/heads/test'))[0]
+ tip_oidx = tip_sref.strip().split()[0]
+ tip_tree_oidx = exo(('git', 'log', '--pretty=%T', '-n1',
+ tip_oidx))[0].strip()
+ tip_tree_oid = tip_tree_oidx.decode('hex')
+ tip_tree = tree_dict(repo, tip_tree_oid)
+
+ name, item = vfs.resolve(repo, '/test/latest')[2]
+ wvpasseq('latest', name)
+ expected = frozenset((x.name, vfs.Item(oid=x.oid, meta=x.meta))
+ for x in (tip_tree[name]
+ for name in ('.', 'foo', 'foo.')))
+ contents = tuple(vfs.contents(repo, item))
+ wvpasseq(expected, frozenset(contents))
+ # Spot check, in case tree_dict shares too much code with the vfs
+ name, item = next(((n, i) for n, i in contents if n == 'foo'))
+ wvpass(S_ISDIR(item.meta))
+ name, item = next(((n, i) for n, i in contents if n == 'foo.'))
+ wvpass(S_ISREG(item.meta.mode))
+
+# FIXME: add tests for the want_meta=False cases.
--- /dev/null
+"""Virtual File System interface to bup repository content.
+
+This module provides a path-based interface to the content of a bup
+repository.
+
+The VFS is structured like this:
+
+ /SAVE-NAME/latest/...
+ /SAVE-NAME/SAVE-DATE/...
+ /.tag/TAG-NAME/...
+
+Each path is represented by an item that has least an item.meta which
+may be either a Metadata object, or an integer mode. Functions like
+item_mode() and item_size() will return the mode and size in either
+case. Any item.meta Metadata instances must not be modified directly.
+Make a copy to modify via item.meta.copy() if needed.
+
+The want_meta argument is advisory for calls that accept it, and it
+may not be honored. Callers must be able to handle an item.meta value
+that is either an instance of Metadata or an integer mode, perhaps
+via item_mode() or augment_item_meta().
+
+Setting want_meta=False is rarely desirable since it can limit the VFS
+to only the metadata that git itself can represent, and so for
+example, fifos and sockets will appear to be regular files
+(e.g. S_ISREG(item_mode(item)) will be true). But the option is still
+provided because it may be more efficient when just the path names or
+the more limited metadata is sufficient.
+
+Any given metadata object's size may be None, in which case the size
+can be computed via item_size() or augment_item_meta(...,
+include_size=True).
+
+When traversing a directory using functions like contents(), the meta
+value for any directories other than '.' will be a default directory
+mode, not a Metadata object. This is because the actual metadata for
+a directory is stored inside the directory.
+
+At the moment tagged commits (e.g. /.tag/some-commit) are represented
+as an item that is indistinguishable from a normal directory, so you
+cannot assume that the oid of an item satisfying
+S_ISDIR(item_mode(item)) refers to a tree.
+
+"""
+
+from __future__ import print_function
+from collections import namedtuple
+from errno import ELOOP, ENOENT, ENOTDIR
+from itertools import chain, dropwhile, izip
+from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISLNK, S_ISREG
+from time import localtime, strftime
+import exceptions, re, sys
+
+from bup import client, git, metadata
+from bup.git import BUP_CHUNKED, cp, get_commit_items, parse_commit, tree_decode
+from bup.helpers import debug2, last
+from bup.metadata import Metadata
+from bup.repo import LocalRepo, RemoteRepo
+
+
+class IOError(exceptions.IOError):
+ def __init__(self, errno, message):
+ exceptions.IOError.__init__(self, errno, message)
+
+class Loop(IOError):
+ def __init__(self, message, terminus=None):
+ IOError.__init__(self, ELOOP, message)
+ self.terminus = terminus
+
+default_file_mode = S_IFREG | 0o644
+default_dir_mode = S_IFDIR | 0o755
+default_symlink_mode = S_IFLNK | 0o755
+
+def _default_mode_for_gitmode(gitmode):
+ if S_ISREG(gitmode):
+ return default_file_mode
+ if S_ISDIR(gitmode):
+ return default_dir_mode
+ if S_ISLNK(gitmode):
+ return default_symlink_mode
+ raise Exception('unexpected git mode ' + oct(gitmode))
+
+def _normal_or_chunked_file_size(repo, oid):
+ """Return the size of the normal or chunked file indicated by oid."""
+ # FIXME: --batch-format CatPipe?
+ it = repo.cat(oid.encode('hex'))
+ _, obj_t, size = next(it)
+ ofs = 0
+ while obj_t == 'tree':
+ mode, name, last_oid = last(tree_decode(''.join(it)))
+ ofs += int(name, 16)
+ it = repo.cat(last_oid.encode('hex'))
+ _, obj_t, size = next(it)
+ return ofs + sum(len(b) for b in it)
+
+def _tree_chunks(repo, tree, startofs):
+ "Tree should be a sequence of (name, mode, hash) as per tree_decode()."
+ assert(startofs >= 0)
+ # name is the chunk's hex offset in the original file
+ tree = dropwhile(lambda (_1, name, _2): int(name, 16) < startofs, tree)
+ for mode, name, oid in tree:
+ ofs = int(name, 16)
+ skipmore = startofs - ofs
+ if skipmore < 0:
+ skipmore = 0
+ it = repo.cat(oid.encode('hex'))
+ _, obj_t, size = next(it)
+ data = ''.join(it)
+ if S_ISDIR(mode):
+ assert obj_t == 'tree'
+ for b in _tree_chunks(repo, tree_decode(data), skipmore):
+ yield b
+ else:
+ assert obj_t == 'blob'
+ yield data[skipmore:]
+
+class _ChunkReader:
+ def __init__(self, repo, oid, startofs):
+ it = repo.cat(oid.encode('hex'))
+ _, obj_t, size = next(it)
+ isdir = obj_t == 'tree'
+ data = ''.join(it)
+ if isdir:
+ self.it = _tree_chunks(repo, tree_decode(data), startofs)
+ self.blob = None
+ else:
+ self.it = None
+ self.blob = data[startofs:]
+ self.ofs = startofs
+
+ def next(self, size):
+ out = ''
+ while len(out) < size:
+ if self.it and not self.blob:
+ try:
+ self.blob = self.it.next()
+ except StopIteration:
+ self.it = None
+ if self.blob:
+ want = size - len(out)
+ out += self.blob[:want]
+ self.blob = self.blob[want:]
+ if not self.it:
+ break
+ debug2('next(%d) returned %d\n' % (size, len(out)))
+ self.ofs += len(out)
+ return out
+
+class _FileReader(object):
+ def __init__(self, repo, oid, known_size=None):
+ self.oid = oid
+ self.ofs = 0
+ self.reader = None
+ self._repo = repo
+ self._size = known_size
+
+ def _compute_size():
+ if not self._size:
+ self._size = _normal_or_chunked_file_size(self._repo, self.oid)
+ return self._size
+
+ def seek(self, ofs):
+ if ofs < 0:
+ raise IOError(errno.EINVAL, 'Invalid argument')
+ if ofs > self._compute_size():
+ raise IOError(errno.EINVAL, 'Invalid argument')
+ self.ofs = ofs
+
+ def tell(self):
+ return self.ofs
+
+ def read(self, count=-1):
+ if count < 0:
+ count = self._compute_size() - self.ofs
+ if not self.reader or self.reader.ofs != self.ofs:
+ self.reader = _ChunkReader(self._repo, self.oid, self.ofs)
+ try:
+ buf = self.reader.next(count)
+ except:
+ self.reader = None
+ raise # our offsets will be all screwed up otherwise
+ self.ofs += len(buf)
+ return buf
+
+ def close(self):
+ pass
+
+ def __enter__(self):
+ return self
+ def __exit__(self, type, value, traceback):
+ self.close()
+ return False
+
+_multiple_slashes_rx = re.compile(r'//+')
+
+def _decompose_path(path):
+ """Return a reversed list of path elements, omitting any occurrences
+ of "." and ignoring any leading or trailing slash."""
+ path = re.sub(_multiple_slashes_rx, '/', path)
+ if path.startswith('/'):
+ path = path[1:]
+ if path.endswith('/'):
+ path = path[:-1]
+ result = [x for x in path.split('/') if x != '.']
+ result.reverse()
+ return result
+
+
+Item = namedtuple('Item', ('meta', 'oid'))
+Chunky = namedtuple('Chunky', ('meta', 'oid'))
+Root = namedtuple('Root', ('meta'))
+Tags = namedtuple('Tags', ('meta'))
+RevList = namedtuple('RevList', ('meta', 'oid'))
+
+_root = Root(meta=default_dir_mode)
+_tags = Tags(meta=default_dir_mode)
+
+def copy_item(item):
+ """Return a completely independent copy of item, such that
+ modifications will not affect the original.
+
+ """
+ meta = getattr(item, 'meta', None)
+ if not meta:
+ return item
+ return(item._replace(meta=meta.copy()))
+
+def item_mode(item):
+ """Return the integer mode (stat st_mode) for item."""
+ m = item.meta
+ if isinstance(m, Metadata):
+ return m.mode
+ return m
+
+def _read_dir_meta(bupm):
+ # This is because save writes unmodified Metadata() entries for
+ # fake parents -- test-save-strip-graft.sh demonstrates.
+ m = Metadata.read(bupm)
+ if not m:
+ return default_dir_mode
+ assert m.mode is not None
+ if m.size is None:
+ m.size = 0
+ return m
+
+def _tree_data_and_bupm(repo, oid):
+ """Return (tree_bytes, bupm_oid) where bupm_oid will be None if the
+ tree has no metadata (i.e. older bup save, or non-bup tree).
+
+ """
+ assert len(oid) == 20
+ it = repo.cat(oid.encode('hex'))
+ _, item_t, size = next(it)
+ data = ''.join(it)
+ if item_t == 'commit':
+ commit = parse_commit(data)
+ it = repo.cat(commit.tree)
+ _, item_t, size = next(it)
+ data = ''.join(it)
+ assert item_t == 'tree'
+ elif item_t != 'tree':
+ raise Exception('%r is not a tree or commit' % oid.encode('hex'))
+ for _, mangled_name, sub_oid in tree_decode(data):
+ if mangled_name == '.bupm':
+ return data, sub_oid
+ if mangled_name > '.bupm':
+ break
+ return data, None
+
+def _find_dir_item_metadata(repo, item):
+ """Return the metadata for the tree or commit item, or None if the
+ tree has no metadata (i.e. older bup save, or non-bup tree).
+
+ """
+ tree_data, bupm_oid = _tree_data_and_bupm(repo, item.oid)
+ if bupm_oid:
+ with _FileReader(repo, bupm_oid) as meta_stream:
+ return _read_dir_meta(meta_stream)
+ return None
+
+def _readlink(repo, oid):
+ return ''.join(repo.join(oid.encode('hex')))
+
+def readlink(repo, item):
+ """Return the link target of item, which must be a symlink. Reads the
+ target from the repository if necessary."""
+ assert repo
+ assert S_ISLNK(item_mode(item))
+ if isinstance(item.meta, Metadata):
+ target = item.meta.symlink_target
+ if target:
+ return target
+ return _readlink(repo, item.oid)
+
+def _compute_item_size(repo, item):
+ mode = item_mode(item)
+ if S_ISREG(mode):
+ size = _normal_or_chunked_file_size(repo, item.oid)
+ return size
+ if S_ISLNK(mode):
+ return len(_readlink(repo, item.oid))
+ return 0
+
+def item_size(repo, item):
+ """Return the size of item, computing it if necessary."""
+ m = item.meta
+ if isinstance(m, Metadata) and m.size is not None:
+ return m.size
+ return _compute_item_size(repo, item)
+
+def fopen(repo, item):
+ """Return an open reader for the given file item."""
+ assert repo
+ assert S_ISREG(item_mode(item))
+ return _FileReader(repo, item.oid)
+
+def augment_item_meta(repo, item, include_size=False):
+ """Ensure item has a Metadata instance for item.meta. If item.meta is
+ currently a mode, replace it with a compatible "fake" Metadata
+ instance. If include_size is true, ensure item.meta.size is
+ correct, computing it if needed. If item.meta is a Metadata
+ instance, this call may modify it in place or replace it.
+
+ """
+ # If we actually had parallelism, we'd need locking...
+ assert repo
+ m = item.meta
+ if isinstance(m, Metadata):
+ if include_size and m.size is None:
+ m.size = _compute_item_size(repo, item)
+ return item._replace(meta=m)
+ return item
+ # m is mode
+ meta = Metadata()
+ meta.mode = m
+ meta.uid = meta.gid = meta.atime = meta.mtime = meta.ctime = 0
+ if S_ISLNK(m):
+ target = _readlink(repo, item.oid)
+ meta.symlink_target = target
+ meta.size = len(target)
+ elif include_size:
+ meta.size = _compute_item_size(repo, item)
+ return item._replace(meta=meta)
+
+def _commit_meta_from_auth_sec(author_sec):
+ m = Metadata()
+ m.mode = default_dir_mode
+ m.uid = m.gid = m.size = 0
+ m.atime = m.mtime = m.ctime = author_sec * 10**9
+ return m
+
+def _commit_meta_from_oidx(repo, oidx):
+ it = repo.cat(oidx)
+ _, typ, size = next(it)
+ assert typ == 'commit'
+ author_sec = parse_commit(''.join(it)).author_sec
+ return _commit_meta_from_auth_sec(author_sec)
+
+def parse_rev_auth_secs(f):
+ tree, author_secs = f.readline().split(None, 2)
+ return tree, int(author_secs)
+
+def root_items(repo, names=None):
+ """Yield (name, item) for the items in '/' in the VFS. Return
+ everything if names is logically false, otherwise return only
+ items with a name in the collection.
+
+ """
+ # FIXME: what about non-leaf refs like 'refs/heads/foo/bar/baz?
+
+ global _root, _tags
+ if not names:
+ yield '.', _root
+ yield '.tag', _tags
+ # FIXME: maybe eventually support repo.clone() or something
+ # and pass in two repos, so we can drop the tuple() and stream
+ # in parallel (i.e. meta vs refs).
+ for name, oid in tuple(repo.refs([], limit_to_heads=True)):
+ assert(name.startswith('refs/heads/'))
+ name = name[11:]
+ m = _commit_meta_from_oidx(repo, oid.encode('hex'))
+ yield name, RevList(meta=m, oid=oid)
+ return
+
+ if '.' in names:
+ yield '.', _root
+ if '.tag' in names:
+ yield '.tag', _tags
+ for ref in names:
+ if ref in ('.', '.tag'):
+ continue
+ it = repo.cat(ref)
+ oidx, typ, size = next(it)
+ if not oidx:
+ for _ in it: pass
+ continue
+ assert typ == 'commit'
+ commit = parse_commit(''.join(it))
+ yield ref, RevList(meta=_commit_meta_from_auth_sec(commit.author_sec),
+ oid=oidx.decode('hex'))
+
+def ordered_tree_entries(tree_data, bupm=None):
+ """Yields (name, mangled_name, kind, gitmode, oid) for each item in
+ tree, sorted by name.
+
+ """
+ # Sadly, the .bupm entries currently aren't in git tree order,
+ # i.e. they don't account for the fact that git sorts trees
+ # (including our chunked trees) as if their names ended with "/",
+ # so "fo" sorts after "fo." iff fo is a directory. This makes
+ # streaming impossible when we need the metadata.
+ def result_from_tree_entry(tree_entry):
+ gitmode, mangled_name, oid = tree_entry
+ name, kind = git.demangle_name(mangled_name, gitmode)
+ return name, mangled_name, kind, gitmode, oid
+
+ tree_ents = (result_from_tree_entry(x) for x in tree_decode(tree_data))
+ if bupm:
+ tree_ents = sorted(tree_ents, key=lambda x: x[0])
+ for ent in tree_ents:
+ yield ent
+
+def tree_items(oid, tree_data, names=frozenset(tuple()), bupm=None):
+
+ def tree_item(ent_oid, kind, gitmode):
+ if kind == BUP_CHUNKED:
+ meta = Metadata.read(bupm) if bupm else default_file_mode
+ return Chunky(oid=ent_oid, meta=meta)
+
+ if S_ISDIR(gitmode):
+ # No metadata here (accessable via '.' inside ent_oid).
+ return Item(meta=default_dir_mode, oid=ent_oid)
+
+ return Item(oid=ent_oid,
+ meta=(Metadata.read(bupm) if bupm \
+ else _default_mode_for_gitmode(gitmode)))
+
+ assert len(oid) == 20
+ if not names:
+ dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode
+ yield '.', Item(oid=oid, meta=dot_meta)
+ tree_entries = ordered_tree_entries(tree_data, bupm)
+ for name, mangled_name, kind, gitmode, ent_oid in tree_entries:
+ if mangled_name == '.bupm':
+ continue
+ assert name != '.'
+ yield name, tree_item(ent_oid, kind, gitmode)
+ return
+
+ # Assumes the tree is properly formed, i.e. there are no
+ # duplicates, and entries will be in git tree order.
+ if type(names) not in (frozenset, set):
+ names = frozenset(names)
+ remaining = len(names)
+
+ # Account for the bupm sort order issue (cf. ordered_tree_entries above)
+ last_name = max(names) if bupm else max(names) + '/'
+
+ if '.' in names:
+ dot_meta = _read_dir_meta(bupm) if bupm else default_dir_mode
+ yield '.', Item(oid=oid, meta=dot_meta)
+ if remaining == 1:
+ return
+ remaining -= 1
+
+ tree_entries = ordered_tree_entries(tree_data, bupm)
+ for name, mangled_name, kind, gitmode, ent_oid in tree_entries:
+ if mangled_name == '.bupm':
+ continue
+ assert name != '.'
+ if name not in names:
+ if bupm:
+ if (name + '/') > last_name:
+ break # given git sort order, we're finished
+ else:
+ if name > last_name:
+ break # given bupm sort order, we're finished
+ if (kind == BUP_CHUNKED or not S_ISDIR(gitmode)) and bupm:
+ Metadata.read(bupm)
+ continue
+ yield name, tree_item(ent_oid, kind, gitmode)
+ if remaining == 1:
+ break
+ remaining -= 1
+
+def tree_items_with_meta(repo, oid, tree_data, names):
+ # For now, the .bupm order doesn't quite match git's, and we don't
+ # load the tree data incrementally anyway, so we just work in RAM
+ # via tree_data.
+ assert len(oid) == 20
+ bupm = None
+ for _, mangled_name, sub_oid in tree_decode(tree_data):
+ if mangled_name == '.bupm':
+ bupm = _FileReader(repo, sub_oid)
+ break
+ if mangled_name > '.bupm':
+ break
+ for item in tree_items(oid, tree_data, names, bupm):
+ yield item
+
+_save_name_rx = re.compile(r'^\d\d\d\d-\d\d-\d\d-\d{6}$')
+
+def revlist_items(repo, oid, names):
+ assert len(oid) == 20
+ oidx = oid.encode('hex')
+
+ # There might well be duplicate names in this dir (time resolution is secs)
+ names = frozenset(name for name in (names or tuple()) \
+ if _save_name_rx.match(name) or name in ('.', 'latest'))
+
+ # Do this before we open the rev_list iterator so we're not nesting
+ if (not names) or ('.' in names):
+ yield '.', RevList(oid=oid, meta=_commit_meta_from_oidx(repo, oidx))
+
+ revs = repo.rev_list((oidx,), format='%T %at', parse=parse_rev_auth_secs)
+ first_rev = next(revs, None)
+ revs = chain((first_rev,), revs)
+
+ if not names:
+ for commit, (tree_oidx, utc) in revs:
+ assert len(tree_oidx) == 40
+ name = strftime('%Y-%m-%d-%H%M%S', localtime(utc))
+ yield name, Item(meta=default_dir_mode, oid=tree_oidx.decode('hex'))
+ if first_rev:
+ commit, (tree_oidx, utc) = first_rev
+ yield 'latest', Item(meta=default_dir_mode,
+ oid=tree_oidx.decode('hex'))
+ return
+
+ last_name = max(names)
+ for commit, (tree_oidx, utc) in revs:
+ assert len(tree_oidx) == 40
+ name = strftime('%Y-%m-%d-%H%M%S', localtime(utc))
+ if name > last_name:
+ break
+ if not name in names:
+ continue
+ yield name, Item(meta=default_dir_mode, oid=tree_oidx.decode('hex'))
+
+ # FIXME: need real short circuit...
+ for _ in revs:
+ pass
+
+ if first_rev and 'latest' in names:
+ commit, (tree_oidx, utc) = first_rev
+ yield 'latest', Item(meta=default_dir_mode, oid=tree_oidx.decode('hex'))
+
+def tags_items(repo, names):
+ global _tags
+
+ def tag_item(oid):
+ assert len(oid) == 20
+ oidx = oid.encode('hex')
+ it = repo.cat(oidx)
+ _, typ, size = next(it)
+ if typ == 'commit':
+ tree_oid = parse_commit(''.join(it)).tree.decode('hex')
+ assert len(tree_oid) == 20
+ # FIXME: more efficient/bulk?
+ return RevList(meta=_commit_meta_from_oidx(repo, oidx), oid=oid)
+ for _ in it: pass
+ if typ == 'blob':
+ return Item(meta=default_file_mode, oid=oid)
+ elif typ == 'tree':
+ return Item(meta=default_dir_mode, oid=oid)
+ raise Exception('unexpected tag type ' + typ + ' for tag ' + name)
+
+ if not names:
+ yield '.', _tags
+ # We have to pull these all into ram because tag_item calls cat()
+ for name, oid in tuple(repo.refs(names, limit_to_tags=True)):
+ assert(name.startswith('refs/tags/'))
+ name = name[10:]
+ yield name, tag_item(oid)
+ return
+
+ # Assumes no duplicate refs
+ if type(names) not in (frozenset, set):
+ names = frozenset(names)
+ remaining = len(names)
+ last_name = max(names)
+ if '.' in names:
+ yield '.', _tags
+ if remaining == 1:
+ return
+ remaining -= 1
+
+ for name, oid in repo.refs(names, limit_to_tags=True):
+ assert(name.startswith('refs/tags/'))
+ name = name[10:]
+ if name > last_name:
+ return
+ if name not in names:
+ continue
+ yield name, tag_item(oid)
+ if remaining == 1:
+ return
+ remaining -= 1
+
+def contents(repo, item, names=None, want_meta=True):
+ """Yields information about the items contained in item. Yields
+ (name, item) for each name in names, if the name exists, in an
+ unspecified order. If there are no names, then yields (name,
+ item) for all items, including, a first item named '.'
+ representing the container itself.
+
+ Any given name might produce more than one result. For example,
+ saves to a branch that happen within the same second currently end
+ up with the same VFS timestmap, i.e. /foo/2017-09-10-150833/.
+
+ Note that want_meta is advisory. For any given item, item.meta
+ might be a Metadata instance or a mode, and if the former,
+ meta.size might be None. Missing sizes can be computed via via
+ item_size() or augment_item_meta(..., include_size=True).
+
+ Do not modify any item.meta Metadata instances directly. If
+ needed, make a copy via item.meta.copy() and modify that instead.
+
+ """
+ # Q: are we comfortable promising '.' first when no names?
+ assert repo
+ assert S_ISDIR(item_mode(item))
+ item_t = type(item)
+ if item_t == Item:
+ it = repo.cat(item.oid.encode('hex'))
+ _, obj_type, size = next(it)
+ data = ''.join(it)
+ if obj_type == 'tree':
+ if want_meta:
+ item_gen = tree_items_with_meta(repo, item.oid, data, names)
+ else:
+ item_gen = tree_items(item.oid, data, names)
+ elif obj_type == 'commit':
+ tree_oidx = parse_commit(data).tree
+ it = repo.cat(tree_oidx)
+ _, obj_type, size = next(it)
+ assert obj_type == 'tree'
+ tree_data = ''.join(it)
+ if want_meta:
+ item_gen = tree_items_with_meta(repo, tree_oidx.decode('hex'),
+ tree_data, names)
+ else:
+ item_gen = tree_items(tree_oidx.decode('hex'), tree_data, names)
+ else:
+ for _ in it: pass
+ raise Exception('unexpected git ' + obj_type)
+ elif item_t == RevList:
+ item_gen = revlist_items(repo, item.oid, names)
+ elif item_t == Root:
+ item_gen = root_items(repo, names)
+ elif item_t == Tags:
+ item_gen = tags_items(repo, names)
+ else:
+ raise Exception('unexpected VFS item ' + str(item))
+ for x in item_gen:
+ yield x
+
+def _resolve_path(repo, path, parent=None, want_meta=True, deref=False):
+ assert repo
+ assert len(path)
+ global _root
+ future = _decompose_path(path)
+ past = []
+ if path.startswith('/'):
+ assert(not parent)
+ past = [('', _root)]
+ if future == ['']: # path was effectively '/'
+ return tuple(past)
+ if not past and not parent:
+ past = [('', _root)]
+ if parent:
+ past = [parent]
+ hops = 0
+ result = None
+ while True:
+ segment = future.pop()
+ if segment == '..':
+ if len(past) > 1: # .. from / is /
+ past.pop()
+ else:
+ parent_name, parent_item = past[-1]
+ wanted = (segment,) if not want_meta else ('.', segment)
+ items = contents(repo, parent_item, names=wanted,
+ want_meta=want_meta)
+ if want_meta: # First item will be '.' and have the metadata
+ dot, parent_item = next(items)
+ assert dot == '.'
+ past[-1] = parent_name, parent_item
+ item = None
+ # FIXME: content no longer returns anything for missing items
+ if items:
+ _, item = next(items, (None, None))
+ if not item:
+ return tuple(past + [(segment, None)])
+ mode = item_mode(item)
+ if not S_ISLNK(mode):
+ if not S_ISDIR(mode):
+ assert(not future)
+ return tuple(past + [(segment, item)])
+ # It's treeish
+ if want_meta and type(item) == Item:
+ dir_meta = _find_dir_item_metadata(repo, item)
+ if dir_meta:
+ item = item._replace(meta=dir_meta)
+ if not future:
+ return tuple(past + [(segment, item)])
+ past.append((segment, item))
+ else: # symlink
+ if not future and not deref:
+ return tuple(past + [(segment, item)])
+ target = readlink(repo, item)
+ target_future = _decompose_path(target)
+ if target.startswith('/'):
+ future = target_future
+ past = [('', _root)]
+ if target_future == ['']: # path was effectively '/'
+ return tuple(past)
+ else:
+ future = future + target_future
+ hops += 1
+ if hops > 100:
+ raise Loop('too many symlinks encountered while resolving %r%s'
+ % (path,
+ 'relative to %r' % parent if parent else ''))
+
+def lresolve(repo, path, parent=None, want_meta=True):
+ """Perform exactly the same function as resolve(), except if the
+ final path element is a symbolic link, don't follow it, just
+ return it in the result."""
+ return _resolve_path(repo, path, parent=parent, want_meta=want_meta,
+ deref=False)
+
+
+def resolve(repo, path, parent=None, want_meta=True):
+ """Follow the path in the virtual filesystem and return a tuple
+ representing the location, if any, denoted by the path. Each
+ element in the result tuple will be (name, info), where info will
+ be a VFS item that can be passed to functions like item_mode().
+
+ If a path segment that does not exist is encountered during
+ resolution, the result will represent the location of the missing
+ item, and that item in the result will be None.
+
+ Any symlinks along the path, including at the end, will be
+ resolved. A Loop exception will be raised if too many symlinks
+ are traversed whiile following the path. raised if too many
+ symlinks are traversed while following the path. That exception
+ is effectively like a normal ELOOP IOError exception, but will
+ include a terminus element describing the location of the failure,
+ which will be a tuple of (name, info) elements.
+
+ Currently, a path ending in '/' will still resolve if it exists,
+ even if not a directory. The parent, if specified, must be a
+ (name, item) tuple, and will provide the starting point for the
+ resolution of the path. Currently, the path must be relative when
+ a parent is provided. The result may include parent directly, so
+ it must not be modified later. If this is a concern, pass in
+ copy_item(parent) instead.
+
+ When want_meta is true, detailed metadata will be included in each
+ result item if it's avaiable, otherwise item.meta will be an
+ integer mode. The metadata size may or may not be provided, but
+ can be computed by item_size() or augment_item_meta(...,
+ include_size=True). Setting want_meta=False is rarely desirable
+ since it can limit the VFS to just the metadata git itself can
+ represent, and so, as an example, fifos and sockets will appear to
+ be regular files (e.g. S_ISREG(item_mode(item)) will be true) .
+ But the option is provided because it may be more efficient when
+ only the path names or the more limited metadata is sufficient.
+
+ Do not modify any item.meta Metadata instances directly. If
+ needed, make a copy via item.meta.copy() and modify that instead.
+
+ """
+ return _resolve_path(repo, path, parent=parent, want_meta=want_meta,
+ deref=True)
projects / bup.git / commitdiff