from random import randrange
from stat import S_IFDIR, S_IFLNK, S_IFREG, S_ISDIR, S_ISLNK, S_ISREG
from time import localtime, strftime
-import exceptions, re, sys
+import re, sys
-from bup import client, git, metadata
-from bup.compat import range
+from bup import git, metadata, vint
+from bup.compat import hexstr, range
from bup.git import BUP_CHUNKED, cp, get_commit_items, parse_commit, tree_decode
from bup.helpers import debug2, last
from bup.metadata import Metadata
-from bup.repo import LocalRepo, RemoteRepo
+from bup.vint import read_bvec, write_bvec
+from bup.vint import read_vint, write_vint
+from bup.vint import read_vuint, write_vuint
+if sys.version_info[0] < 3:
+ from exceptions import IOError as py_IOError
+else:
+ py_IOError = IOError
-class IOError(exceptions.IOError):
+# We currently assume that it's always appropriate to just forward IOErrors
+# to a remote client.
+
+class IOError(py_IOError):
def __init__(self, errno, message, terminus=None):
- exceptions.IOError.__init__(self, errno, message)
+ py_IOError.__init__(self, errno, message)
self.terminus = terminus
+def write_ioerror(port, ex):
+ assert isinstance(ex, IOError)
+ write_vuint(port,
+ (1 if ex.errno is not None else 0)
+ | (2 if ex.message is not None else 0)
+ | (4 if ex.terminus is not None else 0))
+ if ex.errno is not None:
+ write_vint(port, ex.errno)
+ if ex.message is not None:
+ write_bvec(port, ex.message.encode('utf-8'))
+ if ex.terminus is not None:
+ write_resolution(port, ex.terminus)
+
+def read_ioerror(port):
+ mask = read_vuint(port)
+ no = read_vint(port) if 1 & mask else None
+ msg = read_bvec(port).decode('utf-8') if 2 & mask else None
+ term = read_resolution(port) if 4 & mask else None
+ return IOError(errno=no, message=msg, terminus=term)
+
+
default_file_mode = S_IFREG | 0o644
default_dir_mode = S_IFDIR | 0o755
default_symlink_mode = S_IFLNK | 0o755
Item = namedtuple('Item', ('meta', 'oid'))
Chunky = namedtuple('Chunky', ('meta', 'oid'))
+FakeLink = namedtuple('FakeLink', ('meta', 'target'))
Root = namedtuple('Root', ('meta'))
Tags = namedtuple('Tags', ('meta'))
RevList = namedtuple('RevList', ('meta', 'oid'))
item_types = frozenset((Item, Chunky, Root, Tags, RevList, Commit))
real_tree_types = frozenset((Item, Commit))
+def write_item(port, item):
+ kind = type(item)
+ name = bytes(kind.__name__)
+ meta = item.meta
+ has_meta = 1 if isinstance(meta, Metadata) else 0
+ if kind in (Item, Chunky, RevList):
+ assert len(item.oid) == 20
+ if has_meta:
+ vint.send(port, 'sVs', name, has_meta, item.oid)
+ Metadata.write(meta, port, include_path=False)
+ else:
+ vint.send(port, 'sVsV', name, has_meta, item.oid, item.meta)
+ elif kind in (Root, Tags):
+ if has_meta:
+ vint.send(port, 'sV', name, has_meta)
+ Metadata.write(meta, port, include_path=False)
+ else:
+ vint.send(port, 'sVV', name, has_meta, item.meta)
+ elif kind == Commit:
+ assert len(item.oid) == 20
+ assert len(item.coid) == 20
+ if has_meta:
+ vint.send(port, 'sVss', name, has_meta, item.oid, item.coid)
+ Metadata.write(meta, port, include_path=False)
+ else:
+ vint.send(port, 'sVssV', name, has_meta, item.oid, item.coid,
+ item.meta)
+ elif kind == FakeLink:
+ if has_meta:
+ vint.send(port, 'sVs', name, has_meta, item.target)
+ Metadata.write(meta, port, include_path=False)
+ else:
+ vint.send(port, 'sVsV', name, has_meta, item.target, item.meta)
+ else:
+ assert False
+
+def read_item(port):
+ def read_m(port, has_meta):
+ if has_meta:
+ m = Metadata.read(port)
+ return m
+ return read_vuint(port)
+ kind, has_meta = vint.recv(port, 'sV')
+ if kind == b'Item':
+ oid, meta = read_bvec(port), read_m(port, has_meta)
+ return Item(oid=oid, meta=meta)
+ if kind == b'Chunky':
+ oid, meta = read_bvec(port), read_m(port, has_meta)
+ return Chunky(oid=oid, meta=meta)
+ if kind == b'RevList':
+ oid, meta = read_bvec(port), read_m(port, has_meta)
+ return RevList(oid=oid, meta=meta)
+ if kind == b'Root':
+ return Root(meta=read_m(port, has_meta))
+ if kind == b'Tags':
+ return Tags(meta=read_m(port, has_meta))
+ if kind == b'Commit':
+ oid, coid = vint.recv(port, 'ss')
+ meta = read_m(port, has_meta)
+ return Commit(oid=oid, coid=coid, meta=meta)
+ if kind == b'FakeLink':
+ target, meta = read_bvec(port), read_m(port, has_meta)
+ return FakeLink(target=target, meta=meta)
+ assert False
+
+def write_resolution(port, resolution):
+ write_vuint(port, len(resolution))
+ for name, item in resolution:
+ write_bvec(port, name)
+ if item:
+ port.write(b'\1')
+ write_item(port, item)
+ else:
+ port.write(b'\0')
+
+def read_resolution(port):
+ n = read_vuint(port)
+ result = []
+ for i in range(n):
+ name = read_bvec(port)
+ have_item = ord(port.read(1))
+ assert have_item in (0, 1)
+ item = read_item(port) if have_item else None
+ result.append((name, item))
+ return tuple(result)
+
+
_root = Root(meta=default_dir_mode)
_tags = Tags(meta=default_dir_mode)
"""Return logically true if x looks like it could be a valid cache key
(with respect to structure). Current valid cache entries:
res:... -> resolution
- commit_oid -> commit
- commit_oid + ':r' -> rev-list
- i.e. rev-list -> {'.', commit, '2012...', next_commit, ...}
+ itm:OID -> Commit
+ rvl:OID -> {'.', commit, '2012...', next_commit, ...}
"""
# Suspect we may eventually add "(container_oid, name) -> ...", and others.
x_t = type(x)
if x_t is bytes:
- if len(x) == 20:
+ tag = x[:4]
+ if tag in ('itm:', 'rvl:') and len(x) == 24:
return True
- if len(x) == 22 and x.endswith(b':r'):
- return True
- if x.startswith('res:'):
+ if tag == 'res:':
return True
def cache_get(key):
global _cache
- assert is_valid_cache_key(key)
+ if not is_valid_cache_key(key):
+ raise Exception('invalid cache key: ' + repr(key))
return _cache.get(key)
def cache_notice(key, value):
global _cache, _cache_keys, _cache_max_items
- assert is_valid_cache_key(key)
+ if not is_valid_cache_key(key):
+ raise Exception('invalid cache key: ' + repr(key))
if key in _cache:
return
if len(_cache) < _cache_max_items:
When need_meta is true don't return a cached item that only has a
mode."""
# tree might be stored independently, or as '.' with its entries.
- item = cache_get(oid)
+ commit_key = b'itm:' + oid
+ item = cache_get(commit_key)
if item:
if not need_meta:
return item
if isinstance(item.meta, Metadata):
return item
- entries = cache_get(oid + b':r')
+ entries = cache_get(b'rvl:' + oid)
if entries:
return entries['.']
data = ''.join(it)
assert item_t == 'tree'
elif item_t != 'tree':
- raise Exception('%r is not a tree or commit' % oid.encode('hex'))
+ raise Exception('%s is not a tree or commit' % hexstr(oid))
for _, mangled_name, sub_oid in tree_decode(data):
if mangled_name == '.bupm':
return data, sub_oid
target from the repository if necessary."""
assert repo
assert S_ISLNK(item_mode(item))
+ if isinstance(item, FakeLink):
+ return item.target
if isinstance(item.meta, Metadata):
target = item.meta.symlink_target
if target:
size = _normal_or_chunked_file_size(repo, item.oid)
return size
if S_ISLNK(mode):
+ if isinstance(item, FakeLink):
+ return len(item.target)
return len(_readlink(repo, item.oid))
return 0
meta = _find_treeish_oid_metadata(repo, commit.oid)
if meta:
commit = commit._replace(meta=meta)
- cache_notice(oid, commit)
+ commit_key = b'itm:' + oid
+ cache_notice(commit_key, commit)
return commit
def _revlist_item_from_oid(repo, oid, require_meta):
if item:
return item
item = Commit(meta=default_dir_mode, oid=tree_oid, coid=coid)
- cache_notice(item.coid, item)
+ commit_key = b'itm:' + coid
+ cache_notice(commit_key, item)
return item
def cache_commit(repo, oid):
revs = None # Don't disturb the tees
rev_names = _reverse_suffix_duplicates(_name_for_rev(x) for x in rev_names)
rev_items = (_item_for_rev(x) for x in rev_items)
- latest = None
+ tip = None
for item in rev_items:
- latest = latest or item
name = next(rev_names)
+ tip = tip or (name, item)
entries[name] = item
- entries['latest'] = latest
- cache_notice(latest.coid + b':r', entries)
+ entries['latest'] = FakeLink(meta=default_symlink_mode, target=tip[0])
+ revlist_key = b'rvl:' + tip[1].coid
+ cache_notice(revlist_key, entries)
return entries
def revlist_items(repo, oid, names):
# For now, don't worry about the possibility of the contents being
# "too big" for the cache.
- entries = cache_get(oid + b':r')
+ revlist_key = b'rvl:' + oid
+ entries = cache_get(revlist_key)
if not entries:
entries = cache_commit(repo, oid)
for x in item_gen:
yield x
-def _resolve_path(repo, path, parent=None, want_meta=True, deref=False):
+def _resolve_path(repo, path, parent=None, want_meta=True, follow=True):
cache_key = b'res:%d%d%d:%s\0%s' \
- % (bool(want_meta), bool(deref), repo.id(),
+ % (bool(want_meta), bool(follow), repo.id(),
('/'.join(x[0] for x in parent) if parent else ''),
'/'.join(path))
resolution = cache_get(cache_key)
% (parent,))
is_absolute, must_be_dir, future = _decompose_path(path)
if must_be_dir:
- deref = True
+ follow = True
if not future: # path was effectively '.' or '/'
if is_absolute:
return notice_resolution((('', _root),))
item = item._replace(meta=dir_meta)
past.append((segment, item))
else: # symlink
- if not future and not deref:
+ if not future and not follow:
past.append((segment, item),)
continue
if hops > 100:
future.extend(target_future)
hops += 1
-def lresolve(repo, path, parent=None, want_meta=True):
- """Perform exactly the same function as resolve(), except if the final
- path element is a symbolic link, don't follow it, just return it
- in the result.
-
- """
- return _resolve_path(repo, path, parent=parent, want_meta=want_meta,
- deref=False)
-
-def resolve(repo, path, parent=None, want_meta=True):
+def resolve(repo, path, parent=None, want_meta=True, follow=True):
"""Follow the path in the virtual filesystem and return a tuple
representing the location, if any, denoted by the path. Each
element in the result tuple will be (name, info), where info will
be a VFS item that can be passed to functions like item_mode().
+ If follow is false, and if the final path element is a symbolic
+ link, don't follow it, just return it in the result.
+
If a path segment that does not exist is encountered during
resolution, the result will represent the location of the missing
item, and that item in the result will be None.
needed, make a copy via item.meta.copy() and modify that instead.
"""
+ if repo.is_remote():
+ # Redirect to the more efficient remote version
+ return repo.resolve(path, parent=parent, want_meta=want_meta,
+ follow=follow)
result = _resolve_path(repo, path, parent=parent, want_meta=want_meta,
- deref=True)
+ follow=follow)
_, leaf_item = result[-1]
- if leaf_item:
+ if leaf_item and follow:
assert not S_ISLNK(item_mode(leaf_item))
return result
def try_resolve(repo, path, parent=None, want_meta=True):
"""If path does not refer to a symlink, does not exist, or refers to a
- valid symlink, behave exactly like resolve(). If path refers to
- an invalid symlink, behave like lresolve.
+ valid symlink, behave exactly like resolve(..., follow=True). If
+ path refers to an invalid symlink, behave like resolve(...,
+ follow=False).
"""
- res = lresolve(repo, path, parent=parent, want_meta=want_meta)
+ res = resolve(repo, path, parent=parent, want_meta=want_meta, follow=False)
leaf_name, leaf_item = res[-1]
if not leaf_item:
return res
if not S_ISLNK(item_mode(leaf_item)):
return res
- deref = resolve(repo, leaf_name, parent=res[:-1], want_meta=want_meta)
- deref_name, deref_item = deref[-1]
- if deref_item:
- return deref
+ follow = resolve(repo, leaf_name, parent=res[:-1], want_meta=want_meta)
+ follow_name, follow_item = follow[-1]
+ if follow_item:
+ return follow
return res
def augment_item_meta(repo, item, include_size=False):
meta.mode = m
meta.uid = meta.gid = meta.atime = meta.mtime = meta.ctime = 0
if S_ISLNK(m):
- target = _readlink(repo, item.oid)
+ if isinstance(item, FakeLink):
+ target = item.target
+ else:
+ target = _readlink(repo, item.oid)
meta.symlink_target = target
meta.size = len(target)
elif include_size:
projects / bup.git / blobdiff