git archive origin/html | (cd Documentation; tar -xvf -)
git archive origin/man | (cd Documentation; tar -xvf -)
-# tgit.py plays with permissions on lib/bup/t/pybuptest.tmp, so we should
-# ensure that we can delete the directory before doing it.
clean: Documentation/clean
rm -f *.o lib/*/*.o *.so lib/*/*.so *.dll *.exe \
.*~ *~ */*~ lib/*/*~ lib/*/*/*~ \
def update_index(top, excluded_paths):
+ tmax = time.time() - 1
ri = index.Reader(indexfile)
- wi = index.Writer(indexfile)
+ wi = index.Writer(indexfile, tmax)
rig = IterHelper(ri.iter(name=top))
tstart = int(time.time())
check_index(ri)
log('check: before merging: newfile\n')
check_index(wr)
- mi = index.Writer(indexfile)
+ mi = index.Writer(indexfile, tmax)
for e in index.merge(ri, wr):
# FIXME: shouldn't we remove deleted entries eventually? When?
from bup import options
from bup.helpers import handle_ctrl_c, log, saved_errors
+
+def open_input(name):
+ if name != '-':
+ return open(name, 'r')
+ else:
+ return sys.stdin
+
+
optspec = """
bup meta --create [OPTION ...] <PATH ...>
bup meta --extract [OPTION ...]
q,quiet don't show progress meter
"""
-action = None
-target_filename = ''
-should_recurse = False
-restore_numeric_ids = False
-include_paths = True
-handle_symlinks = True
-xdev = False
-
handle_ctrl_c()
o = options.Options(optspec)
-(opt, flags, remainder) = o.parse(sys.argv[1:])
+(opt, flags, remainder) = o.parse(['--paths', '--symlinks'] + sys.argv[1:])
-for flag, value in flags:
- if flag == '--create' or flag == '-c':
- action = 'create'
- elif flag == '--list' or flag == '-t':
- action = 'list'
- elif flag == '--extract' or flag == '-x':
- action = 'extract'
- elif flag == '--start-extract':
- action = 'start-extract'
- elif flag == '--finish-extract':
- action = 'finish-extract'
- elif flag == '--file' or flag == '-f':
- target_filename = value
- elif flag == '--recurse' or flag == '-R':
- should_recurse = True
- elif flag == '--no-recurse':
- should_recurse = False
- elif flag in frozenset(['--xdev', '--one-file-system']):
- xdev = True
- elif flag in frozenset(['--no-xdev', '--no-one-file-system']):
- xdev = False
- elif flag == '--numeric-ids':
- restore_numeric_ids = True
- elif flag == '--no-numeric-ids':
- restore_numeric_ids = False
- elif flag == '--paths':
- include_paths = True
- elif flag == '--no-paths':
- include_paths = False
- elif flag == '--symlinks':
- handle_symlinks = True
- elif flag == '--no-symlinks':
- handle_symlinks = False
- elif flag == '--verbose' or flag == '-v':
- metadata.verbose += 1
- elif flag == '--quiet' or flag == '-q':
- metadata.verbose = 0
+opt.verbose = opt.verbose or 0
+opt.quiet = opt.quiet or 0
+metadata.verbose = opt.verbose - opt.quiet
-if not action:
- o.fatal("no action specified")
+action_count = sum([bool(x) for x in [opt.create, opt.list, opt.extract,
+ opt.start_extract, opt.finish_extract]])
+if action_count > 1:
+ o.fatal("bup: only one action permitted: --create --list --extract")
+if action_count == 0:
+ o.fatal("bup: no action specified")
-if action == 'create':
+if opt.create:
if len(remainder) < 1:
o.fatal("no paths specified for create")
- if target_filename != '-':
- output_file = open(target_filename, 'w')
+ if opt.file != '-':
+ output_file = open(opt.file, 'w')
else:
output_file = sys.stdout
metadata.save_tree(output_file,
remainder,
- recurse=should_recurse,
- write_paths=include_paths,
- save_symlinks=handle_symlinks,
- xdev=xdev)
-
-elif action == 'list':
+ recurse=opt.recurse,
+ write_paths=opt.paths,
+ save_symlinks=opt.symlinks,
+ xdev=opt.xdev)
+elif opt.list:
if len(remainder) > 0:
o.fatal("cannot specify paths for --list")
- if target_filename != '-':
- src = open(target_filename, 'r')
- else:
- src = sys.stdin
+ src = open_input(opt.file)
metadata.display_archive(src)
-
-elif action == 'start-extract':
+elif opt.start_extract:
if len(remainder) > 0:
o.fatal("cannot specify paths for --start-extract")
- if target_filename != '-':
- src = open(target_filename, 'r')
- else:
- src = sys.stdin
- metadata.start_extract(src, create_symlinks=handle_symlinks)
-
-elif action == 'finish-extract':
+ src = open_input(opt.file)
+ metadata.start_extract(src, create_symlinks=opt.symlinks)
+elif opt.finish_extract:
if len(remainder) > 0:
o.fatal("cannot specify paths for --finish-extract")
- if target_filename != '-':
- src = open(target_filename, 'r')
- else:
- src = sys.stdin
- num_ids = restore_numeric_ids
- metadata.finish_extract(src, restore_numeric_ids=num_ids)
-
-elif action == 'extract':
+ src = open_input(opt.file)
+ metadata.finish_extract(src, restore_numeric_ids=opt.numeric_ids)
+elif opt.extract:
if len(remainder) > 0:
o.fatal("cannot specify paths for --extract")
- if target_filename != '-':
- src = open(target_filename, 'r')
- else:
- src = sys.stdin
+ src = open_input(opt.file)
metadata.extract(src,
- restore_numeric_ids=restore_numeric_ids,
- create_symlinks=handle_symlinks)
+ restore_numeric_ids=opt.numeric_ids,
+ create_symlinks=opt.symlinks)
if saved_errors:
log('WARNING: %d errors encountered.\n' % len(saved_errors))
#!/usr/bin/env python
import sys, math, struct, glob, resource
import tempfile
-from bup import options, git, midx, _helpers
+from bup import options, git, midx, _helpers, xstat
from bup.helpers import *
PAGE_SIZE=4096
# sort the biggest+newest midxes first, so that we can eliminate
# smaller (or older) redundant ones that come later in the list
- midxs.sort(key=lambda ix: (-sizes[ix], -os.stat(ix).st_mtime))
+ midxs.sort(key=lambda ix: (-sizes[ix], -xstat.stat(ix).st_mtime))
for mname in midxs:
any = 0
import sys, stat, errno
from bup import metadata, options, xstat
from bup.helpers import handle_ctrl_c, saved_errors, add_error, log
-from bup import _helpers
def fstimestr(fstime):
- (s, ns) = fstime.secs_nsecs()
+ (s, ns) = xstat.fstime_to_timespec(fstime)
+ if(s < 0):
+ s += 1
if ns == 0:
return '%d' % s
else:
treat_include_fields_as_definitive = True
for flag, value in flags:
- if flag == '--verbose' or flag == '-v':
- metadata.verbose += 1
- elif flag == '--quiet' or flag == '-q':
- metadata.verbose = 0
- elif flag == '--exclude-fields':
+ if flag == '--exclude-fields':
exclude_fields = frozenset(value.split(','))
for f in exclude_fields:
if not f in all_fields:
else:
active_fields = active_fields | include_fields
+opt.verbose = opt.verbose or 0
+opt.quiet = opt.quiet or 0
+metadata.verbose = opt.verbose - opt.quiet
+
for path in remainder:
try:
m = metadata.from_path(path, archive_path = path)
if 'group' in active_fields:
print 'group:', m.group
if 'atime' in active_fields:
- # if we don't have_ns_fs_timestamps, that means we have to use
+ # If we don't have utimensat, that means we have to use
# utime(), and utime() has no way to set the mtime/atime of a
- # symlink. Thus, the mtime/atime of a symlink is meaningless, so
- # let's not report it. (That way scripts comparing before/after
- # won't trigger.)
- if _helpers._have_ns_fs_timestamps or not stat.S_ISLNK(m.mode):
+ # symlink. Thus, the mtime/atime of a symlink is meaningless,
+ # so let's not report it. (That way scripts comparing
+ # before/after won't trigger.)
+ if xstat._have_utimensat or not stat.S_ISLNK(m.mode):
print 'atime: ' + fstimestr(m.atime)
else:
print 'atime: 0'
if 'mtime' in active_fields:
- if _helpers._have_ns_fs_timestamps or not stat.S_ISLNK(m.mode):
+ if xstat._have_utimensat or not stat.S_ISLNK(m.mode):
print 'mtime: ' + fstimestr(m.mtime)
else:
print 'mtime: 0'
#include <fcntl.h>
#include <arpa/inet.h>
#include <stdint.h>
+#include <sys/types.h>
+#include <sys/stat.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#ifdef linux
#include <linux/fs.h>
#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <sys/time.h>
#endif
static int istty2 = 0;
}
close(fd);
- return Py_BuildValue("i", 1);
+ return Py_BuildValue("O", Py_None);
}
#endif /* def linux */
if (rc != 0)
return PyErr_SetFromErrnoWithFilename(PyExc_OSError, path);
- return Py_BuildValue("i", 1);
+ return Py_BuildValue("O", Py_None);
}
#endif /* defined(_ATFILE_SOURCE)
|| _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */
#endif /* HAVE_UTIMENSAT */
-#ifdef linux /* and likely others */
+static PyObject *stat_struct_to_py(const struct stat *st)
+{
+ /* Enforce the current timespec nanosecond range expectations. */
+ if (st->st_atim.tv_nsec < 0 || st->st_atim.tv_nsec > 999999999)
+ {
+ PyErr_SetString(PyExc_ValueError, "invalid atime timespec nanoseconds");
+ return NULL;
+ }
+ if (st->st_mtim.tv_nsec < 0 || st->st_mtim.tv_nsec > 999999999)
+ {
+ PyErr_SetString(PyExc_ValueError, "invalid mtime timespec nanoseconds");
+ return NULL;
+ }
+ if (st->st_ctim.tv_nsec < 0 || st->st_ctim.tv_nsec > 999999999)
+ {
+ PyErr_SetString(PyExc_ValueError, "invalid ctime timespec nanoseconds");
+ return NULL;
+ }
+
+ return Py_BuildValue("kkkkkkkk(Ll)(Ll)(Ll)",
+ (unsigned long) st->st_mode,
+ (unsigned long) st->st_ino,
+ (unsigned long) st->st_dev,
+ (unsigned long) st->st_nlink,
+ (unsigned long) st->st_uid,
+ (unsigned long) st->st_gid,
+ (unsigned long) st->st_rdev,
+ (unsigned long) st->st_size,
+ (long long) st->st_atime,
+ (long) st->st_atim.tv_nsec,
+ (long long) st->st_mtime,
+ (long) st->st_mtim.tv_nsec,
+ (long long) st->st_ctime,
+ (long) st->st_ctim.tv_nsec);
+}
+
-#define HAVE_BUP_STAT 1
static PyObject *bup_stat(PyObject *self, PyObject *args)
{
int rc;
rc = stat(filename, &st);
if (rc != 0)
return PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
+ return stat_struct_to_py(&st);
+}
+
- return Py_BuildValue("kkkkkkkk"
- "(ll)"
- "(ll)"
- "(ll)",
- (unsigned long) st.st_mode,
- (unsigned long) st.st_ino,
- (unsigned long) st.st_dev,
- (unsigned long) st.st_nlink,
- (unsigned long) st.st_uid,
- (unsigned long) st.st_gid,
- (unsigned long) st.st_rdev,
- (unsigned long) st.st_size,
- (long) st.st_atime,
- (long) st.st_atim.tv_nsec,
- (long) st.st_mtime,
- (long) st.st_mtim.tv_nsec,
- (long) st.st_ctime,
- (long) st.st_ctim.tv_nsec);
-}
-
-
-#define HAVE_BUP_LSTAT 1
static PyObject *bup_lstat(PyObject *self, PyObject *args)
{
int rc;
rc = lstat(filename, &st);
if (rc != 0)
return PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
+ return stat_struct_to_py(&st);
+}
+
- return Py_BuildValue("kkkkkkkk"
- "(ll)"
- "(ll)"
- "(ll)",
- (unsigned long) st.st_mode,
- (unsigned long) st.st_ino,
- (unsigned long) st.st_dev,
- (unsigned long) st.st_nlink,
- (unsigned long) st.st_uid,
- (unsigned long) st.st_gid,
- (unsigned long) st.st_rdev,
- (unsigned long) st.st_size,
- (long) st.st_atime,
- (long) st.st_atim.tv_nsec,
- (long) st.st_mtime,
- (long) st.st_mtim.tv_nsec,
- (long) st.st_ctime,
- (long) st.st_ctim.tv_nsec);
-}
-
-
-#define HAVE_BUP_FSTAT 1
static PyObject *bup_fstat(PyObject *self, PyObject *args)
{
int rc, fd;
rc = fstat(fd, &st);
if (rc != 0)
return PyErr_SetFromErrno(PyExc_OSError);
-
- return Py_BuildValue("kkkkkkkk"
- "(ll)"
- "(ll)"
- "(ll)",
- (unsigned long) st.st_mode,
- (unsigned long) st.st_ino,
- (unsigned long) st.st_dev,
- (unsigned long) st.st_nlink,
- (unsigned long) st.st_uid,
- (unsigned long) st.st_gid,
- (unsigned long) st.st_rdev,
- (unsigned long) st.st_size,
- (long) st.st_atime,
- (long) st.st_atim.tv_nsec,
- (long) st.st_mtime,
- (long) st.st_mtim.tv_nsec,
- (long) st.st_ctime,
- (long) st.st_ctim.tv_nsec);
+ return stat_struct_to_py(&st);
}
-#endif /* def linux */
-
static PyMethodDef helper_methods[] = {
{ "selftest", selftest, METH_VARARGS,
{ "utimensat", bup_utimensat, METH_VARARGS,
"Change file timestamps with nanosecond precision." },
#endif
-#ifdef HAVE_BUP_STAT
{ "stat", bup_stat, METH_VARARGS,
"Extended version of stat." },
-#endif
-#ifdef HAVE_BUP_LSTAT
{ "lstat", bup_lstat, METH_VARARGS,
"Extended version of lstat." },
-#endif
-#ifdef HAVE_BUP_FSTAT
{ "fstat", bup_fstat, METH_VARARGS,
"Extended version of fstat." },
-#endif
{ NULL, NULL, 0, NULL }, // sentinel
};
PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD));
PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW",
Py_BuildValue("i", AT_SYMLINK_NOFOLLOW));
-#endif
-#ifdef HAVE_BUP_STAT
- PyModule_AddIntConstant(m, "_have_ns_fs_timestamps", 1);
-#else
- PyModule_AddIntConstant(m, "_have_ns_fs_timestamps", 0);
#endif
e = getenv("BUP_FORCE_TTY");
istty2 = isatty(2) || (atoi(e ? e : "0") & 2);
"""
import os, sys, zlib, time, subprocess, struct, stat, re, tempfile, glob
from bup.helpers import *
-from bup import _helpers, path, midx, bloom
+from bup import _helpers, path, midx, bloom, xstat
max_pack_size = 1000*1000*1000 # larger packs will slow down pruning
max_pack_objects = 200*1000 # cache memory usage is about 83 bytes per object
else:
midxl.append(mx)
midxl.sort(key=lambda ix:
- (-len(ix), -os.stat(ix.name).st_mtime))
+ (-len(ix), -xstat.stat(ix.name).st_mtime))
for ix in midxl:
any_needed = False
for sub in ix.idxnames:
import os, stat, struct, tempfile
+from bup import xstat
from bup.helpers import *
EMPTY_SHA = '\0'*20
FAKE_SHA = '\x01'*20
-INDEX_HDR = 'BUPI\0\0\0\2'
+INDEX_HDR = 'BUPI\0\0\0\3'
-# FIXME: guess I should have used 64-bit integers to store the mtime/ctime.
-# NTFS mtime=0 corresponds to the year 1600, which can't be stored in a 32-bit
-# time_t. Next time we update the bupindex format, keep that in mind.
-INDEX_SIG = '!IiiIIQII20sHII'
+# Use 64-bit integers for mtime/ctime to handle NTFS zero (Y1600) and Y2038.
+INDEX_SIG = '!QQQqqIIQII20sHII'
ENTLEN = struct.calcsize(INDEX_SIG)
FOOTER_SIG = '!Q'
return (ofs,n)
-def _golevel(level, f, ename, newentry):
+def _golevel(level, f, ename, newentry, tmax):
# close nodes back up the tree
assert(level)
while ename[:len(level.ename)] != level.ename:
- n = BlankNewEntry(level.ename[-1])
+ n = BlankNewEntry(level.ename[-1], tmax)
n.flags |= IX_EXISTS
(n.children_ofs,n.children_n) = level.write(f)
level.parent.list.append(n)
# are we in precisely the right place?
assert(ename == level.ename)
- n = newentry or BlankNewEntry(ename and level.ename[-1] or None)
+ n = newentry or BlankNewEntry(ename and level.ename[-1] or None, tmax)
(n.children_ofs,n.children_n) = level.write(f)
if level.parent:
level.parent.list.append(n)
class Entry:
- def __init__(self, basename, name):
+ def __init__(self, basename, name, tmax):
self.basename = str(basename)
self.name = str(name)
+ self.tmax = tmax
self.children_ofs = 0
self.children_n = 0
def __repr__(self):
return ("(%s,0x%04x,%d,%d,%d,%d,%d,%s/%s,0x%04x,0x%08x/%d)"
- % (self.name, self.dev,
+ % (self.name, self.dev, self.ino, self.nlink,
self.ctime, self.mtime, self.uid, self.gid,
self.size, oct(self.mode), oct(self.gitmode),
self.flags, self.children_ofs, self.children_n))
def packed(self):
try:
return struct.pack(INDEX_SIG,
- self.dev, self.ctime, self.mtime,
+ self.dev, self.ino, self.nlink,
+ self.ctime, self.mtime,
self.uid, self.gid, self.size, self.mode,
self.gitmode, self.sha, self.flags,
self.children_ofs, self.children_n)
raise
def from_stat(self, st, tstart):
- old = (self.dev, self.ctime, self.mtime,
+ old = (self.dev, self.ino, self.nlink, self.ctime, self.mtime,
self.uid, self.gid, self.size, self.flags & IX_EXISTS)
- new = (st.st_dev,
- int(st.st_ctime.approx_secs()),
- int(st.st_mtime.approx_secs()),
+ new = (st.st_dev, st.st_ino, st.st_nlink,
+ xstat.fstime_floor_secs(st.st_ctime),
+ xstat.fstime_floor_secs(st.st_mtime),
st.st_uid, st.st_gid, st.st_size, IX_EXISTS)
self.dev = st.st_dev
- self.ctime = int(st.st_ctime.approx_secs())
- self.mtime = int(st.st_mtime.approx_secs())
+ self.ino = st.st_ino
+ self.nlink = st.st_nlink
+ self.ctime = xstat.fstime_floor_secs(st.st_ctime)
+ self.mtime = xstat.fstime_floor_secs(st.st_mtime)
self.uid = st.st_uid
self.gid = st.st_gid
self.size = st.st_size
self.mode = st.st_mode
self.flags |= IX_EXISTS
- if int(st.st_ctime.approx_secs()) >= tstart or old != new \
+ if xstat.fstime_floor_secs(st.st_ctime) >= tstart or old != new \
or self.sha == EMPTY_SHA or not self.gitmode:
self.invalidate()
self._fixup()
self.gid += 0x100000000
assert(self.uid >= 0)
assert(self.gid >= 0)
- if self.mtime < -0x80000000: # can happen in NTFS on 64-bit linux
- self.mtime = 0
- if self.ctime < -0x80000000:
- self.ctime = 0
- if self.mtime > 0x7fffffff:
- self.mtime = 0x7fffffff
- if self.ctime > 0x7fffffff:
- self.ctime = 0x7fffffff
+ self.mtime = self._fixup_time(self.mtime)
+ self.ctime = self._fixup_time(self.ctime)
+
+ def _fixup_time(self, t):
+ if self.tmax != None and t > self.tmax:
+ return self.tmax
+ else:
+ return t
def is_valid(self):
f = IX_HASHVALID|IX_EXISTS
class NewEntry(Entry):
- def __init__(self, basename, name, dev, ctime, mtime, uid, gid,
- size, mode, gitmode, sha, flags, children_ofs, children_n):
- Entry.__init__(self, basename, name)
- (self.dev, self.ctime, self.mtime, self.uid, self.gid,
- self.size, self.mode, self.gitmode, self.sha,
+ def __init__(self, basename, name, tmax, dev, ino, nlink, ctime, mtime,
+ uid, gid, size, mode, gitmode, sha, flags,
+ children_ofs, children_n):
+ Entry.__init__(self, basename, name, tmax)
+ (self.dev, self.ino, self.nlink, self.ctime, self.mtime,
+ self.uid, self.gid, self.size, self.mode, self.gitmode, self.sha,
self.flags, self.children_ofs, self.children_n
- ) = (dev, int(ctime), int(mtime), uid, gid,
+ ) = (dev, ino, nlink, int(ctime), int(mtime), uid, gid,
size, mode, gitmode, sha, flags, children_ofs, children_n)
self._fixup()
class BlankNewEntry(NewEntry):
- def __init__(self, basename):
- NewEntry.__init__(self, basename, basename,
- 0, 0, 0, 0, 0, 0, 0,
+ def __init__(self, basename, tmax):
+ NewEntry.__init__(self, basename, basename, tmax,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, EMPTY_SHA, 0, 0, 0)
class ExistingEntry(Entry):
def __init__(self, parent, basename, name, m, ofs):
- Entry.__init__(self, basename, name)
+ Entry.__init__(self, basename, name, None)
self.parent = parent
self._m = m
self._ofs = ofs
- (self.dev, self.ctime, self.mtime, self.uid, self.gid,
- self.size, self.mode, self.gitmode, self.sha,
+ (self.dev, self.ino, self.nlink, self.ctime, self.mtime,
+ self.uid, self.gid, self.size, self.mode, self.gitmode, self.sha,
self.flags, self.children_ofs, self.children_n
) = struct.unpack(INDEX_SIG, str(buffer(m, ofs, ENTLEN)))
class Writer:
- def __init__(self, filename):
+ def __init__(self, filename, tmax):
self.rootlevel = self.level = Level([], None)
self.f = None
self.count = 0
self.lastfile = None
self.filename = None
self.filename = filename = realpath(filename)
+ self.tmax = tmax
(dir,name) = os.path.split(filename)
(ffd,self.tmpname) = tempfile.mkstemp('.tmp', filename, dir)
self.f = os.fdopen(ffd, 'wb', 65536)
def flush(self):
if self.level:
- self.level = _golevel(self.level, self.f, [], None)
+ self.level = _golevel(self.level, self.f, [], None, self.tmax)
self.count = self.rootlevel.count
if self.count:
self.count += 1
raise Error('%r must come before %r'
% (''.join(e.name), ''.join(self.lastfile)))
self.lastfile = e.name
- self.level = _golevel(self.level, self.f, ename, entry)
+ self.level = _golevel(self.level, self.f, ename, entry, self.tmax)
def add(self, name, st, hashgen = None):
endswith = name.endswith('/')
if st:
isdir = stat.S_ISDIR(st.st_mode)
assert(isdir == endswith)
- e = NewEntry(basename, name, st.st_dev,
- int(st.st_ctime.approx_secs()),
- int(st.st_mtime.approx_secs()),
+ e = NewEntry(basename, name, self.tmax,
+ st.st_dev, st.st_ino, st.st_nlink,
+ xstat.fstime_floor_secs(st.st_ctime),
+ xstat.fstime_floor_secs(st.st_mtime),
st.st_uid, st.st_gid,
st.st_size, st.st_mode, gitmode, sha, flags,
0, 0)
else:
assert(endswith)
- e = BlankNewEntry(basename)
+ e = BlankNewEntry(basename, tmax)
e.gitmode = gitmode
e.sha = sha
e.flags = flags
# Public License as described in the bup LICENSE file.
import errno, os, sys, stat, pwd, grp, struct, re
from cStringIO import StringIO
-from bup import vint
+from bup import vint, xstat
from bup.drecurse import recursive_dirlist
from bup.helpers import add_error, mkdirp, log, is_superuser
-from bup.xstat import utime, lutime, lstat, FSTime
+from bup.xstat import utime, lutime, lstat
import bup._helpers as _helpers
try:
add_error("no group name for id %s '%s'" % (st.st_gid, path))
def _encode_common(self):
- atime = self.atime.to_timespec()
- mtime = self.mtime.to_timespec()
- ctime = self.ctime.to_timespec()
+ atime = xstat.nsecs_to_timespec(self.atime)
+ mtime = xstat.nsecs_to_timespec(self.mtime)
+ ctime = xstat.nsecs_to_timespec(self.ctime)
result = vint.pack('VVsVsVvVvVvV',
self.mode,
self.uid,
mtime_ns,
self.ctime,
ctime_ns) = vint.unpack('VVsVsVvVvVvV', data)
- self.atime = FSTime.from_timespec((self.atime, atime_ns))
- self.mtime = FSTime.from_timespec((self.mtime, mtime_ns))
- self.ctime = FSTime.from_timespec((self.ctime, ctime_ns))
+ self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
+ self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
+ self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
def _create_via_common_rec(self, path, create_symlinks=True):
# If the path already exists and is a dir, try rmdir.
# If the path already exists and is anything else, try unlink.
st = None
try:
- st = lstat(path)
+ st = xstat.lstat(path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
def from_path(path, statinfo=None, archive_path=None, save_symlinks=True):
result = Metadata()
result.path = archive_path
- st = statinfo or lstat(path)
+ st = statinfo or xstat.lstat(path)
result._add_common(path, st)
if save_symlinks:
result._add_symlink_target(path, st)
unlink('index.tmp')
ds = xstat.stat('.')
fs = xstat.stat('tindex.py')
- w = index.Writer('index.tmp')
+ w = index.Writer('index.tmp', time.time() - 1)
w.add('/var/tmp/sporky', fs)
w.add('/etc/passwd', fs)
w.add('/etc/', ds)
# Dec 31, 1969
os.utime("foo", (-86400, -86400))
- e = index.BlankNewEntry("foo")
- e.from_stat(xstat.stat("foo"), time.time())
+ now = time.time()
+ e = index.BlankNewEntry("foo", now - 1)
+ e.from_stat(xstat.stat("foo"), now)
assert len(e.packed())
WVPASS()
# Jun 10, 1893
os.utime("foo", (-0x80000000, -0x80000000))
- e = index.BlankNewEntry("foo")
- e.from_stat(xstat.stat("foo"), time.time())
+ e = index.BlankNewEntry("foo", now - 1)
+ e.from_stat(xstat.stat("foo"), now)
assert len(e.packed())
WVPASS()
unlink('index2.tmp')
ds = xstat.stat('.')
fs = xstat.stat('tindex.py')
+ tmax = time.time() - 1
- w1 = index.Writer('index.tmp')
+ w1 = index.Writer('index.tmp', tmax)
w1.add('/a/b/x', fs)
w1.add('/a/b/c', fs)
w1.add('/a/b/', ds)
#w1.close()
WVPASS()
- w2 = index.Writer('index2.tmp')
+ w2 = index.Writer('index2.tmp', tmax)
w2.add('/a/b/n/2', fs)
#w2.close()
WVPASS()
- w3 = index.Writer('index3.tmp')
+ w3 = index.Writer('index3.tmp', tmax)
w3.add('/a/c/n/3', fs)
#w3.close()
WVPASS()
-import math
+import math, tempfile, subprocess
from wvtest import *
import bup._helpers as _helpers
-from bup.xstat import FSTime
+from bup import xstat
-def _test_fstime():
- def approx_eq(x, y):
- return math.fabs(x - y) < 1 / 10e8
- def ts_eq_ish(x, y):
- return approx_eq(x[0], y[0]) and approx_eq(x[1], y[1])
- def fst_eq_ish(x, y):
- return approx_eq(x.approx_secs(), y.approx_secs())
- def s_ns_eq_ish(fst, s, ns):
- (fst_s, fst_ns) = fst.secs_nsecs()
- return approx_eq(fst_s, s) and approx_eq(fst_ns, ns)
- from_secs = FSTime.from_secs
- from_ts = FSTime.from_timespec
- WVPASS(from_secs(0) == from_secs(0))
- WVPASS(from_secs(0) < from_secs(1))
- WVPASS(from_secs(-1) < from_secs(1))
- WVPASS(from_secs(1) > from_secs(0))
- WVPASS(from_secs(1) > from_secs(-1))
+@wvtest
+def test_fstime():
+ WVPASSEQ(xstat.timespec_to_nsecs((0, 0)), 0)
+ WVPASSEQ(xstat.timespec_to_nsecs((1, 0)), 10**9)
+ WVPASSEQ(xstat.timespec_to_nsecs((0, 10**9 / 2)), 500000000)
+ WVPASSEQ(xstat.timespec_to_nsecs((1, 10**9 / 2)), 1500000000)
+ WVPASSEQ(xstat.timespec_to_nsecs((-1, 0)), -10**9)
+ WVPASSEQ(xstat.timespec_to_nsecs((-1, 10**9 / 2)), -500000000)
+ WVPASSEQ(xstat.timespec_to_nsecs((-2, 10**9 / 2)), -1500000000)
+ WVEXCEPT(Exception, xstat.timespec_to_nsecs, (0, -1))
+ WVPASSEQ(type(xstat.timespec_to_nsecs((2, 22222222))), type(0))
+ WVPASSEQ(type(xstat.timespec_to_nsecs((-2, 22222222))), type(0))
- WVPASS(fst_eq_ish(from_secs(0), from_ts((0, 0))))
- WVPASS(fst_eq_ish(from_secs(1), from_ts((1, 0))))
- WVPASS(fst_eq_ish(from_secs(-1), from_ts((-1, 0))))
- WVPASS(fst_eq_ish(from_secs(-0.5), from_ts((-1, 10**9 / 2))))
- WVPASS(fst_eq_ish(from_secs(-1.5), from_ts((-2, 10**9 / 2))))
- WVPASS(fst_eq_ish(from_secs(1 / 10e8), from_ts((0, 1))))
- WVPASS(fst_eq_ish(from_secs(-1 / 10e8), from_ts((-1, 10**9 - 1))))
+ WVPASSEQ(xstat.nsecs_to_timespec(0), (0, 0))
+ WVPASSEQ(xstat.nsecs_to_timespec(10**9), (1, 0))
+ WVPASSEQ(xstat.nsecs_to_timespec(500000000), (0, 10**9 / 2))
+ WVPASSEQ(xstat.nsecs_to_timespec(1500000000), (1, 10**9 / 2))
+ WVPASSEQ(xstat.nsecs_to_timespec(-10**9), (-1, 0))
+ WVPASSEQ(xstat.nsecs_to_timespec(-500000000), (-1, 10**9 / 2))
+ WVPASSEQ(xstat.nsecs_to_timespec(-1500000000), (-2, 10**9 / 2))
+ x = xstat.nsecs_to_timespec(1977777778)
+ WVPASSEQ(type(x[0]), type(0))
+ WVPASSEQ(type(x[1]), type(0))
+ x = xstat.nsecs_to_timespec(-1977777778)
+ WVPASSEQ(type(x[0]), type(0))
+ WVPASSEQ(type(x[1]), type(0))
- WVPASS(ts_eq_ish(from_secs(0).to_timespec(), (0, 0)))
- WVPASS(ts_eq_ish(from_secs(1).to_timespec(), (1, 0)))
- WVPASS(ts_eq_ish(from_secs(-1).to_timespec(), (-1, 0)))
- WVPASS(ts_eq_ish(from_secs(-0.5).to_timespec(), (-1, 10**9 / 2)))
- WVPASS(ts_eq_ish(from_secs(-1.5).to_timespec(), (-2, 10**9 / 2)))
- WVPASS(ts_eq_ish(from_secs(1 / 10e8).to_timespec(), (0, 1)))
- WVPASS(ts_eq_ish(from_secs(-1 / 10e8).to_timespec(), (-1, 10**9 - 1)))
+ WVPASSEQ(xstat.fstime_floor_secs(0), 0)
+ WVPASSEQ(xstat.fstime_floor_secs(10**9 / 2), 0)
+ WVPASSEQ(xstat.fstime_floor_secs(10**9), 1)
+ WVPASSEQ(xstat.fstime_floor_secs(-10**9 / 2), -1)
+ WVPASSEQ(xstat.fstime_floor_secs(-10**9), -1)
+ WVPASSEQ(type(xstat.fstime_floor_secs(10**9 / 2)), type(0))
+ WVPASSEQ(type(xstat.fstime_floor_secs(-10**9 / 2)), type(0))
- WVPASS(s_ns_eq_ish(from_secs(0), 0, 0))
- WVPASS(s_ns_eq_ish(from_secs(1), 1, 0))
- WVPASS(s_ns_eq_ish(from_secs(-1), -1, 0))
- WVPASS(s_ns_eq_ish(from_secs(-0.5), 0, - 10**9 / 2))
- WVPASS(s_ns_eq_ish(from_secs(-1.5), -1, - 10**9 / 2))
- WVPASS(s_ns_eq_ish(from_secs(-1 / 10e8), 0, -1))
@wvtest
-def test_fstime():
- _test_fstime();
- if _helpers._have_ns_fs_timestamps: # Test native python timestamp rep too.
- orig = _helpers._have_ns_fs_timestamps
- try:
- _helpers._have_ns_fs_timestamps = None
- _test_fstime();
- finally:
- _helpers._have_ns_fs_timestamps = orig
+def test_timespec_behavior():
+ tmpdir = tempfile.mkdtemp(prefix='bup-tmetadata-')
+ try:
+ path = tmpdir + '/foo'
+ open(path, 'w').close()
+ frac_ts = (0, 10**9 / 2)
+ _helpers.utimensat(_helpers.AT_FDCWD, path, (frac_ts, frac_ts), 0)
+ st = _helpers.stat(path)
+ atime_ts = st[8]
+ mtime_ts = st[9]
+ WVPASSEQ(atime_ts[0], 0)
+ WVPASS(atime_ts[1] == 0 or atime_ts[1] == frac_ts[1])
+ WVPASSEQ(mtime_ts[0], 0)
+ WVPASS(mtime_ts[1] == 0 or mtime_ts[1] == frac_ts[1])
+ if(mtime_ts[1] == frac_ts[1]):
+ # Sub-second resolution -- check behavior of negative timespecs.
+ neg_ts = (-43, 10**9 / 2)
+ _helpers.utimensat(_helpers.AT_FDCWD, path, (neg_ts, neg_ts), 0)
+ st = _helpers.stat(path)
+ atime_ts = st[8]
+ mtime_ts = st[9]
+ WVPASSEQ(atime_ts, neg_ts)
+ WVPASSEQ(mtime_ts, neg_ts)
+ finally:
+ subprocess.call(['rm', '-rf', tmpdir])
_have_utimensat = False
-class FSTime:
- # Class to represent filesystem timestamps. Use integer
- # nanoseconds on platforms where we have the higher resolution
- # lstat. Use the native python stat representation (floating
- # point seconds) otherwise.
-
- def __cmp__(self, x):
- return self._value.__cmp__(x._value)
-
- def __repr__(self):
- return 'FSTime(%d)' % self._value
-
- def to_timespec(self):
- """Return (s, ns) where ns is always non-negative
- and t = s + ns / 10e8""" # metadata record rep (and libc rep)
- s_ns = self.secs_nsecs()
- if s_ns[0] > 0 or s_ns[1] >= 0:
- return s_ns
- return (s_ns[0] - 1, 10**9 + s_ns[1]) # ns is negative
+def timespec_to_nsecs((ts_s, ts_ns)):
+ # c.f. _helpers.c: timespec_vals_to_py_ns()
+ if ts_ns < 0 or ts_ns > 999999999:
+ raise Exception('invalid timespec nsec value')
+ return ts_s * 10**9 + ts_ns
- @staticmethod
- def from_secs(secs):
- ts = FSTime()
- ts._value = int(round(secs * 10**9))
- return ts
- @staticmethod
- def from_timespec(timespec):
- ts = FSTime()
- ts._value = timespec[0] * 10**9 + timespec[1]
- return ts
-
- def approx_secs(self):
- return self._value / 10e8;
-
- def secs_nsecs(self):
- "Return a (s, ns) pair: -1.5s -> (-1, -10**9 / 2)."
- if self._value >= 0:
- return (self._value / 10**9, self._value % 10**9)
- abs_val = -self._value
- return (- (abs_val / 10**9), - (abs_val % 10**9))
-
- if _helpers._have_ns_fs_timestamps: # Use integer nanoseconds.
- @staticmethod
- def from_stat_time(stat_time):
- return FSTime.from_timespec(stat_time)
- else: # Use python default floating-point seconds.
- @staticmethod
- def from_stat_time(stat_time):
- return FSTime.from_secs(stat_time)
+def nsecs_to_timespec(ns):
+ """Return (s, ns) where ns is always non-negative
+ and t = s + ns / 10e8""" # metadata record rep (and libc rep)
+ ns = int(ns)
+ return (ns / 10**9, ns % 10**9)
+
+
+def fstime_floor_secs(ns):
+ """Return largest integer not greater than ns / 10e8."""
+ return int(ns) / 10**9;
+
+
+def fstime_to_timespec(ns):
+ return nsecs_to_timespec(ns)
if _have_utimensat:
def lutime(path, times):
- atime = times[0].to_timespec()
- mtime = times[1].to_timespec()
- return _helpers.utimensat(_helpers.AT_FDCWD, path, (atime, mtime),
- _helpers.AT_SYMLINK_NOFOLLOW)
+ atime = nsecs_to_timespec(times[0])
+ mtime = nsecs_to_timespec(times[1])
+ _helpers.utimensat(_helpers.AT_FDCWD, path, (atime, mtime),
+ _helpers.AT_SYMLINK_NOFOLLOW)
def utime(path, times):
- atime = times[0].to_timespec()
- mtime = times[1].to_timespec()
- return _helpers.utimensat(_helpers.AT_FDCWD, path, (atime, mtime), 0)
+ atime = nsecs_to_timespec(times[0])
+ mtime = nsecs_to_timespec(times[1])
+ _helpers.utimensat(_helpers.AT_FDCWD, path, (atime, mtime), 0)
else:
def lutime(path, times):
return None
def utime(path, times):
- atime = times[0].approx_secs()
- mtime = times[1].approx_secs()
+ atime = fstime_floor_secs(times[0])
+ mtime = fstime_floor_secs(times[1])
os.utime(path, (atime, mtime))
class stat_result:
@staticmethod
- def from_stat_rep(st):
+ def from_xstat_rep(st):
result = stat_result()
- if _helpers._have_ns_fs_timestamps:
- (result.st_mode,
- result.st_ino,
- result.st_dev,
- result.st_nlink,
- result.st_uid,
- result.st_gid,
- result.st_rdev,
- result.st_size,
- atime,
- mtime,
- ctime) = st
- else:
- result.st_mode = st.st_mode
- result.st_ino = st.st_ino
- result.st_dev = st.st_dev
- result.st_nlink = st.st_nlink
- result.st_uid = st.st_uid
- result.st_gid = st.st_gid
- result.st_rdev = st.st_rdev
- result.st_size = st.st_size
- atime = st.st_atime
- mtime = st.st_mtime
- ctime = st.st_ctime
- result.st_atime = FSTime.from_stat_time(atime)
- result.st_mtime = FSTime.from_stat_time(mtime)
- result.st_ctime = FSTime.from_stat_time(ctime)
+ (result.st_mode,
+ result.st_ino,
+ result.st_dev,
+ result.st_nlink,
+ result.st_uid,
+ result.st_gid,
+ result.st_rdev,
+ result.st_size,
+ result.st_atime,
+ result.st_mtime,
+ result.st_ctime) = st
+ result.st_atime = timespec_to_nsecs(result.st_atime)
+ result.st_mtime = timespec_to_nsecs(result.st_mtime)
+ result.st_ctime = timespec_to_nsecs(result.st_ctime)
return result
-try:
- _stat = _helpers.stat
-except AttributeError, e:
- _stat = os.stat
-
def stat(path):
- return stat_result.from_stat_rep(_stat(path))
-
+ return stat_result.from_xstat_rep(_helpers.stat(path))
-try:
- _fstat = _helpers.fstat
-except AttributeError, e:
- _fstat = os.fstat
def fstat(path):
- return stat_result.from_stat_rep(_fstat(path))
+ return stat_result.from_xstat_rep(_helpers.fstat(path))
-try:
- _lstat = _helpers.lstat
-except AttributeError, e:
- _lstat = os.lstat
-
def lstat(path):
- return stat_result.from_stat_rep(_lstat(path))
+ return stat_result.from_xstat_rep(_helpers.lstat(path))
mkdir testfs/src/foo
touch testfs/src/bar
PYTHONPATH="$TOP/lib" \
- python -c "from bup.xstat import lutime, FSTime; \
- x = FSTime.from_secs(42);\
- lutime('testfs/src/foo', (x, x));\
- lutime('testfs/src/bar', (x, x));"
+ python -c "from bup import xstat; \
+ x = xstat.timespec_to_nsecs((42, 0));\
+ xstat.lutime('testfs/src/foo', (x, x));\
+ xstat.lutime('testfs/src/bar', (x, x));"
cd testfs
WVPASS bup meta -v --create --recurse --file src.meta src
bup meta -tvf src.meta
projects / bup.git / commitdiff