From: Avery Pennarun Date: Mon, 30 May 2011 00:50:25 +0000 (-0400) Subject: Merge branch 'master' into meta X-Git-Tag: bup-0.25-rc1~1^2~14 X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=56dd4712e7d66a8623ec7b85753469f9500c6003;hp=-c;p=bup.git Merge branch 'master' into meta * master: (27 commits) t/test.sh: 'ls' on NetBSD sets -A by default as root; work around it. README: add a list of binary packages README: rework the title hierarchy Clarify the message when the BUP_DIR doesn't exist. Refactor: unify ls/ftp-ls code ftp/ls: Adjust documentation ls: include hidden files when explicitly requested ftp: implement ls -s (show hashes) ftp/ls: columnate output attached to a tty, else don't ftp: don't output trailing line for 'ls' ftp: output a newline on EOF when on a tty config: more config stuff to config/ subdir, call it from Makefile. cmd/{split,save}: support any compression level using the new -# feature. options.py: add support for '-#' style compression options. Add documentation for compression levels Add test case for compression level Add compression level options to bup save and bup split Make zlib compression level a parameter for Client Make zlib compression level a parameter of git.PackWriter Use is_superuser() rather than checking euid directly ... Conflicts: lib/bup/metadata.py --- 56dd4712e7d66a8623ec7b85753469f9500c6003 diff --combined Makefile index 2069099,e04235d..e369f1d --- a/Makefile +++ b/Makefile @@@ -16,10 -16,11 +16,11 @@@ Documentation/all: bu INSTALL=install PYTHON=python - MANDIR=$(DESTDIR)/usr/share/man - DOCDIR=$(DESTDIR)/usr/share/doc/bup - BINDIR=$(DESTDIR)/usr/bin - LIBDIR=$(DESTDIR)/usr/lib/bup + PREFIX=/usr + MANDIR=$(DESTDIR)$(PREFIX)/share/man + DOCDIR=$(DESTDIR)$(PREFIX)/share/doc/bup + BINDIR=$(DESTDIR)$(PREFIX)/bin + LIBDIR=$(DESTDIR)$(PREFIX)/lib/bup install: all $(INSTALL) -d $(MANDIR)/man1 $(DOCDIR) $(BINDIR) \ $(LIBDIR)/bup $(LIBDIR)/cmd $(LIBDIR)/tornado \ @@@ -56,11 -57,17 +57,17 @@@ %/clean: $(MAKE) -C $* clean + + config/config.h: config/Makefile config/configure config/configure.inc \ + $(wildcard config/*.in) + cd config && make config.h lib/bup/_helpers$(SOEXT): \ + config/config.h \ lib/bup/bupsplit.c lib/bup/_helpers.c lib/bup/csetup.py @rm -f $@ - cd lib/bup && LDFLAGS="$(LDFLAGS)" CFLAGS="$(CFLAGS)" $(PYTHON) csetup.py build + cd lib/bup && \ + LDFLAGS="$(LDFLAGS)" CFLAGS="$(CFLAGS)" $(PYTHON) csetup.py build cp lib/bup/build/*/_helpers$(SOEXT) lib/bup/ .PHONY: lib/bup/_version.py @@@ -139,6 -146,8 +146,6 @@@ import-docs: Documentation/clea git archive origin/html | (cd Documentation; tar -xvf -) git archive origin/man | (cd Documentation; tar -xvf -) -# tgit.py plays with permissions on lib/bup/t/pybuptest.tmp, so we should -# ensure that we can delete the directory before doing it. clean: Documentation/clean rm -f *.o lib/*/*.o *.so lib/*/*.so *.dll *.exe \ .*~ *~ */*~ lib/*/*~ lib/*/*/*~ \ diff --combined lib/bup/_helpers.c index 2e7c6bf,a0656cc..5956f5b --- a/lib/bup/_helpers.c +++ b/lib/bup/_helpers.c @@@ -1,5 -1,6 +1,6 @@@ #define _LARGEFILE64_SOURCE 1 #undef NDEBUG + #include "../../config/config.h" #include "bupsplit.h" #include #include @@@ -7,8 -8,6 +8,8 @@@ #include #include #include +#include +#include #include #include #include @@@ -16,6 -15,8 +17,6 @@@ #ifdef linux #include #include -#include -#include #endif static int istty2 = 0; @@@ -679,11 -680,11 +680,11 @@@ static PyObject *bup_set_linux_file_att } close(fd); - return Py_BuildValue("i", 1); + return Py_BuildValue("O", Py_None); } #endif /* def linux */ - + #ifdef HAVE_UTIMENSAT #if defined(_ATFILE_SOURCE) \ || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L #define HAVE_BUP_UTIMENSAT 1 @@@ -754,50 -755,16 +755,50 @@@ static PyObject *bup_utimensat(PyObjec if (rc != 0) return PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); - return Py_BuildValue("i", 1); + return Py_BuildValue("O", Py_None); } #endif /* defined(_ATFILE_SOURCE) || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */ - + #endif /* HAVE_UTIMENSAT */ -#ifdef linux /* and likely others */ +static PyObject *stat_struct_to_py(const struct stat *st) +{ + /* Enforce the current timespec nanosecond range expectations. */ + if (st->st_atim.tv_nsec < 0 || st->st_atim.tv_nsec > 999999999) + { + PyErr_SetString(PyExc_ValueError, "invalid atime timespec nanoseconds"); + return NULL; + } + if (st->st_mtim.tv_nsec < 0 || st->st_mtim.tv_nsec > 999999999) + { + PyErr_SetString(PyExc_ValueError, "invalid mtime timespec nanoseconds"); + return NULL; + } + if (st->st_ctim.tv_nsec < 0 || st->st_ctim.tv_nsec > 999999999) + { + PyErr_SetString(PyExc_ValueError, "invalid ctime timespec nanoseconds"); + return NULL; + } + + return Py_BuildValue("kkkkkkkk(Ll)(Ll)(Ll)", + (unsigned long) st->st_mode, + (unsigned long) st->st_ino, + (unsigned long) st->st_dev, + (unsigned long) st->st_nlink, + (unsigned long) st->st_uid, + (unsigned long) st->st_gid, + (unsigned long) st->st_rdev, + (unsigned long) st->st_size, + (long long) st->st_atime, + (long) st->st_atim.tv_nsec, + (long long) st->st_mtime, + (long) st->st_mtim.tv_nsec, + (long long) st->st_ctime, + (long) st->st_ctim.tv_nsec); +} + -#define HAVE_BUP_STAT 1 static PyObject *bup_stat(PyObject *self, PyObject *args) { int rc; @@@ -810,10 -777,29 +811,10 @@@ rc = stat(filename, &st); if (rc != 0) return PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return stat_struct_to_py(&st); +} + - return Py_BuildValue("kkkkkkkk" - "(ll)" - "(ll)" - "(ll)", - (unsigned long) st.st_mode, - (unsigned long) st.st_ino, - (unsigned long) st.st_dev, - (unsigned long) st.st_nlink, - (unsigned long) st.st_uid, - (unsigned long) st.st_gid, - (unsigned long) st.st_rdev, - (unsigned long) st.st_size, - (long) st.st_atime, - (long) st.st_atim.tv_nsec, - (long) st.st_mtime, - (long) st.st_mtim.tv_nsec, - (long) st.st_ctime, - (long) st.st_ctim.tv_nsec); -} - - -#define HAVE_BUP_LSTAT 1 static PyObject *bup_lstat(PyObject *self, PyObject *args) { int rc; @@@ -826,10 -812,29 +827,10 @@@ rc = lstat(filename, &st); if (rc != 0) return PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return stat_struct_to_py(&st); +} + - return Py_BuildValue("kkkkkkkk" - "(ll)" - "(ll)" - "(ll)", - (unsigned long) st.st_mode, - (unsigned long) st.st_ino, - (unsigned long) st.st_dev, - (unsigned long) st.st_nlink, - (unsigned long) st.st_uid, - (unsigned long) st.st_gid, - (unsigned long) st.st_rdev, - (unsigned long) st.st_size, - (long) st.st_atime, - (long) st.st_atim.tv_nsec, - (long) st.st_mtime, - (long) st.st_mtim.tv_nsec, - (long) st.st_ctime, - (long) st.st_ctim.tv_nsec); -} - - -#define HAVE_BUP_FSTAT 1 static PyObject *bup_fstat(PyObject *self, PyObject *args) { int rc, fd; @@@ -841,9 -846,29 +842,9 @@@ rc = fstat(fd, &st); if (rc != 0) return PyErr_SetFromErrno(PyExc_OSError); - - return Py_BuildValue("kkkkkkkk" - "(ll)" - "(ll)" - "(ll)", - (unsigned long) st.st_mode, - (unsigned long) st.st_ino, - (unsigned long) st.st_dev, - (unsigned long) st.st_nlink, - (unsigned long) st.st_uid, - (unsigned long) st.st_gid, - (unsigned long) st.st_rdev, - (unsigned long) st.st_size, - (long) st.st_atime, - (long) st.st_atim.tv_nsec, - (long) st.st_mtime, - (long) st.st_mtim.tv_nsec, - (long) st.st_ctime, - (long) st.st_ctim.tv_nsec); + return stat_struct_to_py(&st); } -#endif /* def linux */ - static PyMethodDef helper_methods[] = { { "selftest", selftest, METH_VARARGS, @@@ -884,12 -909,18 +885,12 @@@ { "utimensat", bup_utimensat, METH_VARARGS, "Change file timestamps with nanosecond precision." }, #endif -#ifdef HAVE_BUP_STAT { "stat", bup_stat, METH_VARARGS, "Extended version of stat." }, -#endif -#ifdef HAVE_BUP_LSTAT { "lstat", bup_lstat, METH_VARARGS, "Extended version of lstat." }, -#endif -#ifdef HAVE_BUP_FSTAT { "fstat", bup_fstat, METH_VARARGS, "Extended version of fstat." }, -#endif { NULL, NULL, 0, NULL }, // sentinel }; @@@ -904,6 -935,11 +905,6 @@@ PyMODINIT_FUNC init_helpers(void PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD)); PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW", Py_BuildValue("i", AT_SYMLINK_NOFOLLOW)); -#endif -#ifdef HAVE_BUP_STAT - PyModule_AddIntConstant(m, "_have_ns_fs_timestamps", 1); -#else - PyModule_AddIntConstant(m, "_have_ns_fs_timestamps", 0); #endif e = getenv("BUP_FORCE_TTY"); istty2 = isatty(2) || (atoi(e ? e : "0") & 2); diff --combined lib/bup/git.py index eacf407,421c306..5edfe1c --- a/lib/bup/git.py +++ b/lib/bup/git.py @@@ -4,7 -4,7 +4,7 @@@ interact with the Git data structures """ import os, sys, zlib, time, subprocess, struct, stat, re, tempfile, glob from bup.helpers import * -from bup import _helpers, path, midx, bloom +from bup import _helpers, path, midx, bloom, xstat max_pack_size = 1000*1000*1000 # larger packs will slow down pruning max_pack_objects = 200*1000 # cache memory usage is about 83 bytes per object @@@ -165,7 -165,7 +165,7 @@@ def tree_decode(buf) yield (int(mode, 8), name, sha) - def _encode_packobj(type, content): + def _encode_packobj(type, content, compression_level=1): szout = '' sz = len(content) szbits = (sz & 0x0f) | (_typemap[type]<<4) @@@ -177,14 -177,18 +177,18 @@@ break szbits = sz & 0x7f sz >>= 7 - z = zlib.compressobj(1) + if compression_level > 9: + compression_level = 9 + elif compression_level < 0: + compression_level = 0 + z = zlib.compressobj(compression_level) yield szout yield z.compress(content) yield z.flush() - def _encode_looseobj(type, content): - z = zlib.compressobj(1) + def _encode_looseobj(type, content, compression_level=1): + z = zlib.compressobj(compression_level) yield z.compress('%s %d\0' % (type, len(content))) yield z.compress(content) yield z.flush() @@@ -409,7 -413,7 +413,7 @@@ class PackIdxList else: midxl.append(mx) midxl.sort(key=lambda ix: - (-len(ix), -os.stat(ix.name).st_mtime)) + (-len(ix), -xstat.stat(ix.name).st_mtime)) for ix in midxl: any_needed = False for sub in ix.idxnames: @@@ -489,7 -493,7 +493,7 @@@ def _make_objcache() class PackWriter: """Writes Git objects inside a pack file.""" - def __init__(self, objcache_maker=_make_objcache): + def __init__(self, objcache_maker=_make_objcache, compression_level=1): self.count = 0 self.outbytes = 0 self.filename = None @@@ -497,6 -501,7 +501,7 @@@ self.idx = None self.objcache_maker = objcache_maker self.objcache = None + self.compression_level = compression_level def __del__(self): self.close() @@@ -540,7 -545,9 +545,9 @@@ log('>') if not sha: sha = calc_hash(type, content) - size, crc = self._raw_write(_encode_packobj(type, content), sha=sha) + size, crc = self._raw_write(_encode_packobj(type, content, + self.compression_level), + sha=sha) if self.outbytes >= max_pack_size or self.count >= max_pack_objects: self.breakpoint() return sha @@@ -836,7 -843,8 +843,8 @@@ def check_repo_or_die(path=None) except OSError, e: if e.errno == errno.ENOENT: if repodir != home_repodir: - log('error: %r is not a bup/git repository\n' % repo()) + log('error: %r is not a bup repository; run "bup init"\n' + % repo()) sys.exit(15) else: init_repo() diff --combined lib/bup/metadata.py index aecf740,981ad09..46d04b9 --- a/lib/bup/metadata.py +++ b/lib/bup/metadata.py @@@ -6,10 -6,10 +6,10 @@@ # Public License as described in the bup LICENSE file. import errno, os, sys, stat, pwd, grp, struct, re from cStringIO import StringIO -from bup import vint +from bup import vint, xstat from bup.drecurse import recursive_dirlist - from bup.helpers import add_error, mkdirp, log - from bup.xstat import utime, lutime + from bup.helpers import add_error, mkdirp, log, is_superuser -from bup.xstat import utime, lutime, lstat, FSTime ++from bup.xstat import utime, lutime, lstat import bup._helpers as _helpers try: @@@ -201,9 -201,9 +201,9 @@@ class Metadata add_error("no group name for id %s '%s'" % (st.st_gid, path)) def _encode_common(self): - atime = self.atime.to_timespec() - mtime = self.mtime.to_timespec() - ctime = self.ctime.to_timespec() + atime = xstat.nsecs_to_timespec(self.atime) + mtime = xstat.nsecs_to_timespec(self.mtime) + ctime = xstat.nsecs_to_timespec(self.ctime) result = vint.pack('VVsVsVvVvVvV', self.mode, self.uid, @@@ -233,16 -233,16 +233,16 @@@ mtime_ns, self.ctime, ctime_ns) = vint.unpack('VVsVsVvVvVvV', data) - self.atime = FSTime.from_timespec((self.atime, atime_ns)) - self.mtime = FSTime.from_timespec((self.mtime, mtime_ns)) - self.ctime = FSTime.from_timespec((self.ctime, ctime_ns)) + self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns)) + self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns)) + self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns)) def _create_via_common_rec(self, path, create_symlinks=True): # If the path already exists and is a dir, try rmdir. # If the path already exists and is anything else, try unlink. st = None try: - st = lstat(path) + st = xstat.lstat(path) except OSError, e: if e.errno != errno.ENOENT: raise @@@ -312,7 -312,7 +312,7 @@@ uid = -1 add_error('ignoring missing owner for "%s"\n' % path) else: - if os.geteuid() != 0: + if not is_superuser(): uid = -1 # Not root; assume we can't change owner. else: try: @@@ -611,7 -611,7 +611,7 @@@ def from_path(path, statinfo=None, archive_path=None, save_symlinks=True): result = Metadata() result.path = archive_path - st = statinfo or lstat(path) + st = statinfo or xstat.lstat(path) result._add_common(path, st) if save_symlinks: result._add_symlink_target(path, st) diff --combined t/test-meta.sh index 830e2cf,8d565b6..032d158 --- a/t/test-meta.sh +++ b/t/test-meta.sh @@@ -80,7 -80,8 +80,8 @@@ force-delete "$TOP/bupmeta.tmp ( rm -rf "$TOP/bupmeta.tmp/src" mkdir -p "$TOP/bupmeta.tmp/src" - cp -a Documentation cmd lib t "$TOP/bupmeta.tmp"/src + #cp -a Documentation cmd lib t "$TOP/bupmeta.tmp"/src + cp -pPR Documentation cmd lib t "$TOP/bupmeta.tmp"/src ) || WVFAIL # Use the test tree to check bup meta. @@@ -112,7 -113,8 +113,8 @@@ if actually-root; the chown root:root testfs chmod 0700 testfs - cp -a src testfs/src + #cp -a src testfs/src + cp -pPR src testfs/src (cd testfs && test-src-create-extract) WVSTART 'meta - atime (as root)' @@@ -122,10 -124,10 +124,10 @@@ mkdir testfs/src/foo touch testfs/src/bar PYTHONPATH="$TOP/lib" \ - python -c "from bup.xstat import lutime, FSTime; \ - x = FSTime.from_secs(42);\ - lutime('testfs/src/foo', (x, x));\ - lutime('testfs/src/bar', (x, x));" + python -c "from bup import xstat; \ + x = xstat.timespec_to_nsecs((42, 0));\ + xstat.lutime('testfs/src/foo', (x, x));\ + xstat.lutime('testfs/src/bar', (x, x));" cd testfs WVPASS bup meta -v --create --recurse --file src.meta src bup meta -tvf src.meta