From: Avery Pennarun <apenwarr@gmail.com>
Date: Mon, 28 Feb 2011 09:33:09 +0000 (-0800)
Subject: Merge branch 'master' into meta
X-Git-Tag: bup-0.25-rc1~68^2
X-Git-Url: https://arthur.barton.de/gitweb/?a=commitdiff_plain;h=2a191b01e6082564f188d1517241d19e4a6c5c7b;hp=-c;p=bup.git

Merge branch 'master' into meta

* master:
  midx/bloom: use progress() and debug1() for non-critical messages
  helpers: separately determine if stdout and stderr are ttys.
  cmd/newliner: restrict progress lines to the screen width.
  hashsplit: use shorter offset-filenames inside trees.
  Replace 040000 and 0100644 constants with GIT_MODE_{TREE,FILE}
  git.py: rename treeparse to tree_decode() and add tree_encode().
  hashsplit.py: remove PackWriter-specific knowledge.
  cmd/split: fixup progress message, and print -b output incrementally.
  hashsplit.py: convert from 'bits' to 'level' earlier in the sequence.
  hashsplit.py: okay, *really* fix BLOB_MAX.
  hashsplit.py: simplify code and fix BLOB_MAX handling.
  options.py: o.fatal(): print error after, not before, usage message.
  options.py: make --usage just print the usage message.

Conflicts:
	lib/bup/_helpers.c
---

2a191b01e6082564f188d1517241d19e4a6c5c7b
diff --combined lib/bup/_helpers.c
index 78c86f0,af9d06f..0bd4ab0
--- a/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@@ -1,24 -1,15 +1,24 @@@
 +#define _LARGEFILE64_SOURCE 1
  #undef NDEBUG
  #include "bupsplit.h"
  #include <Python.h>
  #include <assert.h>
 -#include <stdint.h>
 +#include <errno.h>
  #include <fcntl.h>
  #include <arpa/inet.h>
 +#include <stdint.h>
  #include <unistd.h>
  #include <stdlib.h>
  #include <stdio.h>
  
 +#ifdef linux
 +#include <linux/fs.h>
 +#include <sys/ioctl.h>
 +#include <sys/stat.h>
 +#include <sys/time.h>
 +#endif
 +
- static int istty = 0;
+ static int istty2 = 0;
  
  // Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV...
  #if __WIN32__ || __CYGWIN__
@@@ -96,6 -87,7 +96,7 @@@ static PyObject *splitbuf(PyObject *sel
      if (!PyArg_ParseTuple(args, "t#", &buf, &len))
  	return NULL;
      out = bupsplit_find_ofs(buf, len, &bits);
+     if (out) assert(bits >= BUP_BLOBBITS);
      return Py_BuildValue("ii", out, bits);
  }
  
@@@ -408,7 -400,7 +409,7 @@@ static PyObject *merge_into(PyObject *s
      {
  	struct idx *idx;
  	uint32_t new_prefix;
- 	if (count % 102424 == 0 && istty)
+ 	if (count % 102424 == 0 && istty2)
  	    fprintf(stderr, "midx: writing %.2f%% (%d/%d)\r",
  		    count*100.0/total, count, total);
  	idx = idxs[last_i];
@@@ -629,247 -621,7 +630,247 @@@ static PyObject *fadvise_done(PyObject 
  }
  
  
 -static PyMethodDef faster_methods[] = {
 +#ifdef linux
 +static PyObject *bup_get_linux_file_attr(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    unsigned long attr;
 +    char *path;
 +    int fd;
 +
 +    if (!PyArg_ParseTuple(args, "s", &path))
 +        return NULL;
 +
 +    fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
 +    if (fd == -1)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +
 +    attr = 0;
 +    rc = ioctl(fd, FS_IOC_GETFLAGS, &attr);
 +    if (rc == -1)
 +    {
 +        close(fd);
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +    }
 +
 +    close(fd);
 +    return Py_BuildValue("k", attr);
 +}
 +
 +
 +static PyObject *bup_set_linux_file_attr(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    unsigned long attr;
 +    char *path;
 +    int fd;
 +
 +    if (!PyArg_ParseTuple(args, "sk", &path, &attr))
 +        return NULL;
 +
 +    fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
 +    if(fd == -1)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +
 +    rc = ioctl(fd, FS_IOC_SETFLAGS, &attr);
 +    if (rc == -1)
 +    {
 +        close(fd);
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +    }
 +
 +    close(fd);
 +    Py_RETURN_TRUE;
 +}
 +#endif /* def linux */
 +
 +
 +#if defined(_ATFILE_SOURCE) \
 +  || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L
 +#define HAVE_BUP_UTIMENSAT 1
 +
 +static PyObject *bup_utimensat(PyObject *self, PyObject *args)
 +{
 +    int rc, dirfd, flags;
 +    char *path;
 +    long access, access_ns, modification, modification_ns;
 +    struct timespec ts[2];
 +
 +    if (!PyArg_ParseTuple(args, "is((ll)(ll))i",
 +                          &dirfd,
 +                          &path,
 +                          &access, &access_ns,
 +                          &modification, &modification_ns,
 +                          &flags))
 +        return NULL;
 +
 +    if (isnan(access))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(access))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time is infinite");
 +        return NULL;
 +    }
 +    else if (isnan(modification))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(modification))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time is infinite");
 +        return NULL;
 +    }
 +
 +    if (isnan(access_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time ns is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(access_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time ns is infinite");
 +        return NULL;
 +    }
 +    else if (isnan(modification_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time ns is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(modification_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time ns is infinite");
 +        return NULL;
 +    }
 +
 +    ts[0].tv_sec = access;
 +    ts[0].tv_nsec = access_ns;
 +    ts[1].tv_sec = modification;
 +    ts[1].tv_nsec = modification_ns;
 +
 +    rc = utimensat(dirfd, path, ts, flags);
 +    if (rc != 0)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +
 +    Py_RETURN_TRUE;
 +}
 +
 +#endif /* defined(_ATFILE_SOURCE)
 +          || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */
 +
 +
 +#ifdef linux /* and likely others */
 +
 +#define HAVE_BUP_STAT 1
 +static PyObject *bup_stat(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    char *filename;
 +
 +    if (!PyArg_ParseTuple(args, "s", &filename))
 +        return NULL;
 +
 +    struct stat st;
 +    rc = stat(filename, &st);
 +    if (rc != 0)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 +
 +    return Py_BuildValue("kkkkkkkk"
 +                         "(ll)"
 +                         "(ll)"
 +                         "(ll)",
 +                         (unsigned long) st.st_mode,
 +                         (unsigned long) st.st_ino,
 +                         (unsigned long) st.st_dev,
 +                         (unsigned long) st.st_nlink,
 +                         (unsigned long) st.st_uid,
 +                         (unsigned long) st.st_gid,
 +                         (unsigned long) st.st_rdev,
 +                         (unsigned long) st.st_size,
 +                         (long) st.st_atime,
 +                         (long) st.st_atim.tv_nsec,
 +                         (long) st.st_mtime,
 +                         (long) st.st_mtim.tv_nsec,
 +                         (long) st.st_ctime,
 +                         (long) st.st_ctim.tv_nsec);
 +}
 +
 +
 +#define HAVE_BUP_LSTAT 1
 +static PyObject *bup_lstat(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    char *filename;
 +
 +    if (!PyArg_ParseTuple(args, "s", &filename))
 +        return NULL;
 +
 +    struct stat st;
 +    rc = lstat(filename, &st);
 +    if (rc != 0)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 +
 +    return Py_BuildValue("kkkkkkkk"
 +                         "(ll)"
 +                         "(ll)"
 +                         "(ll)",
 +                         (unsigned long) st.st_mode,
 +                         (unsigned long) st.st_ino,
 +                         (unsigned long) st.st_dev,
 +                         (unsigned long) st.st_nlink,
 +                         (unsigned long) st.st_uid,
 +                         (unsigned long) st.st_gid,
 +                         (unsigned long) st.st_rdev,
 +                         (unsigned long) st.st_size,
 +                         (long) st.st_atime,
 +                         (long) st.st_atim.tv_nsec,
 +                         (long) st.st_mtime,
 +                         (long) st.st_mtim.tv_nsec,
 +                         (long) st.st_ctime,
 +                         (long) st.st_ctim.tv_nsec);
 +}
 +
 +
 +#define HAVE_BUP_FSTAT 1
 +static PyObject *bup_fstat(PyObject *self, PyObject *args)
 +{
 +    int rc, fd;
 +
 +    if (!PyArg_ParseTuple(args, "i", &fd))
 +        return NULL;
 +
 +    struct stat st;
 +    rc = fstat(fd, &st);
 +    if (rc != 0)
 +        return PyErr_SetFromErrno(PyExc_IOError);
 +
 +    return Py_BuildValue("kkkkkkkk"
 +                         "(ll)"
 +                         "(ll)"
 +                         "(ll)",
 +                         (unsigned long) st.st_mode,
 +                         (unsigned long) st.st_ino,
 +                         (unsigned long) st.st_dev,
 +                         (unsigned long) st.st_nlink,
 +                         (unsigned long) st.st_uid,
 +                         (unsigned long) st.st_gid,
 +                         (unsigned long) st.st_rdev,
 +                         (unsigned long) st.st_size,
 +                         (long) st.st_atime,
 +                         (long) st.st_atim.tv_nsec,
 +                         (long) st.st_mtime,
 +                         (long) st.st_mtim.tv_nsec,
 +                         (long) st.st_ctime,
 +                         (long) st.st_ctim.tv_nsec);
 +}
 +
 +#endif /* def linux */
 +
 +
- static PyMethodDef faster_methods[] = {
++static PyMethodDef helper_methods[] = {
      { "selftest", selftest, METH_VARARGS,
  	"Check that the rolling checksum rolls correctly (for unit tests)." },
      { "blobbits", blobbits, METH_VARARGS,
@@@ -898,49 -650,13 +899,51 @@@
  	"open() the given filename for read with O_NOATIME if possible" },
      { "fadvise_done", fadvise_done, METH_VARARGS,
  	"Inform the kernel that we're finished with earlier parts of a file" },
 +#ifdef linux
 +    { "get_linux_file_attr", bup_get_linux_file_attr, METH_VARARGS,
 +      "Return the Linux attributes for the given file." },
 +    { "set_linux_file_attr", bup_set_linux_file_attr, METH_VARARGS,
 +      "Set the Linux attributes for the given file." },
 +#endif
 +#ifdef HAVE_BUP_UTIMENSAT
 +    { "utimensat", bup_utimensat, METH_VARARGS,
 +      "Change file timestamps with nanosecond precision." },
 +#endif
 +#ifdef HAVE_BUP_STAT
 +    { "stat", bup_stat, METH_VARARGS,
 +      "Extended version of stat." },
 +#endif
 +#ifdef HAVE_BUP_LSTAT
 +    { "lstat", bup_lstat, METH_VARARGS,
 +      "Extended version of lstat." },
 +#endif
 +#ifdef HAVE_BUP_FSTAT
 +    { "fstat", bup_fstat, METH_VARARGS,
 +      "Extended version of fstat." },
 +#endif
      { NULL, NULL, 0, NULL },  // sentinel
  };
  
 +
  PyMODINIT_FUNC init_helpers(void)
  {
-     PyObject *m = Py_InitModule("_helpers", faster_methods);
 -    char *e = getenv("BUP_FORCE_TTY");
 -    Py_InitModule("_helpers", faster_methods);
++    char *e;
++    PyObject *m = Py_InitModule("_helpers", helper_methods);
 +    if (m == NULL)
 +        return;
 +#ifdef HAVE_BUP_UTIMENSAT
 +    PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD));
 +    PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW",
 +                       Py_BuildValue("i", AT_SYMLINK_NOFOLLOW));
 +#endif
 +#ifdef HAVE_BUP_STAT
 +    Py_INCREF(Py_True);
 +    PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_True);
 +#else
 +    Py_INCREF(Py_False);
 +    PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_False);
 +#endif
-     istty = isatty(2) || getenv("BUP_FORCE_TTY");
++    e = getenv("BUP_FORCE_TTY");
+     istty2 = isatty(2) || (atoi(e ? e : "0") & 2);
      unpythonize_argv();
  }
diff --combined lib/bup/helpers.py
index 0cbb1eb,1432b8b..13125af
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@@ -3,7 -3,6 +3,7 @@@
  import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
  import heapq, operator, time
  from bup import _version, _helpers
 +import bup._helpers as _helpers
  
  # This function should really be in helpers, not in bup.options.  But we
  # want options.py to be standalone so people can include it in other projects.
@@@ -67,17 -66,22 +67,22 @@@ def debug2(s)
          log(s)
  
  
- istty = os.isatty(2) or atoi(os.environ.get('BUP_FORCE_TTY'))
+ istty1 = os.isatty(1) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 1)
+ istty2 = os.isatty(2) or (atoi(os.environ.get('BUP_FORCE_TTY')) & 2)
+ _last_progress = ''
  def progress(s):
      """Calls log() if stderr is a TTY.  Does nothing otherwise."""
-     if istty:
+     global _last_progress
+     if istty2:
          log(s)
+         _last_progress = s
  
  
  def qprogress(s):
      """Calls progress() only if we haven't printed progress in a while.
      
-     This avoids overloading the stderr buffer with excess junk."""
+     This avoids overloading the stderr buffer with excess junk.
+     """
      global _last_prog
      now = time.time()
      if now - _last_prog > 0.1:
@@@ -85,6 -89,16 +90,16 @@@
          _last_prog = now
  
  
+ def reprogress():
+     """Calls progress() to redisplay the most recent progress message.
+ 
+     Useful after you've printed some other message that wipes out the
+     progress line.
+     """
+     if _last_progress and _last_progress.endswith('\r'):
+         progress(_last_progress)
+ 
+ 
  def mkdirp(d, mode=None):
      """Recursively create directories on path 'd'.
  
@@@ -183,11 -197,6 +198,11 @@@ def realpath(p)
      return out
  
  
 +def detect_fakeroot():
 +    "Return True if we appear to be running under fakeroot."
 +    return os.getenv("FAKEROOTKEY") != None
 +
 +
  _username = None
  def username():
      """Get the user's login name."""
@@@ -563,11 -572,6 +578,11 @@@ def add_error(e)
      log('%-70s\n' % e)
  
  
 +def clear_errors():
 +    global saved_errors
 +    saved_errors = []
 +
 +
  def handle_ctrl_c():
      """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
  
diff --combined lib/bup/index.py
index c000fbb,7483407..b7e65dd
--- a/lib/bup/index.py
+++ b/lib/bup/index.py
@@@ -95,19 -95,17 +95,19 @@@ class Entry
      def from_stat(self, st, tstart):
          old = (self.dev, self.ctime, self.mtime,
                 self.uid, self.gid, self.size, self.flags & IX_EXISTS)
 -        new = (st.st_dev, int(st.st_ctime), int(st.st_mtime),
 +        new = (st.st_dev,
 +               int(st.st_ctime.approx_secs()),
 +               int(st.st_mtime.approx_secs()),
                 st.st_uid, st.st_gid, st.st_size, IX_EXISTS)
          self.dev = st.st_dev
 -        self.ctime = int(st.st_ctime)
 -        self.mtime = int(st.st_mtime)
 +        self.ctime = int(st.st_ctime.approx_secs())
 +        self.mtime = int(st.st_mtime.approx_secs())
          self.uid = st.st_uid
          self.gid = st.st_gid
          self.size = st.st_size
          self.mode = st.st_mode
          self.flags |= IX_EXISTS
 -        if int(st.st_ctime) >= tstart or old != new \
 +        if int(st.st_ctime.approx_secs()) >= tstart or old != new \
                or self.sha == EMPTY_SHA or not self.gitmode:
              self.invalidate()
          self._fixup()
@@@ -138,6 -136,7 +138,7 @@@
      def validate(self, gitmode, sha):
          assert(sha)
          assert(gitmode)
+         assert(gitmode+0 == gitmode)
          self.gitmode = gitmode
          self.sha = sha
          self.flags |= IX_HASHVALID|IX_EXISTS
@@@ -408,10 -407,8 +409,10 @@@ class Writer
          if st:
              isdir = stat.S_ISDIR(st.st_mode)
              assert(isdir == endswith)
 -            e = NewEntry(basename, name, st.st_dev, int(st.st_ctime),
 -                         int(st.st_mtime), st.st_uid, st.st_gid,
 +            e = NewEntry(basename, name, st.st_dev,
 +                         int(st.st_ctime.approx_secs()),
 +                         int(st.st_mtime.approx_secs()),
 +                         st.st_uid, st.st_gid,
                           st.st_size, st.st_mode, gitmode, sha, flags,
                           0, 0)
          else: