Merge branch 'master' into meta

* master:
  midx/bloom: use progress() and debug1() for non-critical messages
  helpers: separately determine if stdout and stderr are ttys.
  cmd/newliner: restrict progress lines to the screen width.
  hashsplit: use shorter offset-filenames inside trees.
  Replace 040000 and 0100644 constants with GIT_MODE_{TREE,FILE}
  git.py: rename treeparse to tree_decode() and add tree_encode().
  hashsplit.py: remove PackWriter-specific knowledge.
  cmd/split: fixup progress message, and print -b output incrementally.
  hashsplit.py: convert from 'bits' to 'level' earlier in the sequence.
  hashsplit.py: okay, *really* fix BLOB_MAX.
  hashsplit.py: simplify code and fix BLOB_MAX handling.
  options.py: o.fatal(): print error after, not before, usage message.
  options.py: make --usage just print the usage message.

Conflicts:
	lib/bup/_helpers.c

diff --cc lib/bup/_helpers.c
index 78c86f02277d147b76e8000e1a9830b61976134c,af9d06f35273cabe4304d1142eb62d7b6e24cee8..0bd4ab08e3ffaa98ee2f9378a7355f19fa9f22ce
--- 1/lib/bup/_helpers.c
--- 2/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@@ -11,14 -9,7 +11,14 @@@
  #include <stdlib.h>
  #include <stdio.h>
  
- static int istty = 0;
 +#ifdef linux
 +#include <linux/fs.h>
 +#include <sys/ioctl.h>
 +#include <sys/stat.h>
 +#include <sys/time.h>
 +#endif
 +
+ static int istty2 = 0;
  
  // Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV...
  #if __WIN32__ || __CYGWIN__
@@@ -629,247 -621,7 +630,247 @@@ static PyObject *fadvise_done(PyObject 
  }
  
  
 -static PyMethodDef faster_methods[] = {
 +#ifdef linux
 +static PyObject *bup_get_linux_file_attr(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    unsigned long attr;
 +    char *path;
 +    int fd;
 +
 +    if (!PyArg_ParseTuple(args, "s", &path))
 +        return NULL;
 +
 +    fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
 +    if (fd == -1)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +
 +    attr = 0;
 +    rc = ioctl(fd, FS_IOC_GETFLAGS, &attr);
 +    if (rc == -1)
 +    {
 +        close(fd);
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +    }
 +
 +    close(fd);
 +    return Py_BuildValue("k", attr);
 +}
 +
 +
 +static PyObject *bup_set_linux_file_attr(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    unsigned long attr;
 +    char *path;
 +    int fd;
 +
 +    if (!PyArg_ParseTuple(args, "sk", &path, &attr))
 +        return NULL;
 +
 +    fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW);
 +    if(fd == -1)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +
 +    rc = ioctl(fd, FS_IOC_SETFLAGS, &attr);
 +    if (rc == -1)
 +    {
 +        close(fd);
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +    }
 +
 +    close(fd);
 +    Py_RETURN_TRUE;
 +}
 +#endif /* def linux */
 +
 +
 +#if defined(_ATFILE_SOURCE) \
 +  || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L
 +#define HAVE_BUP_UTIMENSAT 1
 +
 +static PyObject *bup_utimensat(PyObject *self, PyObject *args)
 +{
 +    int rc, dirfd, flags;
 +    char *path;
 +    long access, access_ns, modification, modification_ns;
 +    struct timespec ts[2];
 +
 +    if (!PyArg_ParseTuple(args, "is((ll)(ll))i",
 +                          &dirfd,
 +                          &path,
 +                          &access, &access_ns,
 +                          &modification, &modification_ns,
 +                          &flags))
 +        return NULL;
 +
 +    if (isnan(access))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(access))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time is infinite");
 +        return NULL;
 +    }
 +    else if (isnan(modification))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(modification))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time is infinite");
 +        return NULL;
 +    }
 +
 +    if (isnan(access_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time ns is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(access_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "access time ns is infinite");
 +        return NULL;
 +    }
 +    else if (isnan(modification_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time ns is NaN");
 +        return NULL;
 +    }
 +    else if (isinf(modification_ns))
 +    {
 +        PyErr_SetString(PyExc_ValueError, "modification time ns is infinite");
 +        return NULL;
 +    }
 +
 +    ts[0].tv_sec = access;
 +    ts[0].tv_nsec = access_ns;
 +    ts[1].tv_sec = modification;
 +    ts[1].tv_nsec = modification_ns;
 +
 +    rc = utimensat(dirfd, path, ts, flags);
 +    if (rc != 0)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path);
 +
 +    Py_RETURN_TRUE;
 +}
 +
 +#endif /* defined(_ATFILE_SOURCE)
 +          || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */
 +
 +
 +#ifdef linux /* and likely others */
 +
 +#define HAVE_BUP_STAT 1
 +static PyObject *bup_stat(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    char *filename;
 +
 +    if (!PyArg_ParseTuple(args, "s", &filename))
 +        return NULL;
 +
 +    struct stat st;
 +    rc = stat(filename, &st);
 +    if (rc != 0)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 +
 +    return Py_BuildValue("kkkkkkkk"
 +                         "(ll)"
 +                         "(ll)"
 +                         "(ll)",
 +                         (unsigned long) st.st_mode,
 +                         (unsigned long) st.st_ino,
 +                         (unsigned long) st.st_dev,
 +                         (unsigned long) st.st_nlink,
 +                         (unsigned long) st.st_uid,
 +                         (unsigned long) st.st_gid,
 +                         (unsigned long) st.st_rdev,
 +                         (unsigned long) st.st_size,
 +                         (long) st.st_atime,
 +                         (long) st.st_atim.tv_nsec,
 +                         (long) st.st_mtime,
 +                         (long) st.st_mtim.tv_nsec,
 +                         (long) st.st_ctime,
 +                         (long) st.st_ctim.tv_nsec);
 +}
 +
 +
 +#define HAVE_BUP_LSTAT 1
 +static PyObject *bup_lstat(PyObject *self, PyObject *args)
 +{
 +    int rc;
 +    char *filename;
 +
 +    if (!PyArg_ParseTuple(args, "s", &filename))
 +        return NULL;
 +
 +    struct stat st;
 +    rc = lstat(filename, &st);
 +    if (rc != 0)
 +        return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 +
 +    return Py_BuildValue("kkkkkkkk"
 +                         "(ll)"
 +                         "(ll)"
 +                         "(ll)",
 +                         (unsigned long) st.st_mode,
 +                         (unsigned long) st.st_ino,
 +                         (unsigned long) st.st_dev,
 +                         (unsigned long) st.st_nlink,
 +                         (unsigned long) st.st_uid,
 +                         (unsigned long) st.st_gid,
 +                         (unsigned long) st.st_rdev,
 +                         (unsigned long) st.st_size,
 +                         (long) st.st_atime,
 +                         (long) st.st_atim.tv_nsec,
 +                         (long) st.st_mtime,
 +                         (long) st.st_mtim.tv_nsec,
 +                         (long) st.st_ctime,
 +                         (long) st.st_ctim.tv_nsec);
 +}
 +
 +
 +#define HAVE_BUP_FSTAT 1
 +static PyObject *bup_fstat(PyObject *self, PyObject *args)
 +{
 +    int rc, fd;
 +
 +    if (!PyArg_ParseTuple(args, "i", &fd))
 +        return NULL;
 +
 +    struct stat st;
 +    rc = fstat(fd, &st);
 +    if (rc != 0)
 +        return PyErr_SetFromErrno(PyExc_IOError);
 +
 +    return Py_BuildValue("kkkkkkkk"
 +                         "(ll)"
 +                         "(ll)"
 +                         "(ll)",
 +                         (unsigned long) st.st_mode,
 +                         (unsigned long) st.st_ino,
 +                         (unsigned long) st.st_dev,
 +                         (unsigned long) st.st_nlink,
 +                         (unsigned long) st.st_uid,
 +                         (unsigned long) st.st_gid,
 +                         (unsigned long) st.st_rdev,
 +                         (unsigned long) st.st_size,
 +                         (long) st.st_atime,
 +                         (long) st.st_atim.tv_nsec,
 +                         (long) st.st_mtime,
 +                         (long) st.st_mtim.tv_nsec,
 +                         (long) st.st_ctime,
 +                         (long) st.st_ctim.tv_nsec);
 +}
 +
 +#endif /* def linux */
 +
 +
- static PyMethodDef faster_methods[] = {
++static PyMethodDef helper_methods[] = {
      { "selftest", selftest, METH_VARARGS,
        "Check that the rolling checksum rolls correctly (for unit tests)." },
      { "blobbits", blobbits, METH_VARARGS,
@@@ -923,24 -653,10 +924,26 @@@
      { NULL, NULL, 0, NULL },  // sentinel
  };
  
 +
  PyMODINIT_FUNC init_helpers(void)
  {
-     PyObject *m = Py_InitModule("_helpers", faster_methods);
 -    char *e = getenv("BUP_FORCE_TTY");
 -    Py_InitModule("_helpers", faster_methods);
++    char *e;
++    PyObject *m = Py_InitModule("_helpers", helper_methods);
 +    if (m == NULL)
 +        return;
 +#ifdef HAVE_BUP_UTIMENSAT
 +    PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD));
 +    PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW",
 +                       Py_BuildValue("i", AT_SYMLINK_NOFOLLOW));
 +#endif
 +#ifdef HAVE_BUP_STAT
 +    Py_INCREF(Py_True);
 +    PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_True);
 +#else
 +    Py_INCREF(Py_False);
 +    PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_False);
 +#endif
-     istty = isatty(2) || getenv("BUP_FORCE_TTY");
++    e = getenv("BUP_FORCE_TTY");
+     istty2 = isatty(2) || (atoi(e ? e : "0") & 2);
      unpythonize_argv();
  }

diff --cc lib/bup/helpers.py
Simple merge

diff --cc lib/bup/index.py
Simple merge