Merge remote branch 'origin/master' into meta

* origin/master:
  doc/import-rsnapshot: small corrections and clarification
  cmd/midx, git.py: all else being equal, delete older midxes first.
  t/test.sh: a test for the recently-uncovered midx4 problem.
  _helpers.c: midx4 didn't handle multiple index with the same object.
  cmd/midx: add a --check option.
  Add git.shorten_hash(), printing only the first few bytes of a sha1.
  tclient.py: add some additional tests that objcache.refresh() is called.
  cmd/server: add a debug message saying which object caused a suggestion.
  cmd/list-idx: a quick tool for searching the contents of idx/midx files.
  Add tests around the bloom ruin and check options
  Add a bloom --ruin for testing failure cases
  One more constant for header lengths
  Split PackMidx from git.py into a new midx.py.
  bloom.py: move bloom.ShaBloom.create to just bloom.create.
  Move bloom-related stuff from git.py to a new bloom.py.
  cmd/bloom: add a --force option to forget regenerating the bloom.
  Use the new qprogress() function in more places.
  Bail out immediately instead of redownloading .idx
  Add a --check behavior to verify bloom
  Defines/preprocessor lengths > magic numbers
  cmd/{bloom,midx}: clean up progress messages.
  cmd/bloom: by default generate bloom filters in *all* index-cache dirs.
  cmd/newliner: avoid printing a blank line if the final output ended in \r.
  cmd/index: make the default mode '-u'.
  _helpers.c: don't cygwin doesn't set any win32 defines.
  _helpers.c: don'g unpythonize_argv() on win32.
  Remove .c and .o rules, apply flags to csetup.py
  Fix a valid warning that some compilers warned
  Move .idx file writing to C
  main.py: fix whitespace in the usage string.
  cmd/daemon: FD_CLOEXEC the listener socket and don't leak fd for the connection.
  cmd/daemon: close file descriptors correctly in parent process.
  cmd/daemon: use SO_REUSEADDR.
  cmd/daemon: pass extra options along to 'bup server'.
  cmd/daemon: correctly report socket binding/listening errors.
  main.py: use execvp() instead of subprocess.Popen() when possible.
  _helpers.c: Remove ugly 'python' junk from argv[0] so 'ps' is prettier.
  cmd/bloom: fix a message pluralization.
  cmd/join: add a new -o (output filename) option.
  cmd/ls: fix a typo causing 'bup ls foo/latest' to not work.
  cmd/server: add a new 'help' command.
  midx4: Fix the other side of my previous nasty bug

Conflicts:
	lib/bup/_helpers.c
	lib/bup/helpers.py

diff --cc Makefile
Simple merge

diff --cc lib/bup/_helpers.c
index dbe64a7a617c4cb6c4609656f9303b1e5a2caca4,4d12ddfe9a7ea8d7cbf6d1d34f2fd795fbd62799..78c86f02277d147b76e8000e1a9830b61976134c
--- 1/lib/bup/_helpers.c
--- 2/lib/bup/_helpers.c
+++ b/lib/bup/_helpers.c
@@@ -1,5 -1,4 +1,5 @@@
- 
 +#define _LARGEFILE64_SOURCE 1
+ #undef NDEBUG
  #include "bupsplit.h"
  #include <Python.h>
  #include <assert.h>
@@@ -11,15 -9,59 +11,66 @@@
  #include <stdlib.h>
  #include <stdio.h>
  
 +#ifdef linux
 +#include <linux/fs.h>
 +#include <sys/ioctl.h>
 +#include <sys/stat.h>
 +#include <sys/time.h>
 +#endif
 +
  static int istty = 0;
  
+ // Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV...
+ #if __WIN32__ || __CYGWIN__
+ 
+ // There's no 'ps' on win32 anyway, and Py_GetArgcArgv() isn't available.
+ static void unpythonize_argv(void) { }
+ 
+ #else // not __WIN32__
+ 
+ // For some reason this isn't declared in Python.h
+ extern void Py_GetArgcArgv(int *argc, char ***argv);
+ 
+ static void unpythonize_argv(void)
+ {
+     int argc, i;
+     char **argv, *arge;
+     
+     Py_GetArgcArgv(&argc, &argv);
+     
+     for (i = 0; i < argc-1; i++)
+     {
+       if (argv[i] + strlen(argv[i]) + 1 != argv[i+1])
+       {
+           // The argv block doesn't work the way we expected; it's unsafe
+           // to mess with it.
+           return;
+       }
+     }
+     
+     arge = argv[argc-1] + strlen(argv[argc-1]) + 1;
+     
+     if (strstr(argv[0], "python") && argv[1] == argv[0] + strlen(argv[0]) + 1)
+     {
+       char *p;
+       size_t len, diff;
+       p = strrchr(argv[1], '/');
+       if (p)
+       {
+           p++;
+           diff = p - argv[0];
+           len = arge - p;
+           memmove(argv[0], p, len);
+           memset(arge - diff, 0, diff);
+           for (i = 0; i < argc; i++)
+               argv[i] = argv[i+1] ? argv[i+1]-diff : NULL;
+       }
+     }
+ }
+ 
+ #endif // not __WIN32__ or __CYGWIN__
+ 
+ 
  static PyObject *selftest(PyObject *self, PyObject *args)
  {
      if (!PyArg_ParseTuple(args, ""))
@@@ -794,23 -652,9 +923,24 @@@ static PyMethodDef faster_methods[] = 
      { NULL, NULL, 0, NULL },  // sentinel
  };
  
 +
  PyMODINIT_FUNC init_helpers(void)
  {
 -    Py_InitModule("_helpers", faster_methods);
 +    PyObject *m = Py_InitModule("_helpers", faster_methods);
 +    if (m == NULL)
 +        return;
 +#ifdef HAVE_BUP_UTIMENSAT
 +    PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD));
 +    PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW",
 +                       Py_BuildValue("i", AT_SYMLINK_NOFOLLOW));
 +#endif
 +#ifdef HAVE_BUP_STAT
 +    Py_INCREF(Py_True);
 +    PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_True);
 +#else
 +    Py_INCREF(Py_False);
 +    PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_False);
 +#endif
      istty = isatty(2) || getenv("BUP_FORCE_TTY");
+     unpythonize_argv();
  }

diff --cc lib/bup/helpers.py
index 566343d2b0e4518b90e1ccacb9c656d4688c070a,5b0028364bb29f5b04b2b231f8467172a37b4b0a..0cbb1ebc8bff0397a5114e58d7af2cbfaddb15f5
--- 1/lib/bup/helpers.py
--- 2/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@@ -1,9 -1,8 +1,9 @@@
  """Helper functions and classes for bup."""
  
  import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
- import heapq, operator
- from bup import _version
+ import heapq, operator, time
+ from bup import _version, _helpers
 +import bup._helpers as _helpers
  
  # This function should really be in helpers, not in bup.options.  But we
  # want options.py to be standalone so people can include it in other projects.
@@@ -541,18 -557,6 +563,11 @@@ def add_error(e)
      log('%-70s\n' % e)
  
  
- istty = os.isatty(2) or atoi(os.environ.get('BUP_FORCE_TTY'))
- def progress(s):
-     """Calls log(s) if stderr is a TTY.  Does nothing otherwise."""
-     if istty:
-         log(s)
- 
- 
 +def clear_errors():
 +    global saved_errors
 +    saved_errors = []
 +
 +
  def handle_ctrl_c():
      """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
  

diff --cc lib/bup/index.py
Simple merge