Add a batchpipe() command to helpers that behaves somewhat like xargs(1).

Add batchpipe(), which will yield the output produced by calling a
given external command with a given list of arguments.

The resulting output may be provided in chunks, from multiple
invocations of the command, if the limits imposed by ARG_MAX make that
necessary.

See http://www.in-ulm.de/~mascheck/various/argmax/ for details, but
note that batchpipe() takes the additional precaution of adding room
for the argv pointers in addition to the envp pointers.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py
index ef4be682f0152d26c372ca1a8078619a30a5ab49..0923761fb4f5fdb4be224f2c7d7ea9abd94d7cf0 100644 (file)
--- a/lib/bup/helpers.py
+++ b/lib/bup/helpers.py
@@ -1,7 +1,10 @@
 """Helper functions and classes for bup."""
 
+from ctypes import sizeof, c_void_p
+from os import environ
 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct
-import hashlib, heapq, operator, time, grp
+import config, hashlib, heapq, operator, time, grp
+
 from bup import _version, _helpers
 import bup._helpers as _helpers
 import math
@@ -188,9 +191,9 @@ def unlink(f):
             pass  # it doesn't exist, that's what you asked for
 
 
-def readpipe(argv):
+def readpipe(argv, preexec_fn=None):
     """Run a subprocess and return its output."""
-    p = subprocess.Popen(argv, stdout=subprocess.PIPE)
+    p = subprocess.Popen(argv, stdout=subprocess.PIPE, preexec_fn=preexec_fn)
     out, err = p.communicate()
     if p.returncode != 0:
         raise Exception('subprocess %r failed with status %d'
@@ -198,6 +201,39 @@ def readpipe(argv):
     return out
 
 
+def _argmax_base(command):
+    base_size = 2048
+    for c in command:
+        base_size += len(command) + 1
+    for k, v in environ.iteritems():
+        base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
+    return base_size
+
+
+def _argmax_args_size(args):
+    return sum(len(x) + 1 + sizeof(c_void_p) for x in args)
+
+
+def batchpipe(command, args, preexec_fn=None):
+    """If args is not empty, yield the output produced by calling the
+command list with args as a sequence of strings (It may be necessary
+to return multiple strings in order to respect ARG_MAX)."""
+    base_size = _argmax_base(command)
+    while args:
+        room = config.arg_max - base_size
+        i = 0
+        while i < len(args):
+            next_size = _argmax_args_size(args[i:i+1])
+            if room - next_size < 0:
+                break
+            room -= next_size
+            i += 1
+        sub_args = args[:i]
+        args = args[i:]
+        assert(len(sub_args))
+        yield readpipe(command + sub_args, preexec_fn=preexec_fn)
+
+
 def realpath(p):
     """Get the absolute path of a file.
 

diff --git a/lib/bup/t/thelpers.py b/lib/bup/t/thelpers.py
index d260f8e767a57655796cd8e0c02efce306e89879..38596d7447e1e33c416cb399748506a2613e232a 100644 (file)
--- a/lib/bup/t/thelpers.py
+++ b/lib/bup/t/thelpers.py
@@ -1,3 +1,4 @@
+import config
 import helpers
 import math
 import os
@@ -116,3 +117,32 @@ def test_readpipe():
         readpipe(['bash', '-c', 'exit 42'])
     except Exception, ex:
         WVPASSEQ(str(ex), "subprocess 'bash -c exit 42' failed with status 42")
+
+
+@wvtest
+def test_batchpipe():
+    for chunk in batchpipe(['echo'], []):
+        WVPASS(False)
+    out = ''
+    for chunk in batchpipe(['echo'], ['42']):
+        out += chunk
+    WVPASSEQ(out, '42\n')
+    try:
+        batchpipe(['bash', '-c'], ['exit 42'])
+    except Exception, ex:
+        WVPASSEQ(str(ex), "subprocess 'bash -c exit 42' failed with status 42")
+    oldmax = config.arg_max
+    args = [str(x) for x in range(6)]
+    # Force batchpipe to break the args into batches of 3.  This
+    # approach assumes all args are the same length.
+    config.arg_max = \
+        helpers._argmax_base(['echo']) + helpers._argmax_args_size(args[:3])
+    batches = batchpipe(['echo'], args)
+    WVPASSEQ(next(batches), '0 1 2\n')
+    WVPASSEQ(next(batches), '3 4 5\n')
+    WVPASSEQ(next(batches, None), None)
+    batches = batchpipe(['echo'], [str(x) for x in range(5)])
+    WVPASSEQ(next(batches), '0 1 2\n')
+    WVPASSEQ(next(batches), '3 4\n')
+    WVPASSEQ(next(batches, None), None)
+    config.arg_max = oldmax