on: handle remote stdout and stderr via mux

Previously, when running "bup on" (i.e. when in BUP_SERVER_REVERSE
mode), bup would redirect the remote command's stdout to stderr because
stderr is the only remaining avenue back to the local console.  It's the
only avenue because in reverse mode, stdout (and stdin) are connected
back to a local "bup server" instance (hence the "reverse").

Of course that makes it impossible to reliably capture the
non-diagnostic output from the remote commands, i.e.

  commit_id="$(bup on HOST save -t ...)"

To fix that, in the remote "bup on--server" multiplex the stdout and
stderr from all "bup on" subcommands with "bup mux", and then
demultiplex those streams back to the local stdout and stderr via
DemuxConn() in the receiving "bup on".

Thanks to Alexander Barton for pointing out an error in a previous
version of this commit message, and thanks to Gabriel Filion for
pointing out that we could use the existing mux infrastructure instead
of reinventing the wheel.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/cmd/on--server-cmd.py b/cmd/on--server-cmd.py
index 327c81ffd8edc32cc945f7959de6b4cf45a04039..a6f3e2188719d7c9e922f03e07c2a4c3ba33046b 100755 (executable)
--- a/cmd/on--server-cmd.py
+++ b/cmd/on--server-cmd.py
@@ -23,6 +23,7 @@ assert(sz < 1000000)
 buf = sys.stdin.read(sz)
 assert(len(buf) == sz)
 argv = buf.split('\0')
+argv = [argv[0], 'mux', '--'] + argv
 
 # stdin/stdout are supposedly connected to 'bup server' that the caller
 # started for us (often on the other end of an ssh tunnel), so we don't want

diff --git a/cmd/on-cmd.py b/cmd/on-cmd.py
index 60759b64a6d6d85332c489de8a81df04f6d57f94..9d6f683f357c869597bdb45f3ab1d735fb2e1e1b 100755 (executable)
--- a/cmd/on-cmd.py
+++ b/cmd/on-cmd.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import sys, os, struct, getopt, subprocess, signal
+from subprocess import PIPE
 from bup import options, ssh, path
 from bup.helpers import *
 
@@ -33,9 +34,8 @@ try:
         (hostname, port) = (hp[0], None)
     else:
         (hostname, port) = hp
-
     argv = extra[1:]
-    p = ssh.connect(hostname, port, 'on--server')
+    p = ssh.connect(hostname, port, 'on--server', stderr=PIPE)
 
     try:
         argvs = '\0'.join(['bup'] + argv)
@@ -45,6 +45,10 @@ try:
                               stdin=p.stdout, stdout=p.stdin)
         p.stdin.close()
         p.stdout.close()
+        # Demultiplex remote client's stderr (back to stdout/stderr).
+        dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "w"))
+        for line in iter(dmc.readline, ""):
+            sys.stdout.write(line)
     finally:
         while 1:
             # if we get a signal while waiting, we have to keep waiting, just

diff --git a/lib/bup/ssh.py b/lib/bup/ssh.py
index 344355aa1f2b537140b36367d5d4b4d4e56021ab..34b5f3c38a2c30740d97dd097a9906c1a8e00f5f 100644 (file)
--- a/lib/bup/ssh.py
+++ b/lib/bup/ssh.py
@@ -5,7 +5,7 @@ import sys, os, re, subprocess
 from bup import helpers, path
 
 
-def connect(rhost, port, subcmd):
+def connect(rhost, port, subcmd, stderr=None):
     """Connect to 'rhost' and execute the bup subcommand 'subcmd' on it."""
     assert(not re.search(r'[^\w-]', subcmd))
     nicedir = re.sub(r':', "_", path.exedir())
@@ -39,4 +39,5 @@ def connect(rhost, port, subcmd):
                                            os.environ.get('PATH', '')])
         os.setsid()
     return subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                            stderr=stderr,
                             preexec_fn=setup)

diff --git a/t/test-on.sh b/t/test-on.sh
index c3ee46016751d74c627829e4b9bbc9d1d312bd0b..61e6dd847d037556ef97b5d14965e6581f5b3a30 100755 (executable)
--- a/t/test-on.sh
+++ b/t/test-on.sh
@@ -21,13 +21,25 @@ WVPASS mkdir src src/foo
 WVPASS date > src/bar
 WVPASS bup random 1k > src/baz
 WVPASS bup on - index src
-WVPASS bup on - save -n src src
+WVPASS bup on - save -ctn src src > get.log
+WVPASSEQ "$(cat get.log | wc -l)" 2
+tree_id=$(WVPASS awk 'FNR == 1' get.log) || exit $?
+commit_id=$(WVPASS awk 'FNR == 2' get.log) || exit $?
+WVPASS git ls-tree "$tree_id"
+WVPASS git cat-file commit "$commit_id" | head -n 1 \
+    | WVPASS grep "^tree $tree_id\$"
+
 WVPASS bup restore -C restore "src/latest/$(pwd)/src/."
 WVPASS compare-trees src/ restore/
 WVPASS rm -r restore
 
 WVSTART "split"
-WVPASS bup on - split -n baz src/baz
+WVPASS bup on - split -ctn baz src/baz > get.log
+tree_id=$(WVPASS awk 'FNR == 1' get.log) || exit $?
+commit_id=$(WVPASS awk 'FNR == 2' get.log) || exit $?
+WVPASS git ls-tree "$tree_id"
+WVPASS git cat-file commit "$commit_id" | head -n 1 \
+    | WVPASS grep "^tree $tree_id\$"
 WVPASS bup join baz > restore-baz
 WVPASS cmp src/baz restore-baz