]> arthur.barton.de Git - bup.git/commitdiff
Wrap readline oursleves to avoid py3's interference
authorRob Browning <rlb@defaultvalue.org>
Sun, 7 Jun 2020 20:32:10 +0000 (15:32 -0500)
committerRob Browning <rlb@defaultvalue.org>
Sun, 21 Jun 2020 16:20:58 +0000 (11:20 -0500)
We don't want Python to "help" us by guessing and insisting on the
encoding before we can even look at the incoming data, so wrap
readline ourselves, with a bytes-oriented (and more direct) API.  This
will allows us to preserve the status quo for now (and maintain parity
between Python 2 and 3) when using Python 3 as we remove our LC_CTYPE
override.

At least on Linux, readline --cflags currently defines _DEFAULT_SOURCE
and defines _XOPEN_SOURCE to 600, and the latter conflicts with a
setting of 700 via Python.h in some installations, so for now, just
defer to Python as long as it doesn't choose an older version.

Thanks to Johannes Berg for fixes for allocation issues, etc. in an
earler version, and help figuring out the #define arrangement.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Signed-off-by: Johannes Berg <johannes@sipsolutions.net>
Tested-by: Rob Browning <rlb@defaultvalue.org>
Makefile
README.md
config/configure
dev/prep-for-debianish-build
dev/prep-for-freebsd-build
dev/prep-for-macos-build
lib/bup/_helpers.c

index 3d129a4e3f7cc71ca67c7bd9370e0a6009aa01b6..39202e79329fa113812687b4c24b29e854b450af 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -46,6 +46,21 @@ config/config.vars: configure config/configure config/configure.inc \
   $(wildcard config/*.in)
        MAKE="$(MAKE)" ./configure
 
+# On some platforms, Python.h and readline.h fight over the
+# _XOPEN_SOURCE version, i.e. -Werror crashes on a mismatch, so for
+# now, we're just going to let Python's version win.
+readline_cflags += $(shell pkg-config readline --cflags)
+readline_xopen := $(filter -D_XOPEN_SOURCE=%,$(readline_cflags))
+readline_xopen := $(subst -D_XOPEN_SOURCE=,,$(readline_xopen))
+ifneq ($(readline_xopen),600)
+  $(error "Unexpected pkg-config readline _XOPEN_SOURCE --cflags $(readline_cflags)")
+endif
+readline_cflags := $(filter-out -D_XOPEN_SOURCE=%,$(readline_cflags))
+readline_cflags += $(addprefix -DBUP_RL_EXPECTED_XOPEN_SOURCE=,$(readline_xopen))
+
+CFLAGS += $(readline_cflags)
+LDFLAGS += $(shell pkg-config readline --libs)
+
 bup_cmds := cmd/bup-python \
   $(patsubst cmd/%-cmd.py,cmd/bup-%,$(wildcard cmd/*-cmd.py)) \
   $(patsubst cmd/%-cmd.sh,cmd/bup-%,$(wildcard cmd/*-cmd.sh))
index d08ea4ca28f32052a86a75b749ccf4ab2251a758..ae49cce765a82ceb4fc73ce60b8e564f6e25ff98 100644 (file)
--- a/README.md
+++ b/README.md
@@ -147,7 +147,8 @@ From source
     apt-get install python-pyxattr python-pylibacl
     apt-get install linux-libc-dev
     apt-get install acl attr
-    apt-get install python-tornado # optional
+    apt-get install libreadline-dev # optional (bup ftp)
+    apt-get install python-tornado # optional (bup web)
     ```
 
    On CentOS (for CentOS 6, at least), this should be sufficient (run
@@ -158,6 +159,8 @@ From source
     yum install python python-devel
     yum install fuse-python pyxattr pylibacl
     yum install perl-Time-HiRes
+    yum install readline-devel # optional (bup ftp)
+    yum install python-tornado # optional (bup web)
     ```
 
    In addition to the default CentOS repositories, you may need to add
index fbf174fd8c95df5e0c591c917699af805102f445..86ecd761a3057c94e51e99c2c2ccb25e6734efc4 100755 (executable)
@@ -166,6 +166,13 @@ if test "$ac_defined_HAVE_MINCORE"; then
     fi
 fi
 
+TLOGN "checking for readline"
+if pkg-config readline; then
+    AC_DEFINE BUP_HAVE_READLINE 1
+    TLOG ' (yes)'
+else
+    TLOG ' (no)'
+fi
 
 AC_CHECK_FIELD stat st_atim sys/types.h sys/stat.h unistd.h
 AC_CHECK_FIELD stat st_mtim sys/types.h sys/stat.h unistd.h
index a6bc1259ee2da17c2768af813c1ac97647f3443e..73fa2357f1c969abc59aece7e6e29a014059b7e6 100755 (executable)
@@ -12,6 +12,7 @@ apt-get update
 
 common_debs='gcc make linux-libc-dev git rsync eatmydata acl attr par2'
 common_debs="$common_debs duplicity rdiff-backup rsnapshot dosfstools kmod"
+common_debs="$common_debs libreadline-dev"
 
 pyver="${1:-python2}"
 xattr="${2:-pyxattr}"
index 11866dbc44c443f4497d8831d1b3240308cd2bcf..7a8901c7bd30229fe9eae3ae1f00e91b1ca77e90 100755 (executable)
@@ -7,5 +7,5 @@ export ASSUME_ALWAYS_YES=yes
 pkg update
 pkg install \
     gmake git bash rsync curl par2cmdline \
-    python2 python py27-pylibacl py27-tornado \
+    python2 python py27-pylibacl py27-tornado readline \
     duplicity rdiff-backup rsnapshot
index 750b3e1bf55375ef912fdfc990b9255656a5b95e..7099574fa5e9c164e51b819755261ceedeb6d9d9 100755 (executable)
@@ -5,4 +5,4 @@ set -ex
 ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
 
 brew update
-brew install par2 rsync
+brew install par2 readline rsync
index 946ff4ad63763ba61847d44e5092502070a1eb05..d954f1d3266766d353ca9694aa6865e8225164e5 100644 (file)
 #include <time.h>
 #endif
 
+#ifdef BUP_HAVE_READLINE
+#if ! defined(_XOPEN_SOURCE) || _XOPEN_SOURCE < BUP_RL_EXPECTED_XOPEN_SOURCE
+# warning "_XOPEN_SOURCE version is too low for readline"
+#endif
+#include <readline/readline.h>
+#include <readline/history.h>
+#endif
+
 #include "bupsplit.h"
 
 #if defined(FS_IOC_GETFLAGS) && defined(FS_IOC_SETFLAGS)
@@ -1740,6 +1748,7 @@ static PyObject *tuple_from_cstrs(char **cstrs)
     return result;
 }
 
+
 static long getpw_buf_size;
 
 static PyObject *pwd_struct_to_py(const struct passwd *pwd, int rc)
@@ -1876,6 +1885,217 @@ static PyObject *bup_gethostname(PyObject *mod, PyObject *ignore)
     return PyBytes_FromString(buf);
 }
 
+
+#ifdef BUP_HAVE_READLINE
+
+static char *cstr_from_bytes(PyObject *bytes)
+{
+    char *buf;
+    Py_ssize_t length;
+    int rc = PyBytes_AsStringAndSize(bytes, &buf, &length);
+    if (rc == -1)
+        return NULL;
+    char *result = checked_malloc(length, sizeof(char));
+    if (!result)
+        return NULL;
+    memcpy(result, buf, length);
+    return result;
+}
+
+static char **cstrs_from_seq(PyObject *seq)
+{
+    char **result = NULL;
+    seq = PySequence_Fast(seq, "Cannot convert sequence items to C strings");
+    if (!seq)
+        return NULL;
+
+    const Py_ssize_t len = PySequence_Fast_GET_SIZE(seq);
+    if (len > PY_SSIZE_T_MAX - 1) {
+        PyErr_Format(PyExc_OverflowError,
+                     "Sequence length %zd too large for conversion to C array",
+                     len);
+        goto finish;
+    }
+    result = checked_malloc(len + 1, sizeof(char *));
+    if (!result)
+        goto finish;
+    Py_ssize_t i = 0;
+    for (i = 0; i < len; i++)
+    {
+        PyObject *item = PySequence_Fast_GET_ITEM(seq, i);
+        if (!item)
+            goto abandon_result;
+        result[i] = cstr_from_bytes(item);
+        if (!result[i]) {
+            i--;
+            goto abandon_result;
+        }
+    }
+    result[len] = NULL;
+    goto finish;
+
+ abandon_result:
+    if (result) {
+        for (; i > 0; i--)
+            free(result[i]);
+        free(result);
+        result = NULL;
+    }
+ finish:
+    Py_DECREF(seq);
+    return result;
+}
+
+static char* our_word_break_chars = NULL;
+
+static PyObject *
+bup_set_completer_word_break_characters(PyObject *self, PyObject *args)
+{
+    char *bytes;
+    if (!PyArg_ParseTuple(args, cstr_argf, &bytes))
+       return NULL;
+    char *prev = our_word_break_chars;
+    char *next = strdup(bytes);
+    if (!next)
+        return PyErr_NoMemory();
+    our_word_break_chars = next;
+    rl_completer_word_break_characters = next;
+    if (prev)
+        free(prev);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+bup_get_completer_word_break_characters(PyObject *self, PyObject *args)
+{
+    if (!PyArg_ParseTuple(args, ""))
+       return NULL;
+    return PyBytes_FromString(rl_completer_word_break_characters);
+}
+
+static PyObject *bup_get_line_buffer(PyObject *self, PyObject *args)
+{
+    if (!PyArg_ParseTuple(args, ""))
+       return NULL;
+    return PyBytes_FromString(rl_line_buffer);
+}
+
+static PyObject *
+bup_parse_and_bind(PyObject *self, PyObject *args)
+{
+    char *bytes;
+    if (!PyArg_ParseTuple(args, cstr_argf ":parse_and_bind", &bytes))
+       return NULL;
+    char *tmp = strdup(bytes); // Because it may modify the arg
+    if (!tmp)
+        return PyErr_NoMemory();
+    int rc = rl_parse_and_bind(tmp);
+    free(tmp);
+    if (rc != 0)
+        return PyErr_Format(PyExc_OSError,
+                            "system rl_parse_and_bind failed (%d)", rc);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *py_on_attempted_completion;
+static char **prev_completions;
+
+static char **on_attempted_completion(const char *text, int start, int end)
+{
+    if (!py_on_attempted_completion)
+        return NULL;
+
+    char **result = NULL;
+    PyObject *py_result = PyObject_CallFunction(py_on_attempted_completion,
+                                                cstr_argf "ii",
+                                                text, start, end);
+    if (!py_result)
+        return NULL;
+    if (py_result != Py_None) {
+        result = cstrs_from_seq(py_result);
+        free(prev_completions);
+        prev_completions = result;
+    }
+    Py_DECREF(py_result);
+    return result;
+}
+
+static PyObject *
+bup_set_attempted_completion_function(PyObject *self, PyObject *args)
+{
+    PyObject *completer;
+    if (!PyArg_ParseTuple(args, "O", &completer))
+       return NULL;
+
+    PyObject *prev = py_on_attempted_completion;
+    if (completer == Py_None)
+    {
+        py_on_attempted_completion = NULL;
+        rl_attempted_completion_function = NULL;
+    } else {
+        py_on_attempted_completion = completer;
+        rl_attempted_completion_function = on_attempted_completion;
+        Py_INCREF(completer);
+    }
+    Py_XDECREF(prev);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *py_on_completion_entry;
+
+static char *on_completion_entry(const char *text, int state)
+{
+    if (!py_on_completion_entry)
+        return NULL;
+
+    PyObject *py_result = PyObject_CallFunction(py_on_completion_entry,
+                                                cstr_argf "i", text, state);
+    if (!py_result)
+        return NULL;
+    char *result = (py_result == Py_None) ? NULL : cstr_from_bytes(py_result);
+    Py_DECREF(py_result);
+    return result;
+}
+
+static PyObject *
+bup_set_completion_entry_function(PyObject *self, PyObject *args)
+{
+    PyObject *completer;
+    if (!PyArg_ParseTuple(args, "O", &completer))
+       return NULL;
+
+    PyObject *prev = py_on_completion_entry;
+    if (completer == Py_None) {
+        py_on_completion_entry = NULL;
+        rl_completion_entry_function = NULL;
+    } else {
+        py_on_completion_entry = completer;
+        rl_completion_entry_function = on_completion_entry;
+        Py_INCREF(completer);
+    }
+    Py_XDECREF(prev);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+bup_readline(PyObject *self, PyObject *args)
+{
+    char *prompt;
+    if (!PyArg_ParseTuple(args, cstr_argf, &prompt))
+       return NULL;
+    char *line = readline(prompt);
+    if (!line)
+        return PyErr_Format(PyExc_EOFError, "readline EOF");
+    PyObject *result = PyBytes_FromString(line);
+    free(line);
+    return result;
+}
+
+#endif // defined BUP_HAVE_READLINE
+
+
 static PyMethodDef helper_methods[] = {
     { "write_sparsely", bup_write_sparsely, METH_VARARGS,
       "Write buf excepting zeros at the end. Return trailing zero count." },
@@ -1965,6 +2185,22 @@ static PyMethodDef helper_methods[] = {
       " not exist." },
     { "gethostname", bup_gethostname, METH_NOARGS,
       "Return the current hostname (as bytes)" },
+#ifdef BUP_HAVE_READLINE
+    { "set_completion_entry_function", bup_set_completion_entry_function, METH_VARARGS,
+      "Set rl_completion_entry_function.  Called as f(text, state)." },
+    { "set_attempted_completion_function", bup_set_attempted_completion_function, METH_VARARGS,
+      "Set rl_attempted_completion_function.  Called as f(text, start, end)." },
+    { "parse_and_bind", bup_parse_and_bind, METH_VARARGS,
+      "Call rl_parse_and_bind." },
+    { "get_line_buffer", bup_get_line_buffer, METH_NOARGS,
+      "Return rl_line_buffer." },
+    { "get_completer_word_break_characters", bup_get_completer_word_break_characters, METH_NOARGS,
+      "Return rl_completer_word_break_characters." },
+    { "set_completer_word_break_characters", bup_set_completer_word_break_characters, METH_VARARGS,
+      "Set rl_completer_word_break_characters." },
+    { "readline", bup_readline, METH_VARARGS,
+      "Call readline(prompt)." },
+#endif // defined BUP_HAVE_READLINE
     { NULL, NULL, 0, NULL },  // sentinel
 };