From 9277b335d0be832b19f95f5f77f0c3bd29362a5b Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Sun, 7 Jun 2020 15:32:10 -0500 Subject: [PATCH] Wrap readline oursleves to avoid py3's interference We don't want Python to "help" us by guessing and insisting on the encoding before we can even look at the incoming data, so wrap readline ourselves, with a bytes-oriented (and more direct) API. This will allows us to preserve the status quo for now (and maintain parity between Python 2 and 3) when using Python 3 as we remove our LC_CTYPE override. At least on Linux, readline --cflags currently defines _DEFAULT_SOURCE and defines _XOPEN_SOURCE to 600, and the latter conflicts with a setting of 700 via Python.h in some installations, so for now, just defer to Python as long as it doesn't choose an older version. Thanks to Johannes Berg for fixes for allocation issues, etc. in an earler version, and help figuring out the #define arrangement. Signed-off-by: Rob Browning Signed-off-by: Johannes Berg Tested-by: Rob Browning --- Makefile | 15 +++ README.md | 5 +- config/configure | 7 ++ dev/prep-for-debianish-build | 1 + dev/prep-for-freebsd-build | 2 +- dev/prep-for-macos-build | 2 +- lib/bup/_helpers.c | 236 +++++++++++++++++++++++++++++++++++ 7 files changed, 265 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3d129a4..39202e7 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,21 @@ config/config.vars: configure config/configure config/configure.inc \ $(wildcard config/*.in) MAKE="$(MAKE)" ./configure +# On some platforms, Python.h and readline.h fight over the +# _XOPEN_SOURCE version, i.e. -Werror crashes on a mismatch, so for +# now, we're just going to let Python's version win. +readline_cflags += $(shell pkg-config readline --cflags) +readline_xopen := $(filter -D_XOPEN_SOURCE=%,$(readline_cflags)) +readline_xopen := $(subst -D_XOPEN_SOURCE=,,$(readline_xopen)) +ifneq ($(readline_xopen),600) + $(error "Unexpected pkg-config readline _XOPEN_SOURCE --cflags $(readline_cflags)") +endif +readline_cflags := $(filter-out -D_XOPEN_SOURCE=%,$(readline_cflags)) +readline_cflags += $(addprefix -DBUP_RL_EXPECTED_XOPEN_SOURCE=,$(readline_xopen)) + +CFLAGS += $(readline_cflags) +LDFLAGS += $(shell pkg-config readline --libs) + bup_cmds := cmd/bup-python \ $(patsubst cmd/%-cmd.py,cmd/bup-%,$(wildcard cmd/*-cmd.py)) \ $(patsubst cmd/%-cmd.sh,cmd/bup-%,$(wildcard cmd/*-cmd.sh)) diff --git a/README.md b/README.md index d08ea4c..ae49cce 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,8 @@ From source apt-get install python-pyxattr python-pylibacl apt-get install linux-libc-dev apt-get install acl attr - apt-get install python-tornado # optional + apt-get install libreadline-dev # optional (bup ftp) + apt-get install python-tornado # optional (bup web) ``` On CentOS (for CentOS 6, at least), this should be sufficient (run @@ -158,6 +159,8 @@ From source yum install python python-devel yum install fuse-python pyxattr pylibacl yum install perl-Time-HiRes + yum install readline-devel # optional (bup ftp) + yum install python-tornado # optional (bup web) ``` In addition to the default CentOS repositories, you may need to add diff --git a/config/configure b/config/configure index fbf174f..86ecd76 100755 --- a/config/configure +++ b/config/configure @@ -166,6 +166,13 @@ if test "$ac_defined_HAVE_MINCORE"; then fi fi +TLOGN "checking for readline" +if pkg-config readline; then + AC_DEFINE BUP_HAVE_READLINE 1 + TLOG ' (yes)' +else + TLOG ' (no)' +fi AC_CHECK_FIELD stat st_atim sys/types.h sys/stat.h unistd.h AC_CHECK_FIELD stat st_mtim sys/types.h sys/stat.h unistd.h diff --git a/dev/prep-for-debianish-build b/dev/prep-for-debianish-build index a6bc125..73fa235 100755 --- a/dev/prep-for-debianish-build +++ b/dev/prep-for-debianish-build @@ -12,6 +12,7 @@ apt-get update common_debs='gcc make linux-libc-dev git rsync eatmydata acl attr par2' common_debs="$common_debs duplicity rdiff-backup rsnapshot dosfstools kmod" +common_debs="$common_debs libreadline-dev" pyver="${1:-python2}" xattr="${2:-pyxattr}" diff --git a/dev/prep-for-freebsd-build b/dev/prep-for-freebsd-build index 11866db..7a8901c 100755 --- a/dev/prep-for-freebsd-build +++ b/dev/prep-for-freebsd-build @@ -7,5 +7,5 @@ export ASSUME_ALWAYS_YES=yes pkg update pkg install \ gmake git bash rsync curl par2cmdline \ - python2 python py27-pylibacl py27-tornado \ + python2 python py27-pylibacl py27-tornado readline \ duplicity rdiff-backup rsnapshot diff --git a/dev/prep-for-macos-build b/dev/prep-for-macos-build index 750b3e1..7099574 100755 --- a/dev/prep-for-macos-build +++ b/dev/prep-for-macos-build @@ -5,4 +5,4 @@ set -ex ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew update -brew install par2 rsync +brew install par2 readline rsync diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c index 946ff4a..d954f1d 100644 --- a/lib/bup/_helpers.c +++ b/lib/bup/_helpers.c @@ -46,6 +46,14 @@ #include #endif +#ifdef BUP_HAVE_READLINE +#if ! defined(_XOPEN_SOURCE) || _XOPEN_SOURCE < BUP_RL_EXPECTED_XOPEN_SOURCE +# warning "_XOPEN_SOURCE version is too low for readline" +#endif +#include +#include +#endif + #include "bupsplit.h" #if defined(FS_IOC_GETFLAGS) && defined(FS_IOC_SETFLAGS) @@ -1740,6 +1748,7 @@ static PyObject *tuple_from_cstrs(char **cstrs) return result; } + static long getpw_buf_size; static PyObject *pwd_struct_to_py(const struct passwd *pwd, int rc) @@ -1876,6 +1885,217 @@ static PyObject *bup_gethostname(PyObject *mod, PyObject *ignore) return PyBytes_FromString(buf); } + +#ifdef BUP_HAVE_READLINE + +static char *cstr_from_bytes(PyObject *bytes) +{ + char *buf; + Py_ssize_t length; + int rc = PyBytes_AsStringAndSize(bytes, &buf, &length); + if (rc == -1) + return NULL; + char *result = checked_malloc(length, sizeof(char)); + if (!result) + return NULL; + memcpy(result, buf, length); + return result; +} + +static char **cstrs_from_seq(PyObject *seq) +{ + char **result = NULL; + seq = PySequence_Fast(seq, "Cannot convert sequence items to C strings"); + if (!seq) + return NULL; + + const Py_ssize_t len = PySequence_Fast_GET_SIZE(seq); + if (len > PY_SSIZE_T_MAX - 1) { + PyErr_Format(PyExc_OverflowError, + "Sequence length %zd too large for conversion to C array", + len); + goto finish; + } + result = checked_malloc(len + 1, sizeof(char *)); + if (!result) + goto finish; + Py_ssize_t i = 0; + for (i = 0; i < len; i++) + { + PyObject *item = PySequence_Fast_GET_ITEM(seq, i); + if (!item) + goto abandon_result; + result[i] = cstr_from_bytes(item); + if (!result[i]) { + i--; + goto abandon_result; + } + } + result[len] = NULL; + goto finish; + + abandon_result: + if (result) { + for (; i > 0; i--) + free(result[i]); + free(result); + result = NULL; + } + finish: + Py_DECREF(seq); + return result; +} + +static char* our_word_break_chars = NULL; + +static PyObject * +bup_set_completer_word_break_characters(PyObject *self, PyObject *args) +{ + char *bytes; + if (!PyArg_ParseTuple(args, cstr_argf, &bytes)) + return NULL; + char *prev = our_word_break_chars; + char *next = strdup(bytes); + if (!next) + return PyErr_NoMemory(); + our_word_break_chars = next; + rl_completer_word_break_characters = next; + if (prev) + free(prev); + Py_RETURN_NONE; +} + +static PyObject * +bup_get_completer_word_break_characters(PyObject *self, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "")) + return NULL; + return PyBytes_FromString(rl_completer_word_break_characters); +} + +static PyObject *bup_get_line_buffer(PyObject *self, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "")) + return NULL; + return PyBytes_FromString(rl_line_buffer); +} + +static PyObject * +bup_parse_and_bind(PyObject *self, PyObject *args) +{ + char *bytes; + if (!PyArg_ParseTuple(args, cstr_argf ":parse_and_bind", &bytes)) + return NULL; + char *tmp = strdup(bytes); // Because it may modify the arg + if (!tmp) + return PyErr_NoMemory(); + int rc = rl_parse_and_bind(tmp); + free(tmp); + if (rc != 0) + return PyErr_Format(PyExc_OSError, + "system rl_parse_and_bind failed (%d)", rc); + Py_RETURN_NONE; +} + + +static PyObject *py_on_attempted_completion; +static char **prev_completions; + +static char **on_attempted_completion(const char *text, int start, int end) +{ + if (!py_on_attempted_completion) + return NULL; + + char **result = NULL; + PyObject *py_result = PyObject_CallFunction(py_on_attempted_completion, + cstr_argf "ii", + text, start, end); + if (!py_result) + return NULL; + if (py_result != Py_None) { + result = cstrs_from_seq(py_result); + free(prev_completions); + prev_completions = result; + } + Py_DECREF(py_result); + return result; +} + +static PyObject * +bup_set_attempted_completion_function(PyObject *self, PyObject *args) +{ + PyObject *completer; + if (!PyArg_ParseTuple(args, "O", &completer)) + return NULL; + + PyObject *prev = py_on_attempted_completion; + if (completer == Py_None) + { + py_on_attempted_completion = NULL; + rl_attempted_completion_function = NULL; + } else { + py_on_attempted_completion = completer; + rl_attempted_completion_function = on_attempted_completion; + Py_INCREF(completer); + } + Py_XDECREF(prev); + Py_RETURN_NONE; +} + + +static PyObject *py_on_completion_entry; + +static char *on_completion_entry(const char *text, int state) +{ + if (!py_on_completion_entry) + return NULL; + + PyObject *py_result = PyObject_CallFunction(py_on_completion_entry, + cstr_argf "i", text, state); + if (!py_result) + return NULL; + char *result = (py_result == Py_None) ? NULL : cstr_from_bytes(py_result); + Py_DECREF(py_result); + return result; +} + +static PyObject * +bup_set_completion_entry_function(PyObject *self, PyObject *args) +{ + PyObject *completer; + if (!PyArg_ParseTuple(args, "O", &completer)) + return NULL; + + PyObject *prev = py_on_completion_entry; + if (completer == Py_None) { + py_on_completion_entry = NULL; + rl_completion_entry_function = NULL; + } else { + py_on_completion_entry = completer; + rl_completion_entry_function = on_completion_entry; + Py_INCREF(completer); + } + Py_XDECREF(prev); + Py_RETURN_NONE; +} + +static PyObject * +bup_readline(PyObject *self, PyObject *args) +{ + char *prompt; + if (!PyArg_ParseTuple(args, cstr_argf, &prompt)) + return NULL; + char *line = readline(prompt); + if (!line) + return PyErr_Format(PyExc_EOFError, "readline EOF"); + PyObject *result = PyBytes_FromString(line); + free(line); + return result; +} + +#endif // defined BUP_HAVE_READLINE + + static PyMethodDef helper_methods[] = { { "write_sparsely", bup_write_sparsely, METH_VARARGS, "Write buf excepting zeros at the end. Return trailing zero count." }, @@ -1965,6 +2185,22 @@ static PyMethodDef helper_methods[] = { " not exist." }, { "gethostname", bup_gethostname, METH_NOARGS, "Return the current hostname (as bytes)" }, +#ifdef BUP_HAVE_READLINE + { "set_completion_entry_function", bup_set_completion_entry_function, METH_VARARGS, + "Set rl_completion_entry_function. Called as f(text, state)." }, + { "set_attempted_completion_function", bup_set_attempted_completion_function, METH_VARARGS, + "Set rl_attempted_completion_function. Called as f(text, start, end)." }, + { "parse_and_bind", bup_parse_and_bind, METH_VARARGS, + "Call rl_parse_and_bind." }, + { "get_line_buffer", bup_get_line_buffer, METH_NOARGS, + "Return rl_line_buffer." }, + { "get_completer_word_break_characters", bup_get_completer_word_break_characters, METH_NOARGS, + "Return rl_completer_word_break_characters." }, + { "set_completer_word_break_characters", bup_set_completer_word_break_characters, METH_VARARGS, + "Set rl_completer_word_break_characters." }, + { "readline", bup_readline, METH_VARARGS, + "Call readline(prompt)." }, +#endif // defined BUP_HAVE_READLINE { NULL, NULL, 0, NULL }, // sentinel }; -- 2.39.2