From 4d9dd65141326b0f3ffa1658a0535348940ed017 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Sat, 13 Feb 2021 16:36:35 -0600 Subject: [PATCH] Convert top level executables to binaries and clean up clean Signed-off-by: Rob Browning --- .gitignore | 32 ++-- DESIGN | 46 +++-- Makefile | 104 ++++++---- README.md | 15 +- cmd | 1 - config/config.vars.in | 8 +- config/configure | 88 +++++---- dev/bup-python | 10 - dev/cleanup-mounts-under | 2 +- dev/data-size | 12 +- dev/echo-argv-bytes | 18 +- dev/hardlink-sets | 23 +-- dev/id-other-than | 22 +-- dev/install-python-script | 38 ---- dev/lib.sh | 2 +- dev/make-random-paths | 14 +- dev/mksock | 10 +- dev/ns-timestamp-resolutions | 19 +- dev/python.c | 20 ++ dev/root-status | 5 +- dev/sparse-test-data | 15 +- dev/subtree-hash | 11 +- dev/unknown-owner | 5 +- dev/validate-python | 19 ++ lib/bup/_helpers.c | 64 +------ lib/bup/cmd/get.py | 2 +- lib/bup/cmd/split.py | 2 +- lib/bup/compat.py | 52 ++--- lib/bup/csetup.py | 24 --- lib/{cmd/bup => bup/main.py} | 39 +--- lib/cmd/bup.c | 359 +++++++++++++++++++++++++++++++++++ test/ext/test-misc | 2 +- 32 files changed, 674 insertions(+), 409 deletions(-) delete mode 120000 cmd delete mode 100755 dev/bup-python delete mode 100755 dev/install-python-script create mode 100644 dev/python.c create mode 100755 dev/validate-python delete mode 100644 lib/bup/csetup.py rename lib/{cmd/bup => bup/main.py} (94%) create mode 100644 lib/cmd/bup.c diff --git a/.gitignore b/.gitignore index bda8de8..1555c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,20 @@ -\#*# -.#* -randomgen -memtest -*.o -*.so -*.exe -*.dll +*.swp *~ -*.pyc -*.tmp -*.tmp.meta -/build -/config/bin/ +/config/config.h.tmp +/dev/bup-exec +/dev/bup-python +/dev/python +/lib/bup/_helpers.dll +/lib/bup/_helpers.so /lib/bup/checkout_info.py -*.swp -nbproject -/lib/cmd/bup-* +/lib/cmd/bup +/nbproject/ +/test/int/__init__.pyc +/test/lib/__init__.pyc +/test/lib/buptest/__init__.pyc +/test/lib/buptest/vfs.pyc +/test/lib/wvpytest.pyc /test/sampledata/var/ /test/tmp/ +\#*# +__pycache__/ diff --git a/DESIGN b/DESIGN index 89b06b7..d6e8c1b 100644 --- a/DESIGN +++ b/DESIGN @@ -18,17 +18,41 @@ source code to follow along and see what we're talking about. bup's code is written primarily in python with a bit of C code in speed-sensitive places. Here are the most important things to know: - - bup (symlinked to main.py) is the main program that runs when you type - 'bup'. - - - cmd/bup-* (mostly symlinked to cmd/*-cmd.py) are the individual - subcommands, in a way similar to how git breaks all its subcommands into - separate programs. Not all the programs have to be written in python; - they could be in any language, as long as they end up named cmd/bup-*. - We might end up re-coding large parts of bup in C eventually so that it - can be even faster and (perhaps) more portable. - - - lib/bup/*.py are python library files used by the cmd/*.py commands. + - The main program is a fairly small C program that mostly just + initializes the correct Python interpreter and then runs + bup.main.main(). This arrangement was chosen in order to give us + more flexibility. For example: + + - It allows us to avoid + [crashing on some Unicode-unfriendly command line arguments](https://bugs.python.org/issue35883) + which is critical, given that paths can be arbitrary byte + sequences. + + - It allows more flexibility in dealing with upstream changes + like the breakage of our ability to manipulate the + processes arguement list on platforms that support it during + the Python 3.9 series. + + - It means that we'll no longer be affected by any changes to the + `#!/...` path, i.e. if `/usr/bin/python`, or + `/usr/bin/python3`, or whatever we'd previously selected during + `./configure` were to change from 2 to 3, or 3.5 to 3.20. + + The version of python bup uses is determined by the `python-config` + program selected by `./configure`. It tries to find a suitable + default unless `BUP_PYTHON_CONFIG` is set in the environment. + + - bup supports both internal and external subcommands. The former + are the most common, and are all located in lib/bup/cmd/. They + must be python modules named lib/bup/cmd/COMMAND.py, and must + contain a `main(argv)` function that will be passed the *binary* + command line arguments (bytes, not strings). The filename must + have underscores for any dashes in the subcommand name. The + external subcommands are in lib/cmd/. + + - The python code is all in lib/bup. + + - lib/bup/\*.py contains the python code (modules) that bup depends on. That directory name seems a little silly (and worse, redundant) but there seemed to be no better way to let programs write "from bup import index" and have it work. Putting bup in the top level conflicted with diff --git a/Makefile b/Makefile index a9c7fb9..f3205f7 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,12 @@ MAKEFLAGS += --warn-undefined-variables SHELL := bash .DEFAULT_GOAL := all -# See config/config.vars.in (sets bup_python, among other things) --include config/config.vars +clean_paths := + +# See config/config.vars.in (sets bup_python_config, among other things) +include config/config.vars pf := set -o pipefail -cfg_py := $(CURDIR)/config/bin/python define isok && echo " ok" || echo " no" @@ -30,7 +31,7 @@ os := $(call shout,$(os),Unable to determine OS) CFLAGS := -O2 -Wall -Werror -Wformat=2 $(CFLAGS) CFLAGS := -Wno-unknown-pragmas -Wsign-compare $(CFLAGS) -CFLAGS := -D_FILE_OFFSET_BITS=64 $(PYINCLUDE) $(CFLAGS) +CFLAGS := -D_FILE_OFFSET_BITS=64 $(CFLAGS) SOEXT:=.so ifeq ($(os),CYGWIN) @@ -45,6 +46,7 @@ endif initial_setup := $(shell dev/update-checkout-info lib/bup/checkout_info.py $(isok)) initial_setup := $(call shout,$(initial_setup),update-checkout-info failed)) +clean_paths += lib/bup/checkout_info.py config/config.vars: \ configure config/configure config/configure.inc \ @@ -73,11 +75,10 @@ endif bup_ext_cmds := lib/cmd/bup-import-rdiff-backup lib/cmd/bup-import-rsnapshot -config/bin/python: config/config.vars - -bup_deps := lib/bup/_helpers$(SOEXT) +bup_deps := lib/bup/_helpers$(SOEXT) lib/cmd/bup -all: $(bup_deps) Documentation/all $(current_sampledata) +all: dev/bup-exec dev/bup-python dev/python $(bup_deps) Documentation/all \ + $(current_sampledata) $(current_sampledata): dev/configure-sampledata --setup @@ -113,7 +114,7 @@ install: all test -z "$(man_roff)" || $(INSTALL) -m 0644 $(man_roff) $(dest_mandir)/man1 test -z "$(man_html)" || install -d $(dest_docdir) test -z "$(man_html)" || $(INSTALL) -m 0644 $(man_html) $(dest_docdir) - dev/install-python-script lib/cmd/bup "$(dest_libdir)/cmd/bup" + $(INSTALL) -pm 0755 lib/cmd/bup "$(dest_libdir)/cmd/bup" $(INSTALL) -pm 0755 $(bup_ext_cmds) "$(dest_libdir)/cmd/" cd "$(dest_bindir)" && \ ln -sf "$$($(bup_python) -c 'import os; print(os.path.relpath("$(abspath $(dest_libdir))/cmd/bup"))')" @@ -138,32 +139,51 @@ install: all fi config/config.h: config/config.vars +clean_paths += config/config.h.tmp + +dev/python: dev/python.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -o $@-proposed + dev/validate-python $@-proposed + mv $@-proposed $@ +# Do not add to clean_paths - want it available until the very end + +dev/bup-exec: lib/cmd/bup.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -fPIC \ + -D BUP_DEV_BUP_EXEC=1 -o $@ +clean_paths += dev/bup-exec + +dev/bup-python: lib/cmd/bup.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -fPIC \ + -D BUP_DEV_BUP_PYTHON=1 -o $@ +clean_paths += dev/bup-python + +lib/cmd/bup: lib/cmd/bup.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -fPIC -o $@ +clean_paths += lib/cmd/bup + +helper_src := config/config.h lib/bup/bupsplit.h lib/bup/bupsplit.c +helper_src += lib/bup/_helpers.c -lib/bup/_helpers$(SOEXT): \ - config/config.h lib/bup/bupsplit.h \ - lib/bup/bupsplit.c lib/bup/_helpers.c lib/bup/csetup.py - @rm -f $@ - cd lib/bup && $(cfg_py) csetup.py build "$(CFLAGS)" "$(LDFLAGS)" - # Make sure there's just the one file we expect before we copy it. - $(cfg_py) -c \ - "import glob; assert(len(glob.glob('lib/bup/build/*/_helpers*$(SOEXT)')) == 1)" - cp lib/bup/build/*/_helpers*$(SOEXT) "$@" +lib/bup/_helpers$(SOEXT): dev/python $(helper_src) + $(CC) $(bup_python_cflags) $(CFLAGS) -shared -fPIC $(helper_src) \ + $(bup_python_ldflags) $(LDFLAGS) -o $@ +clean_paths += lib/bup/_helpers$(SOEXT) test/tmp: mkdir test/tmp -ifeq (yes,$(shell config/bin/python -c "import xdist; print('yes')" 2>/dev/null)) - # MAKEFLAGS must not be in an immediate := assignment - parallel_opt = $(lastword $(filter -j%,$(MAKEFLAGS))) - get_parallel_n = $(patsubst -j%,%,$(parallel_opt)) - maybe_specific_n = $(if $(filter -j%,$(parallel_opt)),-n$(get_parallel_n)) - xdist_opt = $(if $(filter -j,$(parallel_opt)),-nauto,$(maybe_specific_n)) -else - xdist_opt = -endif +# MAKEFLAGS must not be in an immediate := assignment +parallel_opt = $(lastword $(filter -j%,$(MAKEFLAGS))) +get_parallel_n = $(patsubst -j%,%,$(parallel_opt)) +maybe_specific_n = $(if $(filter -j%,$(parallel_opt)),-n$(get_parallel_n)) +xdist_opt = $(if $(filter -j,$(parallel_opt)),-nauto,$(maybe_specific_n)) -test: all test/tmp - ./pytest $(xdist_opt) +test: all test/tmp dev/python + if test yes = $$(dev/python -c "import xdist; print('yes')" 2>/dev/null); then \ + (set -x; ./pytest $(xdist_opt);) \ + else \ + (set-x; ./pytest;) \ + fi stupid: PATH=/bin:/usr/bin $(MAKE) test @@ -171,7 +191,11 @@ stupid: check: test distcheck: all - ./pytest $(xdist_opt) -m release + if test yes = $$(dev/python -c "import xdist; print('yes')" 2>/dev/null); then \ + (set -x; ./pytest $(xdist_opt) -m release;) \ + else \ + (set -x; ./pytest -m release;) \ + fi long-test: export BUP_TEST_LEVEL=11 long-test: test @@ -181,8 +205,8 @@ long-check: check .PHONY: check-both check-both: - $(MAKE) clean && PYTHON=python3 $(MAKE) check - $(MAKE) clean && PYTHON=python2 $(MAKE) check + $(MAKE) clean && BUP_PYTHON_CONFIG=python3-config $(MAKE) check + $(MAKE) clean && BUP_PYTHON_CONFIG=python2.7-config $(MAKE) check .PHONY: Documentation/all Documentation/all: $(man_roff) $(man_html) @@ -223,24 +247,20 @@ import-docs: Documentation/clean $(pf); git archive origin/html | (cd Documentation && tar -xvf -) $(pf); git archive origin/man | (cd Documentation && tar -xvf -) -clean: Documentation/clean config/bin/python +clean: Documentation/clean dev/python cd config && rm -rf config.var - cd config && rm -f *~ .*~ \ + cd config && rm -f \ ${CONFIGURE_DETRITUS} ${CONFIGURE_FILES} ${GENERATED_FILES} - rm -f *.o lib/*/*.o *.so lib/*/*.so *.dll lib/*/*.dll *.exe \ - .*~ *~ */*~ lib/*/*~ lib/*/*/*~ \ - *.pyc */*.pyc lib/*/*.pyc lib/*/*/*.pyc \ - lib/bup/checkout_info.py \ - randomgen memtest \ - testfs.img test/int/testfs.img + rm -rf $(clean_paths) .pytest_cache + find . -name __pycache__ -exec rm -rf {} + if test -e test/mnt; then dev/cleanup-mounts-under test/mnt; fi if test -e test/mnt; then rm -r test/mnt; fi if test -e test/tmp; then dev/cleanup-mounts-under test/tmp; fi # FIXME: migrate these to test/mnt/ if test -e test/int/testfs; \ then umount test/int/testfs || true; fi - rm -rf *.tmp *.tmp.meta test/*.tmp lib/*/*/*.tmp build lib/bup/build test/int/testfs + rm -rf test/int/testfs test/int/testfs.img testfs.img if test -e test/tmp; then dev/force-delete test/tmp; fi dev/configure-sampledata --clean # Remove last so that cleanup tools can depend on it - rm -rf config/bin + rm -f dev/python diff --git a/README.md b/README.md index 2c00a3f..0e9f472 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ From source pip install tornado ``` - - Build the python module and symlinks: + - Build: ```sh make @@ -244,12 +244,13 @@ From source make install DESTDIR=/opt/bup PREFIX='' ``` - - The Python executable that bup will use is chosen by ./configure, - which will search for a reasonable version unless PYTHON is set in - the environment, in which case, bup will use that path. You can - see which Python executable was chosen by looking at the - configure output, or examining cmd/python-cmd.sh, and you can - change the selection by re-running ./configure. + - The Python version that bup will use is determined by the + `python-config` program chosen by `./configure`, which will search + for a reasonable version unless `BUP_PYTHON_CONFIG` is set in the + environment. You can see which Python executable was chosen by + looking at the configure output, or examining + `config/config.var/bup-python-config`, and you can change the + selection by re-running `./configure`. From binary packages -------------------- diff --git a/cmd b/cmd deleted file mode 120000 index 7819428..0000000 --- a/cmd +++ /dev/null @@ -1 +0,0 @@ -lib/cmd \ No newline at end of file diff --git a/config/config.vars.in b/config/config.vars.in index 8f4769c..6606bfd 100644 --- a/config/config.vars.in +++ b/config/config.vars.in @@ -2,8 +2,12 @@ CONFIGURE_FILES=@CONFIGURE_FILES@ GENERATED_FILES=@GENERATED_FILES@ bup_make=@bup_make@ -bup_python=@bup_python@ -bup_python_majver=@bup_python_majver@ + +bup_python_config=@bup_python_config@ +bup_python_cflags=@bup_python_cflags@ +bup_python_ldflags=@bup_python_ldflags@ +bup_python_cflags_embed=@bup_python_cflags_embed@ +bup_python_ldflags_embed=@bup_python_ldflags_embed@ bup_have_libacl=@bup_have_libacl@ bup_libacl_cflags=@bup_libacl_cflags@ diff --git a/config/configure b/config/configure index 6ef0531..3506c54 100755 --- a/config/configure +++ b/config/configure @@ -37,6 +37,9 @@ TARGET=bup . ./configure.inc +# FIXME: real tmpdir +rm -rf config.var config.var.tmp config.vars + AC_INIT $TARGET if ! AC_PROG_CC; then @@ -65,30 +68,43 @@ expr "$MAKE_VERSION" '>=' '3.81' || AC_FAIL "ERROR: $MAKE must be >= version 3.8 AC_SUB bup_make "$MAKE" -bup_python="$(type -p "$PYTHON")" -test -z "$bup_python" && bup_python="$(bup_find_prog python3.8 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python3.7 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python3.6 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python3 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python2.7 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python2.6 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python2 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python '')" -if test -z "$bup_python"; then - AC_FAIL "ERROR: unable to find python" + +# Haven't seen a documented way to determine the python version via +# python-config right now, so we'll defer version checking until +# later. + +if test "$BUP_PYTHON_CONFIG"; then + bup_python_config="$(type -p "$BUP_PYTHON_CONFIG")" + if test -z "$bup_python_config"; then + AC_FAIL $(printf "ERROR: BUP_PYTHON_CONFIG value %q appears invalid" \ + "$BUP_PYTHON_CONFIG") + fi else - AC_SUB bup_python "$bup_python" - bup_python_majver=$("$bup_python" -c 'import sys; print(sys.version_info[0])') - bup_python_minver=$("$bup_python" -c 'import sys; print(sys.version_info[1])') - AC_SUB bup_python_majver "$bup_python_majver" + test -z "$bup_python_config" \ + && bup_python_config="$(bup_find_prog python3-config '')" + test -z "$bup_python_config" \ + && bup_python_config="$(bup_find_prog python2.7-config '')" + if test -z "$bup_python_config"; then + AC_FAIL "ERROR: unable to find a suitable python-config" + fi fi -# May not be correct yet, i.e. actual requirement may be higher. -if test "$bup_python_majver" -gt 2 -a "$bup_python_minver" -lt 3; then - # utime follow_symlinks >= 3.3 - bup_version_str=$("$bup_python" --version 2>&1) - AC_FAIL "ERROR: found $bup_version_str (must be >= 3.3 if >= 3)" + +bup_python_cflags=$("$bup_python_config" --cflags) || exit $? +bup_python_ldflags=$("$bup_python_config" --ldflags) || exit $? +bup_python_cflags_embed=$("$bup_python_config" --cflags --embed) +if test $? -eq 0; then + bup_python_ldflags_embed=$("$bup_python_config" --ldflags --embed) || exit $? +else # Earlier versions didn't support --embed + bup_python_cflags_embed=$("$bup_python_config" --cflags) || exit $? + bup_python_ldflags_embed=$("$bup_python_config" --ldflags) || exit $? fi +AC_SUB bup_python_config "$bup_python_config" +AC_SUB bup_python_cflags "$bup_python_cflags" +AC_SUB bup_python_ldflags "$bup_python_ldflags" +AC_SUB bup_python_cflags_embed "$bup_python_cflags_embed" +AC_SUB bup_python_ldflags_embed "$bup_python_ldflags_embed" + bup_git="$(bup_find_prog git '')" if test -z "$bup_git"; then @@ -109,16 +125,12 @@ AC_CHECK_HEADERS sys/mman.h AC_CHECK_HEADERS linux/fs.h AC_CHECK_HEADERS sys/ioctl.h -if test "$bup_python_majver" -gt 2; then - AC_DEFINE BUP_USE_PYTHON_UTIME 1 -else # Python 2 - # On GNU/kFreeBSD utimensat is defined in GNU libc, but won't work. - if [ -z "$OS_GNU_KFREEBSD" ]; then - AC_CHECK_FUNCS utimensat - fi - AC_CHECK_FUNCS utimes - AC_CHECK_FUNCS lutimes +# On GNU/kFreeBSD utimensat is defined in GNU libc, but won't work. +if [ -z "$OS_GNU_KFREEBSD" ]; then + AC_CHECK_FUNCS utimensat fi +AC_CHECK_FUNCS utimes +AC_CHECK_FUNCS lutimes builtin_mul_overflow_code=" #include @@ -303,20 +315,20 @@ LIBS="$orig_libs" AC_OUTPUT config.vars -if test -e config.var; then rm -r config.var; fi -mkdir -p config.var -echo -n "$MAKE" > config.var/bup-make -echo -n "$bup_python" > config.var/bup-python -if test -e bin; then rm -r bin; fi -mkdir -p bin -(cd bin && ln -s "$bup_python" python) +set -euo pipefail + +# FIXME: real tmpdir +mkdir -p config.var.tmp +echo -n "$MAKE" > config.var.tmp/bup-make +echo -n "$bup_python_config" > config.var.tmp/bup-python-config +mv config.var.tmp config.var printf " -found: python (%q, $("$bup_python" --version 2>&1)) +found: python-config (%q) found: git (%q, ($("$bup_git" --version)) " \ - "$bup_python" \ + "$bup_python_config" \ "$bup_git" \ 1>&5 diff --git a/dev/bup-python b/dev/bup-python deleted file mode 100755 index 384a8fd..0000000 --- a/dev/bup-python +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -script_home="$(cd "$(dirname "$0")" && pwd -P)" -python="$script_home/../config/bin/python" -libdir="$script_home/../lib" - -export PYTHONPATH="$libdir${PYTHONPATH:+:$PYTHONPATH}" -exec "$python" "$@" diff --git a/dev/cleanup-mounts-under b/dev/cleanup-mounts-under index c0c2671..b04f095 100755 --- a/dev/cleanup-mounts-under +++ b/dev/cleanup-mounts-under @@ -1,6 +1,6 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? +bup_python="$(dirname "$0")/../dev/python" || exit $? exec "$bup_python" "$0" ${1+"$@"} """ diff --git a/dev/data-size b/dev/data-size index e5068da..451498d 100755 --- a/dev/data-size +++ b/dev/data-size @@ -1,16 +1,18 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function from os.path import getsize, isdir -from sys import argv, stderr +from sys import stderr import os +from bup.compat import get_argvb + + def listdir_failure(ex): raise ex @@ -18,7 +20,7 @@ def usage(): print('Usage: data-size PATH ...', file=sys.stderr) total = 0 -for path in argv[1:]: +for path in get_argvb()[1:]: if isdir(path): for root, dirs, files in os.walk(path, onerror=listdir_failure): total += sum(getsize(os.path.join(root, name)) for name in files) diff --git a/dev/echo-argv-bytes b/dev/echo-argv-bytes index d49c26c..f9a71c2 100755 --- a/dev/echo-argv-bytes +++ b/dev/echo-argv-bytes @@ -1,17 +1,8 @@ #!/bin/sh """": # -*-python-*- -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function @@ -19,12 +10,9 @@ from os.path import abspath, dirname from sys import stdout import os, sys -script_home = abspath(dirname(__file__)) -sys.path[:0] = [abspath(script_home + '/../../lib'), abspath(script_home + '/../..')] - from bup import compat -for arg in compat.argvb: +for arg in compat.get_argvb(): os.write(stdout.fileno(), arg) os.write(stdout.fileno(), b'\0\n') stdout.flush() diff --git a/dev/hardlink-sets b/dev/hardlink-sets index e1be742..fb0bdb7 100755 --- a/dev/hardlink-sets +++ b/dev/hardlink-sets @@ -1,24 +1,13 @@ #!/bin/sh """": # -*-python-*- -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function import os, stat, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] - -from bup import compat +from bup.compat import get_argvb from bup.io import byte_stream @@ -29,7 +18,9 @@ from bup.io import byte_stream def usage(): print("Usage: hardlink-sets ", file=sys.stderr) -if len(compat.argv) < 2: +argvb = get_argvb() + +if len(argvb) < 2: usage() sys.exit(1) @@ -41,7 +32,7 @@ out = byte_stream(sys.stdout) hardlink_set = {} -for p in compat.argvb[1:]: +for p in argvb[1:]: for root, dirs, files in os.walk(p, onerror = on_walk_error): for filename in files: full_path = os.path.join(root, filename) diff --git a/dev/id-other-than b/dev/id-other-than index e54696a..a6cab57 100755 --- a/dev/id-other-than +++ b/dev/id-other-than @@ -1,12 +1,8 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble - -# Note: this currently relies on bup-python to handle arbitrary binary -# user/group names. from __future__ import absolute_import, print_function @@ -14,11 +10,15 @@ import grp import pwd import sys +from bup.compat import get_argvb + def usage(): print('Usage: id-other-than <--user|--group> ID [ID ...]', file=sys.stderr) -if len(sys.argv) < 2: +argvb = get_argvb() + +if len(argvb) < 2: usage() sys.exit(1) @@ -29,17 +29,17 @@ def is_integer(x): except ValueError as e: return False -excluded_ids = set(int(x) for x in sys.argv[2:] if is_integer(x)) -excluded_names = (x for x in sys.argv[2:] if not is_integer(x)) +excluded_ids = set(int(x) for x in argvb[2:] if is_integer(x)) +excluded_names = (x for x in argvb[2:] if not is_integer(x)) -if sys.argv[1] == '--user': +if argvb[1] == b'--user': for x in excluded_names: excluded_ids.add(pwd.getpwnam(x).pw_uid) for x in pwd.getpwall(): if x.pw_uid not in excluded_ids: print(x.pw_name + ':' + str(x.pw_uid)) sys.exit(0) -elif sys.argv[1] == '--group': +elif argvb[1] == b'--group': for x in excluded_names: excluded_ids.add(grp.getgrnam(x).gr_gid) for x in grp.getgrall(): diff --git a/dev/install-python-script b/dev/install-python-script deleted file mode 100755 index 83d8861..0000000 --- a/dev/install-python-script +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -"""": # -*-python-*- -export LC_CTYPE=iso-8859-1 -exec "$(dirname "$0")/../config/bin/python" "$0" "$@" -""" - -from __future__ import absolute_import, print_function -from tempfile import NamedTemporaryFile -import os, shutil, sys - -if sys.version_info[0] >= 3: - from shlex import quote -else: - from pipes import quote - -src_path, dest_path = sys.argv[1:] - -with open(b'config/config.var/bup-python', 'rb') as src: - python = src.read() - -with NamedTemporaryFile() as tmp: - # Replace the section between "Here to end..." and the end of the - # preamble with the correct 'exec PYTHON "$0"'. - with open(src_path, 'rb') as src: - for line in src: - if line.startswith(b'# Here to end of preamble replaced during install'): - break - tmp.write(line) - for line in src: - if line == b'"""\n': - break - tmp.write(b'exec %s "$0"\n' % python) - tmp.write(b'"""\n') - for line in src: - tmp.write(line) - tmp.flush() - shutil.copy(tmp.name, dest_path) - os.chmod(dest_path, 0o755) diff --git a/dev/lib.sh b/dev/lib.sh index 20780e2..b89c05d 100644 --- a/dev/lib.sh +++ b/dev/lib.sh @@ -3,7 +3,7 @@ # Assumes this is always loaded while pwd is still the source tree root bup_dev_lib_top=$(pwd) || exit $? -bup-cfg-py() { "$bup_dev_lib_top/config/bin/python" "$@"; } +bup-cfg-py() { "$bup_dev_lib_top/dev/python" "$@"; } bup-python() { "$bup_dev_lib_top/dev/bup-python" "$@"; } force-delete() diff --git a/dev/make-random-paths b/dev/make-random-paths index 05d24f4..c0c6c78 100755 --- a/dev/make-random-paths +++ b/dev/make-random-paths @@ -1,19 +1,19 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")//bup-python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function from os.path import abspath, dirname from random import randint -from sys import argv, exit, stderr, stdout +from sys import stderr, stdout import errno, re, sys -from bup.compat import fsencode, range +from bup.compat import fsencode, get_argv, get_argvb, range +argv = get_argv() def usage(out=stdout): print('Usage:', argv[0], 'NUM', 'DEST_DIR', file=out) @@ -44,10 +44,10 @@ def random_filename(): if len(argv) != 3: misuse() -count, dest = argv[1:] +count, dest = get_argvb()[1:] count = int(count) i = 0 while i < count: - with open(fsencode(dest) + b'/' + random_filename(), 'w') as _: + with open(dest + b'/' + random_filename(), 'w') as _: i += 1 diff --git a/dev/mksock b/dev/mksock index 88372c3..4b14b22 100755 --- a/dev/mksock +++ b/dev/mksock @@ -1,13 +1,13 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import - import socket, sys +from bup.compat import get_argvb + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) -s.bind(sys.argv[1]) +s.bind(get_argvb()[1]) diff --git a/dev/ns-timestamp-resolutions b/dev/ns-timestamp-resolutions index f5b296c..3f0c9d4 100755 --- a/dev/ns-timestamp-resolutions +++ b/dev/ns-timestamp-resolutions @@ -1,24 +1,13 @@ #!/bin/sh """": # -*-python-*- -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import import os.path, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../../lib'] - -from bup.compat import argv_bytes +from bup.compat import argv_bytes, get_argvb from bup.helpers import handle_ctrl_c, saved_errors from bup.io import byte_stream from bup import compat, metadata, options @@ -33,7 +22,7 @@ ns-timestamp-resolutions TEST_FILE_NAME handle_ctrl_c() o = options.Options(optspec) -opt, flags, extra = o.parse(compat.argv[1:]) +opt, flags, extra = o.parse_bytes(get_argvb()[1:]) sys.stdout.flush() out = byte_stream(sys.stdout) diff --git a/dev/python.c b/dev/python.c new file mode 100644 index 0000000..0c41508 --- /dev/null +++ b/dev/python.c @@ -0,0 +1,20 @@ +#define _LARGEFILE64_SOURCE 1 +#define PY_SSIZE_T_CLEAN 1 +#undef NDEBUG +#include "../config/config.h" + +// According to Python, its header has to go first: +// http://docs.python.org/2/c-api/intro.html#include-files +// http://docs.python.org/3/c-api/intro.html#include-files +#include + +#if PY_MAJOR_VERSION > 2 +#define bup_py_main Py_BytesMain +# else +#define bup_py_main Py_Main +#endif + +int main(int argc, char **argv) +{ + return bup_py_main (argc, argv); +} diff --git a/dev/root-status b/dev/root-status index c37806d..36a173f 100755 --- a/dev/root-status +++ b/dev/root-status @@ -1,9 +1,8 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +python="$(dirname "$0")/python" || exit $? +exec "$python" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function import os, sys diff --git a/dev/sparse-test-data b/dev/sparse-test-data index 34bb117..3d9f712 100755 --- a/dev/sparse-test-data +++ b/dev/sparse-test-data @@ -1,7 +1,7 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ from __future__ import absolute_import, print_function @@ -9,8 +9,7 @@ from random import randint from sys import stderr, stdout import os, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] - +from bup.compat import get_argvb from bup.io import byte_stream def smaller_region(max_offset): @@ -48,11 +47,13 @@ def random_region(): global generators return generators[randint(0, len(generators) - 1)]() -if len(sys.argv) == 0: +argv = get_argvb() + +if len(argv) == 0: stdout.flush() out = byte_stream(stdout) -if len(sys.argv) == 2: - out = open(sys.argv[1], 'wb') +if len(argv) == 2: + out = open(argv[1], 'wb') else: print('Usage: sparse-test-data [FILE]', file=stderr) sys.exit(2) diff --git a/dev/subtree-hash b/dev/subtree-hash index e3468fb..f0a3f6f 100755 --- a/dev/subtree-hash +++ b/dev/subtree-hash @@ -1,16 +1,13 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function import os.path, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] - -from bup.compat import argv_bytes +from bup.compat import argv_bytes, get_argvb from bup.helpers import handle_ctrl_c, readpipe from bup.io import byte_stream from bup import options @@ -24,7 +21,7 @@ subtree-hash ROOT_HASH [PATH_ITEM...] handle_ctrl_c() o = options.Options(optspec) -(opt, flags, extra) = o.parse(sys.argv[1:]) +opt, flags, extra = o.parse_bytes(get_argvb()[1:]) if len(extra) < 1: o.fatal('must specify a root hash') diff --git a/dev/unknown-owner b/dev/unknown-owner index 937e708..0077e24 100755 --- a/dev/unknown-owner +++ b/dev/unknown-owner @@ -1,9 +1,8 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +python="$(dirname "$0")/python" || exit $? +exec "$python" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function diff --git a/dev/validate-python b/dev/validate-python new file mode 100755 index 0000000..a64ecfe --- /dev/null +++ b/dev/validate-python @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ueo pipefail + +die () { echo "Usage: validate-python PYTHON_EXECUTABLE"; } + +test $# -eq 1 || { usage 1>&2 ; exit 2; } +python="$1" + +majver=$("$python" -c 'import sys; print(sys.version_info[0])') +minver=$("$python" -c 'import sys; print(sys.version_info[1])') + +# May not be correct yet, i.e. actual requirement may be higher. +if test "$majver" -gt 2 -a "$minver" -lt 3; then + # utime follow_symlinks >= 3.3 + bup_version_str=$("$python" --version 2>&1) + echo "ERROR: found $bup_version_str (must be >= 3.3 if >= 3)" 1>&2 + exit 2 +fi diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c index 2790b07..5b9ace9 100644 --- a/lib/bup/_helpers.c +++ b/lib/bup/_helpers.c @@ -5,6 +5,7 @@ // According to Python, its header has to go first: // http://docs.python.org/2/c-api/intro.html#include-files +// http://docs.python.org/3/c-api/intro.html#include-files #include #include @@ -70,6 +71,10 @@ #define BUP_HAVE_FILE_ATTRS 1 #endif +#if PY_MAJOR_VERSION > 2 +# define BUP_USE_PYTHON_UTIME 1 +#endif + #ifndef BUP_USE_PYTHON_UTIME // just for Python 2 now /* * Check for incomplete UTIMENSAT support (NetBSD 6), and if so, @@ -353,58 +358,6 @@ static PyObject *bup_cat_bytes(PyObject *self, PyObject *args) } - -// Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV... -#if __WIN32__ || __CYGWIN__ || PY_VERSION_HEX >= 0x03090000 - -// There's no 'ps' on win32 anyway, and Py_GetArgcArgv() isn't available. -static void unpythonize_argv(void) { } - -#else // not __WIN32__ - -// For some reason this isn't declared in Python.h -extern void Py_GetArgcArgv(int *argc, char ***argv); - -static void unpythonize_argv(void) -{ - int argc, i; - char **argv, *arge; - - Py_GetArgcArgv(&argc, &argv); - - for (i = 0; i < argc-1; i++) - { - if (argv[i] + strlen(argv[i]) + 1 != argv[i+1]) - { - // The argv block doesn't work the way we expected; it's unsafe - // to mess with it. - return; - } - } - - arge = argv[argc-1] + strlen(argv[argc-1]) + 1; - - if (strstr(argv[0], "python") && argv[1] == argv[0] + strlen(argv[0]) + 1) - { - char *p; - size_t len, diff; - p = strrchr(argv[1], '/'); - if (p) - { - p++; - diff = p - argv[0]; - len = arge - p; - memmove(argv[0], p, len); - memset(arge - diff, 0, diff); - for (i = 0; i < argc; i++) - argv[i] = argv[i+1] ? argv[i+1]-diff : NULL; - } - } -} - -#endif // not __WIN32__ or __CYGWIN__ - - static int write_all(int fd, const void *buf, const size_t count) { size_t written = 0; @@ -2460,7 +2413,6 @@ static int setup_module(PyObject *m) e = getenv("BUP_FORCE_TTY"); get_state(m)->istty2 = isatty(2) || (atoi(e ? e : "0") & 2); - unpythonize_argv(); return 1; } @@ -2470,11 +2422,13 @@ static int setup_module(PyObject *m) PyMODINIT_FUNC init_helpers(void) { PyObject *m = Py_InitModule("_helpers", helper_methods); - if (m == NULL) + if (m == NULL) { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers init failed"); return; - + } if (!setup_module(m)) { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers set up failed"); Py_DECREF(m); return; } diff --git a/lib/bup/cmd/get.py b/lib/bup/cmd/get.py index b3adf0e..8844544 100755 --- a/lib/bup/cmd/get.py +++ b/lib/bup/cmd/get.py @@ -391,7 +391,7 @@ def handle_append(item, src_repo, writer, opt): if item.src.type == 'tree': get_random_item(item.spec.src, src_oidx, src_repo, writer, opt) parent = item.dest.hash - msg = b'bup save\n\nGenerated by command:\n%r\n' % compat.argvb + msg = b'bup save\n\nGenerated by command:\n%r\n' % compat.get_argvb() userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) now = time.time() commit = writer.new_commit(item.src.hash, parent, diff --git a/lib/bup/cmd/split.py b/lib/bup/cmd/split.py index fc354cb..87ad887 100755 --- a/lib/bup/cmd/split.py +++ b/lib/bup/cmd/split.py @@ -205,7 +205,7 @@ def main(argv): if opt.tree: out.write(hexlify(tree) + b'\n') if opt.commit or opt.name: - msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.argvb + msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.get_argvb() ref = opt.name and (b'refs/heads/%s' % opt.name) or None userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) commit = pack_writer.new_commit(tree, oldref, userline, date, None, diff --git a/lib/bup/compat.py b/lib/bup/compat.py index 2cd6fba..a06ffe8 100644 --- a/lib/bup/compat.py +++ b/lib/bup/compat.py @@ -169,41 +169,27 @@ else: # Python 2 buffer = buffer - -argv = None -argvb = None - -def _configure_argv(): - global argv, argvb - assert not argv - assert not argvb - if len(sys.argv) > 1: - if environ.get(b'BUP_ARGV_0'): - print('error: BUP_ARGV* set and sys.argv not empty', file=sys.stderr) - sys.exit(2) - argv = sys.argv - argvb = [argv_bytes(x) for x in argv] - return - args = [] - i = 0 - arg = environ.get(b'BUP_ARGV_%d' % i) - while arg is not None: - args.append(arg) - i += 1 - arg = environ.get(b'BUP_ARGV_%d' % i) - i -= 1 - while i >= 0: - del environ[b'BUP_ARGV_%d' % i] - i -= 1 - argvb = args - # System encoding? +try: + import bup_main +except ModuleNotFoundError: + bup_main = None + +if bup_main: + def get_argvb(): + "Return a new list containing the current process argv bytes." + return bup_main.argv() if py3: - argv = [x.decode(errors='surrogateescape') for x in args] + def get_argv(): + "Return a new list containing the current process argv strings." + return [x.decode(errors='surrogateescape') for x in bup_main.argv()] else: - argv = argvb - -_configure_argv() - + def get_argv(): + return bup_main.argv() +else: + def get_argvb(): + raise Exception('get_argvb requires the bup_main module'); + def get_argv(): + raise Exception('get_argv requires the bup_main module'); def wrap_main(main): """Run main() and raise a SystemExit with the return value if it diff --git a/lib/bup/csetup.py b/lib/bup/csetup.py deleted file mode 100644 index 9dbb4a7..0000000 --- a/lib/bup/csetup.py +++ /dev/null @@ -1,24 +0,0 @@ - -from __future__ import absolute_import, print_function - -import shlex, sys -from distutils.core import setup, Extension -import os - -if len(sys.argv) != 4: - print('Usage: csetup.py CFLAGS LDFLAGS', file=sys.stderr) - sys.exit(2) -_helpers_cflags = shlex.split(sys.argv[2]) -_helpers_ldflags = shlex.split(sys.argv[3]) -sys.argv = sys.argv[:2] - -_helpers_mod = Extension('_helpers', - sources=['_helpers.c', 'bupsplit.c'], - depends=['../../config/config.h', 'bupsplit.h'], - extra_compile_args=_helpers_cflags, - extra_link_args=_helpers_ldflags) - -setup(name='_helpers', - version='0.1', - description='accelerator library for bup', - ext_modules=[_helpers_mod]) diff --git a/lib/cmd/bup b/lib/bup/main.py similarity index 94% rename from lib/cmd/bup rename to lib/bup/main.py index e474b34..16aeaad 100755 --- a/lib/cmd/bup +++ b/lib/bup/main.py @@ -1,35 +1,5 @@ -#!/bin/sh -"""": # -*-python-*- -set -e -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -# Here to end of preamble replaced during install -# Find our directory -top="$(pwd)" -cmdpath="$0" -# loop because macos doesn't have recursive readlink/realpath utils -while test -L "$cmdpath"; do - link="$(readlink "$cmdpath")" - cd "$(dirname "$cmdpath")" - cmdpath="$link" -done -script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)" -cd "$top" -exec "$script_home/../../config/bin/python" "$0" -""" -# end of bup preamble from __future__ import absolute_import, print_function - -import os, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] - from importlib import import_module from pkgutil import iter_modules from subprocess import PIPE @@ -74,7 +44,7 @@ def maybe_import_early(argv): import_module(mod) argv = argv[2:] -maybe_import_early(compat.argv) +maybe_import_early(compat.get_argv()) handle_ctrl_c() @@ -130,7 +100,7 @@ def usage(msg=""): log("\n%s\n" % msg) sys.exit(99) -argv = compat.argv +argv = compat.get_argv() if len(argv) < 2: usage() @@ -461,5 +431,8 @@ def run_subcmd(module, args): else: run_subproc_cmd(args) +def main(): + wrap_main(lambda : run_subcmd(cmd_module, subcmd)) -wrap_main(lambda : run_subcmd(cmd_module, subcmd)) +if __name__ == "__main__": + main() diff --git a/lib/cmd/bup.c b/lib/cmd/bup.c new file mode 100644 index 0000000..1dcb724 --- /dev/null +++ b/lib/cmd/bup.c @@ -0,0 +1,359 @@ + +#define PY_SSIZE_T_CLEAN +#define _GNU_SOURCE 1 // asprintf +#undef NDEBUG + +// According to Python, its header has to go first: +// http://docs.python.org/2/c-api/intro.html#include-files +// http://docs.python.org/3/c-api/intro.html#include-files +#include + +#include +#include +#include +#include +#include +#include +#include +#if defined(__FreeBSD__) || defined(__NetBSD__) +# include +#endif +#include +#include + +__attribute__ ((format(printf, 2, 3))) +static void +die(int exit_status, const char * const msg, ...) +{ + if (fputs("bup: ", stderr) == EOF) + exit(3); + va_list ap; + va_start(ap, msg);; + if (vfprintf(stderr, msg, ap) < 0) + exit(3); + va_end(ap); + exit(exit_status); +} + +static int prog_argc = 0; +static char **prog_argv = NULL; + +static PyObject* +get_argv(PyObject *self, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "")) + return NULL; + + PyObject *result = PyList_New(prog_argc); + for (int i = 0; i < prog_argc; i++) { + PyObject *s = PyBytes_FromString(prog_argv[i]); + if (!s) + die(2, "cannot convert argument to bytes: %s\n", prog_argv[i]); + PyList_SET_ITEM(result, i, s); + } + return result; +} + +static PyMethodDef bup_main_methods[] = { + {"argv", get_argv, METH_VARARGS, + "Return the program's current argv array as a list of byte strings." }, + {NULL, NULL, 0, NULL} +}; + +static int setup_module(PyObject *mod) +{ + return 1; +} + +#if PY_MAJOR_VERSION >= 3 + +static struct PyModuleDef bup_main_module_def = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "bup_main", + .m_doc = "Built-in bup module providing direct access to argv.", + .m_size = -1, + .m_methods = bup_main_methods +}; + +PyObject * +PyInit_bup_main(void) { + PyObject *mod = PyModule_Create(&bup_main_module_def); + if (!setup_module(mod)) + { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +#else // PY_MAJOR_VERSION < 3 + +void PyInit_bup_main(void) +{ + PyObject *mod = Py_InitModule("bup_main", bup_main_methods); + if (mod == NULL) { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers init failed"); + return; + } + if (!setup_module(mod)) + { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers set up failed"); + Py_DECREF(mod); + return; + } +} + +#endif // PY_MAJOR_VERSION < 3 + +static void +setup_bup_main_module(void) +{ + if (PyImport_AppendInittab("bup_main", PyInit_bup_main) == -1) + die(2, "unable to register bup_main module\n"); +} + +#if defined(__APPLE__) && defined(__MACH__) + +static char *exe_parent_dir(const char * const argv_0) { + char path[4096]; // FIXME + uint32_t size = sizeof(path); + if(_NSGetExecutablePath(path, &size) !=0) + die(2, "unable to find executable path\n"); + char * abs_exe = realpath(path, NULL); + if (!abs_exe) + die(2, "cannot resolve path (%s): %s\n", strerror(errno), path); + char * const abs_parent = strdup(dirname(abs_exe)); + assert(abs_parent); + free(abs_exe); + return abs_parent; +} + +#elif defined(__FreeBSD__) || defined(__NetBSD__) + +static char *exe_path () +{ + const int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + size_t path_len; + int rc = sysctl (mib, 4, NULL, &path_len, NULL, 0); + if (rc != 0) die(2, "unable to determine executable path length\n"); + char *path = malloc (path_len); + if (!path) die(2, "unable to allocate memory for executable path\n"); + rc = sysctl (mib, 4, path, &path_len, NULL, 0); + if (rc != 0) die(2, "unable to determine executable path via sysctl\n"); + return path; +} + +static char *exe_parent_dir(const char * const argv_0) +{ + char * const exe = exe_path(); + if (!exe) die(2, "unable to determine executable path\n"); + char * const parent = strdup(dirname(exe)); + if (!parent) die(2, "unable to determine parent directory of executable\n"); + free(exe); + return parent; +} + +#else // not defined(__FreeBSD__) || defined(__NetBSD__) + +/// Use /proc if possible, and if all else fails, search in the PATH + +#if defined(__linux__) +# define PROC_SELF_EXE "/proc/self/exe" +#elif defined(__sun) || defined (sun) +# define PROC_SELF_EXE "/proc/self/path/a.out" +#else +# define PROC_SELF_EXE NULL +#endif + +static char *find_in_path(const char * const name, const char * const path) +{ + char *result = NULL; + char *tmp_path = strdup(path); + assert(tmp_path); + const char *elt; + char *tok_path = tmp_path; + while ((elt = strtok(tok_path, ":")) != NULL) { + tok_path = NULL; + char *candidate; + int rc = asprintf(&candidate, "%s/%s", elt, name); + assert(rc >= 0); + struct stat st; + rc = stat(candidate, &st); + if (rc != 0) { + switch (errno) { + case EACCES: case ELOOP: case ENOENT: case ENAMETOOLONG: + case ENOTDIR: + break; + default: + die(2, "cannot stat %s: %s\n", candidate, strerror(errno)); + break; + } + } else if (S_ISREG(st.st_mode)) { + if (access(candidate, X_OK) == 0) { + result = candidate; + break; + } + switch (errno) { + case EACCES: case ELOOP: case ENOENT: case ENAMETOOLONG: + case ENOTDIR: + break; + default: + die(2, "cannot determine executability of %s: %s\n", + candidate, strerror(errno)); + break; + } + } + free(candidate); + } + free(tmp_path); + return result; +} + +static char *find_exe_parent(const char * const argv_0) +{ + char *candidate = NULL; + const char * const slash = index(argv_0, '/'); + if (slash) { + candidate = strdup(argv_0); + assert(candidate); + } else { + const char * const env_path = getenv("PATH"); + if (!env_path) + die(2, "no PATH and executable isn't relative or absolute: %s\n", + argv_0); + char *path_exe = find_in_path(argv_0, env_path); + if (path_exe) { + char * abs_exe = realpath(path_exe, NULL); + if (!abs_exe) + die(2, "cannot resolve path (%s): %s\n", + strerror(errno), path_exe); + free(path_exe); + candidate = abs_exe; + } + } + if (!candidate) + return NULL; + + char * const abs_exe = realpath(candidate, NULL); + if (!abs_exe) + die(2, "cannot resolve path (%s): %s\n", strerror(errno), candidate); + free(candidate); + char * const abs_parent = strdup(dirname(abs_exe)); + assert(abs_parent); + free(abs_exe); + return abs_parent; +} + +static char *exe_parent_dir(const char * const argv_0) +{ + if (PROC_SELF_EXE != NULL) { + char path[4096]; // FIXME + int len = readlink(PROC_SELF_EXE, path, sizeof(path)); + if (len == sizeof(path)) + die(2, "unable to resolve symlink %s: %s\n", + PROC_SELF_EXE, strerror(errno)); + if (len != -1) { + path[len] = '\0'; + return strdup(dirname(path)); + } + switch (errno) { + case ENOENT: case EACCES: case EINVAL: case ELOOP: case ENOTDIR: + case ENAMETOOLONG: + break; + default: + die(2, "cannot resolve %s: %s\n", path, strerror(errno)); + break; + } + } + return find_exe_parent(argv_0); +} + +#endif // use /proc if possible, and if all else fails, search in the PATh + +static void +setenv_or_die(const char *name, const char *value) +{ + int rc = setenv(name, value, 1); + if (rc != 0) + die(2, "setenv %s=%s failed (%s)\n", name, value, strerror(errno)); +} + +static void +prepend_lib_to_pythonpath(const char * const exec_path, + const char * const relative_path) +{ + char *parent = exe_parent_dir(exec_path); + assert(parent); + char *bupmodpath; + int rc = asprintf(&bupmodpath, "%s/%s", parent, relative_path); + assert(rc >= 0); + struct stat st; + rc = stat(bupmodpath, &st); + if (rc != 0) + die(2, "unable find lib dir (%s): %s\n", strerror(errno), bupmodpath); + if (!S_ISDIR(st.st_mode)) + die(2, "lib path is not dir: %s\n", bupmodpath); + char *curpypath = getenv("PYTHONPATH"); + if (curpypath) { + char *path; + int rc = asprintf(&path, "%s:%s", bupmodpath, curpypath); + assert(rc >= 0); + setenv_or_die("PYTHONPATH", path); + free(path); + } else { + setenv_or_die("PYTHONPATH", bupmodpath); + } + + free(bupmodpath); + free(parent); +} + +#if PY_MAJOR_VERSION > 2 +#define bup_py_main Py_BytesMain +# else +#define bup_py_main Py_Main +#endif + +#if defined(BUP_DEV_BUP_PYTHON) && defined(BUP_DEV_BUP_EXEC) +# error "Both BUP_DEV_BUP_PYTHON and BUP_DEV_BUP_EXEC are defined" +#endif + +#ifdef BUP_DEV_BUP_PYTHON + +int main(int argc, char **argv) +{ + prog_argc = argc; + prog_argv = argv; + setup_bup_main_module(); + prepend_lib_to_pythonpath(argv[0], "../lib"); + return bup_py_main (argc, argv); +} + +#elif defined(BUP_DEV_BUP_EXEC) + +int main(int argc, char **argv) +{ + prog_argc = argc - 1; + prog_argv = argv + 1; + setup_bup_main_module(); + prepend_lib_to_pythonpath(argv[0], "../lib"); + if (argc == 1) + return bup_py_main (1, argv); + // This can't handle a script with a name like "-c", but that's + // python's problem, not ours. + return bup_py_main (2, argv); +} + +#else // normal bup command + +int main(int argc, char **argv) +{ + prog_argc = argc; + prog_argv = argv; + setup_bup_main_module(); + prepend_lib_to_pythonpath(argv[0], ".."); + char *bup_argv[] = { argv[0], "-m", "bup.main" }; + return bup_py_main (3, bup_argv); +} + +#endif // normal bup command diff --git a/test/ext/test-misc b/test/ext/test-misc index 5fb96be..b69eaaa 100755 --- a/test/ext/test-misc +++ b/test/ext/test-misc @@ -123,7 +123,7 @@ c/" WVSTART features -expect_py_ver=$(LC_CTYPE=C "$top/config/bin/python" \ +expect_py_ver=$(LC_CTYPE=C "$top/dev/python" \ -c 'import platform; print(platform.python_version())') \ || exit $? actual_py_ver=$(bup features | grep Python: | sed -Ee 's/ +Python: //') || exit $? -- 2.39.2