From: Rob Browning Date: Sat, 13 Feb 2021 22:36:35 +0000 (-0600) Subject: Convert top level executables to binaries and clean up clean X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?p=bup.git;a=commitdiff_plain;h=4d9dd65141326b0f3ffa1658a0535348940ed017 Convert top level executables to binaries and clean up clean Signed-off-by: Rob Browning --- diff --git a/.gitignore b/.gitignore index bda8de8..1555c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,20 @@ -\#*# -.#* -randomgen -memtest -*.o -*.so -*.exe -*.dll +*.swp *~ -*.pyc -*.tmp -*.tmp.meta -/build -/config/bin/ +/config/config.h.tmp +/dev/bup-exec +/dev/bup-python +/dev/python +/lib/bup/_helpers.dll +/lib/bup/_helpers.so /lib/bup/checkout_info.py -*.swp -nbproject -/lib/cmd/bup-* +/lib/cmd/bup +/nbproject/ +/test/int/__init__.pyc +/test/lib/__init__.pyc +/test/lib/buptest/__init__.pyc +/test/lib/buptest/vfs.pyc +/test/lib/wvpytest.pyc /test/sampledata/var/ /test/tmp/ +\#*# +__pycache__/ diff --git a/DESIGN b/DESIGN index 89b06b7..d6e8c1b 100644 --- a/DESIGN +++ b/DESIGN @@ -18,17 +18,41 @@ source code to follow along and see what we're talking about. bup's code is written primarily in python with a bit of C code in speed-sensitive places. Here are the most important things to know: - - bup (symlinked to main.py) is the main program that runs when you type - 'bup'. - - - cmd/bup-* (mostly symlinked to cmd/*-cmd.py) are the individual - subcommands, in a way similar to how git breaks all its subcommands into - separate programs. Not all the programs have to be written in python; - they could be in any language, as long as they end up named cmd/bup-*. - We might end up re-coding large parts of bup in C eventually so that it - can be even faster and (perhaps) more portable. - - - lib/bup/*.py are python library files used by the cmd/*.py commands. + - The main program is a fairly small C program that mostly just + initializes the correct Python interpreter and then runs + bup.main.main(). This arrangement was chosen in order to give us + more flexibility. For example: + + - It allows us to avoid + [crashing on some Unicode-unfriendly command line arguments](https://bugs.python.org/issue35883) + which is critical, given that paths can be arbitrary byte + sequences. + + - It allows more flexibility in dealing with upstream changes + like the breakage of our ability to manipulate the + processes arguement list on platforms that support it during + the Python 3.9 series. + + - It means that we'll no longer be affected by any changes to the + `#!/...` path, i.e. if `/usr/bin/python`, or + `/usr/bin/python3`, or whatever we'd previously selected during + `./configure` were to change from 2 to 3, or 3.5 to 3.20. + + The version of python bup uses is determined by the `python-config` + program selected by `./configure`. It tries to find a suitable + default unless `BUP_PYTHON_CONFIG` is set in the environment. + + - bup supports both internal and external subcommands. The former + are the most common, and are all located in lib/bup/cmd/. They + must be python modules named lib/bup/cmd/COMMAND.py, and must + contain a `main(argv)` function that will be passed the *binary* + command line arguments (bytes, not strings). The filename must + have underscores for any dashes in the subcommand name. The + external subcommands are in lib/cmd/. + + - The python code is all in lib/bup. + + - lib/bup/\*.py contains the python code (modules) that bup depends on. That directory name seems a little silly (and worse, redundant) but there seemed to be no better way to let programs write "from bup import index" and have it work. Putting bup in the top level conflicted with diff --git a/Makefile b/Makefile index a9c7fb9..f3205f7 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,12 @@ MAKEFLAGS += --warn-undefined-variables SHELL := bash .DEFAULT_GOAL := all -# See config/config.vars.in (sets bup_python, among other things) --include config/config.vars +clean_paths := + +# See config/config.vars.in (sets bup_python_config, among other things) +include config/config.vars pf := set -o pipefail -cfg_py := $(CURDIR)/config/bin/python define isok && echo " ok" || echo " no" @@ -30,7 +31,7 @@ os := $(call shout,$(os),Unable to determine OS) CFLAGS := -O2 -Wall -Werror -Wformat=2 $(CFLAGS) CFLAGS := -Wno-unknown-pragmas -Wsign-compare $(CFLAGS) -CFLAGS := -D_FILE_OFFSET_BITS=64 $(PYINCLUDE) $(CFLAGS) +CFLAGS := -D_FILE_OFFSET_BITS=64 $(CFLAGS) SOEXT:=.so ifeq ($(os),CYGWIN) @@ -45,6 +46,7 @@ endif initial_setup := $(shell dev/update-checkout-info lib/bup/checkout_info.py $(isok)) initial_setup := $(call shout,$(initial_setup),update-checkout-info failed)) +clean_paths += lib/bup/checkout_info.py config/config.vars: \ configure config/configure config/configure.inc \ @@ -73,11 +75,10 @@ endif bup_ext_cmds := lib/cmd/bup-import-rdiff-backup lib/cmd/bup-import-rsnapshot -config/bin/python: config/config.vars - -bup_deps := lib/bup/_helpers$(SOEXT) +bup_deps := lib/bup/_helpers$(SOEXT) lib/cmd/bup -all: $(bup_deps) Documentation/all $(current_sampledata) +all: dev/bup-exec dev/bup-python dev/python $(bup_deps) Documentation/all \ + $(current_sampledata) $(current_sampledata): dev/configure-sampledata --setup @@ -113,7 +114,7 @@ install: all test -z "$(man_roff)" || $(INSTALL) -m 0644 $(man_roff) $(dest_mandir)/man1 test -z "$(man_html)" || install -d $(dest_docdir) test -z "$(man_html)" || $(INSTALL) -m 0644 $(man_html) $(dest_docdir) - dev/install-python-script lib/cmd/bup "$(dest_libdir)/cmd/bup" + $(INSTALL) -pm 0755 lib/cmd/bup "$(dest_libdir)/cmd/bup" $(INSTALL) -pm 0755 $(bup_ext_cmds) "$(dest_libdir)/cmd/" cd "$(dest_bindir)" && \ ln -sf "$$($(bup_python) -c 'import os; print(os.path.relpath("$(abspath $(dest_libdir))/cmd/bup"))')" @@ -138,32 +139,51 @@ install: all fi config/config.h: config/config.vars +clean_paths += config/config.h.tmp + +dev/python: dev/python.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -o $@-proposed + dev/validate-python $@-proposed + mv $@-proposed $@ +# Do not add to clean_paths - want it available until the very end + +dev/bup-exec: lib/cmd/bup.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -fPIC \ + -D BUP_DEV_BUP_EXEC=1 -o $@ +clean_paths += dev/bup-exec + +dev/bup-python: lib/cmd/bup.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -fPIC \ + -D BUP_DEV_BUP_PYTHON=1 -o $@ +clean_paths += dev/bup-python + +lib/cmd/bup: lib/cmd/bup.c config/config.h + $(CC) $(bup_python_cflags_embed) $< $(bup_python_ldflags_embed) -fPIC -o $@ +clean_paths += lib/cmd/bup + +helper_src := config/config.h lib/bup/bupsplit.h lib/bup/bupsplit.c +helper_src += lib/bup/_helpers.c -lib/bup/_helpers$(SOEXT): \ - config/config.h lib/bup/bupsplit.h \ - lib/bup/bupsplit.c lib/bup/_helpers.c lib/bup/csetup.py - @rm -f $@ - cd lib/bup && $(cfg_py) csetup.py build "$(CFLAGS)" "$(LDFLAGS)" - # Make sure there's just the one file we expect before we copy it. - $(cfg_py) -c \ - "import glob; assert(len(glob.glob('lib/bup/build/*/_helpers*$(SOEXT)')) == 1)" - cp lib/bup/build/*/_helpers*$(SOEXT) "$@" +lib/bup/_helpers$(SOEXT): dev/python $(helper_src) + $(CC) $(bup_python_cflags) $(CFLAGS) -shared -fPIC $(helper_src) \ + $(bup_python_ldflags) $(LDFLAGS) -o $@ +clean_paths += lib/bup/_helpers$(SOEXT) test/tmp: mkdir test/tmp -ifeq (yes,$(shell config/bin/python -c "import xdist; print('yes')" 2>/dev/null)) - # MAKEFLAGS must not be in an immediate := assignment - parallel_opt = $(lastword $(filter -j%,$(MAKEFLAGS))) - get_parallel_n = $(patsubst -j%,%,$(parallel_opt)) - maybe_specific_n = $(if $(filter -j%,$(parallel_opt)),-n$(get_parallel_n)) - xdist_opt = $(if $(filter -j,$(parallel_opt)),-nauto,$(maybe_specific_n)) -else - xdist_opt = -endif +# MAKEFLAGS must not be in an immediate := assignment +parallel_opt = $(lastword $(filter -j%,$(MAKEFLAGS))) +get_parallel_n = $(patsubst -j%,%,$(parallel_opt)) +maybe_specific_n = $(if $(filter -j%,$(parallel_opt)),-n$(get_parallel_n)) +xdist_opt = $(if $(filter -j,$(parallel_opt)),-nauto,$(maybe_specific_n)) -test: all test/tmp - ./pytest $(xdist_opt) +test: all test/tmp dev/python + if test yes = $$(dev/python -c "import xdist; print('yes')" 2>/dev/null); then \ + (set -x; ./pytest $(xdist_opt);) \ + else \ + (set-x; ./pytest;) \ + fi stupid: PATH=/bin:/usr/bin $(MAKE) test @@ -171,7 +191,11 @@ stupid: check: test distcheck: all - ./pytest $(xdist_opt) -m release + if test yes = $$(dev/python -c "import xdist; print('yes')" 2>/dev/null); then \ + (set -x; ./pytest $(xdist_opt) -m release;) \ + else \ + (set -x; ./pytest -m release;) \ + fi long-test: export BUP_TEST_LEVEL=11 long-test: test @@ -181,8 +205,8 @@ long-check: check .PHONY: check-both check-both: - $(MAKE) clean && PYTHON=python3 $(MAKE) check - $(MAKE) clean && PYTHON=python2 $(MAKE) check + $(MAKE) clean && BUP_PYTHON_CONFIG=python3-config $(MAKE) check + $(MAKE) clean && BUP_PYTHON_CONFIG=python2.7-config $(MAKE) check .PHONY: Documentation/all Documentation/all: $(man_roff) $(man_html) @@ -223,24 +247,20 @@ import-docs: Documentation/clean $(pf); git archive origin/html | (cd Documentation && tar -xvf -) $(pf); git archive origin/man | (cd Documentation && tar -xvf -) -clean: Documentation/clean config/bin/python +clean: Documentation/clean dev/python cd config && rm -rf config.var - cd config && rm -f *~ .*~ \ + cd config && rm -f \ ${CONFIGURE_DETRITUS} ${CONFIGURE_FILES} ${GENERATED_FILES} - rm -f *.o lib/*/*.o *.so lib/*/*.so *.dll lib/*/*.dll *.exe \ - .*~ *~ */*~ lib/*/*~ lib/*/*/*~ \ - *.pyc */*.pyc lib/*/*.pyc lib/*/*/*.pyc \ - lib/bup/checkout_info.py \ - randomgen memtest \ - testfs.img test/int/testfs.img + rm -rf $(clean_paths) .pytest_cache + find . -name __pycache__ -exec rm -rf {} + if test -e test/mnt; then dev/cleanup-mounts-under test/mnt; fi if test -e test/mnt; then rm -r test/mnt; fi if test -e test/tmp; then dev/cleanup-mounts-under test/tmp; fi # FIXME: migrate these to test/mnt/ if test -e test/int/testfs; \ then umount test/int/testfs || true; fi - rm -rf *.tmp *.tmp.meta test/*.tmp lib/*/*/*.tmp build lib/bup/build test/int/testfs + rm -rf test/int/testfs test/int/testfs.img testfs.img if test -e test/tmp; then dev/force-delete test/tmp; fi dev/configure-sampledata --clean # Remove last so that cleanup tools can depend on it - rm -rf config/bin + rm -f dev/python diff --git a/README.md b/README.md index 2c00a3f..0e9f472 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ From source pip install tornado ``` - - Build the python module and symlinks: + - Build: ```sh make @@ -244,12 +244,13 @@ From source make install DESTDIR=/opt/bup PREFIX='' ``` - - The Python executable that bup will use is chosen by ./configure, - which will search for a reasonable version unless PYTHON is set in - the environment, in which case, bup will use that path. You can - see which Python executable was chosen by looking at the - configure output, or examining cmd/python-cmd.sh, and you can - change the selection by re-running ./configure. + - The Python version that bup will use is determined by the + `python-config` program chosen by `./configure`, which will search + for a reasonable version unless `BUP_PYTHON_CONFIG` is set in the + environment. You can see which Python executable was chosen by + looking at the configure output, or examining + `config/config.var/bup-python-config`, and you can change the + selection by re-running `./configure`. From binary packages -------------------- diff --git a/cmd b/cmd deleted file mode 120000 index 7819428..0000000 --- a/cmd +++ /dev/null @@ -1 +0,0 @@ -lib/cmd \ No newline at end of file diff --git a/config/config.vars.in b/config/config.vars.in index 8f4769c..6606bfd 100644 --- a/config/config.vars.in +++ b/config/config.vars.in @@ -2,8 +2,12 @@ CONFIGURE_FILES=@CONFIGURE_FILES@ GENERATED_FILES=@GENERATED_FILES@ bup_make=@bup_make@ -bup_python=@bup_python@ -bup_python_majver=@bup_python_majver@ + +bup_python_config=@bup_python_config@ +bup_python_cflags=@bup_python_cflags@ +bup_python_ldflags=@bup_python_ldflags@ +bup_python_cflags_embed=@bup_python_cflags_embed@ +bup_python_ldflags_embed=@bup_python_ldflags_embed@ bup_have_libacl=@bup_have_libacl@ bup_libacl_cflags=@bup_libacl_cflags@ diff --git a/config/configure b/config/configure index 6ef0531..3506c54 100755 --- a/config/configure +++ b/config/configure @@ -37,6 +37,9 @@ TARGET=bup . ./configure.inc +# FIXME: real tmpdir +rm -rf config.var config.var.tmp config.vars + AC_INIT $TARGET if ! AC_PROG_CC; then @@ -65,30 +68,43 @@ expr "$MAKE_VERSION" '>=' '3.81' || AC_FAIL "ERROR: $MAKE must be >= version 3.8 AC_SUB bup_make "$MAKE" -bup_python="$(type -p "$PYTHON")" -test -z "$bup_python" && bup_python="$(bup_find_prog python3.8 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python3.7 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python3.6 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python3 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python2.7 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python2.6 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python2 '')" -test -z "$bup_python" && bup_python="$(bup_find_prog python '')" -if test -z "$bup_python"; then - AC_FAIL "ERROR: unable to find python" + +# Haven't seen a documented way to determine the python version via +# python-config right now, so we'll defer version checking until +# later. + +if test "$BUP_PYTHON_CONFIG"; then + bup_python_config="$(type -p "$BUP_PYTHON_CONFIG")" + if test -z "$bup_python_config"; then + AC_FAIL $(printf "ERROR: BUP_PYTHON_CONFIG value %q appears invalid" \ + "$BUP_PYTHON_CONFIG") + fi else - AC_SUB bup_python "$bup_python" - bup_python_majver=$("$bup_python" -c 'import sys; print(sys.version_info[0])') - bup_python_minver=$("$bup_python" -c 'import sys; print(sys.version_info[1])') - AC_SUB bup_python_majver "$bup_python_majver" + test -z "$bup_python_config" \ + && bup_python_config="$(bup_find_prog python3-config '')" + test -z "$bup_python_config" \ + && bup_python_config="$(bup_find_prog python2.7-config '')" + if test -z "$bup_python_config"; then + AC_FAIL "ERROR: unable to find a suitable python-config" + fi fi -# May not be correct yet, i.e. actual requirement may be higher. -if test "$bup_python_majver" -gt 2 -a "$bup_python_minver" -lt 3; then - # utime follow_symlinks >= 3.3 - bup_version_str=$("$bup_python" --version 2>&1) - AC_FAIL "ERROR: found $bup_version_str (must be >= 3.3 if >= 3)" + +bup_python_cflags=$("$bup_python_config" --cflags) || exit $? +bup_python_ldflags=$("$bup_python_config" --ldflags) || exit $? +bup_python_cflags_embed=$("$bup_python_config" --cflags --embed) +if test $? -eq 0; then + bup_python_ldflags_embed=$("$bup_python_config" --ldflags --embed) || exit $? +else # Earlier versions didn't support --embed + bup_python_cflags_embed=$("$bup_python_config" --cflags) || exit $? + bup_python_ldflags_embed=$("$bup_python_config" --ldflags) || exit $? fi +AC_SUB bup_python_config "$bup_python_config" +AC_SUB bup_python_cflags "$bup_python_cflags" +AC_SUB bup_python_ldflags "$bup_python_ldflags" +AC_SUB bup_python_cflags_embed "$bup_python_cflags_embed" +AC_SUB bup_python_ldflags_embed "$bup_python_ldflags_embed" + bup_git="$(bup_find_prog git '')" if test -z "$bup_git"; then @@ -109,16 +125,12 @@ AC_CHECK_HEADERS sys/mman.h AC_CHECK_HEADERS linux/fs.h AC_CHECK_HEADERS sys/ioctl.h -if test "$bup_python_majver" -gt 2; then - AC_DEFINE BUP_USE_PYTHON_UTIME 1 -else # Python 2 - # On GNU/kFreeBSD utimensat is defined in GNU libc, but won't work. - if [ -z "$OS_GNU_KFREEBSD" ]; then - AC_CHECK_FUNCS utimensat - fi - AC_CHECK_FUNCS utimes - AC_CHECK_FUNCS lutimes +# On GNU/kFreeBSD utimensat is defined in GNU libc, but won't work. +if [ -z "$OS_GNU_KFREEBSD" ]; then + AC_CHECK_FUNCS utimensat fi +AC_CHECK_FUNCS utimes +AC_CHECK_FUNCS lutimes builtin_mul_overflow_code=" #include @@ -303,20 +315,20 @@ LIBS="$orig_libs" AC_OUTPUT config.vars -if test -e config.var; then rm -r config.var; fi -mkdir -p config.var -echo -n "$MAKE" > config.var/bup-make -echo -n "$bup_python" > config.var/bup-python -if test -e bin; then rm -r bin; fi -mkdir -p bin -(cd bin && ln -s "$bup_python" python) +set -euo pipefail + +# FIXME: real tmpdir +mkdir -p config.var.tmp +echo -n "$MAKE" > config.var.tmp/bup-make +echo -n "$bup_python_config" > config.var.tmp/bup-python-config +mv config.var.tmp config.var printf " -found: python (%q, $("$bup_python" --version 2>&1)) +found: python-config (%q) found: git (%q, ($("$bup_git" --version)) " \ - "$bup_python" \ + "$bup_python_config" \ "$bup_git" \ 1>&5 diff --git a/dev/bup-python b/dev/bup-python deleted file mode 100755 index 384a8fd..0000000 --- a/dev/bup-python +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -script_home="$(cd "$(dirname "$0")" && pwd -P)" -python="$script_home/../config/bin/python" -libdir="$script_home/../lib" - -export PYTHONPATH="$libdir${PYTHONPATH:+:$PYTHONPATH}" -exec "$python" "$@" diff --git a/dev/cleanup-mounts-under b/dev/cleanup-mounts-under index c0c2671..b04f095 100755 --- a/dev/cleanup-mounts-under +++ b/dev/cleanup-mounts-under @@ -1,6 +1,6 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? +bup_python="$(dirname "$0")/../dev/python" || exit $? exec "$bup_python" "$0" ${1+"$@"} """ diff --git a/dev/data-size b/dev/data-size index e5068da..451498d 100755 --- a/dev/data-size +++ b/dev/data-size @@ -1,16 +1,18 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function from os.path import getsize, isdir -from sys import argv, stderr +from sys import stderr import os +from bup.compat import get_argvb + + def listdir_failure(ex): raise ex @@ -18,7 +20,7 @@ def usage(): print('Usage: data-size PATH ...', file=sys.stderr) total = 0 -for path in argv[1:]: +for path in get_argvb()[1:]: if isdir(path): for root, dirs, files in os.walk(path, onerror=listdir_failure): total += sum(getsize(os.path.join(root, name)) for name in files) diff --git a/dev/echo-argv-bytes b/dev/echo-argv-bytes index d49c26c..f9a71c2 100755 --- a/dev/echo-argv-bytes +++ b/dev/echo-argv-bytes @@ -1,17 +1,8 @@ #!/bin/sh """": # -*-python-*- -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function @@ -19,12 +10,9 @@ from os.path import abspath, dirname from sys import stdout import os, sys -script_home = abspath(dirname(__file__)) -sys.path[:0] = [abspath(script_home + '/../../lib'), abspath(script_home + '/../..')] - from bup import compat -for arg in compat.argvb: +for arg in compat.get_argvb(): os.write(stdout.fileno(), arg) os.write(stdout.fileno(), b'\0\n') stdout.flush() diff --git a/dev/hardlink-sets b/dev/hardlink-sets index e1be742..fb0bdb7 100755 --- a/dev/hardlink-sets +++ b/dev/hardlink-sets @@ -1,24 +1,13 @@ #!/bin/sh """": # -*-python-*- -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function import os, stat, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] - -from bup import compat +from bup.compat import get_argvb from bup.io import byte_stream @@ -29,7 +18,9 @@ from bup.io import byte_stream def usage(): print("Usage: hardlink-sets ", file=sys.stderr) -if len(compat.argv) < 2: +argvb = get_argvb() + +if len(argvb) < 2: usage() sys.exit(1) @@ -41,7 +32,7 @@ out = byte_stream(sys.stdout) hardlink_set = {} -for p in compat.argvb[1:]: +for p in argvb[1:]: for root, dirs, files in os.walk(p, onerror = on_walk_error): for filename in files: full_path = os.path.join(root, filename) diff --git a/dev/id-other-than b/dev/id-other-than index e54696a..a6cab57 100755 --- a/dev/id-other-than +++ b/dev/id-other-than @@ -1,12 +1,8 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble - -# Note: this currently relies on bup-python to handle arbitrary binary -# user/group names. from __future__ import absolute_import, print_function @@ -14,11 +10,15 @@ import grp import pwd import sys +from bup.compat import get_argvb + def usage(): print('Usage: id-other-than <--user|--group> ID [ID ...]', file=sys.stderr) -if len(sys.argv) < 2: +argvb = get_argvb() + +if len(argvb) < 2: usage() sys.exit(1) @@ -29,17 +29,17 @@ def is_integer(x): except ValueError as e: return False -excluded_ids = set(int(x) for x in sys.argv[2:] if is_integer(x)) -excluded_names = (x for x in sys.argv[2:] if not is_integer(x)) +excluded_ids = set(int(x) for x in argvb[2:] if is_integer(x)) +excluded_names = (x for x in argvb[2:] if not is_integer(x)) -if sys.argv[1] == '--user': +if argvb[1] == b'--user': for x in excluded_names: excluded_ids.add(pwd.getpwnam(x).pw_uid) for x in pwd.getpwall(): if x.pw_uid not in excluded_ids: print(x.pw_name + ':' + str(x.pw_uid)) sys.exit(0) -elif sys.argv[1] == '--group': +elif argvb[1] == b'--group': for x in excluded_names: excluded_ids.add(grp.getgrnam(x).gr_gid) for x in grp.getgrall(): diff --git a/dev/install-python-script b/dev/install-python-script deleted file mode 100755 index 83d8861..0000000 --- a/dev/install-python-script +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -"""": # -*-python-*- -export LC_CTYPE=iso-8859-1 -exec "$(dirname "$0")/../config/bin/python" "$0" "$@" -""" - -from __future__ import absolute_import, print_function -from tempfile import NamedTemporaryFile -import os, shutil, sys - -if sys.version_info[0] >= 3: - from shlex import quote -else: - from pipes import quote - -src_path, dest_path = sys.argv[1:] - -with open(b'config/config.var/bup-python', 'rb') as src: - python = src.read() - -with NamedTemporaryFile() as tmp: - # Replace the section between "Here to end..." and the end of the - # preamble with the correct 'exec PYTHON "$0"'. - with open(src_path, 'rb') as src: - for line in src: - if line.startswith(b'# Here to end of preamble replaced during install'): - break - tmp.write(line) - for line in src: - if line == b'"""\n': - break - tmp.write(b'exec %s "$0"\n' % python) - tmp.write(b'"""\n') - for line in src: - tmp.write(line) - tmp.flush() - shutil.copy(tmp.name, dest_path) - os.chmod(dest_path, 0o755) diff --git a/dev/lib.sh b/dev/lib.sh index 20780e2..b89c05d 100644 --- a/dev/lib.sh +++ b/dev/lib.sh @@ -3,7 +3,7 @@ # Assumes this is always loaded while pwd is still the source tree root bup_dev_lib_top=$(pwd) || exit $? -bup-cfg-py() { "$bup_dev_lib_top/config/bin/python" "$@"; } +bup-cfg-py() { "$bup_dev_lib_top/dev/python" "$@"; } bup-python() { "$bup_dev_lib_top/dev/bup-python" "$@"; } force-delete() diff --git a/dev/make-random-paths b/dev/make-random-paths index 05d24f4..c0c6c78 100755 --- a/dev/make-random-paths +++ b/dev/make-random-paths @@ -1,19 +1,19 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")//bup-python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function from os.path import abspath, dirname from random import randint -from sys import argv, exit, stderr, stdout +from sys import stderr, stdout import errno, re, sys -from bup.compat import fsencode, range +from bup.compat import fsencode, get_argv, get_argvb, range +argv = get_argv() def usage(out=stdout): print('Usage:', argv[0], 'NUM', 'DEST_DIR', file=out) @@ -44,10 +44,10 @@ def random_filename(): if len(argv) != 3: misuse() -count, dest = argv[1:] +count, dest = get_argvb()[1:] count = int(count) i = 0 while i < count: - with open(fsencode(dest) + b'/' + random_filename(), 'w') as _: + with open(dest + b'/' + random_filename(), 'w') as _: i += 1 diff --git a/dev/mksock b/dev/mksock index 88372c3..4b14b22 100755 --- a/dev/mksock +++ b/dev/mksock @@ -1,13 +1,13 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import - import socket, sys +from bup.compat import get_argvb + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) -s.bind(sys.argv[1]) +s.bind(get_argvb()[1]) diff --git a/dev/ns-timestamp-resolutions b/dev/ns-timestamp-resolutions index f5b296c..3f0c9d4 100755 --- a/dev/ns-timestamp-resolutions +++ b/dev/ns-timestamp-resolutions @@ -1,24 +1,13 @@ #!/bin/sh """": # -*-python-*- -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import import os.path, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../../lib'] - -from bup.compat import argv_bytes +from bup.compat import argv_bytes, get_argvb from bup.helpers import handle_ctrl_c, saved_errors from bup.io import byte_stream from bup import compat, metadata, options @@ -33,7 +22,7 @@ ns-timestamp-resolutions TEST_FILE_NAME handle_ctrl_c() o = options.Options(optspec) -opt, flags, extra = o.parse(compat.argv[1:]) +opt, flags, extra = o.parse_bytes(get_argvb()[1:]) sys.stdout.flush() out = byte_stream(sys.stdout) diff --git a/dev/python.c b/dev/python.c new file mode 100644 index 0000000..0c41508 --- /dev/null +++ b/dev/python.c @@ -0,0 +1,20 @@ +#define _LARGEFILE64_SOURCE 1 +#define PY_SSIZE_T_CLEAN 1 +#undef NDEBUG +#include "../config/config.h" + +// According to Python, its header has to go first: +// http://docs.python.org/2/c-api/intro.html#include-files +// http://docs.python.org/3/c-api/intro.html#include-files +#include + +#if PY_MAJOR_VERSION > 2 +#define bup_py_main Py_BytesMain +# else +#define bup_py_main Py_Main +#endif + +int main(int argc, char **argv) +{ + return bup_py_main (argc, argv); +} diff --git a/dev/root-status b/dev/root-status index c37806d..36a173f 100755 --- a/dev/root-status +++ b/dev/root-status @@ -1,9 +1,8 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +python="$(dirname "$0")/python" || exit $? +exec "$python" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function import os, sys diff --git a/dev/sparse-test-data b/dev/sparse-test-data index 34bb117..3d9f712 100755 --- a/dev/sparse-test-data +++ b/dev/sparse-test-data @@ -1,7 +1,7 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ from __future__ import absolute_import, print_function @@ -9,8 +9,7 @@ from random import randint from sys import stderr, stdout import os, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] - +from bup.compat import get_argvb from bup.io import byte_stream def smaller_region(max_offset): @@ -48,11 +47,13 @@ def random_region(): global generators return generators[randint(0, len(generators) - 1)]() -if len(sys.argv) == 0: +argv = get_argvb() + +if len(argv) == 0: stdout.flush() out = byte_stream(stdout) -if len(sys.argv) == 2: - out = open(sys.argv[1], 'wb') +if len(argv) == 2: + out = open(argv[1], 'wb') else: print('Usage: sparse-test-data [FILE]', file=stderr) sys.exit(2) diff --git a/dev/subtree-hash b/dev/subtree-hash index e3468fb..f0a3f6f 100755 --- a/dev/subtree-hash +++ b/dev/subtree-hash @@ -1,16 +1,13 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/bup-python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +bup_exec="$(dirname "$0")/bup-exec" || exit $? +exec "$bup_exec" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function import os.path, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] - -from bup.compat import argv_bytes +from bup.compat import argv_bytes, get_argvb from bup.helpers import handle_ctrl_c, readpipe from bup.io import byte_stream from bup import options @@ -24,7 +21,7 @@ subtree-hash ROOT_HASH [PATH_ITEM...] handle_ctrl_c() o = options.Options(optspec) -(opt, flags, extra) = o.parse(sys.argv[1:]) +opt, flags, extra = o.parse_bytes(get_argvb()[1:]) if len(extra) < 1: o.fatal('must specify a root hash') diff --git a/dev/unknown-owner b/dev/unknown-owner index 937e708..0077e24 100755 --- a/dev/unknown-owner +++ b/dev/unknown-owner @@ -1,9 +1,8 @@ #!/bin/sh """": # -*-python-*- -bup_python="$(dirname "$0")/../config/bin/python" || exit $? -exec "$bup_python" "$0" ${1+"$@"} +python="$(dirname "$0")/python" || exit $? +exec "$python" "$0" ${1+"$@"} """ -# end of bup preamble from __future__ import absolute_import, print_function diff --git a/dev/validate-python b/dev/validate-python new file mode 100755 index 0000000..a64ecfe --- /dev/null +++ b/dev/validate-python @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ueo pipefail + +die () { echo "Usage: validate-python PYTHON_EXECUTABLE"; } + +test $# -eq 1 || { usage 1>&2 ; exit 2; } +python="$1" + +majver=$("$python" -c 'import sys; print(sys.version_info[0])') +minver=$("$python" -c 'import sys; print(sys.version_info[1])') + +# May not be correct yet, i.e. actual requirement may be higher. +if test "$majver" -gt 2 -a "$minver" -lt 3; then + # utime follow_symlinks >= 3.3 + bup_version_str=$("$python" --version 2>&1) + echo "ERROR: found $bup_version_str (must be >= 3.3 if >= 3)" 1>&2 + exit 2 +fi diff --git a/lib/bup/_helpers.c b/lib/bup/_helpers.c index 2790b07..5b9ace9 100644 --- a/lib/bup/_helpers.c +++ b/lib/bup/_helpers.c @@ -5,6 +5,7 @@ // According to Python, its header has to go first: // http://docs.python.org/2/c-api/intro.html#include-files +// http://docs.python.org/3/c-api/intro.html#include-files #include #include @@ -70,6 +71,10 @@ #define BUP_HAVE_FILE_ATTRS 1 #endif +#if PY_MAJOR_VERSION > 2 +# define BUP_USE_PYTHON_UTIME 1 +#endif + #ifndef BUP_USE_PYTHON_UTIME // just for Python 2 now /* * Check for incomplete UTIMENSAT support (NetBSD 6), and if so, @@ -353,58 +358,6 @@ static PyObject *bup_cat_bytes(PyObject *self, PyObject *args) } - -// Probably we should use autoconf or something and set HAVE_PY_GETARGCARGV... -#if __WIN32__ || __CYGWIN__ || PY_VERSION_HEX >= 0x03090000 - -// There's no 'ps' on win32 anyway, and Py_GetArgcArgv() isn't available. -static void unpythonize_argv(void) { } - -#else // not __WIN32__ - -// For some reason this isn't declared in Python.h -extern void Py_GetArgcArgv(int *argc, char ***argv); - -static void unpythonize_argv(void) -{ - int argc, i; - char **argv, *arge; - - Py_GetArgcArgv(&argc, &argv); - - for (i = 0; i < argc-1; i++) - { - if (argv[i] + strlen(argv[i]) + 1 != argv[i+1]) - { - // The argv block doesn't work the way we expected; it's unsafe - // to mess with it. - return; - } - } - - arge = argv[argc-1] + strlen(argv[argc-1]) + 1; - - if (strstr(argv[0], "python") && argv[1] == argv[0] + strlen(argv[0]) + 1) - { - char *p; - size_t len, diff; - p = strrchr(argv[1], '/'); - if (p) - { - p++; - diff = p - argv[0]; - len = arge - p; - memmove(argv[0], p, len); - memset(arge - diff, 0, diff); - for (i = 0; i < argc; i++) - argv[i] = argv[i+1] ? argv[i+1]-diff : NULL; - } - } -} - -#endif // not __WIN32__ or __CYGWIN__ - - static int write_all(int fd, const void *buf, const size_t count) { size_t written = 0; @@ -2460,7 +2413,6 @@ static int setup_module(PyObject *m) e = getenv("BUP_FORCE_TTY"); get_state(m)->istty2 = isatty(2) || (atoi(e ? e : "0") & 2); - unpythonize_argv(); return 1; } @@ -2470,11 +2422,13 @@ static int setup_module(PyObject *m) PyMODINIT_FUNC init_helpers(void) { PyObject *m = Py_InitModule("_helpers", helper_methods); - if (m == NULL) + if (m == NULL) { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers init failed"); return; - + } if (!setup_module(m)) { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers set up failed"); Py_DECREF(m); return; } diff --git a/lib/bup/cmd/get.py b/lib/bup/cmd/get.py index b3adf0e..8844544 100755 --- a/lib/bup/cmd/get.py +++ b/lib/bup/cmd/get.py @@ -391,7 +391,7 @@ def handle_append(item, src_repo, writer, opt): if item.src.type == 'tree': get_random_item(item.spec.src, src_oidx, src_repo, writer, opt) parent = item.dest.hash - msg = b'bup save\n\nGenerated by command:\n%r\n' % compat.argvb + msg = b'bup save\n\nGenerated by command:\n%r\n' % compat.get_argvb() userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) now = time.time() commit = writer.new_commit(item.src.hash, parent, diff --git a/lib/bup/cmd/split.py b/lib/bup/cmd/split.py index fc354cb..87ad887 100755 --- a/lib/bup/cmd/split.py +++ b/lib/bup/cmd/split.py @@ -205,7 +205,7 @@ def main(argv): if opt.tree: out.write(hexlify(tree) + b'\n') if opt.commit or opt.name: - msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.argvb + msg = b'bup split\n\nGenerated by command:\n%r\n' % compat.get_argvb() ref = opt.name and (b'refs/heads/%s' % opt.name) or None userline = b'%s <%s@%s>' % (userfullname(), username(), hostname()) commit = pack_writer.new_commit(tree, oldref, userline, date, None, diff --git a/lib/bup/compat.py b/lib/bup/compat.py index 2cd6fba..a06ffe8 100644 --- a/lib/bup/compat.py +++ b/lib/bup/compat.py @@ -169,41 +169,27 @@ else: # Python 2 buffer = buffer - -argv = None -argvb = None - -def _configure_argv(): - global argv, argvb - assert not argv - assert not argvb - if len(sys.argv) > 1: - if environ.get(b'BUP_ARGV_0'): - print('error: BUP_ARGV* set and sys.argv not empty', file=sys.stderr) - sys.exit(2) - argv = sys.argv - argvb = [argv_bytes(x) for x in argv] - return - args = [] - i = 0 - arg = environ.get(b'BUP_ARGV_%d' % i) - while arg is not None: - args.append(arg) - i += 1 - arg = environ.get(b'BUP_ARGV_%d' % i) - i -= 1 - while i >= 0: - del environ[b'BUP_ARGV_%d' % i] - i -= 1 - argvb = args - # System encoding? +try: + import bup_main +except ModuleNotFoundError: + bup_main = None + +if bup_main: + def get_argvb(): + "Return a new list containing the current process argv bytes." + return bup_main.argv() if py3: - argv = [x.decode(errors='surrogateescape') for x in args] + def get_argv(): + "Return a new list containing the current process argv strings." + return [x.decode(errors='surrogateescape') for x in bup_main.argv()] else: - argv = argvb - -_configure_argv() - + def get_argv(): + return bup_main.argv() +else: + def get_argvb(): + raise Exception('get_argvb requires the bup_main module'); + def get_argv(): + raise Exception('get_argv requires the bup_main module'); def wrap_main(main): """Run main() and raise a SystemExit with the return value if it diff --git a/lib/bup/csetup.py b/lib/bup/csetup.py deleted file mode 100644 index 9dbb4a7..0000000 --- a/lib/bup/csetup.py +++ /dev/null @@ -1,24 +0,0 @@ - -from __future__ import absolute_import, print_function - -import shlex, sys -from distutils.core import setup, Extension -import os - -if len(sys.argv) != 4: - print('Usage: csetup.py CFLAGS LDFLAGS', file=sys.stderr) - sys.exit(2) -_helpers_cflags = shlex.split(sys.argv[2]) -_helpers_ldflags = shlex.split(sys.argv[3]) -sys.argv = sys.argv[:2] - -_helpers_mod = Extension('_helpers', - sources=['_helpers.c', 'bupsplit.c'], - depends=['../../config/config.h', 'bupsplit.h'], - extra_compile_args=_helpers_cflags, - extra_link_args=_helpers_ldflags) - -setup(name='_helpers', - version='0.1', - description='accelerator library for bup', - ext_modules=[_helpers_mod]) diff --git a/lib/bup/main.py b/lib/bup/main.py new file mode 100755 index 0000000..16aeaad --- /dev/null +++ b/lib/bup/main.py @@ -0,0 +1,438 @@ + +from __future__ import absolute_import, print_function +from importlib import import_module +from pkgutil import iter_modules +from subprocess import PIPE +from threading import Thread +import errno, getopt, os, re, select, signal, subprocess, sys + +from bup import compat, path, helpers +from bup.compat import ( + ModuleNotFoundError, + add_ex_ctx, + add_ex_tb, + argv_bytes, + environ, + fsdecode, + int_types, + wrap_main +) +from bup.compat import add_ex_tb, add_ex_ctx, argv_bytes, wrap_main +from bup.helpers import ( + columnate, + debug1, + handle_ctrl_c, + log, + merge_dict, + tty_width +) +from bup.git import close_catpipes +from bup.io import byte_stream, path_msg +from bup.options import _tty_width +import bup.cmd + +def maybe_import_early(argv): + """Scan argv and import any modules specified by --import-py-module.""" + while argv: + if argv[0] != '--import-py-module': + argv = argv[1:] + continue + if len(argv) < 2: + log("bup: --import-py-module must have an argument\n") + exit(2) + mod = argv[1] + import_module(mod) + argv = argv[2:] + +maybe_import_early(compat.get_argv()) + +handle_ctrl_c() + +cmdpath = path.cmddir() + +# Remove once we finish the internal command transition +transition_cmdpath = path.libdir() + b'/bup/cmd' + +# We manipulate the subcmds here as strings, but they must be ASCII +# compatible, since we're going to be looking for exactly +# b'bup-SUBCMD' to exec. + +def usage(msg=""): + log('Usage: bup [-?|--help] [-d BUP_DIR] [--debug] [--profile] ' + ' [options...]\n\n') + common = dict( + ftp = 'Browse backup sets using an ftp-like client', + fsck = 'Check backup sets for damage and add redundancy information', + fuse = 'Mount your backup sets as a filesystem', + help = 'Print detailed help for the given command', + index = 'Create or display the index of files to back up', + on = 'Backup a remote machine to the local one', + restore = 'Extract files from a backup set', + save = 'Save files into a backup set (note: run "bup index" first)', + tag = 'Tag commits for easier access', + web = 'Launch a web server to examine backup sets', + ) + + log('Common commands:\n') + for cmd,synopsis in sorted(common.items()): + log(' %-10s %s\n' % (cmd, synopsis)) + log('\n') + + log('Other available commands:\n') + cmds = set() + for c in sorted(os.listdir(cmdpath)): + if c.startswith(b'bup-') and c.find(b'.') < 0: + cname = fsdecode(c[4:]) + if cname not in common: + cmds.add(c[4:].decode(errors='backslashreplace')) + # built-in commands take precedence + for _, name, _ in iter_modules(path=bup.cmd.__path__): + name = name.replace('_','-') + if name not in common: + cmds.add(name) + + log(columnate(sorted(cmds), ' ')) + log('\n') + + log("See 'bup help COMMAND' for more information on " + + "a specific command.\n") + if msg: + log("\n%s\n" % msg) + sys.exit(99) + +argv = compat.get_argv() +if len(argv) < 2: + usage() + +# Handle global options. +try: + optspec = ['help', 'version', 'debug', 'profile', 'bup-dir=', + 'import-py-module='] + global_args, subcmd = getopt.getopt(argv[1:], '?VDd:', optspec) +except getopt.GetoptError as ex: + usage('error: %s' % ex.msg) + +subcmd = [argv_bytes(x) for x in subcmd] +help_requested = None +do_profile = False +bup_dir = None + +for opt in global_args: + if opt[0] in ['-?', '--help']: + help_requested = True + elif opt[0] in ['-V', '--version']: + subcmd = [b'version'] + elif opt[0] in ['-D', '--debug']: + helpers.buglvl += 1 + environ[b'BUP_DEBUG'] = b'%d' % helpers.buglvl + elif opt[0] in ['--profile']: + do_profile = True + elif opt[0] in ['-d', '--bup-dir']: + bup_dir = argv_bytes(opt[1]) + elif opt[0] == '--import-py-module': + pass + else: + usage('error: unexpected option "%s"' % opt[0]) + +# Make BUP_DIR absolute, so we aren't affected by chdir (i.e. save -C, etc.). +if bup_dir: + environ[b'BUP_DIR'] = os.path.abspath(bup_dir) + +if len(subcmd) == 0: + if help_requested: + subcmd = [b'help'] + else: + usage() + +if help_requested and subcmd[0] != b'help': + subcmd = [b'help'] + subcmd + +if len(subcmd) > 1 and subcmd[1] == b'--help' and subcmd[0] != b'help': + subcmd = [b'help', subcmd[0]] + subcmd[2:] + +subcmd_name = subcmd[0] +if not subcmd_name: + usage() + +try: + if subcmd_name not in (b'bloom', + b'cat-file', + b'daemon', + b'drecurse', + b'damage', + b'features', + b'ftp', + b'fsck', + b'fuse', + b'gc', + b'get', + b'help', + b'import-duplicity', + b'index', + b'init', + b'join', + b'list-idx', + b'ls', + b'margin', + b'memtest', + b'meta', + b'midx', + b'mux', + b'on', + b'on--server', + b'prune-older', + b'random', + b'restore', + b'rm', + b'save', + b'server', + b'split', + b'tag', + b'tick', + b'version', + b'web', + b'xstat'): + raise ModuleNotFoundError() + cmd_module = import_module('bup.cmd.' + + subcmd_name.decode('ascii').replace('-', '_')) +except ModuleNotFoundError as ex: + cmd_module = None + +if not cmd_module: + subcmd[0] = os.path.join(cmdpath, b'bup-' + subcmd_name) + if not os.path.exists(subcmd[0]): + subcmd[0] = b'%s/%s.py' % (transition_cmdpath, + subcmd_name.replace(b'-', b'_')) + if not os.path.exists(subcmd[0]): + usage('error: unknown command "%s"' % path_msg(subcmd_name)) + +already_fixed = int(environ.get(b'BUP_FORCE_TTY', 0)) +if subcmd_name in [b'mux', b'ftp', b'help']: + already_fixed = True +fix_stdout = not already_fixed and os.isatty(1) +fix_stderr = not already_fixed and os.isatty(2) + +if fix_stdout or fix_stderr: + tty_env = merge_dict(environ, + {b'BUP_FORCE_TTY': (b'%d' + % ((fix_stdout and 1 or 0) + + (fix_stderr and 2 or 0))), + b'BUP_TTY_WIDTH': b'%d' % _tty_width(), }) +else: + tty_env = environ + + +sep_rx = re.compile(br'([\r\n])') + +def print_clean_line(dest, content, width, sep=None): + """Write some or all of content, followed by sep, to the dest fd after + padding the content with enough spaces to fill the current + terminal width or truncating it to the terminal width if sep is a + carriage return.""" + global sep_rx + assert sep in (b'\r', b'\n', None) + if not content: + if sep: + os.write(dest, sep) + return + for x in content: + assert not sep_rx.match(x) + content = b''.join(content) + if sep == b'\r' and len(content) > width: + content = content[width:] + os.write(dest, content) + if len(content) < width: + os.write(dest, b' ' * (width - len(content))) + if sep: + os.write(dest, sep) + +def filter_output(srcs, dests): + """Transfer data from file descriptors in srcs to the corresponding + file descriptors in dests print_clean_line until all of the srcs + have closed. + + """ + global sep_rx + assert all(type(x) in int_types for x in srcs) + assert all(type(x) in int_types for x in srcs) + assert len(srcs) == len(dests) + srcs = tuple(srcs) + dest_for = dict(zip(srcs, dests)) + pending = {} + pending_ex = None + try: + while srcs: + ready_fds, _, _ = select.select(srcs, [], []) + width = tty_width() + for fd in ready_fds: + buf = os.read(fd, 4096) + dest = dest_for[fd] + if not buf: + srcs = tuple([x for x in srcs if x is not fd]) + print_clean_line(dest, pending.pop(fd, []), width) + else: + split = sep_rx.split(buf) + while len(split) > 1: + content, sep = split[:2] + split = split[2:] + print_clean_line(dest, + pending.pop(fd, []) + [content], + width, + sep) + assert len(split) == 1 + if split[0]: + pending.setdefault(fd, []).extend(split) + except BaseException as ex: + pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) + try: + # Try to finish each of the streams + for fd, pending_items in compat.items(pending): + dest = dest_for[fd] + width = tty_width() + try: + print_clean_line(dest, pending_items, width) + except (EnvironmentError, EOFError) as ex: + pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) + except BaseException as ex: + pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) + if pending_ex: + raise pending_ex + + +def import_and_run_main(module, args): + if do_profile: + import cProfile + f = compile('module.main(args)', __file__, 'exec') + cProfile.runctx(f, globals(), locals()) + else: + module.main(args) + + +def run_module_cmd(module, args): + if not (fix_stdout or fix_stderr): + import_and_run_main(module, args) + return + # Interpose filter_output between all attempts to write to the + # stdout/stderr and the real stdout/stderr (e.g. the fds that + # connect directly to the terminal) via a thread that runs + # filter_output in a pipeline. + srcs = [] + dests = [] + real_out_fd = real_err_fd = stdout_pipe = stderr_pipe = None + filter_thread = filter_thread_started = None + pending_ex = None + try: + if fix_stdout: + sys.stdout.flush() + stdout_pipe = os.pipe() # monitored_by_filter, stdout_everyone_uses + real_out_fd = os.dup(sys.stdout.fileno()) + os.dup2(stdout_pipe[1], sys.stdout.fileno()) + srcs.append(stdout_pipe[0]) + dests.append(real_out_fd) + if fix_stderr: + sys.stderr.flush() + stderr_pipe = os.pipe() # monitored_by_filter, stderr_everyone_uses + real_err_fd = os.dup(sys.stderr.fileno()) + os.dup2(stderr_pipe[1], sys.stderr.fileno()) + srcs.append(stderr_pipe[0]) + dests.append(real_err_fd) + + filter_thread = Thread(name='output filter', + target=lambda : filter_output(srcs, dests)) + filter_thread.start() + filter_thread_started = True + import_and_run_main(module, args) + except Exception as ex: + add_ex_tb(ex) + pending_ex = ex + raise + finally: + # Try to make sure that whatever else happens, we restore + # stdout and stderr here, if that's possible, so that we don't + # risk just losing some output. + try: + real_out_fd is not None and os.dup2(real_out_fd, sys.stdout.fileno()) + except Exception as ex: + add_ex_tb(ex) + add_ex_ctx(ex, pending_ex) + try: + real_err_fd is not None and os.dup2(real_err_fd, sys.stderr.fileno()) + except Exception as ex: + add_ex_tb(ex) + add_ex_ctx(ex, pending_ex) + # Kick filter loose + try: + stdout_pipe is not None and os.close(stdout_pipe[1]) + except Exception as ex: + add_ex_tb(ex) + add_ex_ctx(ex, pending_ex) + try: + stderr_pipe is not None and os.close(stderr_pipe[1]) + except Exception as ex: + add_ex_tb(ex) + add_ex_ctx(ex, pending_ex) + try: + close_catpipes() + except Exception as ex: + add_ex_tb(ex) + add_ex_ctx(ex, pending_ex) + if pending_ex: + raise pending_ex + # There's no point in trying to join unless we finished the finally block. + if filter_thread_started: + filter_thread.join() + + +def run_subproc_cmd(args): + + c = (do_profile and [sys.executable, b'-m', b'cProfile'] or []) + args + if not (fix_stdout or fix_stderr): + os.execvp(c[0], c) + + sys.stdout.flush() + sys.stderr.flush() + out = byte_stream(sys.stdout) + err = byte_stream(sys.stderr) + p = None + try: + p = subprocess.Popen(c, + stdout=PIPE if fix_stdout else out, + stderr=PIPE if fix_stderr else err, + env=tty_env, bufsize=4096, close_fds=True) + # Assume p will receive these signals and quit, which will + # then cause us to quit. + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): + signal.signal(sig, signal.SIG_IGN) + + srcs = [] + dests = [] + if fix_stdout: + srcs.append(p.stdout.fileno()) + dests.append(out.fileno()) + if fix_stderr: + srcs.append(p.stderr.fileno()) + dests.append(err.fileno()) + filter_output(srcs, dests) + return p.wait() + except BaseException as ex: + add_ex_tb(ex) + try: + if p and p.poll() == None: + os.kill(p.pid, signal.SIGTERM) + p.wait() + except BaseException as kill_ex: + raise add_ex_ctx(add_ex_tb(kill_ex), ex) + raise ex + + +def run_subcmd(module, args): + if module: + run_module_cmd(module, args) + else: + run_subproc_cmd(args) + +def main(): + wrap_main(lambda : run_subcmd(cmd_module, subcmd)) + +if __name__ == "__main__": + main() diff --git a/lib/cmd/bup b/lib/cmd/bup deleted file mode 100755 index e474b34..0000000 --- a/lib/cmd/bup +++ /dev/null @@ -1,465 +0,0 @@ -#!/bin/sh -"""": # -*-python-*- -set -e -# https://sourceware.org/bugzilla/show_bug.cgi?id=26034 -export "BUP_ARGV_0"="$0" -arg_i=1 -for arg in "$@"; do - export "BUP_ARGV_${arg_i}"="$arg" - shift - arg_i=$((arg_i + 1)) -done -# Here to end of preamble replaced during install -# Find our directory -top="$(pwd)" -cmdpath="$0" -# loop because macos doesn't have recursive readlink/realpath utils -while test -L "$cmdpath"; do - link="$(readlink "$cmdpath")" - cd "$(dirname "$cmdpath")" - cmdpath="$link" -done -script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)" -cd "$top" -exec "$script_home/../../config/bin/python" "$0" -""" -# end of bup preamble - -from __future__ import absolute_import, print_function - -import os, sys -sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] - -from importlib import import_module -from pkgutil import iter_modules -from subprocess import PIPE -from threading import Thread -import errno, getopt, os, re, select, signal, subprocess, sys - -from bup import compat, path, helpers -from bup.compat import ( - ModuleNotFoundError, - add_ex_ctx, - add_ex_tb, - argv_bytes, - environ, - fsdecode, - int_types, - wrap_main -) -from bup.compat import add_ex_tb, add_ex_ctx, argv_bytes, wrap_main -from bup.helpers import ( - columnate, - debug1, - handle_ctrl_c, - log, - merge_dict, - tty_width -) -from bup.git import close_catpipes -from bup.io import byte_stream, path_msg -from bup.options import _tty_width -import bup.cmd - -def maybe_import_early(argv): - """Scan argv and import any modules specified by --import-py-module.""" - while argv: - if argv[0] != '--import-py-module': - argv = argv[1:] - continue - if len(argv) < 2: - log("bup: --import-py-module must have an argument\n") - exit(2) - mod = argv[1] - import_module(mod) - argv = argv[2:] - -maybe_import_early(compat.argv) - -handle_ctrl_c() - -cmdpath = path.cmddir() - -# Remove once we finish the internal command transition -transition_cmdpath = path.libdir() + b'/bup/cmd' - -# We manipulate the subcmds here as strings, but they must be ASCII -# compatible, since we're going to be looking for exactly -# b'bup-SUBCMD' to exec. - -def usage(msg=""): - log('Usage: bup [-?|--help] [-d BUP_DIR] [--debug] [--profile] ' - ' [options...]\n\n') - common = dict( - ftp = 'Browse backup sets using an ftp-like client', - fsck = 'Check backup sets for damage and add redundancy information', - fuse = 'Mount your backup sets as a filesystem', - help = 'Print detailed help for the given command', - index = 'Create or display the index of files to back up', - on = 'Backup a remote machine to the local one', - restore = 'Extract files from a backup set', - save = 'Save files into a backup set (note: run "bup index" first)', - tag = 'Tag commits for easier access', - web = 'Launch a web server to examine backup sets', - ) - - log('Common commands:\n') - for cmd,synopsis in sorted(common.items()): - log(' %-10s %s\n' % (cmd, synopsis)) - log('\n') - - log('Other available commands:\n') - cmds = set() - for c in sorted(os.listdir(cmdpath)): - if c.startswith(b'bup-') and c.find(b'.') < 0: - cname = fsdecode(c[4:]) - if cname not in common: - cmds.add(c[4:].decode(errors='backslashreplace')) - # built-in commands take precedence - for _, name, _ in iter_modules(path=bup.cmd.__path__): - name = name.replace('_','-') - if name not in common: - cmds.add(name) - - log(columnate(sorted(cmds), ' ')) - log('\n') - - log("See 'bup help COMMAND' for more information on " + - "a specific command.\n") - if msg: - log("\n%s\n" % msg) - sys.exit(99) - -argv = compat.argv -if len(argv) < 2: - usage() - -# Handle global options. -try: - optspec = ['help', 'version', 'debug', 'profile', 'bup-dir=', - 'import-py-module='] - global_args, subcmd = getopt.getopt(argv[1:], '?VDd:', optspec) -except getopt.GetoptError as ex: - usage('error: %s' % ex.msg) - -subcmd = [argv_bytes(x) for x in subcmd] -help_requested = None -do_profile = False -bup_dir = None - -for opt in global_args: - if opt[0] in ['-?', '--help']: - help_requested = True - elif opt[0] in ['-V', '--version']: - subcmd = [b'version'] - elif opt[0] in ['-D', '--debug']: - helpers.buglvl += 1 - environ[b'BUP_DEBUG'] = b'%d' % helpers.buglvl - elif opt[0] in ['--profile']: - do_profile = True - elif opt[0] in ['-d', '--bup-dir']: - bup_dir = argv_bytes(opt[1]) - elif opt[0] == '--import-py-module': - pass - else: - usage('error: unexpected option "%s"' % opt[0]) - -# Make BUP_DIR absolute, so we aren't affected by chdir (i.e. save -C, etc.). -if bup_dir: - environ[b'BUP_DIR'] = os.path.abspath(bup_dir) - -if len(subcmd) == 0: - if help_requested: - subcmd = [b'help'] - else: - usage() - -if help_requested and subcmd[0] != b'help': - subcmd = [b'help'] + subcmd - -if len(subcmd) > 1 and subcmd[1] == b'--help' and subcmd[0] != b'help': - subcmd = [b'help', subcmd[0]] + subcmd[2:] - -subcmd_name = subcmd[0] -if not subcmd_name: - usage() - -try: - if subcmd_name not in (b'bloom', - b'cat-file', - b'daemon', - b'drecurse', - b'damage', - b'features', - b'ftp', - b'fsck', - b'fuse', - b'gc', - b'get', - b'help', - b'import-duplicity', - b'index', - b'init', - b'join', - b'list-idx', - b'ls', - b'margin', - b'memtest', - b'meta', - b'midx', - b'mux', - b'on', - b'on--server', - b'prune-older', - b'random', - b'restore', - b'rm', - b'save', - b'server', - b'split', - b'tag', - b'tick', - b'version', - b'web', - b'xstat'): - raise ModuleNotFoundError() - cmd_module = import_module('bup.cmd.' - + subcmd_name.decode('ascii').replace('-', '_')) -except ModuleNotFoundError as ex: - cmd_module = None - -if not cmd_module: - subcmd[0] = os.path.join(cmdpath, b'bup-' + subcmd_name) - if not os.path.exists(subcmd[0]): - subcmd[0] = b'%s/%s.py' % (transition_cmdpath, - subcmd_name.replace(b'-', b'_')) - if not os.path.exists(subcmd[0]): - usage('error: unknown command "%s"' % path_msg(subcmd_name)) - -already_fixed = int(environ.get(b'BUP_FORCE_TTY', 0)) -if subcmd_name in [b'mux', b'ftp', b'help']: - already_fixed = True -fix_stdout = not already_fixed and os.isatty(1) -fix_stderr = not already_fixed and os.isatty(2) - -if fix_stdout or fix_stderr: - tty_env = merge_dict(environ, - {b'BUP_FORCE_TTY': (b'%d' - % ((fix_stdout and 1 or 0) - + (fix_stderr and 2 or 0))), - b'BUP_TTY_WIDTH': b'%d' % _tty_width(), }) -else: - tty_env = environ - - -sep_rx = re.compile(br'([\r\n])') - -def print_clean_line(dest, content, width, sep=None): - """Write some or all of content, followed by sep, to the dest fd after - padding the content with enough spaces to fill the current - terminal width or truncating it to the terminal width if sep is a - carriage return.""" - global sep_rx - assert sep in (b'\r', b'\n', None) - if not content: - if sep: - os.write(dest, sep) - return - for x in content: - assert not sep_rx.match(x) - content = b''.join(content) - if sep == b'\r' and len(content) > width: - content = content[width:] - os.write(dest, content) - if len(content) < width: - os.write(dest, b' ' * (width - len(content))) - if sep: - os.write(dest, sep) - -def filter_output(srcs, dests): - """Transfer data from file descriptors in srcs to the corresponding - file descriptors in dests print_clean_line until all of the srcs - have closed. - - """ - global sep_rx - assert all(type(x) in int_types for x in srcs) - assert all(type(x) in int_types for x in srcs) - assert len(srcs) == len(dests) - srcs = tuple(srcs) - dest_for = dict(zip(srcs, dests)) - pending = {} - pending_ex = None - try: - while srcs: - ready_fds, _, _ = select.select(srcs, [], []) - width = tty_width() - for fd in ready_fds: - buf = os.read(fd, 4096) - dest = dest_for[fd] - if not buf: - srcs = tuple([x for x in srcs if x is not fd]) - print_clean_line(dest, pending.pop(fd, []), width) - else: - split = sep_rx.split(buf) - while len(split) > 1: - content, sep = split[:2] - split = split[2:] - print_clean_line(dest, - pending.pop(fd, []) + [content], - width, - sep) - assert len(split) == 1 - if split[0]: - pending.setdefault(fd, []).extend(split) - except BaseException as ex: - pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) - try: - # Try to finish each of the streams - for fd, pending_items in compat.items(pending): - dest = dest_for[fd] - width = tty_width() - try: - print_clean_line(dest, pending_items, width) - except (EnvironmentError, EOFError) as ex: - pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) - except BaseException as ex: - pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex) - if pending_ex: - raise pending_ex - - -def import_and_run_main(module, args): - if do_profile: - import cProfile - f = compile('module.main(args)', __file__, 'exec') - cProfile.runctx(f, globals(), locals()) - else: - module.main(args) - - -def run_module_cmd(module, args): - if not (fix_stdout or fix_stderr): - import_and_run_main(module, args) - return - # Interpose filter_output between all attempts to write to the - # stdout/stderr and the real stdout/stderr (e.g. the fds that - # connect directly to the terminal) via a thread that runs - # filter_output in a pipeline. - srcs = [] - dests = [] - real_out_fd = real_err_fd = stdout_pipe = stderr_pipe = None - filter_thread = filter_thread_started = None - pending_ex = None - try: - if fix_stdout: - sys.stdout.flush() - stdout_pipe = os.pipe() # monitored_by_filter, stdout_everyone_uses - real_out_fd = os.dup(sys.stdout.fileno()) - os.dup2(stdout_pipe[1], sys.stdout.fileno()) - srcs.append(stdout_pipe[0]) - dests.append(real_out_fd) - if fix_stderr: - sys.stderr.flush() - stderr_pipe = os.pipe() # monitored_by_filter, stderr_everyone_uses - real_err_fd = os.dup(sys.stderr.fileno()) - os.dup2(stderr_pipe[1], sys.stderr.fileno()) - srcs.append(stderr_pipe[0]) - dests.append(real_err_fd) - - filter_thread = Thread(name='output filter', - target=lambda : filter_output(srcs, dests)) - filter_thread.start() - filter_thread_started = True - import_and_run_main(module, args) - except Exception as ex: - add_ex_tb(ex) - pending_ex = ex - raise - finally: - # Try to make sure that whatever else happens, we restore - # stdout and stderr here, if that's possible, so that we don't - # risk just losing some output. - try: - real_out_fd is not None and os.dup2(real_out_fd, sys.stdout.fileno()) - except Exception as ex: - add_ex_tb(ex) - add_ex_ctx(ex, pending_ex) - try: - real_err_fd is not None and os.dup2(real_err_fd, sys.stderr.fileno()) - except Exception as ex: - add_ex_tb(ex) - add_ex_ctx(ex, pending_ex) - # Kick filter loose - try: - stdout_pipe is not None and os.close(stdout_pipe[1]) - except Exception as ex: - add_ex_tb(ex) - add_ex_ctx(ex, pending_ex) - try: - stderr_pipe is not None and os.close(stderr_pipe[1]) - except Exception as ex: - add_ex_tb(ex) - add_ex_ctx(ex, pending_ex) - try: - close_catpipes() - except Exception as ex: - add_ex_tb(ex) - add_ex_ctx(ex, pending_ex) - if pending_ex: - raise pending_ex - # There's no point in trying to join unless we finished the finally block. - if filter_thread_started: - filter_thread.join() - - -def run_subproc_cmd(args): - - c = (do_profile and [sys.executable, b'-m', b'cProfile'] or []) + args - if not (fix_stdout or fix_stderr): - os.execvp(c[0], c) - - sys.stdout.flush() - sys.stderr.flush() - out = byte_stream(sys.stdout) - err = byte_stream(sys.stderr) - p = None - try: - p = subprocess.Popen(c, - stdout=PIPE if fix_stdout else out, - stderr=PIPE if fix_stderr else err, - env=tty_env, bufsize=4096, close_fds=True) - # Assume p will receive these signals and quit, which will - # then cause us to quit. - for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): - signal.signal(sig, signal.SIG_IGN) - - srcs = [] - dests = [] - if fix_stdout: - srcs.append(p.stdout.fileno()) - dests.append(out.fileno()) - if fix_stderr: - srcs.append(p.stderr.fileno()) - dests.append(err.fileno()) - filter_output(srcs, dests) - return p.wait() - except BaseException as ex: - add_ex_tb(ex) - try: - if p and p.poll() == None: - os.kill(p.pid, signal.SIGTERM) - p.wait() - except BaseException as kill_ex: - raise add_ex_ctx(add_ex_tb(kill_ex), ex) - raise ex - - -def run_subcmd(module, args): - if module: - run_module_cmd(module, args) - else: - run_subproc_cmd(args) - - -wrap_main(lambda : run_subcmd(cmd_module, subcmd)) diff --git a/lib/cmd/bup.c b/lib/cmd/bup.c new file mode 100644 index 0000000..1dcb724 --- /dev/null +++ b/lib/cmd/bup.c @@ -0,0 +1,359 @@ + +#define PY_SSIZE_T_CLEAN +#define _GNU_SOURCE 1 // asprintf +#undef NDEBUG + +// According to Python, its header has to go first: +// http://docs.python.org/2/c-api/intro.html#include-files +// http://docs.python.org/3/c-api/intro.html#include-files +#include + +#include +#include +#include +#include +#include +#include +#include +#if defined(__FreeBSD__) || defined(__NetBSD__) +# include +#endif +#include +#include + +__attribute__ ((format(printf, 2, 3))) +static void +die(int exit_status, const char * const msg, ...) +{ + if (fputs("bup: ", stderr) == EOF) + exit(3); + va_list ap; + va_start(ap, msg);; + if (vfprintf(stderr, msg, ap) < 0) + exit(3); + va_end(ap); + exit(exit_status); +} + +static int prog_argc = 0; +static char **prog_argv = NULL; + +static PyObject* +get_argv(PyObject *self, PyObject *args) +{ + if (!PyArg_ParseTuple(args, "")) + return NULL; + + PyObject *result = PyList_New(prog_argc); + for (int i = 0; i < prog_argc; i++) { + PyObject *s = PyBytes_FromString(prog_argv[i]); + if (!s) + die(2, "cannot convert argument to bytes: %s\n", prog_argv[i]); + PyList_SET_ITEM(result, i, s); + } + return result; +} + +static PyMethodDef bup_main_methods[] = { + {"argv", get_argv, METH_VARARGS, + "Return the program's current argv array as a list of byte strings." }, + {NULL, NULL, 0, NULL} +}; + +static int setup_module(PyObject *mod) +{ + return 1; +} + +#if PY_MAJOR_VERSION >= 3 + +static struct PyModuleDef bup_main_module_def = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "bup_main", + .m_doc = "Built-in bup module providing direct access to argv.", + .m_size = -1, + .m_methods = bup_main_methods +}; + +PyObject * +PyInit_bup_main(void) { + PyObject *mod = PyModule_Create(&bup_main_module_def); + if (!setup_module(mod)) + { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +#else // PY_MAJOR_VERSION < 3 + +void PyInit_bup_main(void) +{ + PyObject *mod = Py_InitModule("bup_main", bup_main_methods); + if (mod == NULL) { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers init failed"); + return; + } + if (!setup_module(mod)) + { + PyErr_SetString(PyExc_RuntimeError, "bup._helpers set up failed"); + Py_DECREF(mod); + return; + } +} + +#endif // PY_MAJOR_VERSION < 3 + +static void +setup_bup_main_module(void) +{ + if (PyImport_AppendInittab("bup_main", PyInit_bup_main) == -1) + die(2, "unable to register bup_main module\n"); +} + +#if defined(__APPLE__) && defined(__MACH__) + +static char *exe_parent_dir(const char * const argv_0) { + char path[4096]; // FIXME + uint32_t size = sizeof(path); + if(_NSGetExecutablePath(path, &size) !=0) + die(2, "unable to find executable path\n"); + char * abs_exe = realpath(path, NULL); + if (!abs_exe) + die(2, "cannot resolve path (%s): %s\n", strerror(errno), path); + char * const abs_parent = strdup(dirname(abs_exe)); + assert(abs_parent); + free(abs_exe); + return abs_parent; +} + +#elif defined(__FreeBSD__) || defined(__NetBSD__) + +static char *exe_path () +{ + const int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + size_t path_len; + int rc = sysctl (mib, 4, NULL, &path_len, NULL, 0); + if (rc != 0) die(2, "unable to determine executable path length\n"); + char *path = malloc (path_len); + if (!path) die(2, "unable to allocate memory for executable path\n"); + rc = sysctl (mib, 4, path, &path_len, NULL, 0); + if (rc != 0) die(2, "unable to determine executable path via sysctl\n"); + return path; +} + +static char *exe_parent_dir(const char * const argv_0) +{ + char * const exe = exe_path(); + if (!exe) die(2, "unable to determine executable path\n"); + char * const parent = strdup(dirname(exe)); + if (!parent) die(2, "unable to determine parent directory of executable\n"); + free(exe); + return parent; +} + +#else // not defined(__FreeBSD__) || defined(__NetBSD__) + +/// Use /proc if possible, and if all else fails, search in the PATH + +#if defined(__linux__) +# define PROC_SELF_EXE "/proc/self/exe" +#elif defined(__sun) || defined (sun) +# define PROC_SELF_EXE "/proc/self/path/a.out" +#else +# define PROC_SELF_EXE NULL +#endif + +static char *find_in_path(const char * const name, const char * const path) +{ + char *result = NULL; + char *tmp_path = strdup(path); + assert(tmp_path); + const char *elt; + char *tok_path = tmp_path; + while ((elt = strtok(tok_path, ":")) != NULL) { + tok_path = NULL; + char *candidate; + int rc = asprintf(&candidate, "%s/%s", elt, name); + assert(rc >= 0); + struct stat st; + rc = stat(candidate, &st); + if (rc != 0) { + switch (errno) { + case EACCES: case ELOOP: case ENOENT: case ENAMETOOLONG: + case ENOTDIR: + break; + default: + die(2, "cannot stat %s: %s\n", candidate, strerror(errno)); + break; + } + } else if (S_ISREG(st.st_mode)) { + if (access(candidate, X_OK) == 0) { + result = candidate; + break; + } + switch (errno) { + case EACCES: case ELOOP: case ENOENT: case ENAMETOOLONG: + case ENOTDIR: + break; + default: + die(2, "cannot determine executability of %s: %s\n", + candidate, strerror(errno)); + break; + } + } + free(candidate); + } + free(tmp_path); + return result; +} + +static char *find_exe_parent(const char * const argv_0) +{ + char *candidate = NULL; + const char * const slash = index(argv_0, '/'); + if (slash) { + candidate = strdup(argv_0); + assert(candidate); + } else { + const char * const env_path = getenv("PATH"); + if (!env_path) + die(2, "no PATH and executable isn't relative or absolute: %s\n", + argv_0); + char *path_exe = find_in_path(argv_0, env_path); + if (path_exe) { + char * abs_exe = realpath(path_exe, NULL); + if (!abs_exe) + die(2, "cannot resolve path (%s): %s\n", + strerror(errno), path_exe); + free(path_exe); + candidate = abs_exe; + } + } + if (!candidate) + return NULL; + + char * const abs_exe = realpath(candidate, NULL); + if (!abs_exe) + die(2, "cannot resolve path (%s): %s\n", strerror(errno), candidate); + free(candidate); + char * const abs_parent = strdup(dirname(abs_exe)); + assert(abs_parent); + free(abs_exe); + return abs_parent; +} + +static char *exe_parent_dir(const char * const argv_0) +{ + if (PROC_SELF_EXE != NULL) { + char path[4096]; // FIXME + int len = readlink(PROC_SELF_EXE, path, sizeof(path)); + if (len == sizeof(path)) + die(2, "unable to resolve symlink %s: %s\n", + PROC_SELF_EXE, strerror(errno)); + if (len != -1) { + path[len] = '\0'; + return strdup(dirname(path)); + } + switch (errno) { + case ENOENT: case EACCES: case EINVAL: case ELOOP: case ENOTDIR: + case ENAMETOOLONG: + break; + default: + die(2, "cannot resolve %s: %s\n", path, strerror(errno)); + break; + } + } + return find_exe_parent(argv_0); +} + +#endif // use /proc if possible, and if all else fails, search in the PATh + +static void +setenv_or_die(const char *name, const char *value) +{ + int rc = setenv(name, value, 1); + if (rc != 0) + die(2, "setenv %s=%s failed (%s)\n", name, value, strerror(errno)); +} + +static void +prepend_lib_to_pythonpath(const char * const exec_path, + const char * const relative_path) +{ + char *parent = exe_parent_dir(exec_path); + assert(parent); + char *bupmodpath; + int rc = asprintf(&bupmodpath, "%s/%s", parent, relative_path); + assert(rc >= 0); + struct stat st; + rc = stat(bupmodpath, &st); + if (rc != 0) + die(2, "unable find lib dir (%s): %s\n", strerror(errno), bupmodpath); + if (!S_ISDIR(st.st_mode)) + die(2, "lib path is not dir: %s\n", bupmodpath); + char *curpypath = getenv("PYTHONPATH"); + if (curpypath) { + char *path; + int rc = asprintf(&path, "%s:%s", bupmodpath, curpypath); + assert(rc >= 0); + setenv_or_die("PYTHONPATH", path); + free(path); + } else { + setenv_or_die("PYTHONPATH", bupmodpath); + } + + free(bupmodpath); + free(parent); +} + +#if PY_MAJOR_VERSION > 2 +#define bup_py_main Py_BytesMain +# else +#define bup_py_main Py_Main +#endif + +#if defined(BUP_DEV_BUP_PYTHON) && defined(BUP_DEV_BUP_EXEC) +# error "Both BUP_DEV_BUP_PYTHON and BUP_DEV_BUP_EXEC are defined" +#endif + +#ifdef BUP_DEV_BUP_PYTHON + +int main(int argc, char **argv) +{ + prog_argc = argc; + prog_argv = argv; + setup_bup_main_module(); + prepend_lib_to_pythonpath(argv[0], "../lib"); + return bup_py_main (argc, argv); +} + +#elif defined(BUP_DEV_BUP_EXEC) + +int main(int argc, char **argv) +{ + prog_argc = argc - 1; + prog_argv = argv + 1; + setup_bup_main_module(); + prepend_lib_to_pythonpath(argv[0], "../lib"); + if (argc == 1) + return bup_py_main (1, argv); + // This can't handle a script with a name like "-c", but that's + // python's problem, not ours. + return bup_py_main (2, argv); +} + +#else // normal bup command + +int main(int argc, char **argv) +{ + prog_argc = argc; + prog_argv = argv; + setup_bup_main_module(); + prepend_lib_to_pythonpath(argv[0], ".."); + char *bup_argv[] = { argv[0], "-m", "bup.main" }; + return bup_py_main (3, bup_argv); +} + +#endif // normal bup command diff --git a/test/ext/test-misc b/test/ext/test-misc index 5fb96be..b69eaaa 100755 --- a/test/ext/test-misc +++ b/test/ext/test-misc @@ -123,7 +123,7 @@ c/" WVSTART features -expect_py_ver=$(LC_CTYPE=C "$top/config/bin/python" \ +expect_py_ver=$(LC_CTYPE=C "$top/dev/python" \ -c 'import platform; print(platform.python_version())') \ || exit $? actual_py_ver=$(bup features | grep Python: | sed -Ee 's/ +Python: //') || exit $?