From 938537c3a9bb76cfa7d62f931ffee0ff3359aa55 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Mon, 25 May 2020 14:46:52 -0500 Subject: [PATCH] Stop forcing LC_CTYPE=ISO-8859-1 Now that we've made adjustments to work around all the Python 3 problems with non-Unicode data (argv, env vars, readline, acls, users, groups, hostname, etc.), and added randomized binary path and argv testing, stop overriding the LC_CTYPE since that should no longer be necessary. Thanks to Johannes Berg for nudging me to consider whether we might now be in a position to do this (with a bit more work), and for quite a bit of help getting all the precursors in place once we thought it was feasible. Signed-off-by: Rob Browning Tested-by: Rob Browning --- Makefile | 25 +++++------------ README.md | 4 +++ bup | 7 ++++- buptest.py | 9 +++---- config/configure | 2 +- dev/install-python-script | 37 +++++++++++++++++++++++++ dev/replace | 29 -------------------- lib/bup/compat.py | 29 -------------------- lib/bup/io.py | 10 ++----- lib/bup/ls.py | 2 +- lib/bup/path.py | 7 ++--- lib/bup/t/thelpers.py | 2 +- lib/cmd/bloom-cmd.py | 2 ++ lib/cmd/bup | 10 ++++--- lib/cmd/cat-file-cmd.py | 4 ++- lib/cmd/daemon-cmd.py | 4 ++- lib/cmd/damage-cmd.py | 4 ++- lib/cmd/drecurse-cmd.py | 4 ++- lib/cmd/fsck-cmd.py | 4 ++- lib/cmd/ftp-cmd.py | 4 ++- lib/cmd/fuse-cmd.py | 4 ++- lib/cmd/gc-cmd.py | 4 ++- lib/cmd/get-cmd.py | 2 ++ lib/cmd/help-cmd.py | 6 ++++- lib/cmd/import-duplicity-cmd.py | 5 +++- lib/cmd/index-cmd.py | 4 ++- lib/cmd/init-cmd.py | 4 ++- lib/cmd/join-cmd.py | 4 ++- lib/cmd/list-idx-cmd.py | 4 ++- lib/cmd/ls-cmd.py | 4 ++- lib/cmd/margin-cmd.py | 4 ++- lib/cmd/memtest-cmd.py | 4 ++- lib/cmd/meta-cmd.py | 5 +++- lib/cmd/midx-cmd.py | 2 ++ lib/cmd/mux-cmd.py | 4 ++- lib/cmd/on--server-cmd.py | 4 ++- lib/cmd/on-cmd.py | 4 ++- lib/cmd/prune-older-cmd.py | 4 ++- lib/cmd/python-cmd.sh | 48 --------------------------------- lib/cmd/random-cmd.py | 2 ++ lib/cmd/restore-cmd.py | 4 ++- lib/cmd/rm-cmd.py | 4 ++- lib/cmd/save-cmd.py | 4 ++- lib/cmd/server-cmd.py | 4 ++- lib/cmd/split-cmd.py | 2 ++ lib/cmd/tag-cmd.py | 2 ++ lib/cmd/tick-cmd.py | 5 +++- lib/cmd/version-cmd.py | 4 ++- lib/cmd/web-cmd.py | 2 ++ lib/cmd/xstat-cmd.py | 4 ++- t/echo-argv-bytes | 1 + t/hardlink-sets | 2 ++ t/ns-timestamp-resolutions | 4 ++- t/sparse-test-data | 4 ++- t/subtree-hash | 4 ++- t/test-ftp | 1 + t/test-get | 2 ++ t/test-on.sh | 2 +- t/test-prune-older | 1 + t/test-save-errors | 8 ++++++ 60 files changed, 204 insertions(+), 181 deletions(-) mode change 120000 => 100755 bup create mode 100755 dev/install-python-script delete mode 100755 dev/replace delete mode 100644 lib/cmd/python-cmd.sh diff --git a/Makefile b/Makefile index d7a076e..7dd7cdf 100644 --- a/Makefile +++ b/Makefile @@ -72,15 +72,6 @@ all: $(bup_deps) Documentation/all $(current_sampledata) $(current_sampledata): t/configure-sampledata --setup - -bup_libdir="$script_home/../lib" # bup_libdir will be adjusted during install - -define install-bup-python - set -e; \ - sed -e 's|.*# bup_libdir will be adjusted during install|bup_libdir="$$script_home/.."|' $1 > $2; \ - chmod 0755 $2; -endef - PANDOC ?= $(shell type -p pandoc) ifeq (,$(PANDOC)) @@ -113,9 +104,11 @@ install: all test -z "$(man_roff)" || $(INSTALL) -m 0644 $(man_roff) $(dest_mandir)/man1 test -z "$(man_html)" || install -d $(dest_docdir) test -z "$(man_html)" || $(INSTALL) -m 0644 $(man_html) $(dest_docdir) - $(INSTALL) -pm 0755 cmd/bup $(dest_libdir)/cmd/ - $(INSTALL) -pm 0755 cmd/bup-* $(dest_libdir)/cmd/ - $(call install-bup-python,cmd/bup-python,"$(dest_libdir)/cmd/bup-python") + dev/install-python-script lib/cmd/bup "$(dest_libdir)/cmd/bup" + set -e; \ + for cmd in $$(ls cmd/bup-* | grep -v cmd/bup-python); do \ + dev/install-python-script "$$cmd" "$(dest_libdir)/$$cmd"; \ + done cd "$(dest_bindir)" && \ ln -sf "$$($(bup_python) -c 'import os; print(os.path.relpath("$(abspath $(dest_libdir))/cmd/bup"))')" set -e; \ @@ -260,12 +253,8 @@ check: test distcheck: all ./wvtest run t/test-release-archive.sh -cmd/bup-python: cmd/python-cmd.sh config/config.var/bup-python - "$$(cat config/config.var/bup-python)" dev/replace -l '@bup_python@' \ - "$$(dev/shquote < config/config.var/bup-python)" \ - < "$<" > "$@".$$PPID.tmp - chmod +x "$@".$$PPID.tmp - mv "$@".$$PPID.tmp "$@" +cmd/bup-python: config/config.var/bup-python + cd cmd && ln -sf "$$(< $(CURDIR)/config/config.var/bup-python)" bup-python long-test: export BUP_TEST_LEVEL=11 long-test: test diff --git a/README.md b/README.md index ae49cce..05fc56f 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,10 @@ Reasons you might want to avoid bup Solaris, or Windows (with Cygwin, and maybe with WSL). Patches to support other platforms are welcome. + - Until resolved, a [glibc bug](https://sourceware.org/bugzilla/show_bug.cgi?id=26034) + might cause bup to crash on startup for some (unusual) command line + argument values, when bup is configured to use Python 3. + - Any items in "Things that are stupid" below. diff --git a/bup b/bup deleted file mode 120000 index e1c0ca8..0000000 --- a/bup +++ /dev/null @@ -1 +0,0 @@ -lib/cmd/bup \ No newline at end of file diff --git a/bup b/bup new file mode 100755 index 0000000..0d15289 --- /dev/null +++ b/bup @@ -0,0 +1,6 @@ +#!/bin/sh + +# This is just a convenience wrapper. It's not a symlink because bup +# finds its code relative to dirname(__file__). + +exec $(dirname "$0")/cmd/bup "$@" diff --git a/buptest.py b/buptest.py index 7c7cc2f..2089fcf 100644 --- a/buptest.py +++ b/buptest.py @@ -11,7 +11,7 @@ import errno, os, subprocess, sys, tempfile from wvtest import WVPASSEQ, wvfailure_count from bup import helpers -from bup.compat import str_type +from bup.compat import fsencode, str_type from bup.io import byte_stream @@ -35,7 +35,7 @@ def no_lingering_errors(): # Assumes (of course) this file is at the top-level of the source tree -_bup_tmp = realpath(dirname(__file__.encode('iso-8859-1')) + b'/t/tmp') +_bup_tmp = realpath(dirname(fsencode(__file__))) + b'/t/tmp' try: os.makedirs(_bup_tmp) except OSError as e: @@ -78,9 +78,8 @@ def logcmd(cmd): if isinstance(cmd, str_type): print(s, file=sys.stderr) else: - # bytes - for now just continue to pass it through given - # bup-python wrapper - print(s.decode('iso-8859-1'), file=sys.stderr) + # bytes - for now just escape it + print(s.decode(errors='backslashreplace'), file=sys.stderr) def ex(cmd, **kwargs): """Print cmd to stderr and then run it as per ex(...). diff --git a/config/configure b/config/configure index 86ecd76..3eab5d3 100755 --- a/config/configure +++ b/config/configure @@ -58,7 +58,7 @@ expr "$MAKE_VERSION" '>=' '3.81' || AC_FAIL "ERROR: $MAKE must be >= version 3.8 AC_SUB bup_make "$MAKE" -bup_python="$PYTHON" +bup_python="$(type -p "$PYTHON")" test -z "$bup_python" && bup_python="$(bup_find_prog python2.7 '')" test -z "$bup_python" && bup_python="$(bup_find_prog python2.6 '')" test -z "$bup_python" && bup_python="$(bup_find_prog python2 '')" diff --git a/dev/install-python-script b/dev/install-python-script new file mode 100755 index 0000000..251a9f8 --- /dev/null +++ b/dev/install-python-script @@ -0,0 +1,37 @@ +#!/bin/sh +"""": # -*-python-*- +exec env LC_CTYPE=iso-8859-1 python "$0" ${1+"$@"} +""" + +from __future__ import absolute_import, print_function +from tempfile import NamedTemporaryFile +import os, shutil, sys + +if sys.version_info[0] >= 3: + from shlex import quote +else: + from pipes import quote + +src_path, dest_path = sys.argv[1:] + +with open(b'config/config.var/bup-python', 'rb') as src: + python = src.read() + +with NamedTemporaryFile() as tmp: + # Replace the section between "Here to end..." and the end of the + # preamble with the correct 'exec PYTHON "$0"'. + with open(src_path, 'rb') as src: + for line in src: + if line.startswith(b'# Here to end of preamble replaced during install'): + break + tmp.write(line) + for line in src: + if line == b'"""\n': + break + tmp.write(b'exec %s "$0"\n' % python) + tmp.write(b'"""\n') + for line in src: + tmp.write(line) + tmp.flush() + shutil.copy(tmp.name, dest_path) + os.chmod(dest_path, 0o755) diff --git a/dev/replace b/dev/replace deleted file mode 100755 index 4ac0c0e..0000000 --- a/dev/replace +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -"""": # -*-python-*- -exec env LC_CTYPE=iso-8859-1 python "$0" ${1+"$@"} -""" - -from __future__ import absolute_import, print_function -import sys - -py_maj = sys.version_info[0] - -import argparse -if py_maj > 2: - byte_stream = lambda x: x.buffer - from os import fsencode -else: - byte_stream = lambda x: x - fsencode = lambda x: x - -parser = argparse.ArgumentParser() -parser.add_argument('-l', nargs=2, metavar=('ORIG', 'NEW'), action='append', - help='literally replace ORIG with NEW') -opt = parser.parse_args() - -sys.stdout.flush() -with byte_stream(sys.stdin) as stdin: - content = stdin.read() -for orig, new in opt.l: - content = content.replace(fsencode(orig), fsencode(new)) -byte_stream(sys.stdout).write(content) diff --git a/lib/bup/compat.py b/lib/bup/compat.py index 37b31f1..1ed2d0f 100644 --- a/lib/bup/compat.py +++ b/lib/bup/compat.py @@ -14,17 +14,6 @@ py3 = py_maj >= 3 if py3: from os import environb as environ - - lc_ctype = environ.get(b'LC_CTYPE') - if lc_ctype and lc_ctype.lower() != b'iso-8859-1': - # Because of argv, options.py, pwd, grp, and any number of other issues - print('error: bup currently only works with ISO-8859-1, not LC_CTYPE=%s' - % lc_ctype.decode(), - file=sys.stderr) - print('error: this should already have been arranged, so indicates a bug', - file=sys.stderr) - sys.exit(2) - from os import fsdecode, fsencode from shlex import quote input = input @@ -179,24 +168,6 @@ def _configure_argv(): _configure_argv() -def restore_lc_env(): - # Once we're up and running with iso-8859-1, undo the bup-python - # LC_CTYPE hackery, so we don't affect unrelated subprocesses. - bup_lc_all = environ.get(b'BUP_LC_ALL') - if bup_lc_all: - del environ[b'LC_COLLATE'] - del environ[b'LC_CTYPE'] - del environ[b'LC_MONETARY'] - del environ[b'LC_NUMERIC'] - del environ[b'LC_TIME'] - del environ[b'LC_MESSAGES'] - del environ[b'LC_MESSAGES'] - environ[b'LC_ALL'] = bup_lc_all - return - bup_lc_ctype = environ.get(b'BUP_LC_CTYPE') - if bup_lc_ctype: - environ[b'LC_CTYPE'] = bup_lc_ctype - def wrap_main(main): """Run main() and raise a SystemExit with the return value if it returns, pass along any SystemExit it raises, convert diff --git a/lib/bup/io.py b/lib/bup/io.py index 571e9b9..256f562 100644 --- a/lib/bup/io.py +++ b/lib/bup/io.py @@ -13,12 +13,6 @@ else: def path_msg(x): - """Return a string representation of a path. - - For now, assume that the destination encoding is going to be - ISO-8859-1, which it should be, for the primary current - destination, stderr, given the current bup-python. - - """ + """Return a string representation of a path.""" # FIXME: configurability (might git-config quotePath be involved?) - return x.decode(encoding='iso-8859-1') + return x.decode(errors='backslashreplace') diff --git a/lib/bup/ls.py b/lib/bup/ls.py index bff0401..8117bd7 100644 --- a/lib/bup/ls.py +++ b/lib/bup/ls.py @@ -51,7 +51,7 @@ def item_info(item, name, if classification: cls = xstat.classification_str(vfs.item_mode(item), classification == 'all') - result += cls.encode('iso-8859-1') + result += cls.encode('ascii') return result diff --git a/lib/bup/path.py b/lib/bup/path.py index 7dd3905..fd1c09b 100644 --- a/lib/bup/path.py +++ b/lib/bup/path.py @@ -1,12 +1,13 @@ from __future__ import absolute_import -import os - +import os, sys # Eventually, if we physically move the source tree cmd/ to lib/, then # we could use realpath here and save some stats... -_libdir = os.path.abspath(os.path.dirname(__file__.encode('iso-8859-1')) + b'/..') +fsencode = os.fsencode if sys.version_info[0] >= 3 else lambda x: x + +_libdir = os.path.abspath(os.path.dirname(fsencode(__file__)) + b'/..') _resdir = _libdir _exedir = os.path.abspath(_libdir + b'/cmd') _exe = os.path.join(_exedir, b'bup') diff --git a/lib/bup/t/thelpers.py b/lib/bup/t/thelpers.py index 17ee635..e05c1f6 100644 --- a/lib/bup/t/thelpers.py +++ b/lib/bup/t/thelpers.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from time import tzset -import helpers, math, os, os.path, re, subprocess +import helpers, math, os, re, subprocess from wvtest import * diff --git a/lib/cmd/bloom-cmd.py b/lib/cmd/bloom-cmd.py index 16d8f46..fdf8145 100755 --- a/lib/cmd/bloom-cmd.py +++ b/lib/cmd/bloom-cmd.py @@ -17,6 +17,8 @@ exec "$bup_python" "$0" from __future__ import absolute_import import glob, os, sys, tempfile +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, git, bloom from bup.compat import argv_bytes, hexstr from bup.helpers import (add_error, debug1, handle_ctrl_c, log, progress, qprogress, diff --git a/lib/cmd/bup b/lib/cmd/bup index 92807a5..3e9b919 100755 --- a/lib/cmd/bup +++ b/lib/cmd/bup @@ -26,10 +26,14 @@ exec "$script_home/bup-python" "$0" # end of bup preamble from __future__ import absolute_import, print_function + +import os, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + import errno, getopt, os, re, select, signal, subprocess, sys from subprocess import PIPE -from bup.compat import environ, restore_lc_env +from bup.compat import environ, fsdecode from bup.io import path_msg if sys.version_info[0] != 2 \ @@ -38,8 +42,6 @@ if sys.version_info[0] != 2 \ file=sys.stderr) sys.exit(2) -restore_lc_env() - from bup import compat, path, helpers from bup.compat import add_ex_tb, add_ex_ctx, argv_bytes, wrap_main from bup.helpers import atoi, columnate, debug1, log, merge_dict, tty_width @@ -76,7 +78,7 @@ def usage(msg=""): cmds = [] for c in sorted(os.listdir(cmdpath)): if c.startswith(b'bup-') and c.find(b'.') < 0: - cname = c[4:].decode('iso-8859-1') + cname = fsdecode(c[4:]) if cname not in common: cmds.append(c[4:].decode(errors='backslashreplace')) log(columnate(cmds, ' ')) diff --git a/lib/cmd/cat-file-cmd.py b/lib/cmd/cat-file-cmd.py index 849cb45..7fbc256 100755 --- a/lib/cmd/cat-file-cmd.py +++ b/lib/cmd/cat-file-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import re, stat, sys +import os.path, re, stat, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, git, vfs from bup.compat import argv_bytes diff --git a/lib/cmd/daemon-cmd.py b/lib/cmd/daemon-cmd.py index d47638e..5506ea1 100755 --- a/lib/cmd/daemon-cmd.py +++ b/lib/cmd/daemon-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys, getopt, socket, subprocess, fcntl +import os, sys, getopt, socket, subprocess, fcntl + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, path from bup.helpers import * diff --git a/lib/cmd/damage-cmd.py b/lib/cmd/damage-cmd.py index d9e46da..c7c712a 100755 --- a/lib/cmd/damage-cmd.py +++ b/lib/cmd/damage-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys, os, random +import os, random, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options from bup.compat import argv_bytes, bytes_from_uint, range diff --git a/lib/cmd/drecurse-cmd.py b/lib/cmd/drecurse-cmd.py index 22dc3e1..6cb0fc4 100755 --- a/lib/cmd/drecurse-cmd.py +++ b/lib/cmd/drecurse-cmd.py @@ -16,7 +16,9 @@ exec "$bup_python" "$0" from __future__ import absolute_import, print_function from os.path import relpath -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, drecurse from bup.compat import argv_bytes diff --git a/lib/cmd/fsck-cmd.py b/lib/cmd/fsck-cmd.py index 035300d..e5eea32 100755 --- a/lib/cmd/fsck-cmd.py +++ b/lib/cmd/fsck-cmd.py @@ -15,12 +15,14 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import, print_function -import sys, os, glob, subprocess +import os, glob, subprocess, sys from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp from binascii import hexlify +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, git from bup.compat import argv_bytes from bup.helpers import Sha1, chunkyreader, istty2, log, progress diff --git a/lib/cmd/ftp-cmd.py b/lib/cmd/ftp-cmd.py index 87fd3ce..1aada9a 100755 --- a/lib/cmd/ftp-cmd.py +++ b/lib/cmd/ftp-cmd.py @@ -20,7 +20,9 @@ exec "$bup_python" "$0" # (e.g. ISO-8859-1). from __future__ import absolute_import, print_function -import sys, os, stat, fnmatch +import os, fnmatch, stat, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import _helpers, compat, options, git, shquote, ls, vfs from bup.compat import argv_bytes diff --git a/lib/cmd/fuse-cmd.py b/lib/cmd/fuse-cmd.py index eb5daf8..bb6d98d 100755 --- a/lib/cmd/fuse-cmd.py +++ b/lib/cmd/fuse-cmd.py @@ -15,7 +15,7 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import, print_function -import sys, os, errno +import errno, os, sys try: import fuse @@ -41,6 +41,8 @@ if sys.version_info[0] > 2: file=sys.stderr) sys.exit(2) +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, git, vfs, xstat from bup.compat import argv_bytes, fsdecode, py_maj from bup.helpers import log diff --git a/lib/cmd/gc-cmd.py b/lib/cmd/gc-cmd.py index d8f7bc9..7368eca 100755 --- a/lib/cmd/gc-cmd.py +++ b/lib/cmd/gc-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, options from bup.gc import bup_gc diff --git a/lib/cmd/get-cmd.py b/lib/cmd/get-cmd.py index 88a919e..c319824 100755 --- a/lib/cmd/get-cmd.py +++ b/lib/cmd/get-cmd.py @@ -21,6 +21,8 @@ from collections import namedtuple from functools import partial from stat import S_ISDIR +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, git, client, helpers, vfs from bup.compat import argv_bytes, environ, hexstr, items, wrap_main from bup.git import get_cat_data, parse_commit, walk_object diff --git a/lib/cmd/help-cmd.py b/lib/cmd/help-cmd.py index c078e82..aeacc70 100755 --- a/lib/cmd/help-cmd.py +++ b/lib/cmd/help-cmd.py @@ -15,9 +15,13 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys, os, glob +import os, glob, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, path + optspec = """ bup help """ diff --git a/lib/cmd/import-duplicity-cmd.py b/lib/cmd/import-duplicity-cmd.py index bfb5183..77b53eb 100755 --- a/lib/cmd/import-duplicity-cmd.py +++ b/lib/cmd/import-duplicity-cmd.py @@ -20,9 +20,12 @@ from pipes import quote from subprocess import check_call from time import strftime, strptime import os +import os.path import sys import tempfile +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, git, helpers, options from bup.compat import argv_bytes, str_type from bup.helpers import (handle_ctrl_c, @@ -40,7 +43,7 @@ n,dry-run don't do anything; just print what would be done """ def logcmd(cmd): - log(shstr(cmd).decode('iso-8859-1', errors='replace') + '\n') + log(shstr(cmd).decode(errors='backslashreplace') + '\n') def exc(cmd, shell=False): global opt diff --git a/lib/cmd/index-cmd.py b/lib/cmd/index-cmd.py index 030451e..a47e208 100755 --- a/lib/cmd/index-cmd.py +++ b/lib/cmd/index-cmd.py @@ -16,7 +16,9 @@ exec "$bup_python" "$0" from __future__ import absolute_import, print_function from binascii import hexlify -import sys, stat, time, os, errno, re +import errno, os, re, stat, sys, time + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, metadata, options, git, index, drecurse, hlinkdb from bup.compat import argv_bytes diff --git a/lib/cmd/init-cmd.py b/lib/cmd/init-cmd.py index cff5c5c..13bff7d 100755 --- a/lib/cmd/init-cmd.py +++ b/lib/cmd/init-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, options, client from bup.helpers import log, saved_errors diff --git a/lib/cmd/join-cmd.py b/lib/cmd/join-cmd.py index e17c315..55d9eed 100755 --- a/lib/cmd/join-cmd.py +++ b/lib/cmd/join-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, options from bup.compat import argv_bytes diff --git a/lib/cmd/list-idx-cmd.py b/lib/cmd/list-idx-cmd.py index d9d318f..ebafdfa 100755 --- a/lib/cmd/list-idx-cmd.py +++ b/lib/cmd/list-idx-cmd.py @@ -16,7 +16,9 @@ exec "$bup_python" "$0" from __future__ import absolute_import, print_function from binascii import hexlify, unhexlify -import sys, os +import os, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, options from bup.compat import argv_bytes diff --git a/lib/cmd/ls-cmd.py b/lib/cmd/ls-cmd.py index b6900b9..6cc047b 100755 --- a/lib/cmd/ls-cmd.py +++ b/lib/cmd/ls-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, ls from bup.io import byte_stream diff --git a/lib/cmd/margin-cmd.py b/lib/cmd/margin-cmd.py index 9dc854a..825102b 100755 --- a/lib/cmd/margin-cmd.py +++ b/lib/cmd/margin-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys, struct, math +import math, os.path, struct, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, git, _helpers from bup.helpers import log diff --git a/lib/cmd/memtest-cmd.py b/lib/cmd/memtest-cmd.py index 85fd2f6..6f4dfd0 100755 --- a/lib/cmd/memtest-cmd.py +++ b/lib/cmd/memtest-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import, print_function -import sys, re, struct, time, resource +import os.path, re, resource, struct, sys, time + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, bloom, midx, options, _helpers from bup.compat import range diff --git a/lib/cmd/meta-cmd.py b/lib/cmd/meta-cmd.py index e006c0b..15eed43 100755 --- a/lib/cmd/meta-cmd.py +++ b/lib/cmd/meta-cmd.py @@ -22,7 +22,10 @@ exec "$bup_python" "$0" # TODO: Add tar-like -C option. from __future__ import absolute_import -import sys +import os, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, metadata from bup import options from bup.compat import argv_bytes diff --git a/lib/cmd/midx-cmd.py b/lib/cmd/midx-cmd.py index 53ea79c..b5ed7c4 100755 --- a/lib/cmd/midx-cmd.py +++ b/lib/cmd/midx-cmd.py @@ -18,6 +18,8 @@ from __future__ import absolute_import, print_function from binascii import hexlify import glob, math, os, resource, struct, sys, tempfile +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, git, midx, _helpers, xstat from bup.compat import argv_bytes, hexstr, range from bup.helpers import (Sha1, add_error, atomically_replaced_file, debug1, fdatasync, diff --git a/lib/cmd/mux-cmd.py b/lib/cmd/mux-cmd.py index 53fe7e6..1050262 100755 --- a/lib/cmd/mux-cmd.py +++ b/lib/cmd/mux-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import os, sys, subprocess, struct +import os, struct, subprocess, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options from bup.helpers import debug1, debug2, mux diff --git a/lib/cmd/on--server-cmd.py b/lib/cmd/on--server-cmd.py index 71ed4d1..12fee08 100755 --- a/lib/cmd/on--server-cmd.py +++ b/lib/cmd/on--server-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys, os, struct +import os, struct, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, helpers, path from bup.compat import environ, py_maj diff --git a/lib/cmd/on-cmd.py b/lib/cmd/on-cmd.py index fd89da4..0644a46 100755 --- a/lib/cmd/on-cmd.py +++ b/lib/cmd/on-cmd.py @@ -16,7 +16,9 @@ exec "$bup_python" "$0" from __future__ import absolute_import from subprocess import PIPE -import sys, os, struct, getopt, subprocess, signal +import getopt, os, signal, struct, subprocess, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, ssh, path from bup.compat import argv_bytes diff --git a/lib/cmd/prune-older-cmd.py b/lib/cmd/prune-older-cmd.py index 4780009..07bf7da 100755 --- a/lib/cmd/prune-older-cmd.py +++ b/lib/cmd/prune-older-cmd.py @@ -20,7 +20,9 @@ from collections import defaultdict from itertools import groupby from sys import stderr from time import localtime, strftime, time -import re, sys +import os.path, re, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, git, options from bup.compat import argv_bytes, int_types diff --git a/lib/cmd/python-cmd.sh b/lib/cmd/python-cmd.sh deleted file mode 100644 index b0a8d3d..0000000 --- a/lib/cmd/python-cmd.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh - -set -e - -top="$(pwd)" -cmdpath="$0" -# loop because macos has no recursive resolution -while test -L "$cmdpath"; do - link="$(readlink "$cmdpath")" - cd "$(dirname "$cmdpath")" - cmdpath="$link" -done -script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)" -cd "$top" - -bup_libdir="$script_home/.." # bup_libdir will be adjusted during install -export PYTHONPATH="$bup_libdir${PYTHONPATH:+:$PYTHONPATH}" - -# Force python to use ISO-8859-1 (aka Latin 1), a single-byte -# encoding, to help avoid any manipulation of data from system APIs -# (paths, users, groups, command line arguments, etc.) - -export PYTHONCOERCECLOCALE=0 # Perhaps not necessary, but shouldn't hurt - -# We can't just export LC_CTYPE directly here because the locale might -# not exist outside python, and then bash (at least) may be cranky. - -if [ "${LC_ALL+x}" ]; then - unset LC_ALL - exec env \ - BUP_LC_ALL="$LC_ALL" \ - LC_COLLATE="$LC_ALL" \ - LC_MONETARY="$LC_ALL" \ - LC_NUMERIC="$LC_ALL" \ - LC_TIME="$LC_ALL" \ - LC_MESSAGES="$LC_ALL" \ - LC_CTYPE=ISO-8859-1 \ - @bup_python@ "$@" -elif [ "${LC_CTYPE+x}" ]; then - exec env \ - BUP_LC_CTYPE="$LC_CTYPE" \ - LC_CTYPE=ISO-8859-1 \ - @bup_python@ "$@" -else - exec env \ - LC_CTYPE=ISO-8859-1 \ - @bup_python@ "$@" -fi diff --git a/lib/cmd/random-cmd.py b/lib/cmd/random-cmd.py index d939fb8..086dfff 100755 --- a/lib/cmd/random-cmd.py +++ b/lib/cmd/random-cmd.py @@ -17,6 +17,8 @@ exec "$bup_python" "$0" from __future__ import absolute_import import os, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, _helpers from bup.helpers import atoi, handle_ctrl_c, log, parse_num diff --git a/lib/cmd/restore-cmd.py b/lib/cmd/restore-cmd.py index f955cca..43071d1 100755 --- a/lib/cmd/restore-cmd.py +++ b/lib/cmd/restore-cmd.py @@ -16,7 +16,9 @@ exec "$bup_python" "$0" from __future__ import absolute_import from stat import S_ISDIR -import copy, errno, os, sys, stat, re +import copy, errno, os, re, stat, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, git, metadata, vfs from bup._helpers import write_sparsely diff --git a/lib/cmd/rm-cmd.py b/lib/cmd/rm-cmd.py index 6c753e2..4016c00 100755 --- a/lib/cmd/rm-cmd.py +++ b/lib/cmd/rm-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat from bup.compat import argv_bytes diff --git a/lib/cmd/save-cmd.py b/lib/cmd/save-cmd.py index 02e6841..c67e1ed 100755 --- a/lib/cmd/save-cmd.py +++ b/lib/cmd/save-cmd.py @@ -18,7 +18,9 @@ from __future__ import absolute_import, print_function from binascii import hexlify from errno import EACCES from io import BytesIO -import os, sys, stat, time, math +import math, os, stat, sys, time + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, hashsplit, git, options, index, client, metadata from bup import hlinkdb diff --git a/lib/cmd/server-cmd.py b/lib/cmd/server-cmd.py index 5b91814..a818445 100755 --- a/lib/cmd/server-cmd.py +++ b/lib/cmd/server-cmd.py @@ -16,7 +16,9 @@ exec "$bup_python" "$0" from __future__ import absolute_import from binascii import hexlify, unhexlify -import os, sys, struct, subprocess +import os, struct, subprocess, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, git, vfs, vint from bup.compat import environ, hexstr diff --git a/lib/cmd/split-cmd.py b/lib/cmd/split-cmd.py index d9cdc7e..6735d70 100755 --- a/lib/cmd/split-cmd.py +++ b/lib/cmd/split-cmd.py @@ -18,6 +18,8 @@ from __future__ import absolute_import, division, print_function from binascii import hexlify import os, sys, time +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, hashsplit, git, options, client from bup.compat import argv_bytes, environ from bup.helpers import (add_error, handle_ctrl_c, hostname, log, parse_num, diff --git a/lib/cmd/tag-cmd.py b/lib/cmd/tag-cmd.py index 2561aba..c2b6fae 100755 --- a/lib/cmd/tag-cmd.py +++ b/lib/cmd/tag-cmd.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from binascii import hexlify import os, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, git, options from bup.compat import argv_bytes from bup.helpers import debug1, handle_ctrl_c, log diff --git a/lib/cmd/tick-cmd.py b/lib/cmd/tick-cmd.py index 1a76350..dcafc81 100755 --- a/lib/cmd/tick-cmd.py +++ b/lib/cmd/tick-cmd.py @@ -15,10 +15,13 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import sys, time +import os, sys, time + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options + optspec = """ bup tick """ diff --git a/lib/cmd/version-cmd.py b/lib/cmd/version-cmd.py index 20fd218..7dcd279 100755 --- a/lib/cmd/version-cmd.py +++ b/lib/cmd/version-cmd.py @@ -15,7 +15,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import, print_function -import re, sys +import os.path, re, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, options, version diff --git a/lib/cmd/web-cmd.py b/lib/cmd/web-cmd.py index 8c084c1..7b8b7b6 100755 --- a/lib/cmd/web-cmd.py +++ b/lib/cmd/web-cmd.py @@ -19,6 +19,8 @@ from collections import namedtuple import mimetypes, os, posixpath, signal, stat, sys, time, urllib, webbrowser from binascii import hexlify +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] + from bup import compat, options, git, vfs from bup.helpers import (chunkyreader, debug1, format_filesize, handle_ctrl_c, log, saved_errors) diff --git a/lib/cmd/xstat-cmd.py b/lib/cmd/xstat-cmd.py index 297df72..8a56ba5 100755 --- a/lib/cmd/xstat-cmd.py +++ b/lib/cmd/xstat-cmd.py @@ -20,7 +20,9 @@ exec "$bup_python" "$0" # Public License as described in the bup LICENSE file. from __future__ import absolute_import, print_function -import sys, stat, errno +import errno, os.path, sys, stat + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..'] from bup import compat, metadata, options, xstat from bup.compat import argv_bytes diff --git a/t/echo-argv-bytes b/t/echo-argv-bytes index 66c5c63..127f85d 100755 --- a/t/echo-argv-bytes +++ b/t/echo-argv-bytes @@ -27,3 +27,4 @@ from bup import compat for arg in compat.argvb: os.write(stdout.fileno(), arg) os.write(stdout.fileno(), b'\0\n') + stdout.flush() diff --git a/t/hardlink-sets b/t/hardlink-sets index 4769154..0e6458a 100755 --- a/t/hardlink-sets +++ b/t/hardlink-sets @@ -16,6 +16,8 @@ exec "$bup_python" "$0" from __future__ import absolute_import, print_function import os, stat, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] + from bup import compat from bup.io import byte_stream diff --git a/t/ns-timestamp-resolutions b/t/ns-timestamp-resolutions index 574fad5..b9970f8 100755 --- a/t/ns-timestamp-resolutions +++ b/t/ns-timestamp-resolutions @@ -14,7 +14,9 @@ exec "$bup_python" "$0" # end of bup preamble from __future__ import absolute_import -import os, sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] from bup.compat import argv_bytes from bup.helpers import handle_ctrl_c, saved_errors diff --git a/t/sparse-test-data b/t/sparse-test-data index 8f982e0..d777f3c 100755 --- a/t/sparse-test-data +++ b/t/sparse-test-data @@ -7,7 +7,9 @@ exec "$bup_python" "$0" ${1+"$@"} from __future__ import absolute_import, print_function from random import randint from sys import stderr, stdout -import sys +import os, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] from bup.io import byte_stream diff --git a/t/subtree-hash b/t/subtree-hash index 1ca9e86..bcc21bb 100755 --- a/t/subtree-hash +++ b/t/subtree-hash @@ -6,7 +6,9 @@ exec "$bup_python" "$0" ${1+"$@"} # end of bup preamble from __future__ import absolute_import, print_function -import sys +import os.path, sys + +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] from bup.compat import argv_bytes from bup.helpers import handle_ctrl_c, readpipe diff --git a/t/test-ftp b/t/test-ftp index aa4b0a5..6aed724 100755 --- a/t/test-ftp +++ b/t/test-ftp @@ -14,6 +14,7 @@ import os, sys # For buptest, wvtest, ... sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),) +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] from buptest import ex, exo, logcmd, test_tempdir from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart diff --git a/t/test-get b/t/test-get index 3f42a0c..4c28fe5 100755 --- a/t/test-get +++ b/t/test-get @@ -23,6 +23,8 @@ import os, re, sys # For buptest, wvtest, ... sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),) +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] + from bup import compat, path from bup.compat import environ, getcwd, items diff --git a/t/test-on.sh b/t/test-on.sh index 24c2c38..871bf2e 100755 --- a/t/test-on.sh +++ b/t/test-on.sh @@ -47,7 +47,7 @@ WVSTART "index-cache" # the 'a-zA-Z0-9_' is '\w' from python, # the trailing _ is because there's no dir specified # and that should thus be empty -hostname=$(bup python -c "from bup import helpers ; print(helpers.hostname().decode('iso8859-1'))") +hostname=$(uname -n) idxcache=$(echo "$hostname" | sed 's/[^@a-zA-Z0-9_]/_/g')_ # there should be an index-cache now for idx in "$tmpdir"/bup/objects/pack/*.idx ; do diff --git a/t/test-prune-older b/t/test-prune-older index a2ea4df..5b07993 100755 --- a/t/test-prune-older +++ b/t/test-prune-older @@ -20,6 +20,7 @@ import os, random, sys # For buptest, wvtest, ... sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),) +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../lib'] from buptest import ex, exo, test_tempdir from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart diff --git a/t/test-save-errors b/t/test-save-errors index cafc306..312a9f5 100755 --- a/t/test-save-errors +++ b/t/test-save-errors @@ -34,7 +34,10 @@ WVPASS bup index "$tmpdir/save" # that gets an error for the .../5 file in metadata.from_path() cat > "$tmpdir/bup-save" << EOF #!/usr/bin/env $top/cmd/bup-python +import os.path, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../../../lib'] from bup import metadata + orig_from_path = metadata.from_path def from_path(path, *args, **kw): if path.endswith(b'/5'): @@ -70,7 +73,10 @@ WVPASS bup index "$tmpdir/save" cat > "$tmpdir/bup-save" << EOF #!/usr/bin/env $top/cmd/bup-python +import os.path, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../../../lib'] from bup import metadata + orig_from_path = metadata.from_path def from_path(path, *args, **kw): if path.endswith(b'/a'): @@ -102,6 +108,8 @@ WVPASS bup index "$tmpdir/save" cat > "$tmpdir/bup-save" << EOF #!/usr/bin/env $top/cmd/bup-python +import os.path, sys +sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/../../../lib'] from bup import index Reader = index.Reader -- 2.39.2