From: Rob Browning Date: Thu, 2 Jan 2020 18:28:09 +0000 (-0600) Subject: ftp: accommodate python 3 and test there X-Git-Tag: 0.31~123 X-Git-Url: https://arthur.barton.de/gitweb/?p=bup.git;a=commitdiff_plain;h=b213782c10c0e86ce7c9064e99c40965cfb79c9f ftp: accommodate python 3 and test there For now, completely rely on the bup-python LC_CTYPE=iso-8859-1 setting since at a minimum, bup.shquote can't handle bytes. Signed-off-by: Rob Browning Tested-by: Rob Browning --- diff --git a/Makefile b/Makefile index 67cc4b5..a8541b5 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,7 @@ cmdline_tests := \ t/test-compression.sh \ t/test-drecurse.sh \ t/test-fsck.sh \ + t/test-ftp \ t/test-gc.sh \ t/test-import-duplicity.sh \ t/test-import-rdiff-backup.sh \ @@ -202,7 +203,6 @@ cmdline_tests := \ ifeq "2" "$(bup_python_majver)" cmdline_tests += \ - t/test-ftp \ t/test-web.sh \ t/test-fuse.sh \ t/test-index-check-device.sh \ diff --git a/cmd/ftp-cmd.py b/cmd/ftp-cmd.py index f1e48bd..53b8c22 100755 --- a/cmd/ftp-cmd.py +++ b/cmd/ftp-cmd.py @@ -5,12 +5,18 @@ exec "$bup_python" "$0" ${1+"$@"} """ # end of bup preamble +# For now, this completely relies on the assumption that the current +# encoding (LC_CTYPE, etc.) is ASCII compatible, and that it returns +# the exact same bytes from a decode/encode round-trip (or the reverse +# (e.g. ISO-8859-1). + from __future__ import absolute_import, print_function import sys, os, stat, fnmatch from bup import options, git, shquote, ls, vfs -from bup.io import byte_stream +from bup.compat import argv_bytes, input from bup.helpers import chunkyreader, handle_ctrl_c, log +from bup.io import byte_stream, path_msg from bup.repo import LocalRepo handle_ctrl_c() @@ -20,6 +26,10 @@ class OptionError(Exception): pass +def input_bytes(s): + return s.encode('iso-8859-1') + + def do_ls(repo, args, out): try: opt = ls.opts_from_cmdline(args, onabort=OptionError) @@ -38,7 +48,7 @@ def inputiter(): if os.isatty(sys.stdin.fileno()): while 1: try: - yield raw_input('bup> ') + yield input('bup> ') except EOFError: print() # Clear the line for the terminal's next prompt break @@ -49,16 +59,16 @@ def inputiter(): def _completer_get_subs(repo, line): (qtype, lastword) = shquote.unfinished_word(line) - (dir,name) = os.path.split(lastword) - dir_path = vfs.resolve(repo, dir or '/') + dir, name = os.path.split(lastword.encode('iso-8859-1')) + dir_path = vfs.resolve(repo, dir or b'/') _, dir_item = dir_path[-1] if not dir_item: subs = tuple() else: subs = tuple(dir_path + (entry,) for entry in vfs.contents(repo, dir_item) - if (entry[0] != '.' and entry[0].startswith(name))) - return dir, name, qtype, lastword, subs + if (entry[0] != b'.' and entry[0].startswith(name))) + return qtype, lastword, subs _last_line = None @@ -72,7 +82,7 @@ def completer(text, iteration): if _last_line != line: _last_res = _completer_get_subs(repo, line) _last_line = line - (dir, name, qtype, lastword, subs) = _last_res + qtype, lastword, subs = _last_res if iteration < len(subs): path = subs[iteration] leaf_name, leaf_item = path[-1] @@ -80,11 +90,13 @@ def completer(text, iteration): leaf_name, leaf_item = res[-1] fullname = os.path.join(*(name for name, item in res)) if stat.S_ISDIR(vfs.item_mode(leaf_item)): - ret = shquote.what_to_add(qtype, lastword, fullname+'/', + ret = shquote.what_to_add(qtype, lastword, + fullname.decode('iso-8859-1') + '/', terminate=False) else: - ret = shquote.what_to_add(qtype, lastword, fullname, - terminate=True) + ' ' + ret = shquote.what_to_add(qtype, lastword, + fullname.decode('iso-8859-1'), + terminate=True) + b' ' return text + ret except Exception as e: log('\n') @@ -107,7 +119,7 @@ git.check_repo_or_die() sys.stdout.flush() out = byte_stream(sys.stdout) repo = LocalRepo() -pwd = vfs.resolve(repo, '/') +pwd = vfs.resolve(repo, b'/') rv = 0 if extra: @@ -137,72 +149,73 @@ for line in lines: try: if cmd == 'ls': # FIXME: respect pwd (perhaps via ls accepting resolve path/parent) - sys.stdout.flush() # FIXME: remove when we finish py3 support do_ls(repo, words[1:], out) elif cmd == 'cd': np = pwd for parm in words[1:]: - res = vfs.resolve(repo, parm, parent=np) + res = vfs.resolve(repo, input_bytes(parm), parent=np) _, leaf_item = res[-1] if not leaf_item: - raise Exception('%r does not exist' - % '/'.join(name for name, item in res)) + raise Exception('%s does not exist' + % path_msg(b'/'.join(name for name, item + in res))) if not stat.S_ISDIR(vfs.item_mode(leaf_item)): - raise Exception('%r is not a directory' % parm) + raise Exception('%s is not a directory' % path_msg(parm)) np = res pwd = np elif cmd == 'pwd': if len(pwd) == 1: - sys.stdout.write('/') - print('/'.join(name for name, item in pwd)) + out.write(b'/') + out.write(b'/'.join(name for name, item in pwd) + b'\n') elif cmd == 'cat': for parm in words[1:]: - res = vfs.resolve(repo, parm, parent=pwd) + res = vfs.resolve(repo, input_bytes(parm), parent=pwd) _, leaf_item = res[-1] if not leaf_item: - raise Exception('%r does not exist' % - '/'.join(name for name, item in res)) + raise Exception('%s does not exist' % + path_msg(b'/'.join(name for name, item + in res))) with vfs.fopen(repo, leaf_item) as srcfile: - write_to_file(srcfile, sys.stdout) + write_to_file(srcfile, out) elif cmd == 'get': if len(words) not in [2,3]: rv = 1 raise Exception('Usage: get [localname]') - rname = words[1] + rname = input_bytes(words[1]) (dir,base) = os.path.split(rname) - lname = len(words)>2 and words[2] or base + lname = input_bytes(len(words) > 2 and words[2] or base) res = vfs.resolve(repo, rname, parent=pwd) _, leaf_item = res[-1] if not leaf_item: - raise Exception('%r does not exist' % - '/'.join(name for name, item in res)) + raise Exception('%s does not exist' % + path_msg(b'/'.join(name for name, item in res))) with vfs.fopen(repo, leaf_item) as srcfile: with open(lname, 'wb') as destfile: - log('Saving %r\n' % lname) + log('Saving %s\n' % path_msg(lname)) write_to_file(srcfile, destfile) elif cmd == 'mget': for parm in words[1:]: - (dir,base) = os.path.split(parm) + dir, base = os.path.split(input_bytes(parm)) res = vfs.resolve(repo, dir, parent=pwd) _, dir_item = res[-1] if not dir_item: - raise Exception('%r does not exist' % dir) + raise Exception('%s does not exist' % path_msg(dir)) for name, item in vfs.contents(repo, dir_item): - if name == '.': + if name == b'.': continue if fnmatch.fnmatch(name, base): if stat.S_ISLNK(vfs.item_mode(item)): deref = vfs.resolve(repo, name, parent=res) deref_name, deref_item = deref[-1] if not deref_item: - raise Exception('%r does not exist' % - '/'.join(name for name, item - in deref)) + raise Exception('%s does not exist' % + path_msg('/'.join(name for name, item + in deref))) item = deref_item with vfs.fopen(repo, item) as srcfile: with open(name, 'wb') as destfile: - log('Saving %r\n' % name) + log('Saving %s\n' % path_msg(name)) write_to_file(srcfile, destfile) elif cmd == 'help' or cmd == '?': # FIXME: move to stdout diff --git a/lib/bup/compat.py b/lib/bup/compat.py index 692bd9c..03041f3 100644 --- a/lib/bup/compat.py +++ b/lib/bup/compat.py @@ -27,6 +27,7 @@ if py3: from os import fsencode from shlex import quote + input = input range = range str_type = str int_types = (int,) @@ -83,6 +84,7 @@ else: # Python 2 from bup.py2raise import reraise + input = raw_input range = xrange str_type = basestring int_types = (int, long) diff --git a/t/test-ftp b/t/test-ftp index 3a8e5db..e9f4166 100755 --- a/t/test-ftp +++ b/t/test-ftp @@ -6,21 +6,23 @@ exec "$bup_python" "$0" ${1+"$@"} # end of bup preamble from __future__ import absolute_import, print_function -from os import environ, chdir, mkdir, symlink, unlink +from os import chdir, mkdir, symlink, unlink from os.path import abspath, dirname from subprocess import PIPE from time import localtime, strftime import os, sys -script_home = abspath(dirname(sys.argv[0] or '.')) -sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')] -top = os.getcwd() -bup_cmd = top + '/bup' +# For buptest, wvtest, ... +sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),) from buptest import ex, exo, logcmd, test_tempdir from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart +from bup.compat import environ from bup.helpers import unlink as unlink_if_exists +import bup.path + +bup_cmd = bup.path.exe() def bup(*args, **kwargs): if 'stdout' not in kwargs: @@ -28,106 +30,108 @@ def bup(*args, **kwargs): return ex((bup_cmd,) + args, **kwargs) def jl(*lines): - return ''.join(line + '\n' for line in lines) + return b''.join(line + b'\n' for line in lines) -environ['GIT_AUTHOR_NAME'] = 'bup test' -environ['GIT_COMMITTER_NAME'] = 'bup test' -environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f' -environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f' +environ[b'GIT_AUTHOR_NAME'] = b'bup test' +environ[b'GIT_COMMITTER_NAME'] = b'bup test' +environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f' +environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f' -with test_tempdir('ftp-') as tmpdir: - environ['BUP_DIR'] = tmpdir + '/repo' - environ['GIT_DIR'] = tmpdir + '/repo' +with test_tempdir(b'ftp-') as tmpdir: + environ[b'BUP_DIR'] = tmpdir + b'/repo' + environ[b'GIT_DIR'] = tmpdir + b'/repo' chdir(tmpdir) - mkdir('src') - chdir('src') - mkdir('dir') - with open('file-1', 'wb') as f: - print('excitement!', file=f) - with open('dir/file-2', 'wb') as f: - print('more excitement!', file=f) - symlink('file-1', 'file-symlink') - symlink('dir', 'dir-symlink') - symlink('not-there', 'bad-symlink') + mkdir(b'src') + chdir(b'src') + mkdir(b'dir') + with open(b'file-1', 'wb') as f: + f.write(b'excitement!\n') + with open(b'dir/file-2', 'wb') as f: + f.write(b'more excitement!\n') + symlink(b'file-1', b'file-symlink') + symlink(b'dir', b'dir-symlink') + symlink(b'not-there', b'bad-symlink') chdir(tmpdir) - bup('init') - bup('index', 'src') - bup('save', '-n', 'src', '--strip', 'src') - save_utc = int(exo(('git', 'show', '-s', '--format=%at', 'src')).out.strip()) - save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)) + bup(b'init') + bup(b'index', b'src') + bup(b'save', b'-n', b'src', b'--strip', b'src') + save_utc = int(exo((b'git', b'show', + b'-s', b'--format=%at', b'src')).out.strip()) + save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)).encode('ascii') wvstart('help') - wvpasseq('Commands: ls cd pwd cat get mget help quit\n', - exo((bup_cmd, 'ftp'), input='help\n', stderr=PIPE).err) + wvpasseq(b'Commands: ls cd pwd cat get mget help quit\n', + exo((bup_cmd, b'ftp'), input=b'help\n', stderr=PIPE).err) wvstart('pwd/cd') - wvpasseq('/\n', bup('ftp', input='pwd\n').out) - wvpasseq('', bup('ftp', input='cd src\n').out) - wvpasseq('/src\n', bup('ftp', input=jl('cd src', 'pwd')).out) - wvpasseq('/src\n/\n', bup('ftp', input=jl('cd src', 'pwd', - 'cd ..', 'pwd')).out) - wvpasseq('/src\n/\n', bup('ftp', input=jl('cd src', 'pwd', - 'cd ..', 'cd ..', 'pwd')).out) - wvpasseq('/src/%s/dir\n' % save_name, - bup('ftp', input=jl('cd src/latest/dir-symlink', 'pwd')).out) - wvpasseq('/src/%s/dir\n' % save_name, - bup('ftp', input=jl('cd src latest dir-symlink', 'pwd')).out) - wvpassne(0, bup('ftp', - input=jl('cd src/latest/bad-symlink', 'pwd'), + wvpasseq(b'/\n', bup(b'ftp', input=b'pwd\n').out) + wvpasseq(b'', bup(b'ftp', input=b'cd src\n').out) + wvpasseq(b'/src\n', bup(b'ftp', input=jl(b'cd src', b'pwd')).out) + wvpasseq(b'/src\n/\n', bup(b'ftp', input=jl(b'cd src', b'pwd', + b'cd ..', b'pwd')).out) + wvpasseq(b'/src\n/\n', bup(b'ftp', input=jl(b'cd src', b'pwd', + b'cd ..', b'cd ..', + b'pwd')).out) + wvpasseq(b'/src/%s/dir\n' % save_name, + bup(b'ftp', input=jl(b'cd src/latest/dir-symlink', b'pwd')).out) + wvpasseq(b'/src/%s/dir\n' % save_name, + bup(b'ftp', input=jl(b'cd src latest dir-symlink', b'pwd')).out) + wvpassne(0, bup(b'ftp', + input=jl(b'cd src/latest/bad-symlink', b'pwd'), check=False, stdout=None).rc) - wvpassne(0, bup('ftp', - input=jl('cd src/latest/not-there', 'pwd'), + wvpassne(0, bup(b'ftp', + input=jl(b'cd src/latest/not-there', b'pwd'), check=False, stdout=None).rc) wvstart('ls') # FIXME: elaborate - wvpasseq('src\n', bup('ftp', input='ls\n').out) - wvpasseq(save_name + '\nlatest\n', - bup('ftp', input='ls src\n').out) + wvpasseq(b'src\n', bup(b'ftp', input=b'ls\n').out) + wvpasseq(save_name + b'\nlatest\n', + bup(b'ftp', input=b'ls src\n').out) wvstart('cat') - wvpasseq('excitement!\n', - bup('ftp', input='cat src/latest/file-1\n').out) - wvpasseq('excitement!\nmore excitement!\n', - bup('ftp', - input='cat src/latest/file-1 src/latest/dir/file-2\n').out) + wvpasseq(b'excitement!\n', + bup(b'ftp', input=b'cat src/latest/file-1\n').out) + wvpasseq(b'excitement!\nmore excitement!\n', + bup(b'ftp', + input=b'cat src/latest/file-1 src/latest/dir/file-2\n').out) wvstart('get') - bup('ftp', input=jl('get src/latest/file-1 dest')) - with open('dest', 'rb') as f: - wvpasseq('excitement!\n', f.read()) - unlink('dest') - bup('ftp', input=jl('get src/latest/file-symlink dest')) - with open('dest', 'rb') as f: - wvpasseq('excitement!\n', f.read()) - unlink('dest') - wvpassne(0, bup('ftp', - input=jl('get src/latest/bad-symlink dest'), + bup(b'ftp', input=jl(b'get src/latest/file-1 dest')) + with open(b'dest', 'rb') as f: + wvpasseq(b'excitement!\n', f.read()) + unlink(b'dest') + bup(b'ftp', input=jl(b'get src/latest/file-symlink dest')) + with open(b'dest', 'rb') as f: + wvpasseq(b'excitement!\n', f.read()) + unlink(b'dest') + wvpassne(0, bup(b'ftp', + input=jl(b'get src/latest/bad-symlink dest'), check=False, stdout=None).rc) - wvpassne(0, bup('ftp', - input=jl('get src/latest/not-there'), + wvpassne(0, bup(b'ftp', + input=jl(b'get src/latest/not-there'), check=False, stdout=None).rc) wvstart('mget') - unlink_if_exists('file-1') - bup('ftp', input=jl('mget src/latest/file-1')) - with open('file-1', 'rb') as f: - wvpasseq('excitement!\n', f.read()) - unlink_if_exists('file-1') - unlink_if_exists('file-2') - bup('ftp', input=jl('mget src/latest/file-1 src/latest/dir/file-2')) - with open('file-1', 'rb') as f: - wvpasseq('excitement!\n', f.read()) - with open('file-2', 'rb') as f: - wvpasseq('more excitement!\n', f.read()) - unlink_if_exists('file-symlink') - bup('ftp', input=jl('mget src/latest/file-symlink')) - with open('file-symlink', 'rb') as f: - wvpasseq('excitement!\n', f.read()) - wvpassne(0, bup('ftp', - input=jl('mget src/latest/bad-symlink dest'), + unlink_if_exists(b'file-1') + bup(b'ftp', input=jl(b'mget src/latest/file-1')) + with open(b'file-1', 'rb') as f: + wvpasseq(b'excitement!\n', f.read()) + unlink_if_exists(b'file-1') + unlink_if_exists(b'file-2') + bup(b'ftp', input=jl(b'mget src/latest/file-1 src/latest/dir/file-2')) + with open(b'file-1', 'rb') as f: + wvpasseq(b'excitement!\n', f.read()) + with open(b'file-2', 'rb') as f: + wvpasseq(b'more excitement!\n', f.read()) + unlink_if_exists(b'file-symlink') + bup(b'ftp', input=jl(b'mget src/latest/file-symlink')) + with open(b'file-symlink', 'rb') as f: + wvpasseq(b'excitement!\n', f.read()) + wvpassne(0, bup(b'ftp', + input=jl(b'mget src/latest/bad-symlink dest'), check=False, stdout=None).rc) # bup mget currently always does pattern matching - bup('ftp', input='mget src/latest/not-there\n') + bup(b'ftp', input=b'mget src/latest/not-there\n')