ftp: accommodate python 3 and test there
authorRob Browning <rlb@defaultvalue.org>
Thu, 2 Jan 2020 18:28:09 +0000 (12:28 -0600)
committerRob Browning <rlb@defaultvalue.org>
Sun, 2 Feb 2020 19:30:12 +0000 (13:30 -0600)
For now, completely rely on the bup-python LC_CTYPE=iso-8859-1 setting
since at a minimum, bup.shquote can't handle bytes.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>
Makefile
cmd/ftp-cmd.py
lib/bup/compat.py
t/test-ftp

index 67cc4b5..a8541b5 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -174,6 +174,7 @@ cmdline_tests := \
   t/test-compression.sh \
   t/test-drecurse.sh \
   t/test-fsck.sh \
+  t/test-ftp \
   t/test-gc.sh \
   t/test-import-duplicity.sh \
   t/test-import-rdiff-backup.sh \
@@ -202,7 +203,6 @@ cmdline_tests := \
 
 ifeq "2" "$(bup_python_majver)"
   cmdline_tests += \
-    t/test-ftp \
     t/test-web.sh \
     t/test-fuse.sh \
     t/test-index-check-device.sh \
index f1e48bd..53b8c22 100755 (executable)
@@ -5,12 +5,18 @@ exec "$bup_python" "$0" ${1+"$@"}
 """
 # end of bup preamble
 
+# For now, this completely relies on the assumption that the current
+# encoding (LC_CTYPE, etc.) is ASCII compatible, and that it returns
+# the exact same bytes from a decode/encode round-trip (or the reverse
+# (e.g. ISO-8859-1).
+
 from __future__ import absolute_import, print_function
 import sys, os, stat, fnmatch
 
 from bup import options, git, shquote, ls, vfs
-from bup.io import byte_stream
+from bup.compat import argv_bytes, input
 from bup.helpers import chunkyreader, handle_ctrl_c, log
+from bup.io import byte_stream, path_msg
 from bup.repo import LocalRepo
 
 handle_ctrl_c()
@@ -20,6 +26,10 @@ class OptionError(Exception):
     pass
 
 
+def input_bytes(s):
+    return s.encode('iso-8859-1')
+
+
 def do_ls(repo, args, out):
     try:
         opt = ls.opts_from_cmdline(args, onabort=OptionError)
@@ -38,7 +48,7 @@ def inputiter():
     if os.isatty(sys.stdin.fileno()):
         while 1:
             try:
-                yield raw_input('bup> ')
+                yield input('bup> ')
             except EOFError:
                 print()  # Clear the line for the terminal's next prompt
                 break
@@ -49,16 +59,16 @@ def inputiter():
 
 def _completer_get_subs(repo, line):
     (qtype, lastword) = shquote.unfinished_word(line)
-    (dir,name) = os.path.split(lastword)
-    dir_path = vfs.resolve(repo, dir or '/')
+    dir, name = os.path.split(lastword.encode('iso-8859-1'))
+    dir_path = vfs.resolve(repo, dir or b'/')
     _, dir_item = dir_path[-1]
     if not dir_item:
         subs = tuple()
     else:
         subs = tuple(dir_path + (entry,)
                      for entry in vfs.contents(repo, dir_item)
-                     if (entry[0] != '.' and entry[0].startswith(name)))
-    return dir, name, qtype, lastword, subs
+                     if (entry[0] != b'.' and entry[0].startswith(name)))
+    return qtype, lastword, subs
 
 
 _last_line = None
@@ -72,7 +82,7 @@ def completer(text, iteration):
         if _last_line != line:
             _last_res = _completer_get_subs(repo, line)
             _last_line = line
-        (dir, name, qtype, lastword, subs) = _last_res
+        qtype, lastword, subs = _last_res
         if iteration < len(subs):
             path = subs[iteration]
             leaf_name, leaf_item = path[-1]
@@ -80,11 +90,13 @@ def completer(text, iteration):
             leaf_name, leaf_item = res[-1]
             fullname = os.path.join(*(name for name, item in res))
             if stat.S_ISDIR(vfs.item_mode(leaf_item)):
-                ret = shquote.what_to_add(qtype, lastword, fullname+'/',
+                ret = shquote.what_to_add(qtype, lastword,
+                                          fullname.decode('iso-8859-1') + '/',
                                           terminate=False)
             else:
-                ret = shquote.what_to_add(qtype, lastword, fullname,
-                                          terminate=True) + ' '
+                ret = shquote.what_to_add(qtype, lastword,
+                                          fullname.decode('iso-8859-1'),
+                                          terminate=True) + b' '
             return text + ret
     except Exception as e:
         log('\n')
@@ -107,7 +119,7 @@ git.check_repo_or_die()
 sys.stdout.flush()
 out = byte_stream(sys.stdout)
 repo = LocalRepo()
-pwd = vfs.resolve(repo, '/')
+pwd = vfs.resolve(repo, b'/')
 rv = 0
 
 if extra:
@@ -137,72 +149,73 @@ for line in lines:
     try:
         if cmd == 'ls':
             # FIXME: respect pwd (perhaps via ls accepting resolve path/parent)
-            sys.stdout.flush()  # FIXME: remove when we finish py3 support
             do_ls(repo, words[1:], out)
         elif cmd == 'cd':
             np = pwd
             for parm in words[1:]:
-                res = vfs.resolve(repo, parm, parent=np)
+                res = vfs.resolve(repo, input_bytes(parm), parent=np)
                 _, leaf_item = res[-1]
                 if not leaf_item:
-                    raise Exception('%r does not exist'
-                                    % '/'.join(name for name, item in res))
+                    raise Exception('%s does not exist'
+                                    % path_msg(b'/'.join(name for name, item
+                                                         in res)))
                 if not stat.S_ISDIR(vfs.item_mode(leaf_item)):
-                    raise Exception('%r is not a directory' % parm)
+                    raise Exception('%s is not a directory' % path_msg(parm))
                 np = res
             pwd = np
         elif cmd == 'pwd':
             if len(pwd) == 1:
-                sys.stdout.write('/')
-            print('/'.join(name for name, item in pwd))
+                out.write(b'/')
+            out.write(b'/'.join(name for name, item in pwd) + b'\n')
         elif cmd == 'cat':
             for parm in words[1:]:
-                res = vfs.resolve(repo, parm, parent=pwd)
+                res = vfs.resolve(repo, input_bytes(parm), parent=pwd)
                 _, leaf_item = res[-1]
                 if not leaf_item:
-                    raise Exception('%r does not exist' %
-                                    '/'.join(name for name, item in res))
+                    raise Exception('%s does not exist' %
+                                    path_msg(b'/'.join(name for name, item
+                                                       in res)))
                 with vfs.fopen(repo, leaf_item) as srcfile:
-                    write_to_file(srcfile, sys.stdout)
+                    write_to_file(srcfile, out)
         elif cmd == 'get':
             if len(words) not in [2,3]:
                 rv = 1
                 raise Exception('Usage: get <filename> [localname]')
-            rname = words[1]
+            rname = input_bytes(words[1])
             (dir,base) = os.path.split(rname)
-            lname = len(words)>2 and words[2] or base
+            lname = input_bytes(len(words) > 2 and words[2] or base)
             res = vfs.resolve(repo, rname, parent=pwd)
             _, leaf_item = res[-1]
             if not leaf_item:
-                raise Exception('%r does not exist' %
-                                '/'.join(name for name, item in res))
+                raise Exception('%s does not exist' %
+                                path_msg(b'/'.join(name for name, item in res)))
             with vfs.fopen(repo, leaf_item) as srcfile:
                 with open(lname, 'wb') as destfile:
-                    log('Saving %r\n' % lname)
+                    log('Saving %s\n' % path_msg(lname))
                     write_to_file(srcfile, destfile)
         elif cmd == 'mget':
             for parm in words[1:]:
-                (dir,base) = os.path.split(parm)
+                dir, base = os.path.split(input_bytes(parm))
 
                 res = vfs.resolve(repo, dir, parent=pwd)
                 _, dir_item = res[-1]
                 if not dir_item:
-                    raise Exception('%r does not exist' % dir)
+                    raise Exception('%s does not exist' % path_msg(dir))
                 for name, item in vfs.contents(repo, dir_item):
-                    if name == '.':
+                    if name == b'.':
                         continue
                     if fnmatch.fnmatch(name, base):
                         if stat.S_ISLNK(vfs.item_mode(item)):
                             deref = vfs.resolve(repo, name, parent=res)
                             deref_name, deref_item = deref[-1]
                             if not deref_item:
-                                raise Exception('%r does not exist' %
-                                                '/'.join(name for name, item
-                                                         in deref))
+                                raise Exception('%s does not exist' %
+                                                path_msg('/'.join(name for name, item
+                                                                  in deref)))
                             item = deref_item
                         with vfs.fopen(repo, item) as srcfile:
                             with open(name, 'wb') as destfile:
-                                log('Saving %r\n' % name)
+                                log('Saving %s\n' % path_msg(name))
                                 write_to_file(srcfile, destfile)
         elif cmd == 'help' or cmd == '?':
             # FIXME: move to stdout
index 692bd9c..03041f3 100644 (file)
@@ -27,6 +27,7 @@ if py3:
 
     from os import fsencode
     from shlex import quote
+    input = input
     range = range
     str_type = str
     int_types = (int,)
@@ -83,6 +84,7 @@ else:  # Python 2
 
     from bup.py2raise import reraise
 
+    input = raw_input
     range = xrange
     str_type = basestring
     int_types = (int, long)
index 3a8e5db..e9f4166 100755 (executable)
@@ -6,21 +6,23 @@ exec "$bup_python" "$0" ${1+"$@"}
 # end of bup preamble
 
 from __future__ import absolute_import, print_function
-from os import environ, chdir, mkdir, symlink, unlink
+from os import chdir, mkdir, symlink, unlink
 from os.path import abspath, dirname
 from subprocess import PIPE
 from time import localtime, strftime
 import os, sys
 
-script_home = abspath(dirname(sys.argv[0] or '.'))
-sys.path[:0] = [abspath(script_home + '/../lib'), abspath(script_home + '/..')]
-top = os.getcwd()
-bup_cmd = top + '/bup'
+# For buptest, wvtest, ...
+sys.path[:0] = (abspath(os.path.dirname(__file__) + '/..'),)
 
 from buptest import ex, exo, logcmd, test_tempdir
 from wvtest import wvfail, wvpass, wvpasseq, wvpassne, wvstart
 
+from bup.compat import environ
 from bup.helpers import unlink as unlink_if_exists
+import bup.path
+
+bup_cmd = bup.path.exe()
 
 def bup(*args, **kwargs):
     if 'stdout' not in kwargs:
@@ -28,106 +30,108 @@ def bup(*args, **kwargs):
     return ex((bup_cmd,) + args, **kwargs)
 
 def jl(*lines):
-    return ''.join(line + '\n' for line in lines)
+    return b''.join(line + b'\n' for line in lines)
 
-environ['GIT_AUTHOR_NAME'] = 'bup test'
-environ['GIT_COMMITTER_NAME'] = 'bup test'
-environ['GIT_AUTHOR_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
-environ['GIT_COMMITTER_EMAIL'] = 'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_AUTHOR_NAME'] = b'bup test'
+environ[b'GIT_COMMITTER_NAME'] = b'bup test'
+environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
+environ[b'GIT_COMMITTER_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
 
-with test_tempdir('ftp-') as tmpdir:
-    environ['BUP_DIR'] = tmpdir + '/repo'
-    environ['GIT_DIR'] = tmpdir + '/repo'
+with test_tempdir(b'ftp-') as tmpdir:
+    environ[b'BUP_DIR'] = tmpdir + b'/repo'
+    environ[b'GIT_DIR'] = tmpdir + b'/repo'
 
     chdir(tmpdir)
-    mkdir('src')
-    chdir('src')
-    mkdir('dir')
-    with open('file-1', 'wb') as f:
-        print('excitement!', file=f)
-    with open('dir/file-2', 'wb') as f:
-        print('more excitement!', file=f)
-    symlink('file-1', 'file-symlink')
-    symlink('dir', 'dir-symlink')
-    symlink('not-there', 'bad-symlink')
+    mkdir(b'src')
+    chdir(b'src')
+    mkdir(b'dir')
+    with open(b'file-1', 'wb') as f:
+        f.write(b'excitement!\n')
+    with open(b'dir/file-2', 'wb') as f:
+        f.write(b'more excitement!\n')
+    symlink(b'file-1', b'file-symlink')
+    symlink(b'dir', b'dir-symlink')
+    symlink(b'not-there', b'bad-symlink')
 
     chdir(tmpdir)    
-    bup('init')
-    bup('index', 'src')
-    bup('save', '-n', 'src', '--strip', 'src')
-    save_utc = int(exo(('git', 'show', '-s', '--format=%at', 'src')).out.strip())
-    save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc))
+    bup(b'init')
+    bup(b'index', b'src')
+    bup(b'save', b'-n', b'src', b'--strip', b'src')
+    save_utc = int(exo((b'git', b'show',
+                        b'-s', b'--format=%at', b'src')).out.strip())
+    save_name = strftime('%Y-%m-%d-%H%M%S', localtime(save_utc)).encode('ascii')
     
     wvstart('help')
-    wvpasseq('Commands: ls cd pwd cat get mget help quit\n',
-             exo((bup_cmd, 'ftp'), input='help\n', stderr=PIPE).err)
+    wvpasseq(b'Commands: ls cd pwd cat get mget help quit\n',
+             exo((bup_cmd, b'ftp'), input=b'help\n', stderr=PIPE).err)
 
     wvstart('pwd/cd')
-    wvpasseq('/\n', bup('ftp', input='pwd\n').out)
-    wvpasseq('', bup('ftp', input='cd src\n').out)
-    wvpasseq('/src\n', bup('ftp', input=jl('cd src', 'pwd')).out)
-    wvpasseq('/src\n/\n', bup('ftp', input=jl('cd src', 'pwd',
-                                              'cd ..', 'pwd')).out)
-    wvpasseq('/src\n/\n', bup('ftp', input=jl('cd src', 'pwd',
-                                              'cd ..', 'cd ..', 'pwd')).out)
-    wvpasseq('/src/%s/dir\n' % save_name,
-             bup('ftp', input=jl('cd src/latest/dir-symlink', 'pwd')).out)
-    wvpasseq('/src/%s/dir\n' % save_name,
-             bup('ftp', input=jl('cd src latest dir-symlink', 'pwd')).out)
-    wvpassne(0, bup('ftp',
-                    input=jl('cd src/latest/bad-symlink', 'pwd'),
+    wvpasseq(b'/\n', bup(b'ftp', input=b'pwd\n').out)
+    wvpasseq(b'', bup(b'ftp', input=b'cd src\n').out)
+    wvpasseq(b'/src\n', bup(b'ftp', input=jl(b'cd src', b'pwd')).out)
+    wvpasseq(b'/src\n/\n', bup(b'ftp', input=jl(b'cd src', b'pwd',
+                                                b'cd ..', b'pwd')).out)
+    wvpasseq(b'/src\n/\n', bup(b'ftp', input=jl(b'cd src', b'pwd',
+                                                b'cd ..', b'cd ..',
+                                                b'pwd')).out)
+    wvpasseq(b'/src/%s/dir\n' % save_name,
+             bup(b'ftp', input=jl(b'cd src/latest/dir-symlink', b'pwd')).out)
+    wvpasseq(b'/src/%s/dir\n' % save_name,
+             bup(b'ftp', input=jl(b'cd src latest dir-symlink', b'pwd')).out)
+    wvpassne(0, bup(b'ftp',
+                    input=jl(b'cd src/latest/bad-symlink', b'pwd'),
                     check=False, stdout=None).rc)
-    wvpassne(0, bup('ftp',
-                    input=jl('cd src/latest/not-there', 'pwd'),
+    wvpassne(0, bup(b'ftp',
+                    input=jl(b'cd src/latest/not-there', b'pwd'),
                     check=False, stdout=None).rc)
 
     wvstart('ls')
     # FIXME: elaborate
-    wvpasseq('src\n', bup('ftp', input='ls\n').out)
-    wvpasseq(save_name + '\nlatest\n',
-             bup('ftp', input='ls src\n').out)
+    wvpasseq(b'src\n', bup(b'ftp', input=b'ls\n').out)
+    wvpasseq(save_name + b'\nlatest\n',
+             bup(b'ftp', input=b'ls src\n').out)
 
     wvstart('cat')
-    wvpasseq('excitement!\n',
-             bup('ftp', input='cat src/latest/file-1\n').out)
-    wvpasseq('excitement!\nmore excitement!\n',
-             bup('ftp',
-                 input='cat src/latest/file-1 src/latest/dir/file-2\n').out)
+    wvpasseq(b'excitement!\n',
+             bup(b'ftp', input=b'cat src/latest/file-1\n').out)
+    wvpasseq(b'excitement!\nmore excitement!\n',
+             bup(b'ftp',
+                 input=b'cat src/latest/file-1 src/latest/dir/file-2\n').out)
     
     wvstart('get')
-    bup('ftp', input=jl('get src/latest/file-1 dest'))
-    with open('dest', 'rb') as f:
-        wvpasseq('excitement!\n', f.read())
-    unlink('dest')
-    bup('ftp', input=jl('get src/latest/file-symlink dest'))
-    with open('dest', 'rb') as f:
-        wvpasseq('excitement!\n', f.read())
-    unlink('dest')
-    wvpassne(0, bup('ftp',
-                    input=jl('get src/latest/bad-symlink dest'),
+    bup(b'ftp', input=jl(b'get src/latest/file-1 dest'))
+    with open(b'dest', 'rb') as f:
+        wvpasseq(b'excitement!\n', f.read())
+    unlink(b'dest')
+    bup(b'ftp', input=jl(b'get src/latest/file-symlink dest'))
+    with open(b'dest', 'rb') as f:
+        wvpasseq(b'excitement!\n', f.read())
+    unlink(b'dest')
+    wvpassne(0, bup(b'ftp',
+                    input=jl(b'get src/latest/bad-symlink dest'),
                     check=False, stdout=None).rc)
-    wvpassne(0, bup('ftp',
-                    input=jl('get src/latest/not-there'),
+    wvpassne(0, bup(b'ftp',
+                    input=jl(b'get src/latest/not-there'),
                     check=False, stdout=None).rc)
     
     wvstart('mget')
-    unlink_if_exists('file-1')
-    bup('ftp', input=jl('mget src/latest/file-1'))
-    with open('file-1', 'rb') as f:
-        wvpasseq('excitement!\n', f.read())
-    unlink_if_exists('file-1')
-    unlink_if_exists('file-2')
-    bup('ftp', input=jl('mget src/latest/file-1 src/latest/dir/file-2'))
-    with open('file-1', 'rb') as f:
-        wvpasseq('excitement!\n', f.read())
-    with open('file-2', 'rb') as f:
-        wvpasseq('more excitement!\n', f.read())
-    unlink_if_exists('file-symlink')
-    bup('ftp', input=jl('mget src/latest/file-symlink'))
-    with open('file-symlink', 'rb') as f:
-        wvpasseq('excitement!\n', f.read())
-    wvpassne(0, bup('ftp',
-                    input=jl('mget src/latest/bad-symlink dest'),
+    unlink_if_exists(b'file-1')
+    bup(b'ftp', input=jl(b'mget src/latest/file-1'))
+    with open(b'file-1', 'rb') as f:
+        wvpasseq(b'excitement!\n', f.read())
+    unlink_if_exists(b'file-1')
+    unlink_if_exists(b'file-2')
+    bup(b'ftp', input=jl(b'mget src/latest/file-1 src/latest/dir/file-2'))
+    with open(b'file-1', 'rb') as f:
+        wvpasseq(b'excitement!\n', f.read())
+    with open(b'file-2', 'rb') as f:
+        wvpasseq(b'more excitement!\n', f.read())
+    unlink_if_exists(b'file-symlink')
+    bup(b'ftp', input=jl(b'mget src/latest/file-symlink'))
+    with open(b'file-symlink', 'rb') as f:
+        wvpasseq(b'excitement!\n', f.read())
+    wvpassne(0, bup(b'ftp',
+                    input=jl(b'mget src/latest/bad-symlink dest'),
                     check=False, stdout=None).rc)
     # bup mget currently always does pattern matching
-    bup('ftp', input='mget src/latest/not-there\n')
+    bup(b'ftp', input=b'mget src/latest/not-there\n')