From 453fcdac8595cf93c65c7deebf1f6557c0e410db Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Tue, 29 Dec 2015 01:31:02 -0600 Subject: [PATCH] Use BytesIO instead of StringIO More accurate, and Python 3 compatible. Signed-off-by: Rob Browning Tested-by: Rob Browning --- cmd/save-cmd.py | 4 ++-- lib/bup/hashsplit.py | 20 ++++++++++++-------- lib/bup/metadata.py | 8 +++++--- lib/bup/t/thashsplit.py | 12 ++++++------ lib/bup/t/tvint.py | 30 ++++++++++++++++-------------- lib/bup/vint.py | 6 +++--- 6 files changed, 44 insertions(+), 36 deletions(-) diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py index e48170a..7f72d00 100755 --- a/cmd/save-cmd.py +++ b/cmd/save-cmd.py @@ -5,8 +5,8 @@ exec "$bup_python" "$0" ${1+"$@"} """ # end of bup preamble import sys, stat, time, math -from cStringIO import StringIO from errno import EACCES +from io import BytesIO from bup import hashsplit, git, options, index, client, metadata, hlinkdb from bup.helpers import * @@ -141,7 +141,7 @@ def _pop(force_tree, dir_metadata=None): metalist = [('', dir_metadata)] + metalist[1:] sorted_metalist = sorted(metalist, key = lambda x : x[0]) metadata = ''.join([m[1].encode() for m in sorted_metalist]) - metadata_f = StringIO(metadata) + metadata_f = BytesIO(metadata) mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree, [metadata_f], keep_boundaries=False) diff --git a/lib/bup/hashsplit.py b/lib/bup/hashsplit.py index 33e91a6..ddf8d0c 100644 --- a/lib/bup/hashsplit.py +++ b/lib/bup/hashsplit.py @@ -1,4 +1,4 @@ -import math, os +import io, math, os from bup import _helpers, helpers from bup.helpers import sc_page_size @@ -95,13 +95,17 @@ def readfile_iter(files, progress=None): b = '' fd = rpr = rstart = rlen = None if _fmincore and hasattr(f, 'fileno'): - fd = f.fileno() - mcore = _fmincore(fd) - if mcore: - max_chunk = max(1, (8 * 1024 * 1024) / sc_page_size) - rpr = _nonresident_page_regions(mcore, helpers.MINCORE_INCORE, - max_chunk) - rstart, rlen = next(rpr, (None, None)) + try: + fd = f.fileno() + except io.UnsupportedOperation: + pass + if fd: + mcore = _fmincore(fd) + if mcore: + max_chunk = max(1, (8 * 1024 * 1024) / sc_page_size) + rpr = _nonresident_page_regions(mcore, helpers.MINCORE_INCORE, + max_chunk) + rstart, rlen = next(rpr, (None, None)) while 1: if progress: progress(filenum, len(b)) diff --git a/lib/bup/metadata.py b/lib/bup/metadata.py index 2243936..bdae4d3 100644 --- a/lib/bup/metadata.py +++ b/lib/bup/metadata.py @@ -4,8 +4,10 @@ # # This code is covered under the terms of the GNU Library General # Public License as described in the bup LICENSE file. + +from io import BytesIO import errno, os, sys, stat, time, pwd, grp, socket, struct -from cStringIO import StringIO + from bup import vint, xstat from bup.drecurse import recursive_dirlist from bup.helpers import add_error, mkdirp, log, is_superuser, format_filesize @@ -646,7 +648,7 @@ class Metadata: def _load_linux_xattr_rec(self, file): data = vint.read_bvec(file) - memfile = StringIO(data) + memfile = BytesIO(data) result = [] for i in range(vint.read_vuint(memfile)): key = vint.read_bvec(memfile) @@ -746,7 +748,7 @@ class Metadata: vint.write_vuint(port, _rec_tag_end) def encode(self, include_path=True): - port = StringIO() + port = BytesIO() self.write(port, include_path) return port.getvalue() diff --git a/lib/bup/t/thashsplit.py b/lib/bup/t/thashsplit.py index ca2cb6b..2a8604c 100644 --- a/lib/bup/t/thashsplit.py +++ b/lib/bup/t/thashsplit.py @@ -1,4 +1,4 @@ -from cStringIO import StringIO +from io import BytesIO from wvtest import * @@ -113,11 +113,11 @@ def test_fanout_behaviour(): # Return a byte which will be split with a level of n sb = lambda n: chr(basebits + n) - split_never = StringIO(z(16)) - split_first = StringIO(z(1) + sb(3) + z(14)) - split_end = StringIO(z(13) + sb(1) + z(2)) - split_many = StringIO(sb(1) + z(3) + sb(2) + z(4) + - sb(0) + z(4) + sb(5) + z(1)) + split_never = BytesIO(z(16)) + split_first = BytesIO(z(1) + sb(3) + z(14)) + split_end = BytesIO(z(13) + sb(1) + z(2)) + split_many = BytesIO(sb(1) + z(3) + sb(2) + z(4) + + sb(0) + z(4) + sb(5) + z(1)) WVPASSEQ(levels(split_never), [(4, 0), (4, 0), (4, 0), (4, 0)]) WVPASSEQ(levels(split_first), [(2, 3), (4, 0), (4, 0), (4, 0), (2, 0)]) WVPASSEQ(levels(split_end), [(4, 0), (4, 0), (4, 0), (2, 1), (2, 0)]) diff --git a/lib/bup/t/tvint.py b/lib/bup/t/tvint.py index 1be4c42..0fc41bd 100644 --- a/lib/bup/t/tvint.py +++ b/lib/bup/t/tvint.py @@ -1,26 +1,28 @@ -from bup import vint +from io import BytesIO + from wvtest import * -from cStringIO import StringIO + +from bup import vint def encode_and_decode_vuint(x): - f = StringIO() + f = BytesIO() vint.write_vuint(f, x) - return vint.read_vuint(StringIO(f.getvalue())) + return vint.read_vuint(BytesIO(f.getvalue())) @wvtest def test_vuint(): for x in (0, 1, 42, 128, 10**16): WVPASSEQ(encode_and_decode_vuint(x), x) - WVEXCEPT(Exception, vint.write_vuint, StringIO(), -1) - WVEXCEPT(EOFError, vint.read_vuint, StringIO()) + WVEXCEPT(Exception, vint.write_vuint, BytesIO(), -1) + WVEXCEPT(EOFError, vint.read_vuint, BytesIO()) def encode_and_decode_vint(x): - f = StringIO() + f = BytesIO() vint.write_vint(f, x) - return vint.read_vint(StringIO(f.getvalue())) + return vint.read_vint(BytesIO(f.getvalue())) @wvtest @@ -30,13 +32,13 @@ def test_vint(): WVPASSEQ(encode_and_decode_vint(x), x) for x in [-x for x in values]: WVPASSEQ(encode_and_decode_vint(x), x) - WVEXCEPT(EOFError, vint.read_vint, StringIO()) + WVEXCEPT(EOFError, vint.read_vint, BytesIO()) def encode_and_decode_bvec(x): - f = StringIO() + f = BytesIO() vint.write_bvec(f, x) - return vint.read_bvec(StringIO(f.getvalue())) + return vint.read_bvec(BytesIO(f.getvalue())) @wvtest @@ -44,11 +46,11 @@ def test_bvec(): values = ('', 'x', 'foo', '\0', '\0foo', 'foo\0bar\0') for x in values: WVPASSEQ(encode_and_decode_bvec(x), x) - WVEXCEPT(EOFError, vint.read_bvec, StringIO()) - outf = StringIO() + WVEXCEPT(EOFError, vint.read_bvec, BytesIO()) + outf = BytesIO() for x in ('foo', 'bar', 'baz', 'bax'): vint.write_bvec(outf, x) - inf = StringIO(outf.getvalue()) + inf = BytesIO(outf.getvalue()) WVPASSEQ(vint.read_bvec(inf), 'foo') WVPASSEQ(vint.read_bvec(inf), 'bar') vint.skip_bvec(inf) diff --git a/lib/bup/vint.py b/lib/bup/vint.py index bca93bd..378234c 100644 --- a/lib/bup/vint.py +++ b/lib/bup/vint.py @@ -5,7 +5,7 @@ # This code is covered under the terms of the GNU Library General # Public License as described in the bup LICENSE file. -from cStringIO import StringIO +from io import BytesIO # Variable length integers are encoded as vints -- see jakarta lucene. @@ -113,7 +113,7 @@ def skip_bvec(port): def pack(types, *args): if len(types) != len(args): raise Exception('number of arguments does not match format string') - port = StringIO() + port = BytesIO() for (type, value) in zip(types, args): if type == 'V': write_vuint(port, value) @@ -128,7 +128,7 @@ def pack(types, *args): def unpack(types, data): result = [] - port = StringIO(data) + port = BytesIO(data) for type in types: if type == 'V': result.append(read_vuint(port)) -- 2.39.2