From: Rob Browning Date: Sun, 22 Dec 2019 19:03:37 +0000 (-0600) Subject: Fix tindex for python 3 X-Git-Tag: 0.31~237 X-Git-Url: https://arthur.barton.de/gitweb/?p=bup.git;a=commitdiff_plain;h=aac44c48f74679f9f14cd1e98b7e117fe8a61de0 Fix tindex for python 3 Signed-off-by: Rob Browning --- diff --git a/Makefile b/Makefile index 560652b..c270e97 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,7 @@ runtests: runtests-python runtests-cmdline python_tests := \ lib/bup/t/tbloom.py \ lib/bup/t/thashsplit.py \ + lib/bup/t/tindex.py \ lib/bup/t/toptions.py \ lib/bup/t/tshquote.py \ lib/bup/t/tvint.py \ @@ -156,7 +157,6 @@ ifeq "2" "$(bup_python_majver)" lib/bup/t/tclient.py \ lib/bup/t/tgit.py \ lib/bup/t/thelpers.py \ - lib/bup/t/tindex.py \ lib/bup/t/tmetadata.py \ lib/bup/t/tresolve.py \ lib/bup/t/tvfs.py diff --git a/cmd/index-cmd.py b/cmd/index-cmd.py index 51a43bb..61a679a 100755 --- a/cmd/index-cmd.py +++ b/cmd/index-cmd.py @@ -37,13 +37,13 @@ def check_index(reader): log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n, e.name)) assert(e.children_ofs) - assert(e.name.endswith('/')) + assert e.name[-1] == b'/' assert(not d.get(e.children_ofs)) d[e.children_ofs] = 1 if e.flags & index.IX_HASHVALID: assert(e.sha != index.EMPTY_SHA) assert(e.gitmode) - assert(not e or e.name == '/') # last entry is *always* / + assert not e or bytes(e.name) == b'/' # last entry is *always* / log('check: checking normal iteration...\n') last = None for e in reader: diff --git a/lib/bup/compat.py b/lib/bup/compat.py index 3707ceb..7b20ebc 100644 --- a/lib/bup/compat.py +++ b/lib/bup/compat.py @@ -63,6 +63,9 @@ if py3: else: # Python 2 + def fsencode(x): + return x + from pipes import quote from os import environ range = xrange diff --git a/lib/bup/index.py b/lib/bup/index.py index a03a2dd..2e7e5ec 100644 --- a/lib/bup/index.py +++ b/lib/bup/index.py @@ -1,17 +1,22 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function import errno, os, stat, struct, tempfile -from bup import metadata, xstat +from bup import compat, metadata, xstat from bup._helpers import UINT_MAX, bytescmp from bup.compat import range from bup.helpers import (add_error, log, merge_iter, mmap_readwrite, progress, qprogress, resolve_parent, slashappend) -EMPTY_SHA = '\0'*20 -FAKE_SHA = '\x01'*20 +if compat.py_maj > 2: + from bup.compat import buffer -INDEX_HDR = 'BUPI\0\0\0\7' +import sys + +EMPTY_SHA = b'\0' * 20 +FAKE_SHA = b'\x01' * 20 + +INDEX_HDR = b'BUPI\0\0\0\7' # Time values are handled as integer nanoseconds since the epoch in # memory, but are written as xstat/metadata timespecs. This behavior @@ -175,7 +180,7 @@ class Entry: self.children_n = 0 def __repr__(self): - return ("(%s,0x%04x,%d,%d,%d,%d,%d,%d,%s/%s,0x%04x,%d,0x%08x/%d)" + return ("(%r,0x%04x,%d,%d,%d,%d,%d,%d,%s/%s,0x%04x,%d,0x%08x/%d)" % (self.name, self.dev, self.ino, self.nlink, self.ctime, self.mtime, self.atime, self.size, self.mode, self.gitmode, @@ -316,7 +321,7 @@ class Entry: return self._cmp(other) >= 0 def write(self, f): - f.write(self.basename + '\0' + self.packed()) + f.write(self.basename + b'\0' + self.packed()) class NewEntry(Entry): @@ -381,13 +386,13 @@ class ExistingEntry(Entry): def iter(self, name=None, wantrecurse=None): dname = name - if dname and not dname.endswith('/'): - dname += '/' + if dname and not dname.endswith(b'/'): + dname += b'/' ofs = self.children_ofs assert(ofs <= len(self._m)) assert(self.children_n <= UINT_MAX) # i.e. python struct 'I' for i in range(self.children_n): - eon = self._m.find('\0', ofs) + eon = self._m.find(b'\0', ofs) assert(eon >= 0) assert(eon >= ofs) assert(eon > ofs) @@ -396,7 +401,7 @@ class ExistingEntry(Entry): self._m, eon+1) if (not dname or child.name.startswith(dname) - or child.name.endswith('/') and dname.startswith(child.name)): + or child.name.endswith(b'/') and dname.startswith(child.name)): if not wantrecurse or wantrecurse(child): for e in child.iter(name=name, wantrecurse=wantrecurse): yield e @@ -411,12 +416,12 @@ class ExistingEntry(Entry): class Reader: def __init__(self, filename): self.filename = filename - self.m = '' + self.m = b'' self.writable = False self.count = 0 f = None try: - f = open(filename, 'r+') + f = open(filename, 'rb+') except IOError as e: if e.errno == errno.ENOENT: pass @@ -445,7 +450,7 @@ class Reader: def forward_iter(self): ofs = len(INDEX_HDR) while ofs+ENTLEN <= len(self.m)-FOOTLEN: - eon = self.m.find('\0', ofs) + eon = self.m.find(b'\0', ofs) assert(eon >= 0) assert(eon >= ofs) assert(eon > ofs) @@ -456,9 +461,9 @@ class Reader: def iter(self, name=None, wantrecurse=None): if len(self.m) > len(INDEX_HDR)+ENTLEN: dname = name - if dname and not dname.endswith('/'): - dname += '/' - root = ExistingEntry(None, '/', '/', + if dname and not dname.endswith(b'/'): + dname += b'/' + root = ExistingEntry(None, b'/', b'/', self.m, len(self.m)-FOOTLEN-ENTLEN) for sub in root.iter(name=name, wantrecurse=wantrecurse): yield sub @@ -508,9 +513,9 @@ class Reader: # in an odd way and depends on a terminating '/' to indicate directories. def pathsplit(p): """Split a path into a list of elements of the file system hierarchy.""" - l = p.split('/') - l = [i+'/' for i in l[:-1]] + l[-1:] - if l[-1] == '': + l = p.split(b'/') + l = [i + b'/' for i in l[:-1]] + l[-1:] + if l[-1] == b'': l.pop() # extra blank caused by terminating '/' return l @@ -526,7 +531,7 @@ class Writer: self.metastore = metastore self.tmax = tmax (dir,name) = os.path.split(filename) - (ffd,self.tmpname) = tempfile.mkstemp('.tmp', filename, dir) + ffd, self.tmpname = tempfile.mkstemp(b'.tmp', filename, dir) self.f = os.fdopen(ffd, 'wb', 65536) self.f.write(INDEX_HDR) @@ -568,7 +573,7 @@ class Writer: self.metastore, self.tmax) def add(self, name, st, meta_ofs, hashgen = None): - endswith = name.endswith('/') + endswith = name.endswith(b'/') ename = pathsplit(name) basename = ename[-1] #log('add: %r %r\n' % (basename, name)) @@ -632,14 +637,14 @@ def reduce_paths(paths): for p in paths: rp = _slashappend_or_add_error(resolve_parent(p), 'reduce_paths') if rp: - xpaths.append((rp, slashappend(p) if rp.endswith('/') else p)) + xpaths.append((rp, slashappend(p) if rp.endswith(b'/') else p)) xpaths.sort() paths = [] prev = None for (rp, p) in xpaths: if prev and (prev == rp - or (prev.endswith('/') and rp.startswith(prev))): + or (prev.endswith(b'/') and rp.startswith(prev))): continue # already superceded by previous path paths.append((rp, p)) prev = rp diff --git a/lib/bup/t/tindex.py b/lib/bup/t/tindex.py index 0314e18..dea7cd8 100644 --- a/lib/bup/t/tindex.py +++ b/lib/bup/t/tindex.py @@ -5,44 +5,45 @@ import os, time from wvtest import * from bup import index, metadata +from bup.compat import fsencode from bup.helpers import mkdirp, resolve_parent from buptest import no_lingering_errors, test_tempdir import bup.xstat as xstat -lib_t_dir = os.path.dirname(__file__) +lib_t_dir = os.path.dirname(fsencode(__file__)) @wvtest def index_basic(): with no_lingering_errors(): - cd = os.path.realpath('../../../t') + cd = os.path.realpath(b'../../../t') WVPASS(cd) - sd = os.path.realpath(cd + '/sampledata') - WVPASSEQ(resolve_parent(cd + '/sampledata'), sd) - WVPASSEQ(os.path.realpath(cd + '/sampledata/x'), sd + '/x') - WVPASSEQ(os.path.realpath(cd + '/sampledata/var/abs-symlink'), - sd + '/var/abs-symlink-target') - WVPASSEQ(resolve_parent(cd + '/sampledata/var/abs-symlink'), - sd + '/var/abs-symlink') + sd = os.path.realpath(cd + b'/sampledata') + WVPASSEQ(resolve_parent(cd + b'/sampledata'), sd) + WVPASSEQ(os.path.realpath(cd + b'/sampledata/x'), sd + b'/x') + WVPASSEQ(os.path.realpath(cd + b'/sampledata/var/abs-symlink'), + sd + b'/var/abs-symlink-target') + WVPASSEQ(resolve_parent(cd + b'/sampledata/var/abs-symlink'), + sd + b'/var/abs-symlink') @wvtest def index_writer(): with no_lingering_errors(): - with test_tempdir('bup-tindex-') as tmpdir: + with test_tempdir(b'bup-tindex-') as tmpdir: orig_cwd = os.getcwd() try: os.chdir(tmpdir) - ds = xstat.stat('.') - fs = xstat.stat(lib_t_dir + '/tindex.py') - ms = index.MetaStoreWriter('index.meta.tmp'); + ds = xstat.stat(b'.') + fs = xstat.stat(lib_t_dir + b'/tindex.py') + ms = index.MetaStoreWriter(b'index.meta.tmp'); tmax = (time.time() - 1) * 10**9 - w = index.Writer('index.tmp', ms, tmax) - w.add('/var/tmp/sporky', fs, 0) - w.add('/etc/passwd', fs, 0) - w.add('/etc/', ds, 0) - w.add('/', ds, 0) + w = index.Writer(b'index.tmp', ms, tmax) + w.add(b'/var/tmp/sporky', fs, 0) + w.add(b'/etc/passwd', fs, 0) + w.add(b'/etc/', ds, 0) + w.add(b'/', ds, 0) ms.close() w.close() finally: @@ -69,9 +70,9 @@ def eget(l, ename): @wvtest def index_negative_timestamps(): with no_lingering_errors(): - with test_tempdir('bup-tindex-') as tmpdir: + with test_tempdir(b'bup-tindex-') as tmpdir: # Makes 'foo' exist - foopath = tmpdir + '/foo' + foopath = tmpdir + b'/foo' f = open(foopath, 'wb') f.close() @@ -93,37 +94,37 @@ def index_negative_timestamps(): @wvtest def index_dirty(): with no_lingering_errors(): - with test_tempdir('bup-tindex-') as tmpdir: + with test_tempdir(b'bup-tindex-') as tmpdir: orig_cwd = os.getcwd() try: os.chdir(tmpdir) default_meta = metadata.Metadata() - ms1 = index.MetaStoreWriter('index.meta.tmp') - ms2 = index.MetaStoreWriter('index2.meta.tmp') - ms3 = index.MetaStoreWriter('index3.meta.tmp') + ms1 = index.MetaStoreWriter(b'index.meta.tmp') + ms2 = index.MetaStoreWriter(b'index2.meta.tmp') + ms3 = index.MetaStoreWriter(b'index3.meta.tmp') meta_ofs1 = ms1.store(default_meta) meta_ofs2 = ms2.store(default_meta) meta_ofs3 = ms3.store(default_meta) ds = xstat.stat(lib_t_dir) - fs = xstat.stat(lib_t_dir + '/tindex.py') + fs = xstat.stat(lib_t_dir + b'/tindex.py') tmax = (time.time() - 1) * 10**9 - w1 = index.Writer('index.tmp', ms1, tmax) - w1.add('/a/b/x', fs, meta_ofs1) - w1.add('/a/b/c', fs, meta_ofs1) - w1.add('/a/b/', ds, meta_ofs1) - w1.add('/a/', ds, meta_ofs1) + w1 = index.Writer(b'index.tmp', ms1, tmax) + w1.add(b'/a/b/x', fs, meta_ofs1) + w1.add(b'/a/b/c', fs, meta_ofs1) + w1.add(b'/a/b/', ds, meta_ofs1) + w1.add(b'/a/', ds, meta_ofs1) #w1.close() WVPASS() - w2 = index.Writer('index2.tmp', ms2, tmax) - w2.add('/a/b/n/2', fs, meta_ofs2) + w2 = index.Writer(b'index2.tmp', ms2, tmax) + w2.add(b'/a/b/n/2', fs, meta_ofs2) #w2.close() WVPASS() - w3 = index.Writer('index3.tmp', ms3, tmax) - w3.add('/a/c/n/3', fs, meta_ofs3) + w3 = index.Writer(b'index3.tmp', ms3, tmax) + w3.add(b'/a/c/n/3', fs, meta_ofs3) #w3.close() WVPASS() @@ -134,18 +135,18 @@ def index_dirty(): r1all = [e.name for e in r1] WVPASSEQ(r1all, - ['/a/b/x', '/a/b/c', '/a/b/', '/a/', '/']) + [b'/a/b/x', b'/a/b/c', b'/a/b/', b'/a/', b'/']) r2all = [e.name for e in r2] WVPASSEQ(r2all, - ['/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/']) + [b'/a/b/n/2', b'/a/b/n/', b'/a/b/', b'/a/', b'/']) r3all = [e.name for e in r3] WVPASSEQ(r3all, - ['/a/c/n/3', '/a/c/n/', '/a/c/', '/a/', '/']) + [b'/a/c/n/3', b'/a/c/n/', b'/a/c/', b'/a/', b'/']) all = [e.name for e in index.merge(r2, r1, r3)] WVPASSEQ(all, - ['/a/c/n/3', '/a/c/n/', '/a/c/', - '/a/b/x', '/a/b/n/2', '/a/b/n/', '/a/b/c', - '/a/b/', '/a/', '/']) + [b'/a/c/n/3', b'/a/c/n/', b'/a/c/', + b'/a/b/x', b'/a/b/n/2', b'/a/b/n/', b'/a/b/c', + b'/a/b/', b'/a/', b'/']) fake_validate(r1) dump(r1) @@ -153,12 +154,12 @@ def index_dirty(): WVPASSEQ([e.name for e in r1 if e.is_valid()], r1all) WVPASSEQ([e.name for e in r1 if not e.is_valid()], []) WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()], - ['/a/c/n/3', '/a/c/n/', '/a/c/', - '/a/b/n/2', '/a/b/n/', '/a/b/', '/a/', '/']) + [b'/a/c/n/3', b'/a/c/n/', b'/a/c/', + b'/a/b/n/2', b'/a/b/n/', b'/a/b/', b'/a/', b'/']) - expect_invalid = ['/'] + r2all + r3all + expect_invalid = [b'/'] + r2all + r3all expect_real = (set(r1all) - set(r2all) - set(r3all)) \ - | set(['/a/b/n/2', '/a/c/n/3']) + | set([b'/a/b/n/2', b'/a/c/n/3']) dump(index.merge(r2, r1, r3)) for e in index.merge(r2, r1, r3): print(e.name, hex(e.flags), e.ctime) @@ -170,12 +171,12 @@ def index_dirty(): dump(index.merge(r2, r1, r3)) WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()], []) - e = eget(index.merge(r2, r1, r3), '/a/b/c') + e = eget(index.merge(r2, r1, r3), b'/a/b/c') e.invalidate() e.repack() dump(index.merge(r2, r1, r3)) WVPASSEQ([e.name for e in index.merge(r2, r1, r3) if not e.is_valid()], - ['/a/b/c', '/a/b/', '/a/', '/']) + [b'/a/b/c', b'/a/b/', b'/a/', b'/']) w1.close() w2.close() w3.close()