From db8a2b95e2cc44f7070ae6ac0bbc086e51696c72 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Sun, 18 Jun 2017 15:40:41 -0500 Subject: [PATCH] Support catpipe get(...,size=True); require git >= 1.5.6 Add a new size argument to get() that requests the object size in addition to the type. To support this, require git 1.5.6 (circa 2008) or newer so that we'll have cat-file --batch. Remove the _slow_get() fallback since it's no longer needed. Signed-off-by: Rob Browning Tested-by: Rob Browning --- lib/bup/git.py | 48 ++++++++++++++++++----------------------------- lib/bup/t/tgit.py | 27 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 30 deletions(-) diff --git a/lib/bup/git.py b/lib/bup/git.py index 950a2d4..94016a4 100644 --- a/lib/bup/git.py +++ b/lib/bup/git.py @@ -1138,14 +1138,9 @@ class CatPipe: self.repo_dir = repo_dir wanted = ('1','5','6') if ver() < wanted: - if not _ver_warned: - log('warning: git version < %s; bup will be slow.\n' - % '.'.join(wanted)) - _ver_warned = 1 - self.get = self._slow_get - else: - self.p = self.inprogress = None - self.get = self._fast_get + log('error: git version must be at least 1.5.6\n') + sys.exit(1) + self.p = self.inprogress = None def _abort(self): if self.p: @@ -1163,15 +1158,19 @@ class CatPipe: bufsize = 4096, preexec_fn = _gitenv(self.repo_dir)) - def _fast_get(self, id): + def get(self, id, size=False): + """Yield the object type, and then an iterator over the data referred + to by the id ref. If size is true, yield (obj_type, obj_size) + instead of just the type. + + """ if not self.p or self.p.poll() != None: self.restart() assert(self.p) poll_result = self.p.poll() assert(poll_result == None) if self.inprogress: - log('_fast_get: opening %r while %r is open\n' - % (id, self.inprogress)) + log('get: opening %r while %r is open\n' % (id, self.inprogress)) assert(not self.inprogress) assert(id.find('\n') < 0) assert(id.find('\r') < 0) @@ -1186,12 +1185,15 @@ class CatPipe: spl = hdr.split(' ') if len(spl) != 3 or len(spl[0]) != 40: raise GitError('expected blob, got %r' % spl) - (hex, type, size) = spl - - it = _AbortableIter(chunkyreader(self.p.stdout, int(spl[2])), - onabort = self._abort) + hex, typ, sz = spl + sz = int(sz) + it = _AbortableIter(chunkyreader(self.p.stdout, sz), + onabort=self._abort) try: - yield type + if size: + yield typ, sz + else: + yield typ for blob in it: yield blob readline_result = self.p.stdout.readline() @@ -1201,20 +1203,6 @@ class CatPipe: it.abort() raise - def _slow_get(self, id): - assert(id.find('\n') < 0) - assert(id.find('\r') < 0) - assert(id[0] != '-') - type = _git_capture(['git', 'cat-file', '-t', id]).strip() - yield type - - p = subprocess.Popen(['git', 'cat-file', type, id], - stdout=subprocess.PIPE, - preexec_fn = _gitenv(self.repo_dir)) - for blob in chunkyreader(p.stdout): - yield blob - _git_wait('git cat-file', p) - def _join(self, it): type = it.next() if type == 'blob': diff --git a/lib/bup/t/tgit.py b/lib/bup/t/tgit.py index 0293153..d12924c 100644 --- a/lib/bup/t/tgit.py +++ b/lib/bup/t/tgit.py @@ -414,3 +414,30 @@ def test__git_date_str(): WVPASSEQ('0 +0000', git._git_date_str(0, 0)) WVPASSEQ('0 -0130', git._git_date_str(0, -90 * 60)) WVPASSEQ('0 +0130', git._git_date_str(0, 90 * 60)) + + +@wvtest +def test_cat_pipe(): + with no_lingering_errors(): + with test_tempdir('bup-tgit-') as tmpdir: + os.environ['BUP_MAIN_EXE'] = bup_exe + os.environ['BUP_DIR'] = bupdir = tmpdir + "/bup" + src = tmpdir + '/src' + mkdirp(src) + with open(src + '/1', 'w+') as f: + print f, 'something' + with open(src + '/2', 'w+') as f: + print f, 'something else' + git.init_repo(bupdir) + exc(bup_exe, 'index', src) + exc(bup_exe, 'save', '-n', 'src', '--strip', src) + git_type = exo('git', '--git-dir', bupdir, + 'cat-file', '-t', 'src').strip() + git_size = int(exo('git', '--git-dir', bupdir, + 'cat-file', '-s', 'src')) + it = git.cp().get('src', size=True) + get_type, get_size = it.next() + for buf in it.next(): + pass + WVPASSEQ(get_type, git_type) + WVPASSEQ(get_size, git_size) -- 2.39.2