lib/bup/t/tgit.py

   1
   2 from __future__ import absolute_import, print_function
   3 from binascii import hexlify, unhexlify
   4 from subprocess import check_call
   5 import struct, os, time
   6
   7 from wvtest import *
   8
   9 from bup import git, path
  10 from bup.compat import bytes_from_byte, environ, range
  11 from bup.helpers import localtime, log, mkdirp, readpipe
  12 from buptest import no_lingering_errors, test_tempdir
  13
  14
  15 bup_exe = path.exe()
  16
  17
  18 def exc(*cmd):
  19     print(repr(cmd), file=sys.stderr)
  20     check_call(cmd)
  21
  22
  23 def exo(*cmd):
  24     print(repr(cmd), file=sys.stderr)
  25     return readpipe(cmd)
  26
  27
  28 @wvtest
  29 def test_git_version_detection():
  30     with no_lingering_errors():
  31         # Test version types from git's tag history
  32         for expected, ver in \
  33             (('insufficient', b'git version 0.99'),
  34              ('insufficient', b'git version 0.99.1'),
  35              ('insufficient', b'git version 0.99.7a'),
  36              ('insufficient', b'git version 1.0rc1'),
  37              ('insufficient', b'git version 1.0.1'),
  38              ('insufficient', b'git version 1.4.2.1'),
  39              ('insufficient', b'git version 1.5.5'),
  40              ('insufficient', b'git version 1.5.6-rc0'),
  41              ('suitable', b'git version 1.5.6'),
  42              ('suitable', b'git version 1.5.6.1'),
  43              ('suitable', b'git version 2.14.0-rc0'),
  44              ('suitable', b'git version 2.14.0 (something ...)'),
  45              ('suitable', b'git version 111.222.333.444-rc555'),
  46              ('unrecognized', b'huh?')):
  47             WVMSG('Checking version validation: %r' % ver)
  48             WVPASSEQ(expected, git.is_suitable_git(ver_str=ver))
  49             try:
  50                 if expected == 'insufficient':
  51                     WVEXCEPT(SystemExit, git.require_suitable_git, ver)
  52                 elif expected == 'suitable':
  53                     git.require_suitable_git(ver_str=ver)
  54                 elif expected == 'unrecognized':
  55                     WVEXCEPT(git.GitError, git.require_suitable_git, ver)
  56                 else:
  57                     WVPASS(False)
  58             finally:
  59                 git._git_great = None
  60             try:
  61                 environ[b'BUP_GIT_VERSION_IS_FINE'] = b'true'
  62                 git.require_suitable_git(ver_str=ver)
  63             finally:
  64                 del environ[b'BUP_GIT_VERSION_IS_FINE']
  65                 git._git_great = None
  66
  67
  68 @wvtest
  69 def testmangle():
  70     with no_lingering_errors():
  71         afile  = 0o100644
  72         afile2 = 0o100770
  73         alink  = 0o120000
  74         adir   = 0o040000
  75         adir2  = 0o040777
  76         WVPASSEQ(git.mangle_name(b'a', adir2, adir), b'a')
  77         WVPASSEQ(git.mangle_name(b'.bup', adir2, adir), b'.bup.bupl')
  78         WVPASSEQ(git.mangle_name(b'a.bupa', adir2, adir), b'a.bupa.bupl')
  79         WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
  80         WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
  81         WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
  82         WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
  83         WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
  84         WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
  85
  86         WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
  87         WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
  88         WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
  89
  90         WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
  91         WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
  92
  93         # for safety, we ignore .bup? suffixes we don't recognize.  Future
  94         # versions might implement a .bup[a-z] extension as something other
  95         # than BUP_NORMAL.
  96         WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
  97
  98
  99 @wvtest
 100 def testencode():
 101     with no_lingering_errors():
 102         s = b'hello world'
 103         looseb = b''.join(git._encode_looseobj(b'blob', s))
 104         looset = b''.join(git._encode_looseobj(b'tree', s))
 105         loosec = b''.join(git._encode_looseobj(b'commit', s))
 106         packb = b''.join(git._encode_packobj(b'blob', s))
 107         packt = b''.join(git._encode_packobj(b'tree', s))
 108         packc = b''.join(git._encode_packobj(b'commit', s))
 109         packlb = b''.join(git._encode_packobj(b'blob', s * 200))
 110         WVPASSEQ(git._decode_looseobj(looseb), (b'blob', s))
 111         WVPASSEQ(git._decode_looseobj(looset), (b'tree', s))
 112         WVPASSEQ(git._decode_looseobj(loosec), (b'commit', s))
 113         WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
 114         WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
 115         WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
 116         WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
 117         for i in range(10):
 118             WVPASS(git._encode_looseobj(b'blob', s, compression_level=i))
 119         def encode_pobj(n):
 120             return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
 121         WVEXCEPT(ValueError, encode_pobj, -1)
 122         WVEXCEPT(ValueError, encode_pobj, 10)
 123         WVEXCEPT(ValueError, encode_pobj, b'x')
 124
 125
 126 @wvtest
 127 def testpacks():
 128     with no_lingering_errors():
 129         with test_tempdir(b'bup-tgit-') as tmpdir:
 130             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 131             git.init_repo(bupdir)
 132             git.verbose = 1
 133
 134             w = git.PackWriter()
 135             w.new_blob(os.urandom(100))
 136             w.new_blob(os.urandom(100))
 137             w.abort()
 138
 139             w = git.PackWriter()
 140             hashes = []
 141             nobj = 1000
 142             for i in range(nobj):
 143                 hashes.append(w.new_blob(b'%d' % i))
 144             log('\n')
 145             nameprefix = w.close()
 146             print(repr(nameprefix))
 147             WVPASS(os.path.exists(nameprefix + b'.pack'))
 148             WVPASS(os.path.exists(nameprefix + b'.idx'))
 149
 150             r = git.open_idx(nameprefix + b'.idx')
 151             print(repr(r.fanout))
 152
 153             for i in range(nobj):
 154                 WVPASS(r.find_offset(hashes[i]) > 0)
 155             WVPASS(r.exists(hashes[99]))
 156             WVFAIL(r.exists(b'\0'*20))
 157
 158             pi = iter(r)
 159             for h in sorted(hashes):
 160                 WVPASSEQ(hexlify(next(pi)), hexlify(h))
 161
 162             WVFAIL(r.find_offset(b'\0'*20))
 163
 164             r = git.PackIdxList(bupdir + b'/objects/pack')
 165             WVPASS(r.exists(hashes[5]))
 166             WVPASS(r.exists(hashes[6]))
 167             WVFAIL(r.exists(b'\0'*20))
 168
 169
 170 @wvtest
 171 def test_pack_name_lookup():
 172     with no_lingering_errors():
 173         with test_tempdir(b'bup-tgit-') as tmpdir:
 174             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 175             git.init_repo(bupdir)
 176             git.verbose = 1
 177             packdir = git.repo(b'objects/pack')
 178
 179             idxnames = []
 180             hashes = []
 181
 182             for start in range(0,28,2):
 183                 w = git.PackWriter()
 184                 for i in range(start, start+2):
 185                     hashes.append(w.new_blob(b'%d' % i))
 186                 log('\n')
 187                 idxnames.append(os.path.basename(w.close() + b'.idx'))
 188
 189             r = git.PackIdxList(packdir)
 190             WVPASSEQ(len(r.packs), 2)
 191             for e,idxname in enumerate(idxnames):
 192                 for i in range(e*2, (e+1)*2):
 193                     WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
 194
 195
 196 @wvtest
 197 def test_long_index():
 198     with no_lingering_errors():
 199         with test_tempdir(b'bup-tgit-') as tmpdir:
 200             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 201             git.init_repo(bupdir)
 202             idx = git.PackIdxV2Writer()
 203             obj_bin = struct.pack('!IIIII',
 204                     0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
 205             obj2_bin = struct.pack('!IIIII',
 206                     0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
 207             obj3_bin = struct.pack('!IIIII',
 208                     0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
 209             pack_bin = struct.pack('!IIIII',
 210                     0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
 211             idx.add(obj_bin, 1, 0xfffffffff)
 212             idx.add(obj2_bin, 2, 0xffffffffff)
 213             idx.add(obj3_bin, 3, 0xff)
 214             name = tmpdir + b'/tmp.idx'
 215             r = idx.write(name, pack_bin)
 216             i = git.PackIdxV2(name, open(name, 'rb'))
 217             WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
 218             WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
 219             WVPASSEQ(i.find_offset(obj3_bin), 0xff)
 220
 221
 222 @wvtest
 223 def test_check_repo_or_die():
 224     with no_lingering_errors():
 225         with test_tempdir(b'bup-tgit-') as tmpdir:
 226             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 227             orig_cwd = os.getcwd()
 228             try:
 229                 os.chdir(tmpdir)
 230                 git.init_repo(bupdir)
 231                 git.check_repo_or_die()
 232                 # if we reach this point the call above passed
 233                 WVPASS('check_repo_or_die')
 234
 235                 os.rename(bupdir + b'/objects/pack',
 236                           bupdir + b'/objects/pack.tmp')
 237                 open(bupdir + b'/objects/pack', 'w').close()
 238                 try:
 239                     git.check_repo_or_die()
 240                 except SystemExit as e:
 241                     WVPASSEQ(e.code, 14)
 242                 else:
 243                     WVFAIL()
 244                 os.unlink(bupdir + b'/objects/pack')
 245                 os.rename(bupdir + b'/objects/pack.tmp',
 246                           bupdir + b'/objects/pack')
 247
 248                 try:
 249                     git.check_repo_or_die(b'nonexistantbup.tmp')
 250                 except SystemExit as e:
 251                     WVPASSEQ(e.code, 15)
 252                 else:
 253                     WVFAIL()
 254             finally:
 255                 os.chdir(orig_cwd)
 256
 257
 258 @wvtest
 259 def test_commit_parsing():
 260
 261     def restore_env_var(name, val):
 262         if val is None:
 263             del environ[name]
 264         else:
 265             environ[name] = val
 266
 267     def showval(commit, val):
 268         return readpipe([b'git', b'show', b'-s',
 269                          b'--pretty=format:%s' % val, commit]).strip()
 270
 271     with no_lingering_errors():
 272         with test_tempdir(b'bup-tgit-') as tmpdir:
 273             orig_cwd = os.getcwd()
 274             workdir = tmpdir + b'/work'
 275             repodir = workdir + b'/.git'
 276             orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
 277             orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
 278             orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
 279             orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
 280             environ[b'GIT_AUTHOR_NAME'] = b'bup test'
 281             environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
 282             environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
 283             environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
 284             try:
 285                 readpipe([b'git', b'init', workdir])
 286                 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
 287                 git.check_repo_or_die(repodir)
 288                 os.chdir(workdir)
 289                 with open('foo', 'w') as f:
 290                     print('bar', file=f)
 291                 readpipe([b'git', b'add', b'.'])
 292                 readpipe([b'git', b'commit', b'-am', b'Do something',
 293                           b'--author', b'Someone <someone@somewhere>',
 294                           b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
 295                 commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
 296                 parents = showval(commit, b'%P')
 297                 tree = showval(commit, b'%T')
 298                 cname = showval(commit, b'%cn')
 299                 cmail = showval(commit, b'%ce')
 300                 cdate = showval(commit, b'%ct')
 301                 coffs = showval(commit, b'%ci')
 302                 coffs = coffs[-5:]
 303                 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
 304                 if bytes_from_byte(coffs[-5]) == b'-':
 305                     coff = - coff
 306                 commit_items = git.get_commit_items(commit, git.cp())
 307                 WVPASSEQ(commit_items.parents, [])
 308                 WVPASSEQ(commit_items.tree, tree)
 309                 WVPASSEQ(commit_items.author_name, b'Someone')
 310                 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
 311                 WVPASSEQ(commit_items.author_sec, 1254613729)
 312                 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
 313                 WVPASSEQ(commit_items.committer_name, cname)
 314                 WVPASSEQ(commit_items.committer_mail, cmail)
 315                 WVPASSEQ(commit_items.committer_sec, int(cdate))
 316                 WVPASSEQ(commit_items.committer_offset, coff)
 317                 WVPASSEQ(commit_items.message, b'Do something\n')
 318                 with open(b'bar', 'wb') as f:
 319                     f.write(b'baz\n')
 320                 readpipe([b'git', b'add', '.'])
 321                 readpipe([b'git', b'commit', b'-am', b'Do something else'])
 322                 child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
 323                 parents = showval(child, b'%P')
 324                 commit_items = git.get_commit_items(child, git.cp())
 325                 WVPASSEQ(commit_items.parents, [commit])
 326             finally:
 327                 os.chdir(orig_cwd)
 328                 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
 329                 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
 330                 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
 331                 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
 332
 333
 334 @wvtest
 335 def test_new_commit():
 336     with no_lingering_errors():
 337         with test_tempdir(b'bup-tgit-') as tmpdir:
 338             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 339             git.init_repo(bupdir)
 340             git.verbose = 1
 341
 342             w = git.PackWriter()
 343             tree = os.urandom(20)
 344             parent = os.urandom(20)
 345             author_name = b'Author'
 346             author_mail = b'author@somewhere'
 347             adate_sec = 1439657836
 348             cdate_sec = adate_sec + 1
 349             committer_name = b'Committer'
 350             committer_mail = b'committer@somewhere'
 351             adate_tz_sec = cdate_tz_sec = None
 352             commit = w.new_commit(tree, parent,
 353                                   b'%s <%s>' % (author_name, author_mail),
 354                                   adate_sec, adate_tz_sec,
 355                                   b'%s <%s>' % (committer_name, committer_mail),
 356                                   cdate_sec, cdate_tz_sec,
 357                                   b'There is a small mailbox here')
 358             adate_tz_sec = -60 * 60
 359             cdate_tz_sec = 120 * 60
 360             commit_off = w.new_commit(tree, parent,
 361                                       b'%s <%s>' % (author_name, author_mail),
 362                                       adate_sec, adate_tz_sec,
 363                                       b'%s <%s>' % (committer_name, committer_mail),
 364                                       cdate_sec, cdate_tz_sec,
 365                                       b'There is a small mailbox here')
 366             w.close()
 367
 368             commit_items = git.get_commit_items(hexlify(commit), git.cp())
 369             local_author_offset = localtime(adate_sec).tm_gmtoff
 370             local_committer_offset = localtime(cdate_sec).tm_gmtoff
 371             WVPASSEQ(tree, unhexlify(commit_items.tree))
 372             WVPASSEQ(1, len(commit_items.parents))
 373             WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
 374             WVPASSEQ(author_name, commit_items.author_name)
 375             WVPASSEQ(author_mail, commit_items.author_mail)
 376             WVPASSEQ(adate_sec, commit_items.author_sec)
 377             WVPASSEQ(local_author_offset, commit_items.author_offset)
 378             WVPASSEQ(committer_name, commit_items.committer_name)
 379             WVPASSEQ(committer_mail, commit_items.committer_mail)
 380             WVPASSEQ(cdate_sec, commit_items.committer_sec)
 381             WVPASSEQ(local_committer_offset, commit_items.committer_offset)
 382
 383             commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
 384             WVPASSEQ(tree, unhexlify(commit_items.tree))
 385             WVPASSEQ(1, len(commit_items.parents))
 386             WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
 387             WVPASSEQ(author_name, commit_items.author_name)
 388             WVPASSEQ(author_mail, commit_items.author_mail)
 389             WVPASSEQ(adate_sec, commit_items.author_sec)
 390             WVPASSEQ(adate_tz_sec, commit_items.author_offset)
 391             WVPASSEQ(committer_name, commit_items.committer_name)
 392             WVPASSEQ(committer_mail, commit_items.committer_mail)
 393             WVPASSEQ(cdate_sec, commit_items.committer_sec)
 394             WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
 395
 396
 397 @wvtest
 398 def test_list_refs():
 399     with no_lingering_errors():
 400         with test_tempdir(b'bup-tgit-') as tmpdir:
 401             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 402             src = tmpdir + b'/src'
 403             mkdirp(src)
 404             with open(src + b'/1', 'wb+') as f:
 405                 f.write(b'something\n')
 406             with open(src + b'/2', 'wb+') as f:
 407                 f.write(b'something else\n')
 408             git.init_repo(bupdir)
 409             emptyset = frozenset()
 410             WVPASSEQ(frozenset(git.list_refs()), emptyset)
 411             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
 412             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
 413             exc(bup_exe, b'index', src)
 414             exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
 415             src_hash = exo(b'git', b'--git-dir', bupdir,
 416                            b'rev-parse', b'src').strip().split(b'\n')
 417             assert(len(src_hash) == 1)
 418             src_hash = unhexlify(src_hash[0])
 419             tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
 420                                       b'rev-parse',
 421                                       b'src:').strip().split(b'\n')[0])
 422             blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
 423                                       b'rev-parse',
 424                                       b'src:1').strip().split(b'\n')[0])
 425             WVPASSEQ(frozenset(git.list_refs()),
 426                      frozenset([(b'refs/heads/src', src_hash)]))
 427             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
 428             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
 429                      frozenset([(b'refs/heads/src', src_hash)]))
 430             exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
 431             WVPASSEQ(frozenset(git.list_refs()),
 432                      frozenset([(b'refs/heads/src', src_hash),
 433                                 (b'refs/tags/commit-tag', src_hash)]))
 434             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
 435                      frozenset([(b'refs/tags/commit-tag', src_hash)]))
 436             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
 437                      frozenset([(b'refs/heads/src', src_hash)]))
 438             exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
 439             exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
 440             os.unlink(bupdir + b'/refs/heads/src')
 441             expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
 442                                        (b'refs/tags/tree-tag', tree_hash),
 443                                        (b'refs/tags/blob-tag', blob_hash)])
 444             WVPASSEQ(frozenset(git.list_refs()), expected_tags)
 445             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
 446             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
 447
 448
 449 @wvtest
 450 def test__git_date_str():
 451     with no_lingering_errors():
 452         WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
 453         WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
 454         WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
 455
 456
 457 @wvtest
 458 def test_cat_pipe():
 459     with no_lingering_errors():
 460         with test_tempdir(b'bup-tgit-') as tmpdir:
 461             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 462             src = tmpdir + b'/src'
 463             mkdirp(src)
 464             with open(src + b'/1', 'wb+') as f:
 465                 f.write(b'something\n')
 466             with open(src + b'/2', 'wb+') as f:
 467                 f.write(b'something else\n')
 468             git.init_repo(bupdir)
 469             exc(bup_exe, b'index', src)
 470             oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
 471                        src).strip()
 472             typ = exo(b'git', b'--git-dir', bupdir,
 473                       b'cat-file', b'-t', b'src').strip()
 474             size = int(exo(b'git', b'--git-dir', bupdir,
 475                                b'cat-file', b'-s', b'src'))
 476             it = git.cp().get(b'src')
 477             get_info = next(it)
 478             for buf in next(it):
 479                 pass
 480             WVPASSEQ((oidx, typ, size), get_info)