lib/bup/t/tgit.py

   1
   2 from __future__ import absolute_import, print_function
   3 from binascii import hexlify, unhexlify
   4 from subprocess import check_call
   5 import struct, os, time
   6
   7 from wvtest import *
   8
   9 from bup import git, path
  10 from bup.compat import bytes_from_byte, environ, range
  11 from bup.helpers import localtime, log, mkdirp, readpipe
  12 from buptest import no_lingering_errors, test_tempdir
  13
  14
  15 bup_exe = path.exe()
  16
  17
  18 def exc(*cmd):
  19     print(repr(cmd), file=sys.stderr)
  20     check_call(cmd)
  21
  22
  23 def exo(*cmd):
  24     print(repr(cmd), file=sys.stderr)
  25     return readpipe(cmd)
  26
  27
  28 @wvtest
  29 def test_git_version_detection():
  30     with no_lingering_errors():
  31         # Test version types from git's tag history
  32         for expected, ver in \
  33             (('insufficient', b'git version 0.99'),
  34              ('insufficient', b'git version 0.99.1'),
  35              ('insufficient', b'git version 0.99.7a'),
  36              ('insufficient', b'git version 1.0rc1'),
  37              ('insufficient', b'git version 1.0.1'),
  38              ('insufficient', b'git version 1.4.2.1'),
  39              ('insufficient', b'git version 1.5.5'),
  40              ('insufficient', b'git version 1.5.6-rc0'),
  41              ('suitable', b'git version 1.5.6'),
  42              ('suitable', b'git version 1.5.6.1'),
  43              ('suitable', b'git version 2.14.0-rc0'),
  44              ('suitable', b'git version 2.14.0 (something ...)'),
  45              ('suitable', b'git version 111.222.333.444-rc555'),
  46              ('unrecognized', b'huh?')):
  47             WVMSG('Checking version validation: %r' % ver)
  48             WVPASSEQ(expected, git.is_suitable_git(ver_str=ver))
  49             try:
  50                 if expected == 'insufficient':
  51                     WVEXCEPT(SystemExit, git.require_suitable_git, ver)
  52                 elif expected == 'suitable':
  53                     git.require_suitable_git(ver_str=ver)
  54                 elif expected == 'unrecognized':
  55                     WVEXCEPT(git.GitError, git.require_suitable_git, ver)
  56                 else:
  57                     WVPASS(False)
  58             finally:
  59                 git._git_great = None
  60             try:
  61                 environ[b'BUP_GIT_VERSION_IS_FINE'] = b'true'
  62                 git.require_suitable_git(ver_str=ver)
  63             finally:
  64                 del environ[b'BUP_GIT_VERSION_IS_FINE']
  65                 git._git_great = None
  66
  67
  68 @wvtest
  69 def testmangle():
  70     with no_lingering_errors():
  71         afile  = 0o100644
  72         afile2 = 0o100770
  73         alink  = 0o120000
  74         adir   = 0o040000
  75         adir2  = 0o040777
  76         WVPASSEQ(git.mangle_name(b'a', adir2, adir), b'a')
  77         WVPASSEQ(git.mangle_name(b'.bup', adir2, adir), b'.bup.bupl')
  78         WVPASSEQ(git.mangle_name(b'a.bupa', adir2, adir), b'a.bupa.bupl')
  79         WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
  80         WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
  81         WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
  82         WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
  83         WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
  84         WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
  85
  86         WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
  87         WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
  88         WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
  89
  90         WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
  91         WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
  92
  93         # for safety, we ignore .bup? suffixes we don't recognize.  Future
  94         # versions might implement a .bup[a-z] extension as something other
  95         # than BUP_NORMAL.
  96         WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
  97
  98
  99 @wvtest
 100 def testencode():
 101     with no_lingering_errors():
 102         s = b'hello world'
 103         looseb = b''.join(git._encode_looseobj(b'blob', s))
 104         looset = b''.join(git._encode_looseobj(b'tree', s))
 105         loosec = b''.join(git._encode_looseobj(b'commit', s))
 106         packb = b''.join(git._encode_packobj(b'blob', s))
 107         packt = b''.join(git._encode_packobj(b'tree', s))
 108         packc = b''.join(git._encode_packobj(b'commit', s))
 109         packlb = b''.join(git._encode_packobj(b'blob', s * 200))
 110         WVPASSEQ(git._decode_looseobj(looseb), (b'blob', s))
 111         WVPASSEQ(git._decode_looseobj(looset), (b'tree', s))
 112         WVPASSEQ(git._decode_looseobj(loosec), (b'commit', s))
 113         WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
 114         WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
 115         WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
 116         WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
 117         for i in range(10):
 118             WVPASS(git._encode_looseobj(b'blob', s, compression_level=i))
 119         def encode_pobj(n):
 120             return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
 121         WVEXCEPT(ValueError, encode_pobj, -1)
 122         WVEXCEPT(ValueError, encode_pobj, 10)
 123         WVEXCEPT(ValueError, encode_pobj, b'x')
 124
 125
 126 @wvtest
 127 def testpacks():
 128     with no_lingering_errors():
 129         with test_tempdir(b'bup-tgit-') as tmpdir:
 130             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 131             git.init_repo(bupdir)
 132             git.verbose = 1
 133
 134             w = git.PackWriter()
 135             w.new_blob(os.urandom(100))
 136             w.new_blob(os.urandom(100))
 137             w.abort()
 138
 139             w = git.PackWriter()
 140             hashes = []
 141             nobj = 1000
 142             for i in range(nobj):
 143                 hashes.append(w.new_blob(b'%d' % i))
 144             log('\n')
 145             nameprefix = w.close()
 146             print(repr(nameprefix))
 147             WVPASS(os.path.exists(nameprefix + b'.pack'))
 148             WVPASS(os.path.exists(nameprefix + b'.idx'))
 149
 150             r = git.open_idx(nameprefix + b'.idx')
 151             print(repr(r.fanout))
 152
 153             for i in range(nobj):
 154                 WVPASS(r.find_offset(hashes[i]) > 0)
 155             WVPASS(r.exists(hashes[99]))
 156             WVFAIL(r.exists(b'\0'*20))
 157
 158             pi = iter(r)
 159             for h in sorted(hashes):
 160                 WVPASSEQ(hexlify(next(pi)), hexlify(h))
 161
 162             WVFAIL(r.find_offset(b'\0'*20))
 163
 164             r = git.PackIdxList(bupdir + b'/objects/pack')
 165             WVPASS(r.exists(hashes[5]))
 166             WVPASS(r.exists(hashes[6]))
 167             WVFAIL(r.exists(b'\0'*20))
 168
 169
 170 @wvtest
 171 def test_pack_name_lookup():
 172     with no_lingering_errors():
 173         with test_tempdir(b'bup-tgit-') as tmpdir:
 174             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 175             git.init_repo(bupdir)
 176             git.verbose = 1
 177             packdir = git.repo(b'objects/pack')
 178
 179             idxnames = []
 180             hashes = []
 181
 182             for start in range(0,28,2):
 183                 w = git.PackWriter()
 184                 for i in range(start, start+2):
 185                     hashes.append(w.new_blob(b'%d' % i))
 186                 log('\n')
 187                 idxnames.append(os.path.basename(w.close() + b'.idx'))
 188
 189             r = git.PackIdxList(packdir)
 190             WVPASSEQ(len(r.packs), 2)
 191             for e,idxname in enumerate(idxnames):
 192                 for i in range(e*2, (e+1)*2):
 193                     WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
 194
 195
 196 @wvtest
 197 def test_long_index():
 198     with no_lingering_errors():
 199         with test_tempdir(b'bup-tgit-') as tmpdir:
 200             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 201             git.init_repo(bupdir)
 202             w = git.PackWriter()
 203             obj_bin = struct.pack('!IIIII',
 204                     0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
 205             obj2_bin = struct.pack('!IIIII',
 206                     0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
 207             obj3_bin = struct.pack('!IIIII',
 208                     0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
 209             pack_bin = struct.pack('!IIIII',
 210                     0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
 211             idx = list(list() for i in range(256))
 212             idx[0].append((obj_bin, 1, 0xfffffffff))
 213             idx[0x11].append((obj2_bin, 2, 0xffffffffff))
 214             idx[0x22].append((obj3_bin, 3, 0xff))
 215             w.count = 3
 216             name = tmpdir + b'/tmp.idx'
 217             r = w._write_pack_idx_v2(name, idx, pack_bin)
 218             i = git.PackIdxV2(name, open(name, 'rb'))
 219             WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
 220             WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
 221             WVPASSEQ(i.find_offset(obj3_bin), 0xff)
 222
 223
 224 @wvtest
 225 def test_check_repo_or_die():
 226     with no_lingering_errors():
 227         with test_tempdir(b'bup-tgit-') as tmpdir:
 228             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 229             orig_cwd = os.getcwd()
 230             try:
 231                 os.chdir(tmpdir)
 232                 git.init_repo(bupdir)
 233                 git.check_repo_or_die()
 234                 # if we reach this point the call above passed
 235                 WVPASS('check_repo_or_die')
 236
 237                 os.rename(bupdir + b'/objects/pack',
 238                           bupdir + b'/objects/pack.tmp')
 239                 open(bupdir + b'/objects/pack', 'w').close()
 240                 try:
 241                     git.check_repo_or_die()
 242                 except SystemExit as e:
 243                     WVPASSEQ(e.code, 14)
 244                 else:
 245                     WVFAIL()
 246                 os.unlink(bupdir + b'/objects/pack')
 247                 os.rename(bupdir + b'/objects/pack.tmp',
 248                           bupdir + b'/objects/pack')
 249
 250                 try:
 251                     git.check_repo_or_die(b'nonexistantbup.tmp')
 252                 except SystemExit as e:
 253                     WVPASSEQ(e.code, 15)
 254                 else:
 255                     WVFAIL()
 256             finally:
 257                 os.chdir(orig_cwd)
 258
 259
 260 @wvtest
 261 def test_commit_parsing():
 262
 263     def restore_env_var(name, val):
 264         if val is None:
 265             del environ[name]
 266         else:
 267             environ[name] = val
 268
 269     def showval(commit, val):
 270         return readpipe([b'git', b'show', b'-s',
 271                          b'--pretty=format:%s' % val, commit]).strip()
 272
 273     with no_lingering_errors():
 274         with test_tempdir(b'bup-tgit-') as tmpdir:
 275             orig_cwd = os.getcwd()
 276             workdir = tmpdir + b'/work'
 277             repodir = workdir + b'/.git'
 278             orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
 279             orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
 280             orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
 281             orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
 282             environ[b'GIT_AUTHOR_NAME'] = b'bup test'
 283             environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
 284             environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
 285             environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
 286             try:
 287                 readpipe([b'git', b'init', workdir])
 288                 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
 289                 git.check_repo_or_die(repodir)
 290                 os.chdir(workdir)
 291                 with open('foo', 'w') as f:
 292                     print('bar', file=f)
 293                 readpipe([b'git', b'add', b'.'])
 294                 readpipe([b'git', b'commit', b'-am', b'Do something',
 295                           b'--author', b'Someone <someone@somewhere>',
 296                           b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
 297                 commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
 298                 parents = showval(commit, b'%P')
 299                 tree = showval(commit, b'%T')
 300                 cname = showval(commit, b'%cn')
 301                 cmail = showval(commit, b'%ce')
 302                 cdate = showval(commit, b'%ct')
 303                 coffs = showval(commit, b'%ci')
 304                 coffs = coffs[-5:]
 305                 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
 306                 if bytes_from_byte(coffs[-5]) == b'-':
 307                     coff = - coff
 308                 commit_items = git.get_commit_items(commit, git.cp())
 309                 WVPASSEQ(commit_items.parents, [])
 310                 WVPASSEQ(commit_items.tree, tree)
 311                 WVPASSEQ(commit_items.author_name, b'Someone')
 312                 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
 313                 WVPASSEQ(commit_items.author_sec, 1254613729)
 314                 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
 315                 WVPASSEQ(commit_items.committer_name, cname)
 316                 WVPASSEQ(commit_items.committer_mail, cmail)
 317                 WVPASSEQ(commit_items.committer_sec, int(cdate))
 318                 WVPASSEQ(commit_items.committer_offset, coff)
 319                 WVPASSEQ(commit_items.message, b'Do something\n')
 320                 with open(b'bar', 'wb') as f:
 321                     f.write(b'baz\n')
 322                 readpipe([b'git', b'add', '.'])
 323                 readpipe([b'git', b'commit', b'-am', b'Do something else'])
 324                 child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
 325                 parents = showval(child, b'%P')
 326                 commit_items = git.get_commit_items(child, git.cp())
 327                 WVPASSEQ(commit_items.parents, [commit])
 328             finally:
 329                 os.chdir(orig_cwd)
 330                 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
 331                 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
 332                 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
 333                 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
 334
 335
 336 @wvtest
 337 def test_new_commit():
 338     with no_lingering_errors():
 339         with test_tempdir(b'bup-tgit-') as tmpdir:
 340             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 341             git.init_repo(bupdir)
 342             git.verbose = 1
 343
 344             w = git.PackWriter()
 345             tree = os.urandom(20)
 346             parent = os.urandom(20)
 347             author_name = b'Author'
 348             author_mail = b'author@somewhere'
 349             adate_sec = 1439657836
 350             cdate_sec = adate_sec + 1
 351             committer_name = b'Committer'
 352             committer_mail = b'committer@somewhere'
 353             adate_tz_sec = cdate_tz_sec = None
 354             commit = w.new_commit(tree, parent,
 355                                   b'%s <%s>' % (author_name, author_mail),
 356                                   adate_sec, adate_tz_sec,
 357                                   b'%s <%s>' % (committer_name, committer_mail),
 358                                   cdate_sec, cdate_tz_sec,
 359                                   b'There is a small mailbox here')
 360             adate_tz_sec = -60 * 60
 361             cdate_tz_sec = 120 * 60
 362             commit_off = w.new_commit(tree, parent,
 363                                       b'%s <%s>' % (author_name, author_mail),
 364                                       adate_sec, adate_tz_sec,
 365                                       b'%s <%s>' % (committer_name, committer_mail),
 366                                       cdate_sec, cdate_tz_sec,
 367                                       b'There is a small mailbox here')
 368             w.close()
 369
 370             commit_items = git.get_commit_items(hexlify(commit), git.cp())
 371             local_author_offset = localtime(adate_sec).tm_gmtoff
 372             local_committer_offset = localtime(cdate_sec).tm_gmtoff
 373             WVPASSEQ(tree, unhexlify(commit_items.tree))
 374             WVPASSEQ(1, len(commit_items.parents))
 375             WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
 376             WVPASSEQ(author_name, commit_items.author_name)
 377             WVPASSEQ(author_mail, commit_items.author_mail)
 378             WVPASSEQ(adate_sec, commit_items.author_sec)
 379             WVPASSEQ(local_author_offset, commit_items.author_offset)
 380             WVPASSEQ(committer_name, commit_items.committer_name)
 381             WVPASSEQ(committer_mail, commit_items.committer_mail)
 382             WVPASSEQ(cdate_sec, commit_items.committer_sec)
 383             WVPASSEQ(local_committer_offset, commit_items.committer_offset)
 384
 385             commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
 386             WVPASSEQ(tree, unhexlify(commit_items.tree))
 387             WVPASSEQ(1, len(commit_items.parents))
 388             WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
 389             WVPASSEQ(author_name, commit_items.author_name)
 390             WVPASSEQ(author_mail, commit_items.author_mail)
 391             WVPASSEQ(adate_sec, commit_items.author_sec)
 392             WVPASSEQ(adate_tz_sec, commit_items.author_offset)
 393             WVPASSEQ(committer_name, commit_items.committer_name)
 394             WVPASSEQ(committer_mail, commit_items.committer_mail)
 395             WVPASSEQ(cdate_sec, commit_items.committer_sec)
 396             WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
 397
 398
 399 @wvtest
 400 def test_list_refs():
 401     with no_lingering_errors():
 402         with test_tempdir(b'bup-tgit-') as tmpdir:
 403             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 404             src = tmpdir + b'/src'
 405             mkdirp(src)
 406             with open(src + b'/1', 'wb+') as f:
 407                 f.write(b'something\n')
 408             with open(src + b'/2', 'wb+') as f:
 409                 f.write(b'something else\n')
 410             git.init_repo(bupdir)
 411             emptyset = frozenset()
 412             WVPASSEQ(frozenset(git.list_refs()), emptyset)
 413             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
 414             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
 415             exc(bup_exe, b'index', src)
 416             exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
 417             src_hash = exo(b'git', b'--git-dir', bupdir,
 418                            b'rev-parse', b'src').strip().split(b'\n')
 419             assert(len(src_hash) == 1)
 420             src_hash = unhexlify(src_hash[0])
 421             tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
 422                                       b'rev-parse',
 423                                       b'src:').strip().split(b'\n')[0])
 424             blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
 425                                       b'rev-parse',
 426                                       b'src:1').strip().split(b'\n')[0])
 427             WVPASSEQ(frozenset(git.list_refs()),
 428                      frozenset([(b'refs/heads/src', src_hash)]))
 429             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
 430             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
 431                      frozenset([(b'refs/heads/src', src_hash)]))
 432             exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
 433             WVPASSEQ(frozenset(git.list_refs()),
 434                      frozenset([(b'refs/heads/src', src_hash),
 435                                 (b'refs/tags/commit-tag', src_hash)]))
 436             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
 437                      frozenset([(b'refs/tags/commit-tag', src_hash)]))
 438             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
 439                      frozenset([(b'refs/heads/src', src_hash)]))
 440             exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
 441             exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
 442             os.unlink(bupdir + b'/refs/heads/src')
 443             expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
 444                                        (b'refs/tags/tree-tag', tree_hash),
 445                                        (b'refs/tags/blob-tag', blob_hash)])
 446             WVPASSEQ(frozenset(git.list_refs()), expected_tags)
 447             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
 448             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
 449
 450
 451 @wvtest
 452 def test__git_date_str():
 453     with no_lingering_errors():
 454         WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
 455         WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
 456         WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
 457
 458
 459 @wvtest
 460 def test_cat_pipe():
 461     with no_lingering_errors():
 462         with test_tempdir(b'bup-tgit-') as tmpdir:
 463             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 464             src = tmpdir + b'/src'
 465             mkdirp(src)
 466             with open(src + b'/1', 'wb+') as f:
 467                 f.write(b'something\n')
 468             with open(src + b'/2', 'wb+') as f:
 469                 f.write(b'something else\n')
 470             git.init_repo(bupdir)
 471             exc(bup_exe, b'index', src)
 472             oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
 473                        src).strip()
 474             typ = exo(b'git', b'--git-dir', bupdir,
 475                       b'cat-file', b'-t', b'src').strip()
 476             size = int(exo(b'git', b'--git-dir', bupdir,
 477                                b'cat-file', b'-s', b'src'))
 478             it = git.cp().get(b'src')
 479             get_info = next(it)
 480             for buf in next(it):
 481                 pass
 482             WVPASSEQ((oidx, typ, size), get_info)