lib/bup/t/tgit.py

   1
   2 from __future__ import absolute_import, print_function
   3 from binascii import hexlify, unhexlify
   4 from subprocess import check_call
   5 import struct, os, time
   6
   7 from wvtest import *
   8
   9 from bup import git, path
  10 from bup.compat import bytes_from_byte, environ, range
  11 from bup.helpers import localtime, log, mkdirp, readpipe
  12 from buptest import no_lingering_errors, test_tempdir
  13
  14
  15 bup_exe = path.exe()
  16
  17
  18 def exc(*cmd):
  19     print(repr(cmd), file=sys.stderr)
  20     check_call(cmd)
  21
  22
  23 def exo(*cmd):
  24     print(repr(cmd), file=sys.stderr)
  25     return readpipe(cmd)
  26
  27
  28 @wvtest
  29 def testmangle():
  30     with no_lingering_errors():
  31         afile  = 0o100644
  32         afile2 = 0o100770
  33         alink  = 0o120000
  34         adir   = 0o040000
  35         adir2  = 0o040777
  36         WVPASSEQ(git.mangle_name(b'a', adir2, adir), b'a')
  37         WVPASSEQ(git.mangle_name(b'.bup', adir2, adir), b'.bup.bupl')
  38         WVPASSEQ(git.mangle_name(b'a.bupa', adir2, adir), b'a.bupa.bupl')
  39         WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
  40         WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
  41         WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
  42         WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
  43         WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
  44         WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
  45
  46         WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
  47         WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
  48         WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
  49
  50         WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
  51         WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
  52
  53         # for safety, we ignore .bup? suffixes we don't recognize.  Future
  54         # versions might implement a .bup[a-z] extension as something other
  55         # than BUP_NORMAL.
  56         WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
  57
  58
  59 @wvtest
  60 def testencode():
  61     with no_lingering_errors():
  62         s = b'hello world'
  63         looseb = b''.join(git._encode_looseobj(b'blob', s))
  64         looset = b''.join(git._encode_looseobj(b'tree', s))
  65         loosec = b''.join(git._encode_looseobj(b'commit', s))
  66         packb = b''.join(git._encode_packobj(b'blob', s))
  67         packt = b''.join(git._encode_packobj(b'tree', s))
  68         packc = b''.join(git._encode_packobj(b'commit', s))
  69         packlb = b''.join(git._encode_packobj(b'blob', s * 200))
  70         WVPASSEQ(git._decode_looseobj(looseb), (b'blob', s))
  71         WVPASSEQ(git._decode_looseobj(looset), (b'tree', s))
  72         WVPASSEQ(git._decode_looseobj(loosec), (b'commit', s))
  73         WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
  74         WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
  75         WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
  76         WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
  77         for i in range(10):
  78             WVPASS(git._encode_looseobj(b'blob', s, compression_level=i))
  79         def encode_pobj(n):
  80             return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
  81         WVEXCEPT(ValueError, encode_pobj, -1)
  82         WVEXCEPT(ValueError, encode_pobj, 10)
  83         WVEXCEPT(ValueError, encode_pobj, b'x')
  84
  85
  86 @wvtest
  87 def testpacks():
  88     with no_lingering_errors():
  89         with test_tempdir(b'bup-tgit-') as tmpdir:
  90             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
  91             git.init_repo(bupdir)
  92             git.verbose = 1
  93
  94             w = git.PackWriter()
  95             w.new_blob(os.urandom(100))
  96             w.new_blob(os.urandom(100))
  97             w.abort()
  98
  99             w = git.PackWriter()
 100             hashes = []
 101             nobj = 1000
 102             for i in range(nobj):
 103                 hashes.append(w.new_blob(b'%d' % i))
 104             log('\n')
 105             nameprefix = w.close()
 106             print(repr(nameprefix))
 107             WVPASS(os.path.exists(nameprefix + b'.pack'))
 108             WVPASS(os.path.exists(nameprefix + b'.idx'))
 109
 110             r = git.open_idx(nameprefix + b'.idx')
 111             print(repr(r.fanout))
 112
 113             for i in range(nobj):
 114                 WVPASS(r.find_offset(hashes[i]) > 0)
 115             WVPASS(r.exists(hashes[99]))
 116             WVFAIL(r.exists(b'\0'*20))
 117
 118             pi = iter(r)
 119             for h in sorted(hashes):
 120                 WVPASSEQ(hexlify(next(pi)), hexlify(h))
 121
 122             WVFAIL(r.find_offset(b'\0'*20))
 123
 124             r = git.PackIdxList(bupdir + b'/objects/pack')
 125             WVPASS(r.exists(hashes[5]))
 126             WVPASS(r.exists(hashes[6]))
 127             WVFAIL(r.exists(b'\0'*20))
 128
 129
 130 @wvtest
 131 def test_pack_name_lookup():
 132     with no_lingering_errors():
 133         with test_tempdir(b'bup-tgit-') as tmpdir:
 134             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 135             git.init_repo(bupdir)
 136             git.verbose = 1
 137             packdir = git.repo(b'objects/pack')
 138
 139             idxnames = []
 140             hashes = []
 141
 142             for start in range(0,28,2):
 143                 w = git.PackWriter()
 144                 for i in range(start, start+2):
 145                     hashes.append(w.new_blob(b'%d' % i))
 146                 log('\n')
 147                 idxnames.append(os.path.basename(w.close() + b'.idx'))
 148
 149             r = git.PackIdxList(packdir)
 150             WVPASSEQ(len(r.packs), 2)
 151             for e,idxname in enumerate(idxnames):
 152                 for i in range(e*2, (e+1)*2):
 153                     WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
 154
 155
 156 @wvtest
 157 def test_long_index():
 158     with no_lingering_errors():
 159         with test_tempdir(b'bup-tgit-') as tmpdir:
 160             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 161             git.init_repo(bupdir)
 162             w = git.PackWriter()
 163             obj_bin = struct.pack('!IIIII',
 164                     0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
 165             obj2_bin = struct.pack('!IIIII',
 166                     0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
 167             obj3_bin = struct.pack('!IIIII',
 168                     0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
 169             pack_bin = struct.pack('!IIIII',
 170                     0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
 171             idx = list(list() for i in range(256))
 172             idx[0].append((obj_bin, 1, 0xfffffffff))
 173             idx[0x11].append((obj2_bin, 2, 0xffffffffff))
 174             idx[0x22].append((obj3_bin, 3, 0xff))
 175             w.count = 3
 176             name = tmpdir + b'/tmp.idx'
 177             r = w._write_pack_idx_v2(name, idx, pack_bin)
 178             i = git.PackIdxV2(name, open(name, 'rb'))
 179             WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
 180             WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
 181             WVPASSEQ(i.find_offset(obj3_bin), 0xff)
 182
 183
 184 @wvtest
 185 def test_check_repo_or_die():
 186     with no_lingering_errors():
 187         with test_tempdir(b'bup-tgit-') as tmpdir:
 188             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 189             orig_cwd = os.getcwd()
 190             try:
 191                 os.chdir(tmpdir)
 192                 git.init_repo(bupdir)
 193                 git.check_repo_or_die()
 194                 # if we reach this point the call above passed
 195                 WVPASS('check_repo_or_die')
 196
 197                 os.rename(bupdir + b'/objects/pack',
 198                           bupdir + b'/objects/pack.tmp')
 199                 open(bupdir + b'/objects/pack', 'w').close()
 200                 try:
 201                     git.check_repo_or_die()
 202                 except SystemExit as e:
 203                     WVPASSEQ(e.code, 14)
 204                 else:
 205                     WVFAIL()
 206                 os.unlink(bupdir + b'/objects/pack')
 207                 os.rename(bupdir + b'/objects/pack.tmp',
 208                           bupdir + b'/objects/pack')
 209
 210                 try:
 211                     git.check_repo_or_die(b'nonexistantbup.tmp')
 212                 except SystemExit as e:
 213                     WVPASSEQ(e.code, 15)
 214                 else:
 215                     WVFAIL()
 216             finally:
 217                 os.chdir(orig_cwd)
 218
 219
 220 @wvtest
 221 def test_commit_parsing():
 222
 223     def restore_env_var(name, val):
 224         if val is None:
 225             del environ[name]
 226         else:
 227             environ[name] = val
 228
 229     def showval(commit, val):
 230         return readpipe([b'git', b'show', b'-s',
 231                          b'--pretty=format:%s' % val, commit]).strip()
 232
 233     with no_lingering_errors():
 234         with test_tempdir(b'bup-tgit-') as tmpdir:
 235             orig_cwd = os.getcwd()
 236             workdir = tmpdir + b'/work'
 237             repodir = workdir + b'/.git'
 238             orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
 239             orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
 240             orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
 241             orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
 242             environ[b'GIT_AUTHOR_NAME'] = b'bup test'
 243             environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
 244             environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
 245             environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
 246             try:
 247                 readpipe([b'git', b'init', workdir])
 248                 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
 249                 git.check_repo_or_die(repodir)
 250                 os.chdir(workdir)
 251                 with open('foo', 'w') as f:
 252                     print('bar', file=f)
 253                 readpipe([b'git', b'add', b'.'])
 254                 readpipe([b'git', b'commit', b'-am', b'Do something',
 255                           b'--author', b'Someone <someone@somewhere>',
 256                           b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
 257                 commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
 258                 parents = showval(commit, b'%P')
 259                 tree = showval(commit, b'%T')
 260                 cname = showval(commit, b'%cn')
 261                 cmail = showval(commit, b'%ce')
 262                 cdate = showval(commit, b'%ct')
 263                 coffs = showval(commit, b'%ci')
 264                 coffs = coffs[-5:]
 265                 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
 266                 if bytes_from_byte(coffs[-5]) == b'-':
 267                     coff = - coff
 268                 commit_items = git.get_commit_items(commit, git.cp())
 269                 WVPASSEQ(commit_items.parents, [])
 270                 WVPASSEQ(commit_items.tree, tree)
 271                 WVPASSEQ(commit_items.author_name, b'Someone')
 272                 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
 273                 WVPASSEQ(commit_items.author_sec, 1254613729)
 274                 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
 275                 WVPASSEQ(commit_items.committer_name, cname)
 276                 WVPASSEQ(commit_items.committer_mail, cmail)
 277                 WVPASSEQ(commit_items.committer_sec, int(cdate))
 278                 WVPASSEQ(commit_items.committer_offset, coff)
 279                 WVPASSEQ(commit_items.message, b'Do something\n')
 280                 with open(b'bar', 'wb') as f:
 281                     f.write(b'baz\n')
 282                 readpipe([b'git', b'add', '.'])
 283                 readpipe([b'git', b'commit', b'-am', b'Do something else'])
 284                 child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
 285                 parents = showval(child, b'%P')
 286                 commit_items = git.get_commit_items(child, git.cp())
 287                 WVPASSEQ(commit_items.parents, [commit])
 288             finally:
 289                 os.chdir(orig_cwd)
 290                 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
 291                 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
 292                 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
 293                 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
 294
 295
 296 @wvtest
 297 def test_new_commit():
 298     with no_lingering_errors():
 299         with test_tempdir(b'bup-tgit-') as tmpdir:
 300             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 301             git.init_repo(bupdir)
 302             git.verbose = 1
 303
 304             w = git.PackWriter()
 305             tree = os.urandom(20)
 306             parent = os.urandom(20)
 307             author_name = b'Author'
 308             author_mail = b'author@somewhere'
 309             adate_sec = 1439657836
 310             cdate_sec = adate_sec + 1
 311             committer_name = b'Committer'
 312             committer_mail = b'committer@somewhere'
 313             adate_tz_sec = cdate_tz_sec = None
 314             commit = w.new_commit(tree, parent,
 315                                   b'%s <%s>' % (author_name, author_mail),
 316                                   adate_sec, adate_tz_sec,
 317                                   b'%s <%s>' % (committer_name, committer_mail),
 318                                   cdate_sec, cdate_tz_sec,
 319                                   b'There is a small mailbox here')
 320             adate_tz_sec = -60 * 60
 321             cdate_tz_sec = 120 * 60
 322             commit_off = w.new_commit(tree, parent,
 323                                       b'%s <%s>' % (author_name, author_mail),
 324                                       adate_sec, adate_tz_sec,
 325                                       b'%s <%s>' % (committer_name, committer_mail),
 326                                       cdate_sec, cdate_tz_sec,
 327                                       b'There is a small mailbox here')
 328             w.close()
 329
 330             commit_items = git.get_commit_items(hexlify(commit), git.cp())
 331             local_author_offset = localtime(adate_sec).tm_gmtoff
 332             local_committer_offset = localtime(cdate_sec).tm_gmtoff
 333             WVPASSEQ(tree, unhexlify(commit_items.tree))
 334             WVPASSEQ(1, len(commit_items.parents))
 335             WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
 336             WVPASSEQ(author_name, commit_items.author_name)
 337             WVPASSEQ(author_mail, commit_items.author_mail)
 338             WVPASSEQ(adate_sec, commit_items.author_sec)
 339             WVPASSEQ(local_author_offset, commit_items.author_offset)
 340             WVPASSEQ(committer_name, commit_items.committer_name)
 341             WVPASSEQ(committer_mail, commit_items.committer_mail)
 342             WVPASSEQ(cdate_sec, commit_items.committer_sec)
 343             WVPASSEQ(local_committer_offset, commit_items.committer_offset)
 344
 345             commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
 346             WVPASSEQ(tree, unhexlify(commit_items.tree))
 347             WVPASSEQ(1, len(commit_items.parents))
 348             WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
 349             WVPASSEQ(author_name, commit_items.author_name)
 350             WVPASSEQ(author_mail, commit_items.author_mail)
 351             WVPASSEQ(adate_sec, commit_items.author_sec)
 352             WVPASSEQ(adate_tz_sec, commit_items.author_offset)
 353             WVPASSEQ(committer_name, commit_items.committer_name)
 354             WVPASSEQ(committer_mail, commit_items.committer_mail)
 355             WVPASSEQ(cdate_sec, commit_items.committer_sec)
 356             WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
 357
 358
 359 @wvtest
 360 def test_list_refs():
 361     with no_lingering_errors():
 362         with test_tempdir(b'bup-tgit-') as tmpdir:
 363             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 364             src = tmpdir + b'/src'
 365             mkdirp(src)
 366             with open(src + b'/1', 'wb+') as f:
 367                 f.write(b'something\n')
 368             with open(src + b'/2', 'wb+') as f:
 369                 f.write(b'something else\n')
 370             git.init_repo(bupdir)
 371             emptyset = frozenset()
 372             WVPASSEQ(frozenset(git.list_refs()), emptyset)
 373             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
 374             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
 375             exc(bup_exe, b'index', src)
 376             exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
 377             src_hash = exo(b'git', b'--git-dir', bupdir,
 378                            b'rev-parse', b'src').strip().split(b'\n')
 379             assert(len(src_hash) == 1)
 380             src_hash = unhexlify(src_hash[0])
 381             tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
 382                                       b'rev-parse',
 383                                       b'src:').strip().split(b'\n')[0])
 384             blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
 385                                       b'rev-parse',
 386                                       b'src:1').strip().split(b'\n')[0])
 387             WVPASSEQ(frozenset(git.list_refs()),
 388                      frozenset([(b'refs/heads/src', src_hash)]))
 389             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
 390             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
 391                      frozenset([(b'refs/heads/src', src_hash)]))
 392             exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
 393             WVPASSEQ(frozenset(git.list_refs()),
 394                      frozenset([(b'refs/heads/src', src_hash),
 395                                 (b'refs/tags/commit-tag', src_hash)]))
 396             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
 397                      frozenset([(b'refs/tags/commit-tag', src_hash)]))
 398             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
 399                      frozenset([(b'refs/heads/src', src_hash)]))
 400             exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
 401             exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
 402             os.unlink(bupdir + b'/refs/heads/src')
 403             expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
 404                                        (b'refs/tags/tree-tag', tree_hash),
 405                                        (b'refs/tags/blob-tag', blob_hash)])
 406             WVPASSEQ(frozenset(git.list_refs()), expected_tags)
 407             WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
 408             WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
 409
 410
 411 @wvtest
 412 def test__git_date_str():
 413     with no_lingering_errors():
 414         WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
 415         WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
 416         WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
 417
 418
 419 @wvtest
 420 def test_cat_pipe():
 421     with no_lingering_errors():
 422         with test_tempdir(b'bup-tgit-') as tmpdir:
 423             environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
 424             src = tmpdir + b'/src'
 425             mkdirp(src)
 426             with open(src + b'/1', 'wb+') as f:
 427                 f.write(b'something\n')
 428             with open(src + b'/2', 'wb+') as f:
 429                 f.write(b'something else\n')
 430             git.init_repo(bupdir)
 431             exc(bup_exe, b'index', src)
 432             oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
 433                        src).strip()
 434             typ = exo(b'git', b'--git-dir', bupdir,
 435                       b'cat-file', b'-t', b'src').strip()
 436             size = int(exo(b'git', b'--git-dir', bupdir,
 437                                b'cat-file', b'-s', b'src'))
 438             it = git.cp().get(b'src')
 439             get_info = next(it)
 440             for buf in next(it):
 441                 pass
 442             WVPASSEQ((oidx, typ, size), get_info)