2 from __future__ import absolute_import, print_function
4 from binascii import hexlify, unhexlify
5 from subprocess import check_call
6 from functools import partial
7 import struct, os, time
10 from wvpytest import *
12 from bup import git, path
13 from bup.compat import bytes_from_byte, environ, range
14 from bup.helpers import localtime, log, mkdirp, readpipe
21 print(repr(cmd), file=sys.stderr)
26 print(repr(cmd), file=sys.stderr)
30 def test_git_version_detection():
31 # Test version types from git's tag history
32 for expected, ver in \
33 (('insufficient', b'git version 0.99'),
34 ('insufficient', b'git version 0.99.1'),
35 ('insufficient', b'git version 0.99.7a'),
36 ('insufficient', b'git version 1.0rc1'),
37 ('insufficient', b'git version 1.0.1'),
38 ('insufficient', b'git version 1.4.2.1'),
39 ('insufficient', b'git version 1.5.5'),
40 ('insufficient', b'git version 1.5.6-rc0'),
41 ('suitable', b'git version 1.5.6'),
42 ('suitable', b'git version 1.5.6.1'),
43 ('suitable', b'git version 2.14.0-rc0'),
44 ('suitable', b'git version 2.14.0 (something ...)'),
45 ('suitable', b'git version 111.222.333.444-rc555'),
46 ('unrecognized', b'huh?')):
47 assert expected == git.is_suitable_git(ver_str=ver)
49 if expected == 'insufficient':
50 with pytest.raises(SystemExit):
51 git.require_suitable_git(ver)
52 elif expected == 'suitable':
53 git.require_suitable_git(ver_str=ver)
54 elif expected == 'unrecognized':
55 with pytest.raises(git.GitError):
56 git.require_suitable_git(ver)
62 environ[b'BUP_GIT_VERSION_IS_FINE'] = b'true'
63 git.require_suitable_git(ver_str=ver)
65 del environ[b'BUP_GIT_VERSION_IS_FINE']
75 assert git.mangle_name(b'a', adir2, adir) == b'a'
76 assert git.mangle_name(b'.bup', adir2, adir) == b'.bup.bupl'
77 assert git.mangle_name(b'a.bupa', adir2, adir) == b'a.bupa.bupl'
78 WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
79 WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
80 WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
81 WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
82 WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
83 WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
85 WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
86 WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
87 WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
89 WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
90 WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
92 # for safety, we ignore .bup? suffixes we don't recognize. Future
93 # versions might implement a .bup[a-z] extension as something other
95 WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
100 packb = b''.join(git._encode_packobj(b'blob', s))
101 packt = b''.join(git._encode_packobj(b'tree', s))
102 packc = b''.join(git._encode_packobj(b'commit', s))
103 packlb = b''.join(git._encode_packobj(b'blob', s * 200))
104 WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
105 WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
106 WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
107 WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
109 return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
110 WVEXCEPT(ValueError, encode_pobj, -1)
111 WVEXCEPT(ValueError, encode_pobj, 10)
112 WVEXCEPT(ValueError, encode_pobj, b'x')
115 def test_packs(tmpdir):
116 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
117 git.init_repo(bupdir)
121 w.new_blob(os.urandom(100))
122 w.new_blob(os.urandom(100))
128 for i in range(nobj):
129 hashes.append(w.new_blob(b'%d' % i))
131 nameprefix = w.close()
132 print(repr(nameprefix))
133 WVPASS(os.path.exists(nameprefix + b'.pack'))
134 WVPASS(os.path.exists(nameprefix + b'.idx'))
136 r = git.open_idx(nameprefix + b'.idx')
137 print(repr(r.fanout))
139 for i in range(nobj):
140 WVPASS(r.find_offset(hashes[i]) > 0)
141 WVPASS(r.exists(hashes[99]))
142 WVFAIL(r.exists(b'\0'*20))
145 for h in sorted(hashes):
146 WVPASSEQ(hexlify(next(pi)), hexlify(h))
148 WVFAIL(r.find_offset(b'\0'*20))
150 r = git.PackIdxList(bupdir + b'/objects/pack')
151 WVPASS(r.exists(hashes[5]))
152 WVPASS(r.exists(hashes[6]))
153 WVFAIL(r.exists(b'\0'*20))
156 def test_pack_name_lookup(tmpdir):
157 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
158 git.init_repo(bupdir)
160 packdir = git.repo(b'objects/pack')
165 for start in range(0,28,2):
167 for i in range(start, start+2):
168 hashes.append(w.new_blob(b'%d' % i))
170 idxnames.append(os.path.basename(w.close() + b'.idx'))
172 r = git.PackIdxList(packdir)
173 WVPASSEQ(len(r.packs), 2)
174 for e,idxname in enumerate(idxnames):
175 for i in range(e*2, (e+1)*2):
176 WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
179 def test_long_index(tmpdir):
180 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
181 git.init_repo(bupdir)
182 idx = git.PackIdxV2Writer()
183 obj_bin = struct.pack('!IIIII',
184 0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
185 obj2_bin = struct.pack('!IIIII',
186 0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
187 obj3_bin = struct.pack('!IIIII',
188 0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
189 pack_bin = struct.pack('!IIIII',
190 0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
191 idx.add(obj_bin, 1, 0xfffffffff)
192 idx.add(obj2_bin, 2, 0xffffffffff)
193 idx.add(obj3_bin, 3, 0xff)
194 name = tmpdir + b'/tmp.idx'
195 r = idx.write(name, pack_bin)
196 i = git.PackIdxV2(name, open(name, 'rb'))
197 WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
198 WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
199 WVPASSEQ(i.find_offset(obj3_bin), 0xff)
202 def test_check_repo_or_die(tmpdir):
203 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
204 orig_cwd = os.getcwd()
207 git.init_repo(bupdir)
208 git.check_repo_or_die()
209 # if we reach this point the call above passed
210 WVPASS('check_repo_or_die')
212 os.rename(bupdir + b'/objects/pack',
213 bupdir + b'/objects/pack.tmp')
214 open(bupdir + b'/objects/pack', 'w').close()
216 git.check_repo_or_die()
217 except SystemExit as e:
221 os.unlink(bupdir + b'/objects/pack')
222 os.rename(bupdir + b'/objects/pack.tmp',
223 bupdir + b'/objects/pack')
226 git.check_repo_or_die(b'nonexistantbup.tmp')
227 except SystemExit as e:
235 def test_commit_parsing(tmpdir):
236 def restore_env_var(name, val):
242 def showval(commit, val):
243 return readpipe([b'git', b'show', b'-s',
244 b'--pretty=format:%s' % val, commit]).strip()
246 orig_cwd = os.getcwd()
247 workdir = tmpdir + b'/work'
248 repodir = workdir + b'/.git'
249 orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
250 orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
251 orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
252 orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
253 environ[b'GIT_AUTHOR_NAME'] = b'bup test'
254 environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
255 environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
256 environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
258 readpipe([b'git', b'init', workdir])
259 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
260 git.check_repo_or_die(repodir)
262 with open('foo', 'w') as f:
264 readpipe([b'git', b'add', b'.'])
265 readpipe([b'git', b'commit', b'-am', b'Do something',
266 b'--author', b'Someone <someone@somewhere>',
267 b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
268 commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
269 parents = showval(commit, b'%P')
270 tree = showval(commit, b'%T')
271 cname = showval(commit, b'%cn')
272 cmail = showval(commit, b'%ce')
273 cdate = showval(commit, b'%ct')
274 coffs = showval(commit, b'%ci')
276 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
277 if bytes_from_byte(coffs[-5]) == b'-':
279 commit_items = git.get_commit_items(commit, git.cp())
280 WVPASSEQ(commit_items.parents, [])
281 WVPASSEQ(commit_items.tree, tree)
282 WVPASSEQ(commit_items.author_name, b'Someone')
283 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
284 WVPASSEQ(commit_items.author_sec, 1254613729)
285 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
286 WVPASSEQ(commit_items.committer_name, cname)
287 WVPASSEQ(commit_items.committer_mail, cmail)
288 WVPASSEQ(commit_items.committer_sec, int(cdate))
289 WVPASSEQ(commit_items.committer_offset, coff)
290 WVPASSEQ(commit_items.message, b'Do something\n')
291 with open(b'bar', 'wb') as f:
293 readpipe([b'git', b'add', '.'])
294 readpipe([b'git', b'commit', b'-am', b'Do something else'])
295 child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
296 parents = showval(child, b'%P')
297 commit_items = git.get_commit_items(child, git.cp())
298 WVPASSEQ(commit_items.parents, [commit])
301 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
302 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
303 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
304 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
307 def test_new_commit(tmpdir):
308 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
309 git.init_repo(bupdir)
313 tree = os.urandom(20)
314 parent = os.urandom(20)
315 author_name = b'Author'
316 author_mail = b'author@somewhere'
317 adate_sec = 1439657836
318 cdate_sec = adate_sec + 1
319 committer_name = b'Committer'
320 committer_mail = b'committer@somewhere'
321 adate_tz_sec = cdate_tz_sec = None
322 commit = w.new_commit(tree, parent,
323 b'%s <%s>' % (author_name, author_mail),
324 adate_sec, adate_tz_sec,
325 b'%s <%s>' % (committer_name, committer_mail),
326 cdate_sec, cdate_tz_sec,
327 b'There is a small mailbox here')
328 adate_tz_sec = -60 * 60
329 cdate_tz_sec = 120 * 60
330 commit_off = w.new_commit(tree, parent,
331 b'%s <%s>' % (author_name, author_mail),
332 adate_sec, adate_tz_sec,
333 b'%s <%s>' % (committer_name, committer_mail),
334 cdate_sec, cdate_tz_sec,
335 b'There is a small mailbox here')
338 commit_items = git.get_commit_items(hexlify(commit), git.cp())
339 local_author_offset = localtime(adate_sec).tm_gmtoff
340 local_committer_offset = localtime(cdate_sec).tm_gmtoff
341 WVPASSEQ(tree, unhexlify(commit_items.tree))
342 WVPASSEQ(1, len(commit_items.parents))
343 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
344 WVPASSEQ(author_name, commit_items.author_name)
345 WVPASSEQ(author_mail, commit_items.author_mail)
346 WVPASSEQ(adate_sec, commit_items.author_sec)
347 WVPASSEQ(local_author_offset, commit_items.author_offset)
348 WVPASSEQ(committer_name, commit_items.committer_name)
349 WVPASSEQ(committer_mail, commit_items.committer_mail)
350 WVPASSEQ(cdate_sec, commit_items.committer_sec)
351 WVPASSEQ(local_committer_offset, commit_items.committer_offset)
353 commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
354 WVPASSEQ(tree, unhexlify(commit_items.tree))
355 WVPASSEQ(1, len(commit_items.parents))
356 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
357 WVPASSEQ(author_name, commit_items.author_name)
358 WVPASSEQ(author_mail, commit_items.author_mail)
359 WVPASSEQ(adate_sec, commit_items.author_sec)
360 WVPASSEQ(adate_tz_sec, commit_items.author_offset)
361 WVPASSEQ(committer_name, commit_items.committer_name)
362 WVPASSEQ(committer_mail, commit_items.committer_mail)
363 WVPASSEQ(cdate_sec, commit_items.committer_sec)
364 WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
367 def test_list_refs(tmpdir):
368 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
369 src = tmpdir + b'/src'
371 with open(src + b'/1', 'wb+') as f:
372 f.write(b'something\n')
373 with open(src + b'/2', 'wb+') as f:
374 f.write(b'something else\n')
375 git.init_repo(bupdir)
376 emptyset = frozenset()
377 WVPASSEQ(frozenset(git.list_refs()), emptyset)
378 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
379 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
380 exc(bup_exe, b'index', src)
381 exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
382 src_hash = exo(b'git', b'--git-dir', bupdir,
383 b'rev-parse', b'src').strip().split(b'\n')
384 assert(len(src_hash) == 1)
385 src_hash = unhexlify(src_hash[0])
386 tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
388 b'src:').strip().split(b'\n')[0])
389 blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
391 b'src:1').strip().split(b'\n')[0])
392 WVPASSEQ(frozenset(git.list_refs()),
393 frozenset([(b'refs/heads/src', src_hash)]))
394 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
395 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
396 frozenset([(b'refs/heads/src', src_hash)]))
397 exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
398 WVPASSEQ(frozenset(git.list_refs()),
399 frozenset([(b'refs/heads/src', src_hash),
400 (b'refs/tags/commit-tag', src_hash)]))
401 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
402 frozenset([(b'refs/tags/commit-tag', src_hash)]))
403 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
404 frozenset([(b'refs/heads/src', src_hash)]))
405 exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
406 exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
407 os.unlink(bupdir + b'/refs/heads/src')
408 expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
409 (b'refs/tags/tree-tag', tree_hash),
410 (b'refs/tags/blob-tag', blob_hash)])
411 WVPASSEQ(frozenset(git.list_refs()), expected_tags)
412 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
413 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
416 def test_git_date_str():
417 WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
418 WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
419 WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
422 def test_cat_pipe(tmpdir):
423 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
424 src = tmpdir + b'/src'
426 with open(src + b'/1', 'wb+') as f:
427 f.write(b'something\n')
428 with open(src + b'/2', 'wb+') as f:
429 f.write(b'something else\n')
430 git.init_repo(bupdir)
431 exc(bup_exe, b'index', src)
432 oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
434 typ = exo(b'git', b'--git-dir', bupdir,
435 b'cat-file', b'-t', b'src').strip()
436 size = int(exo(b'git', b'--git-dir', bupdir,
437 b'cat-file', b'-s', b'src'))
438 it = git.cp().get(b'src')
442 WVPASSEQ((oidx, typ, size), get_info)
444 def _create_idx(d, i):
445 idx = git.PackIdxV2Writer()
446 # add 255 vaguely reasonable entries
448 idx.add(struct.pack('18xBB', i, s), s, 100 * s)
449 packbin = struct.pack('B19x', i)
450 packname = os.path.join(d, b'pack-%s.idx' % hexlify(packbin))
451 idx.write(packname, packbin)
453 def test_midx_close(tmpdir):
454 fddir = b'/proc/self/fd'
458 # not supported, not Linux, I guess
462 for fd in os.listdir(fddir):
464 yield os.readlink(os.path.join(fddir, fd))
468 def force_midx(objdir):
469 args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
472 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
473 git.init_repo(bupdir)
474 # create a few dummy idxes
476 _create_idx(tmpdir, i)
477 git.auto_midx(tmpdir)
478 l = git.PackIdxList(tmpdir)
479 # this doesn't exist (yet)
480 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
481 for i in range(10, 15):
482 _create_idx(tmpdir, i)
483 # delete the midx ...
484 # TODO: why do we need to? git.auto_midx() below doesn't?!
485 for fn in os.listdir(tmpdir):
486 if fn.endswith(b'.midx'):
487 os.unlink(os.path.join(tmpdir, fn))
489 git.auto_midx(tmpdir)
490 # check it still doesn't exist - we haven't refreshed
491 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
492 # check that we still have the midx open, this really
493 # just checks more for the kernel API ('deleted' string)
494 for fn in openfiles():
495 if not b'midx-' in fn:
497 WVPASSEQ(True, b'deleted' in fn)
498 # refresh the PackIdxList
500 # and check that an object in pack 10 exists now
501 WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
502 for fn in openfiles():
503 if not b'midx-' in fn:
505 # check that we don't have it open anymore
506 WVPASSEQ(False, b'deleted' in fn)
509 cfg_file = os.path.join(os.path.dirname(__file__), 'sample.conf')
510 no_such_file = os.path.join(os.path.dirname(__file__), 'nosuch.conf')
511 git_config_get = partial(git.git_config_get, cfg_file=cfg_file)
512 WVPASSEQ(git_config_get(b'bup.foo'), b'bar')
513 WVPASSEQ(git_config_get(b'bup.bup'), b'is great')
514 WVPASSEQ(git_config_get(b'bup.end'), b'end')
515 WVPASSEQ(git_config_get(b'bup.comments'), None)
516 WVPASSEQ(git_config_get(b'bup.;comments'), None)
517 WVPASSEQ(git_config_get(b'bup.and'), None)
518 WVPASSEQ(git_config_get(b'bup.#and'), None)
520 WVPASSEQ(git.git_config_get(b'bup.foo', cfg_file=no_such_file), None)
522 WVEXCEPT(git.GitError, git_config_get, b'bup.isbad', opttype='bool')
523 WVEXCEPT(git.GitError, git_config_get, b'bup.isbad', opttype='int')
524 WVPASSEQ(git_config_get(b'bup.isbad'), b'ok')
525 WVPASSEQ(True, git_config_get(b'bup.istrue1', opttype='bool'))
526 WVPASSEQ(True, git_config_get(b'bup.istrue2', opttype='bool'))
527 WVPASSEQ(True, git_config_get(b'bup.istrue3', opttype='bool'))
528 WVPASSEQ(False, git_config_get(b'bup.isfalse1', opttype='bool'))
529 WVPASSEQ(False, git_config_get(b'bup.isfalse2', opttype='bool'))
530 WVPASSEQ(None, git_config_get(b'bup.nosuchkey', opttype='bool'))
531 WVPASSEQ(1, git_config_get(b'bup.istrue1', opttype='int'))
532 WVPASSEQ(2, git_config_get(b'bup.istrue2', opttype='int'))
533 WVPASSEQ(0, git_config_get(b'bup.isfalse2', opttype='int'))
534 WVPASSEQ(0x777, git_config_get(b'bup.hex', opttype='int'))