2 from __future__ import absolute_import, print_function
4 from binascii import hexlify, unhexlify
5 from subprocess import check_call
6 from functools import partial
7 import struct, os, time
10 from wvpytest import *
12 from bup import git, path
13 from bup.compat import bytes_from_byte, environ, range
14 from bup.helpers import localtime, log, mkdirp, readpipe
21 print(repr(cmd), file=sys.stderr)
26 print(repr(cmd), file=sys.stderr)
30 def test_git_version_detection():
31 # Test version types from git's tag history
32 for expected, ver in \
33 (('insufficient', b'git version 0.99'),
34 ('insufficient', b'git version 0.99.1'),
35 ('insufficient', b'git version 0.99.7a'),
36 ('insufficient', b'git version 1.0rc1'),
37 ('insufficient', b'git version 1.0.1'),
38 ('insufficient', b'git version 1.4.2.1'),
39 ('insufficient', b'git version 1.5.5'),
40 ('insufficient', b'git version 1.5.6-rc0'),
41 ('suitable', b'git version 1.5.6'),
42 ('suitable', b'git version 1.5.6.1'),
43 ('suitable', b'git version 2.14.0-rc0'),
44 ('suitable', b'git version 2.14.0 (something ...)'),
45 ('suitable', b'git version 111.222.333.444-rc555'),
46 ('unrecognized', b'huh?')):
47 assert expected == git.is_suitable_git(ver_str=ver)
49 if expected == 'insufficient':
50 with pytest.raises(SystemExit):
51 git.require_suitable_git(ver)
52 elif expected == 'suitable':
53 git.require_suitable_git(ver_str=ver)
54 elif expected == 'unrecognized':
55 with pytest.raises(git.GitError):
56 git.require_suitable_git(ver)
62 environ[b'BUP_GIT_VERSION_IS_FINE'] = b'true'
63 git.require_suitable_git(ver_str=ver)
65 del environ[b'BUP_GIT_VERSION_IS_FINE']
75 assert git.mangle_name(b'a', adir2, adir) == b'a'
76 assert git.mangle_name(b'.bup', adir2, adir) == b'.bup.bupl'
77 assert git.mangle_name(b'a.bupa', adir2, adir) == b'a.bupa.bupl'
78 WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
79 WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
80 WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
81 WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
82 WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
83 WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
85 WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
86 WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
87 WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
89 WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
90 WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
92 # for safety, we ignore .bup? suffixes we don't recognize. Future
93 # versions might implement a .bup[a-z] extension as something other
95 WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
100 packb = b''.join(git._encode_packobj(b'blob', s))
101 packt = b''.join(git._encode_packobj(b'tree', s))
102 packc = b''.join(git._encode_packobj(b'commit', s))
103 packlb = b''.join(git._encode_packobj(b'blob', s * 200))
104 WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
105 WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
106 WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
107 WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
109 return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
110 WVEXCEPT(ValueError, encode_pobj, -1)
111 WVEXCEPT(ValueError, encode_pobj, 10)
112 WVEXCEPT(ValueError, encode_pobj, b'x')
115 def test_packs(tmpdir):
116 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
117 git.init_repo(bupdir)
121 w.new_blob(os.urandom(100))
122 w.new_blob(os.urandom(100))
128 for i in range(nobj):
129 hashes.append(w.new_blob(b'%d' % i))
131 nameprefix = w.close()
132 print(repr(nameprefix))
133 WVPASS(os.path.exists(nameprefix + b'.pack'))
134 WVPASS(os.path.exists(nameprefix + b'.idx'))
136 r = git.open_idx(nameprefix + b'.idx')
137 print(repr(r.fanout))
139 for i in range(nobj):
140 WVPASS(r.find_offset(hashes[i]) > 0)
141 WVPASS(r.exists(hashes[99]))
142 WVFAIL(r.exists(b'\0'*20))
145 for h in sorted(hashes):
146 WVPASSEQ(hexlify(next(pi)), hexlify(h))
148 WVFAIL(r.find_offset(b'\0'*20))
150 r = git.PackIdxList(bupdir + b'/objects/pack')
151 WVPASS(r.exists(hashes[5]))
152 WVPASS(r.exists(hashes[6]))
153 WVFAIL(r.exists(b'\0'*20))
156 def test_pack_name_lookup(tmpdir):
157 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
158 git.init_repo(bupdir)
160 packdir = git.repo(b'objects/pack')
165 for start in range(0,28,2):
167 for i in range(start, start+2):
168 hashes.append(w.new_blob(b'%d' % i))
170 idxnames.append(os.path.basename(w.close() + b'.idx'))
172 r = git.PackIdxList(packdir)
173 WVPASSEQ(len(r.packs), 2)
174 for e,idxname in enumerate(idxnames):
175 for i in range(e*2, (e+1)*2):
176 WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
179 def test_long_index(tmpdir):
180 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
181 git.init_repo(bupdir)
182 idx = git.PackIdxV2Writer()
183 obj_bin = struct.pack('!IIIII',
184 0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
185 obj2_bin = struct.pack('!IIIII',
186 0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
187 obj3_bin = struct.pack('!IIIII',
188 0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
189 pack_bin = struct.pack('!IIIII',
190 0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
191 idx.add(obj_bin, 1, 0xfffffffff)
192 idx.add(obj2_bin, 2, 0xffffffffff)
193 idx.add(obj3_bin, 3, 0xff)
194 name = tmpdir + b'/tmp.idx'
195 r = idx.write(name, pack_bin)
196 i = git.PackIdxV2(name, open(name, 'rb'))
197 WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
198 WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
199 WVPASSEQ(i.find_offset(obj3_bin), 0xff)
202 def test_check_repo_or_die(tmpdir):
203 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
204 orig_cwd = os.getcwd()
207 git.init_repo(bupdir)
208 git.check_repo_or_die()
209 # if we reach this point the call above passed
210 WVPASS('check_repo_or_die')
212 os.rename(bupdir + b'/objects/pack',
213 bupdir + b'/objects/pack.tmp')
214 open(bupdir + b'/objects/pack', 'w').close()
216 git.check_repo_or_die()
217 except SystemExit as e:
221 os.unlink(bupdir + b'/objects/pack')
222 os.rename(bupdir + b'/objects/pack.tmp',
223 bupdir + b'/objects/pack')
226 git.check_repo_or_die(b'nonexistantbup.tmp')
227 except SystemExit as e:
235 def test_commit_parsing(tmpdir):
236 def restore_env_var(name, val):
242 def showval(commit, val):
243 return readpipe([b'git', b'show', b'-s',
244 b'--pretty=format:%s' % val, commit]).strip()
246 orig_cwd = os.getcwd()
247 workdir = tmpdir + b'/work'
248 repodir = workdir + b'/.git'
249 orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
250 orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
251 orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
252 orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
253 environ[b'GIT_AUTHOR_NAME'] = b'bup test'
254 environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
255 environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
256 environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
258 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
259 readpipe([b'git', b'init', workdir])
260 exc(b'git', b'symbolic-ref', b'HEAD', b'refs/heads/main')
261 git.check_repo_or_die(repodir)
263 with open('foo', 'w') as f:
265 readpipe([b'git', b'add', b'.'])
266 readpipe([b'git', b'commit', b'-am', b'Do something',
267 b'--author', b'Someone <someone@somewhere>',
268 b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
269 commit = readpipe([b'git', b'show-ref', b'-s', b'main']).strip()
270 parents = showval(commit, b'%P')
271 tree = showval(commit, b'%T')
272 cname = showval(commit, b'%cn')
273 cmail = showval(commit, b'%ce')
274 cdate = showval(commit, b'%ct')
275 coffs = showval(commit, b'%ci')
277 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
278 if bytes_from_byte(coffs[-5]) == b'-':
280 commit_items = git.get_commit_items(commit, git.cp())
281 WVPASSEQ(commit_items.parents, [])
282 WVPASSEQ(commit_items.tree, tree)
283 WVPASSEQ(commit_items.author_name, b'Someone')
284 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
285 WVPASSEQ(commit_items.author_sec, 1254613729)
286 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
287 WVPASSEQ(commit_items.committer_name, cname)
288 WVPASSEQ(commit_items.committer_mail, cmail)
289 WVPASSEQ(commit_items.committer_sec, int(cdate))
290 WVPASSEQ(commit_items.committer_offset, coff)
291 WVPASSEQ(commit_items.message, b'Do something\n')
292 with open(b'bar', 'wb') as f:
294 readpipe([b'git', b'add', '.'])
295 readpipe([b'git', b'commit', b'-am', b'Do something else'])
296 child = readpipe([b'git', b'show-ref', b'-s', b'main']).strip()
297 parents = showval(child, b'%P')
298 commit_items = git.get_commit_items(child, git.cp())
299 WVPASSEQ(commit_items.parents, [commit])
302 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
303 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
304 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
305 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
308 def test_new_commit(tmpdir):
309 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
310 git.init_repo(bupdir)
314 tree = os.urandom(20)
315 parent = os.urandom(20)
316 author_name = b'Author'
317 author_mail = b'author@somewhere'
318 adate_sec = 1439657836
319 cdate_sec = adate_sec + 1
320 committer_name = b'Committer'
321 committer_mail = b'committer@somewhere'
322 adate_tz_sec = cdate_tz_sec = None
323 commit = w.new_commit(tree, parent,
324 b'%s <%s>' % (author_name, author_mail),
325 adate_sec, adate_tz_sec,
326 b'%s <%s>' % (committer_name, committer_mail),
327 cdate_sec, cdate_tz_sec,
328 b'There is a small mailbox here')
329 adate_tz_sec = -60 * 60
330 cdate_tz_sec = 120 * 60
331 commit_off = w.new_commit(tree, parent,
332 b'%s <%s>' % (author_name, author_mail),
333 adate_sec, adate_tz_sec,
334 b'%s <%s>' % (committer_name, committer_mail),
335 cdate_sec, cdate_tz_sec,
336 b'There is a small mailbox here')
339 commit_items = git.get_commit_items(hexlify(commit), git.cp())
340 local_author_offset = localtime(adate_sec).tm_gmtoff
341 local_committer_offset = localtime(cdate_sec).tm_gmtoff
342 WVPASSEQ(tree, unhexlify(commit_items.tree))
343 WVPASSEQ(1, len(commit_items.parents))
344 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
345 WVPASSEQ(author_name, commit_items.author_name)
346 WVPASSEQ(author_mail, commit_items.author_mail)
347 WVPASSEQ(adate_sec, commit_items.author_sec)
348 WVPASSEQ(local_author_offset, commit_items.author_offset)
349 WVPASSEQ(committer_name, commit_items.committer_name)
350 WVPASSEQ(committer_mail, commit_items.committer_mail)
351 WVPASSEQ(cdate_sec, commit_items.committer_sec)
352 WVPASSEQ(local_committer_offset, commit_items.committer_offset)
354 commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
355 WVPASSEQ(tree, unhexlify(commit_items.tree))
356 WVPASSEQ(1, len(commit_items.parents))
357 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
358 WVPASSEQ(author_name, commit_items.author_name)
359 WVPASSEQ(author_mail, commit_items.author_mail)
360 WVPASSEQ(adate_sec, commit_items.author_sec)
361 WVPASSEQ(adate_tz_sec, commit_items.author_offset)
362 WVPASSEQ(committer_name, commit_items.committer_name)
363 WVPASSEQ(committer_mail, commit_items.committer_mail)
364 WVPASSEQ(cdate_sec, commit_items.committer_sec)
365 WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
368 def test_list_refs(tmpdir):
369 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
370 src = tmpdir + b'/src'
372 with open(src + b'/1', 'wb+') as f:
373 f.write(b'something\n')
374 with open(src + b'/2', 'wb+') as f:
375 f.write(b'something else\n')
376 git.init_repo(bupdir)
377 emptyset = frozenset()
378 WVPASSEQ(frozenset(git.list_refs()), emptyset)
379 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
380 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
381 exc(bup_exe, b'index', src)
382 exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
383 src_hash = exo(b'git', b'--git-dir', bupdir,
384 b'rev-parse', b'src').strip().split(b'\n')
385 assert(len(src_hash) == 1)
386 src_hash = unhexlify(src_hash[0])
387 tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
389 b'src:').strip().split(b'\n')[0])
390 blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
392 b'src:1').strip().split(b'\n')[0])
393 WVPASSEQ(frozenset(git.list_refs()),
394 frozenset([(b'refs/heads/src', src_hash)]))
395 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
396 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
397 frozenset([(b'refs/heads/src', src_hash)]))
398 exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
399 WVPASSEQ(frozenset(git.list_refs()),
400 frozenset([(b'refs/heads/src', src_hash),
401 (b'refs/tags/commit-tag', src_hash)]))
402 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
403 frozenset([(b'refs/tags/commit-tag', src_hash)]))
404 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
405 frozenset([(b'refs/heads/src', src_hash)]))
406 exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
407 exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
408 os.unlink(bupdir + b'/refs/heads/src')
409 expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
410 (b'refs/tags/tree-tag', tree_hash),
411 (b'refs/tags/blob-tag', blob_hash)])
412 WVPASSEQ(frozenset(git.list_refs()), expected_tags)
413 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
414 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
417 def test_git_date_str():
418 WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
419 WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
420 WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
423 def test_cat_pipe(tmpdir):
424 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
425 src = tmpdir + b'/src'
427 with open(src + b'/1', 'wb+') as f:
428 f.write(b'something\n')
429 with open(src + b'/2', 'wb+') as f:
430 f.write(b'something else\n')
431 git.init_repo(bupdir)
432 exc(bup_exe, b'index', src)
433 oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
435 typ = exo(b'git', b'--git-dir', bupdir,
436 b'cat-file', b'-t', b'src').strip()
437 size = int(exo(b'git', b'--git-dir', bupdir,
438 b'cat-file', b'-s', b'src'))
439 it = git.cp().get(b'src')
443 WVPASSEQ((oidx, typ, size), get_info)
445 def _create_idx(d, i):
446 idx = git.PackIdxV2Writer()
447 # add 255 vaguely reasonable entries
449 idx.add(struct.pack('18xBB', i, s), s, 100 * s)
450 packbin = struct.pack('B19x', i)
451 packname = os.path.join(d, b'pack-%s.idx' % hexlify(packbin))
452 idx.write(packname, packbin)
454 def test_midx_close(tmpdir):
455 fddir = b'/proc/self/fd'
459 # not supported, not Linux, I guess
463 for fd in os.listdir(fddir):
465 yield os.readlink(os.path.join(fddir, fd))
469 def force_midx(objdir):
470 args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
473 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
474 git.init_repo(bupdir)
475 # create a few dummy idxes
477 _create_idx(tmpdir, i)
478 git.auto_midx(tmpdir)
479 l = git.PackIdxList(tmpdir)
480 # this doesn't exist (yet)
481 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
482 for i in range(10, 15):
483 _create_idx(tmpdir, i)
484 # delete the midx ...
485 # TODO: why do we need to? git.auto_midx() below doesn't?!
486 for fn in os.listdir(tmpdir):
487 if fn.endswith(b'.midx'):
488 os.unlink(os.path.join(tmpdir, fn))
490 git.auto_midx(tmpdir)
491 # check it still doesn't exist - we haven't refreshed
492 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
493 # check that we still have the midx open, this really
494 # just checks more for the kernel API ('deleted' string)
495 for fn in openfiles():
496 if not b'midx-' in fn:
498 WVPASSEQ(True, b'deleted' in fn)
499 # refresh the PackIdxList
501 # and check that an object in pack 10 exists now
502 WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
503 for fn in openfiles():
504 if not b'midx-' in fn:
506 # check that we don't have it open anymore
507 WVPASSEQ(False, b'deleted' in fn)
510 cfg_file = os.path.join(os.path.dirname(__file__), 'sample.conf')
511 no_such_file = os.path.join(os.path.dirname(__file__), 'nosuch.conf')
512 git_config_get = partial(git.git_config_get, cfg_file=cfg_file)
513 WVPASSEQ(git_config_get(b'bup.foo'), b'bar')
514 WVPASSEQ(git_config_get(b'bup.bup'), b'is great')
515 WVPASSEQ(git_config_get(b'bup.end'), b'end')
516 WVPASSEQ(git_config_get(b'bup.comments'), None)
517 WVPASSEQ(git_config_get(b'bup.;comments'), None)
518 WVPASSEQ(git_config_get(b'bup.and'), None)
519 WVPASSEQ(git_config_get(b'bup.#and'), None)
521 WVPASSEQ(git.git_config_get(b'bup.foo', cfg_file=no_such_file), None)
523 WVEXCEPT(git.GitError, git_config_get, b'bup.isbad', opttype='bool')
524 WVEXCEPT(git.GitError, git_config_get, b'bup.isbad', opttype='int')
525 WVPASSEQ(git_config_get(b'bup.isbad'), b'ok')
526 WVPASSEQ(True, git_config_get(b'bup.istrue1', opttype='bool'))
527 WVPASSEQ(True, git_config_get(b'bup.istrue2', opttype='bool'))
528 WVPASSEQ(True, git_config_get(b'bup.istrue3', opttype='bool'))
529 WVPASSEQ(False, git_config_get(b'bup.isfalse1', opttype='bool'))
530 WVPASSEQ(False, git_config_get(b'bup.isfalse2', opttype='bool'))
531 WVPASSEQ(None, git_config_get(b'bup.nosuchkey', opttype='bool'))
532 WVPASSEQ(1, git_config_get(b'bup.istrue1', opttype='int'))
533 WVPASSEQ(2, git_config_get(b'bup.istrue2', opttype='int'))
534 WVPASSEQ(0, git_config_get(b'bup.isfalse2', opttype='int'))
535 WVPASSEQ(0x777, git_config_get(b'bup.hex', opttype='int'))