2 from __future__ import absolute_import, print_function
4 from binascii import hexlify, unhexlify
5 from subprocess import check_call
6 import struct, os, time
11 from bup import git, path
12 from bup.compat import bytes_from_byte, environ, range
13 from bup.helpers import localtime, log, mkdirp, readpipe
20 print(repr(cmd), file=sys.stderr)
25 print(repr(cmd), file=sys.stderr)
29 def test_git_version_detection():
30 # Test version types from git's tag history
31 for expected, ver in \
32 (('insufficient', b'git version 0.99'),
33 ('insufficient', b'git version 0.99.1'),
34 ('insufficient', b'git version 0.99.7a'),
35 ('insufficient', b'git version 1.0rc1'),
36 ('insufficient', b'git version 1.0.1'),
37 ('insufficient', b'git version 1.4.2.1'),
38 ('insufficient', b'git version 1.5.5'),
39 ('insufficient', b'git version 1.5.6-rc0'),
40 ('suitable', b'git version 1.5.6'),
41 ('suitable', b'git version 1.5.6.1'),
42 ('suitable', b'git version 2.14.0-rc0'),
43 ('suitable', b'git version 2.14.0 (something ...)'),
44 ('suitable', b'git version 111.222.333.444-rc555'),
45 ('unrecognized', b'huh?')):
46 assert expected == git.is_suitable_git(ver_str=ver)
48 if expected == 'insufficient':
49 with pytest.raises(SystemExit):
50 git.require_suitable_git(ver)
51 elif expected == 'suitable':
52 git.require_suitable_git(ver_str=ver)
53 elif expected == 'unrecognized':
54 with pytest.raises(git.GitError):
55 git.require_suitable_git(ver)
61 environ[b'BUP_GIT_VERSION_IS_FINE'] = b'true'
62 git.require_suitable_git(ver_str=ver)
64 del environ[b'BUP_GIT_VERSION_IS_FINE']
74 assert git.mangle_name(b'a', adir2, adir) == b'a'
75 assert git.mangle_name(b'.bup', adir2, adir) == b'.bup.bupl'
76 assert git.mangle_name(b'a.bupa', adir2, adir) == b'a.bupa.bupl'
77 WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
78 WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
79 WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
80 WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
81 WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
82 WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
84 WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
85 WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
86 WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
88 WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
89 WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
91 # for safety, we ignore .bup? suffixes we don't recognize. Future
92 # versions might implement a .bup[a-z] extension as something other
94 WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
99 looseb = b''.join(git._encode_looseobj(b'blob', s))
100 looset = b''.join(git._encode_looseobj(b'tree', s))
101 loosec = b''.join(git._encode_looseobj(b'commit', s))
102 packb = b''.join(git._encode_packobj(b'blob', s))
103 packt = b''.join(git._encode_packobj(b'tree', s))
104 packc = b''.join(git._encode_packobj(b'commit', s))
105 packlb = b''.join(git._encode_packobj(b'blob', s * 200))
106 WVPASSEQ(git._decode_looseobj(looseb), (b'blob', s))
107 WVPASSEQ(git._decode_looseobj(looset), (b'tree', s))
108 WVPASSEQ(git._decode_looseobj(loosec), (b'commit', s))
109 WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
110 WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
111 WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
112 WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
114 WVPASS(git._encode_looseobj(b'blob', s, compression_level=i))
116 return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
117 WVEXCEPT(ValueError, encode_pobj, -1)
118 WVEXCEPT(ValueError, encode_pobj, 10)
119 WVEXCEPT(ValueError, encode_pobj, b'x')
122 def test_packs(tmpdir):
123 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
124 git.init_repo(bupdir)
128 w.new_blob(os.urandom(100))
129 w.new_blob(os.urandom(100))
135 for i in range(nobj):
136 hashes.append(w.new_blob(b'%d' % i))
138 nameprefix = w.close()
139 print(repr(nameprefix))
140 WVPASS(os.path.exists(nameprefix + b'.pack'))
141 WVPASS(os.path.exists(nameprefix + b'.idx'))
143 r = git.open_idx(nameprefix + b'.idx')
144 print(repr(r.fanout))
146 for i in range(nobj):
147 WVPASS(r.find_offset(hashes[i]) > 0)
148 WVPASS(r.exists(hashes[99]))
149 WVFAIL(r.exists(b'\0'*20))
152 for h in sorted(hashes):
153 WVPASSEQ(hexlify(next(pi)), hexlify(h))
155 WVFAIL(r.find_offset(b'\0'*20))
157 r = git.PackIdxList(bupdir + b'/objects/pack')
158 WVPASS(r.exists(hashes[5]))
159 WVPASS(r.exists(hashes[6]))
160 WVFAIL(r.exists(b'\0'*20))
163 def test_pack_name_lookup(tmpdir):
164 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
165 git.init_repo(bupdir)
167 packdir = git.repo(b'objects/pack')
172 for start in range(0,28,2):
174 for i in range(start, start+2):
175 hashes.append(w.new_blob(b'%d' % i))
177 idxnames.append(os.path.basename(w.close() + b'.idx'))
179 r = git.PackIdxList(packdir)
180 WVPASSEQ(len(r.packs), 2)
181 for e,idxname in enumerate(idxnames):
182 for i in range(e*2, (e+1)*2):
183 WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
186 def test_long_index(tmpdir):
187 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
188 git.init_repo(bupdir)
189 idx = git.PackIdxV2Writer()
190 obj_bin = struct.pack('!IIIII',
191 0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
192 obj2_bin = struct.pack('!IIIII',
193 0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
194 obj3_bin = struct.pack('!IIIII',
195 0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
196 pack_bin = struct.pack('!IIIII',
197 0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
198 idx.add(obj_bin, 1, 0xfffffffff)
199 idx.add(obj2_bin, 2, 0xffffffffff)
200 idx.add(obj3_bin, 3, 0xff)
201 name = tmpdir + b'/tmp.idx'
202 r = idx.write(name, pack_bin)
203 i = git.PackIdxV2(name, open(name, 'rb'))
204 WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
205 WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
206 WVPASSEQ(i.find_offset(obj3_bin), 0xff)
209 def test_check_repo_or_die(tmpdir):
210 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
211 orig_cwd = os.getcwd()
214 git.init_repo(bupdir)
215 git.check_repo_or_die()
216 # if we reach this point the call above passed
217 WVPASS('check_repo_or_die')
219 os.rename(bupdir + b'/objects/pack',
220 bupdir + b'/objects/pack.tmp')
221 open(bupdir + b'/objects/pack', 'w').close()
223 git.check_repo_or_die()
224 except SystemExit as e:
228 os.unlink(bupdir + b'/objects/pack')
229 os.rename(bupdir + b'/objects/pack.tmp',
230 bupdir + b'/objects/pack')
233 git.check_repo_or_die(b'nonexistantbup.tmp')
234 except SystemExit as e:
242 def test_commit_parsing(tmpdir):
243 def restore_env_var(name, val):
249 def showval(commit, val):
250 return readpipe([b'git', b'show', b'-s',
251 b'--pretty=format:%s' % val, commit]).strip()
253 orig_cwd = os.getcwd()
254 workdir = tmpdir + b'/work'
255 repodir = workdir + b'/.git'
256 orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
257 orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
258 orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
259 orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
260 environ[b'GIT_AUTHOR_NAME'] = b'bup test'
261 environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
262 environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
263 environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
265 readpipe([b'git', b'init', workdir])
266 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
267 git.check_repo_or_die(repodir)
269 with open('foo', 'w') as f:
271 readpipe([b'git', b'add', b'.'])
272 readpipe([b'git', b'commit', b'-am', b'Do something',
273 b'--author', b'Someone <someone@somewhere>',
274 b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
275 commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
276 parents = showval(commit, b'%P')
277 tree = showval(commit, b'%T')
278 cname = showval(commit, b'%cn')
279 cmail = showval(commit, b'%ce')
280 cdate = showval(commit, b'%ct')
281 coffs = showval(commit, b'%ci')
283 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
284 if bytes_from_byte(coffs[-5]) == b'-':
286 commit_items = git.get_commit_items(commit, git.cp())
287 WVPASSEQ(commit_items.parents, [])
288 WVPASSEQ(commit_items.tree, tree)
289 WVPASSEQ(commit_items.author_name, b'Someone')
290 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
291 WVPASSEQ(commit_items.author_sec, 1254613729)
292 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
293 WVPASSEQ(commit_items.committer_name, cname)
294 WVPASSEQ(commit_items.committer_mail, cmail)
295 WVPASSEQ(commit_items.committer_sec, int(cdate))
296 WVPASSEQ(commit_items.committer_offset, coff)
297 WVPASSEQ(commit_items.message, b'Do something\n')
298 with open(b'bar', 'wb') as f:
300 readpipe([b'git', b'add', '.'])
301 readpipe([b'git', b'commit', b'-am', b'Do something else'])
302 child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
303 parents = showval(child, b'%P')
304 commit_items = git.get_commit_items(child, git.cp())
305 WVPASSEQ(commit_items.parents, [commit])
308 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
309 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
310 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
311 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
314 def test_new_commit(tmpdir):
315 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
316 git.init_repo(bupdir)
320 tree = os.urandom(20)
321 parent = os.urandom(20)
322 author_name = b'Author'
323 author_mail = b'author@somewhere'
324 adate_sec = 1439657836
325 cdate_sec = adate_sec + 1
326 committer_name = b'Committer'
327 committer_mail = b'committer@somewhere'
328 adate_tz_sec = cdate_tz_sec = None
329 commit = w.new_commit(tree, parent,
330 b'%s <%s>' % (author_name, author_mail),
331 adate_sec, adate_tz_sec,
332 b'%s <%s>' % (committer_name, committer_mail),
333 cdate_sec, cdate_tz_sec,
334 b'There is a small mailbox here')
335 adate_tz_sec = -60 * 60
336 cdate_tz_sec = 120 * 60
337 commit_off = w.new_commit(tree, parent,
338 b'%s <%s>' % (author_name, author_mail),
339 adate_sec, adate_tz_sec,
340 b'%s <%s>' % (committer_name, committer_mail),
341 cdate_sec, cdate_tz_sec,
342 b'There is a small mailbox here')
345 commit_items = git.get_commit_items(hexlify(commit), git.cp())
346 local_author_offset = localtime(adate_sec).tm_gmtoff
347 local_committer_offset = localtime(cdate_sec).tm_gmtoff
348 WVPASSEQ(tree, unhexlify(commit_items.tree))
349 WVPASSEQ(1, len(commit_items.parents))
350 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
351 WVPASSEQ(author_name, commit_items.author_name)
352 WVPASSEQ(author_mail, commit_items.author_mail)
353 WVPASSEQ(adate_sec, commit_items.author_sec)
354 WVPASSEQ(local_author_offset, commit_items.author_offset)
355 WVPASSEQ(committer_name, commit_items.committer_name)
356 WVPASSEQ(committer_mail, commit_items.committer_mail)
357 WVPASSEQ(cdate_sec, commit_items.committer_sec)
358 WVPASSEQ(local_committer_offset, commit_items.committer_offset)
360 commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
361 WVPASSEQ(tree, unhexlify(commit_items.tree))
362 WVPASSEQ(1, len(commit_items.parents))
363 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
364 WVPASSEQ(author_name, commit_items.author_name)
365 WVPASSEQ(author_mail, commit_items.author_mail)
366 WVPASSEQ(adate_sec, commit_items.author_sec)
367 WVPASSEQ(adate_tz_sec, commit_items.author_offset)
368 WVPASSEQ(committer_name, commit_items.committer_name)
369 WVPASSEQ(committer_mail, commit_items.committer_mail)
370 WVPASSEQ(cdate_sec, commit_items.committer_sec)
371 WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
374 def test_list_refs(tmpdir):
375 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
376 src = tmpdir + b'/src'
378 with open(src + b'/1', 'wb+') as f:
379 f.write(b'something\n')
380 with open(src + b'/2', 'wb+') as f:
381 f.write(b'something else\n')
382 git.init_repo(bupdir)
383 emptyset = frozenset()
384 WVPASSEQ(frozenset(git.list_refs()), emptyset)
385 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
386 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
387 exc(bup_exe, b'index', src)
388 exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
389 src_hash = exo(b'git', b'--git-dir', bupdir,
390 b'rev-parse', b'src').strip().split(b'\n')
391 assert(len(src_hash) == 1)
392 src_hash = unhexlify(src_hash[0])
393 tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
395 b'src:').strip().split(b'\n')[0])
396 blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
398 b'src:1').strip().split(b'\n')[0])
399 WVPASSEQ(frozenset(git.list_refs()),
400 frozenset([(b'refs/heads/src', src_hash)]))
401 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
402 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
403 frozenset([(b'refs/heads/src', src_hash)]))
404 exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
405 WVPASSEQ(frozenset(git.list_refs()),
406 frozenset([(b'refs/heads/src', src_hash),
407 (b'refs/tags/commit-tag', src_hash)]))
408 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
409 frozenset([(b'refs/tags/commit-tag', src_hash)]))
410 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
411 frozenset([(b'refs/heads/src', src_hash)]))
412 exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
413 exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
414 os.unlink(bupdir + b'/refs/heads/src')
415 expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
416 (b'refs/tags/tree-tag', tree_hash),
417 (b'refs/tags/blob-tag', blob_hash)])
418 WVPASSEQ(frozenset(git.list_refs()), expected_tags)
419 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
420 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
423 def test_git_date_str():
424 WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
425 WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
426 WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
429 def test_cat_pipe(tmpdir):
430 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
431 src = tmpdir + b'/src'
433 with open(src + b'/1', 'wb+') as f:
434 f.write(b'something\n')
435 with open(src + b'/2', 'wb+') as f:
436 f.write(b'something else\n')
437 git.init_repo(bupdir)
438 exc(bup_exe, b'index', src)
439 oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
441 typ = exo(b'git', b'--git-dir', bupdir,
442 b'cat-file', b'-t', b'src').strip()
443 size = int(exo(b'git', b'--git-dir', bupdir,
444 b'cat-file', b'-s', b'src'))
445 it = git.cp().get(b'src')
449 WVPASSEQ((oidx, typ, size), get_info)
451 def _create_idx(d, i):
452 idx = git.PackIdxV2Writer()
453 # add 255 vaguely reasonable entries
455 idx.add(struct.pack('18xBB', i, s), s, 100 * s)
456 packbin = struct.pack('B19x', i)
457 packname = os.path.join(d, b'pack-%s.idx' % hexlify(packbin))
458 idx.write(packname, packbin)
460 def test_midx_close(tmpdir):
461 fddir = b'/proc/self/fd'
465 # not supported, not Linux, I guess
469 for fd in os.listdir(fddir):
471 yield os.readlink(os.path.join(fddir, fd))
475 def force_midx(objdir):
476 args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
479 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
480 git.init_repo(bupdir)
481 # create a few dummy idxes
483 _create_idx(tmpdir, i)
484 git.auto_midx(tmpdir)
485 l = git.PackIdxList(tmpdir)
486 # this doesn't exist (yet)
487 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
488 for i in range(10, 15):
489 _create_idx(tmpdir, i)
490 # delete the midx ...
491 # TODO: why do we need to? git.auto_midx() below doesn't?!
492 for fn in os.listdir(tmpdir):
493 if fn.endswith(b'.midx'):
494 os.unlink(os.path.join(tmpdir, fn))
496 git.auto_midx(tmpdir)
497 # check it still doesn't exist - we haven't refreshed
498 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
499 # check that we still have the midx open, this really
500 # just checks more for the kernel API ('deleted' string)
501 for fn in openfiles():
502 if not b'midx-' in fn:
504 WVPASSEQ(True, b'deleted' in fn)
505 # refresh the PackIdxList
507 # and check that an object in pack 10 exists now
508 WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
509 for fn in openfiles():
510 if not b'midx-' in fn:
512 # check that we don't have it open anymore
513 WVPASSEQ(False, b'deleted' in fn)