2 from __future__ import absolute_import, print_function
4 from binascii import hexlify, unhexlify
5 from subprocess import check_call
6 import struct, os, time
11 from bup import git, path
12 from bup.compat import bytes_from_byte, environ, range
13 from bup.helpers import localtime, log, mkdirp, readpipe
20 print(repr(cmd), file=sys.stderr)
25 print(repr(cmd), file=sys.stderr)
29 def test_git_version_detection():
30 # Test version types from git's tag history
31 for expected, ver in \
32 (('insufficient', b'git version 0.99'),
33 ('insufficient', b'git version 0.99.1'),
34 ('insufficient', b'git version 0.99.7a'),
35 ('insufficient', b'git version 1.0rc1'),
36 ('insufficient', b'git version 1.0.1'),
37 ('insufficient', b'git version 1.4.2.1'),
38 ('insufficient', b'git version 1.5.5'),
39 ('insufficient', b'git version 1.5.6-rc0'),
40 ('suitable', b'git version 1.5.6'),
41 ('suitable', b'git version 1.5.6.1'),
42 ('suitable', b'git version 2.14.0-rc0'),
43 ('suitable', b'git version 2.14.0 (something ...)'),
44 ('suitable', b'git version 111.222.333.444-rc555'),
45 ('unrecognized', b'huh?')):
46 assert expected == git.is_suitable_git(ver_str=ver)
48 if expected == 'insufficient':
49 with pytest.raises(SystemExit):
50 git.require_suitable_git(ver)
51 elif expected == 'suitable':
52 git.require_suitable_git(ver_str=ver)
53 elif expected == 'unrecognized':
54 with pytest.raises(git.GitError):
55 git.require_suitable_git(ver)
61 environ[b'BUP_GIT_VERSION_IS_FINE'] = b'true'
62 git.require_suitable_git(ver_str=ver)
64 del environ[b'BUP_GIT_VERSION_IS_FINE']
74 assert git.mangle_name(b'a', adir2, adir) == b'a'
75 assert git.mangle_name(b'.bup', adir2, adir) == b'.bup.bupl'
76 assert git.mangle_name(b'a.bupa', adir2, adir) == b'a.bupa.bupl'
77 WVPASSEQ(git.mangle_name(b'b.bup', alink, alink), b'b.bup.bupl')
78 WVPASSEQ(git.mangle_name(b'b.bu', alink, alink), b'b.bu')
79 WVPASSEQ(git.mangle_name(b'f', afile, afile2), b'f')
80 WVPASSEQ(git.mangle_name(b'f.bup', afile, afile2), b'f.bup.bupl')
81 WVPASSEQ(git.mangle_name(b'f.bup', afile, adir), b'f.bup.bup')
82 WVPASSEQ(git.mangle_name(b'f', afile, adir), b'f.bup')
84 WVPASSEQ(git.demangle_name(b'f.bup', afile), (b'f', git.BUP_CHUNKED))
85 WVPASSEQ(git.demangle_name(b'f.bupl', afile), (b'f', git.BUP_NORMAL))
86 WVPASSEQ(git.demangle_name(b'f.bup.bupl', afile), (b'f.bup', git.BUP_NORMAL))
88 WVPASSEQ(git.demangle_name(b'.bupm', afile), (b'', git.BUP_NORMAL))
89 WVPASSEQ(git.demangle_name(b'.bupm', adir), (b'', git.BUP_CHUNKED))
91 # for safety, we ignore .bup? suffixes we don't recognize. Future
92 # versions might implement a .bup[a-z] extension as something other
94 WVPASSEQ(git.demangle_name(b'f.bupa', afile), (b'f.bupa', git.BUP_NORMAL))
99 packb = b''.join(git._encode_packobj(b'blob', s))
100 packt = b''.join(git._encode_packobj(b'tree', s))
101 packc = b''.join(git._encode_packobj(b'commit', s))
102 packlb = b''.join(git._encode_packobj(b'blob', s * 200))
103 WVPASSEQ(git._decode_packobj(packb), (b'blob', s))
104 WVPASSEQ(git._decode_packobj(packt), (b'tree', s))
105 WVPASSEQ(git._decode_packobj(packc), (b'commit', s))
106 WVPASSEQ(git._decode_packobj(packlb), (b'blob', s * 200))
108 return b''.join(git._encode_packobj(b'blob', s, compression_level=n))
109 WVEXCEPT(ValueError, encode_pobj, -1)
110 WVEXCEPT(ValueError, encode_pobj, 10)
111 WVEXCEPT(ValueError, encode_pobj, b'x')
114 def test_packs(tmpdir):
115 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
116 git.init_repo(bupdir)
120 w.new_blob(os.urandom(100))
121 w.new_blob(os.urandom(100))
127 for i in range(nobj):
128 hashes.append(w.new_blob(b'%d' % i))
130 nameprefix = w.close()
131 print(repr(nameprefix))
132 WVPASS(os.path.exists(nameprefix + b'.pack'))
133 WVPASS(os.path.exists(nameprefix + b'.idx'))
135 r = git.open_idx(nameprefix + b'.idx')
136 print(repr(r.fanout))
138 for i in range(nobj):
139 WVPASS(r.find_offset(hashes[i]) > 0)
140 WVPASS(r.exists(hashes[99]))
141 WVFAIL(r.exists(b'\0'*20))
144 for h in sorted(hashes):
145 WVPASSEQ(hexlify(next(pi)), hexlify(h))
147 WVFAIL(r.find_offset(b'\0'*20))
149 r = git.PackIdxList(bupdir + b'/objects/pack')
150 WVPASS(r.exists(hashes[5]))
151 WVPASS(r.exists(hashes[6]))
152 WVFAIL(r.exists(b'\0'*20))
155 def test_pack_name_lookup(tmpdir):
156 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
157 git.init_repo(bupdir)
159 packdir = git.repo(b'objects/pack')
164 for start in range(0,28,2):
166 for i in range(start, start+2):
167 hashes.append(w.new_blob(b'%d' % i))
169 idxnames.append(os.path.basename(w.close() + b'.idx'))
171 r = git.PackIdxList(packdir)
172 WVPASSEQ(len(r.packs), 2)
173 for e,idxname in enumerate(idxnames):
174 for i in range(e*2, (e+1)*2):
175 WVPASSEQ(idxname, r.exists(hashes[i], want_source=True))
178 def test_long_index(tmpdir):
179 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
180 git.init_repo(bupdir)
181 idx = git.PackIdxV2Writer()
182 obj_bin = struct.pack('!IIIII',
183 0x00112233, 0x44556677, 0x88990011, 0x22334455, 0x66778899)
184 obj2_bin = struct.pack('!IIIII',
185 0x11223344, 0x55667788, 0x99001122, 0x33445566, 0x77889900)
186 obj3_bin = struct.pack('!IIIII',
187 0x22334455, 0x66778899, 0x00112233, 0x44556677, 0x88990011)
188 pack_bin = struct.pack('!IIIII',
189 0x99887766, 0x55443322, 0x11009988, 0x77665544, 0x33221100)
190 idx.add(obj_bin, 1, 0xfffffffff)
191 idx.add(obj2_bin, 2, 0xffffffffff)
192 idx.add(obj3_bin, 3, 0xff)
193 name = tmpdir + b'/tmp.idx'
194 r = idx.write(name, pack_bin)
195 i = git.PackIdxV2(name, open(name, 'rb'))
196 WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
197 WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
198 WVPASSEQ(i.find_offset(obj3_bin), 0xff)
201 def test_check_repo_or_die(tmpdir):
202 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
203 orig_cwd = os.getcwd()
206 git.init_repo(bupdir)
207 git.check_repo_or_die()
208 # if we reach this point the call above passed
209 WVPASS('check_repo_or_die')
211 os.rename(bupdir + b'/objects/pack',
212 bupdir + b'/objects/pack.tmp')
213 open(bupdir + b'/objects/pack', 'w').close()
215 git.check_repo_or_die()
216 except SystemExit as e:
220 os.unlink(bupdir + b'/objects/pack')
221 os.rename(bupdir + b'/objects/pack.tmp',
222 bupdir + b'/objects/pack')
225 git.check_repo_or_die(b'nonexistantbup.tmp')
226 except SystemExit as e:
234 def test_commit_parsing(tmpdir):
235 def restore_env_var(name, val):
241 def showval(commit, val):
242 return readpipe([b'git', b'show', b'-s',
243 b'--pretty=format:%s' % val, commit]).strip()
245 orig_cwd = os.getcwd()
246 workdir = tmpdir + b'/work'
247 repodir = workdir + b'/.git'
248 orig_author_name = environ.get(b'GIT_AUTHOR_NAME')
249 orig_author_email = environ.get(b'GIT_AUTHOR_EMAIL')
250 orig_committer_name = environ.get(b'GIT_COMMITTER_NAME')
251 orig_committer_email = environ.get(b'GIT_COMMITTER_EMAIL')
252 environ[b'GIT_AUTHOR_NAME'] = b'bup test'
253 environ[b'GIT_COMMITTER_NAME'] = environ[b'GIT_AUTHOR_NAME']
254 environ[b'GIT_AUTHOR_EMAIL'] = b'bup@a425bc70a02811e49bdf73ee56450e6f'
255 environ[b'GIT_COMMITTER_EMAIL'] = environ[b'GIT_AUTHOR_EMAIL']
257 readpipe([b'git', b'init', workdir])
258 environ[b'GIT_DIR'] = environ[b'BUP_DIR'] = repodir
259 git.check_repo_or_die(repodir)
261 with open('foo', 'w') as f:
263 readpipe([b'git', b'add', b'.'])
264 readpipe([b'git', b'commit', b'-am', b'Do something',
265 b'--author', b'Someone <someone@somewhere>',
266 b'--date', b'Sat Oct 3 19:48:49 2009 -0400'])
267 commit = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
268 parents = showval(commit, b'%P')
269 tree = showval(commit, b'%T')
270 cname = showval(commit, b'%cn')
271 cmail = showval(commit, b'%ce')
272 cdate = showval(commit, b'%ct')
273 coffs = showval(commit, b'%ci')
275 coff = (int(coffs[-4:-2]) * 60 * 60) + (int(coffs[-2:]) * 60)
276 if bytes_from_byte(coffs[-5]) == b'-':
278 commit_items = git.get_commit_items(commit, git.cp())
279 WVPASSEQ(commit_items.parents, [])
280 WVPASSEQ(commit_items.tree, tree)
281 WVPASSEQ(commit_items.author_name, b'Someone')
282 WVPASSEQ(commit_items.author_mail, b'someone@somewhere')
283 WVPASSEQ(commit_items.author_sec, 1254613729)
284 WVPASSEQ(commit_items.author_offset, -(4 * 60 * 60))
285 WVPASSEQ(commit_items.committer_name, cname)
286 WVPASSEQ(commit_items.committer_mail, cmail)
287 WVPASSEQ(commit_items.committer_sec, int(cdate))
288 WVPASSEQ(commit_items.committer_offset, coff)
289 WVPASSEQ(commit_items.message, b'Do something\n')
290 with open(b'bar', 'wb') as f:
292 readpipe([b'git', b'add', '.'])
293 readpipe([b'git', b'commit', b'-am', b'Do something else'])
294 child = readpipe([b'git', b'show-ref', b'-s', b'master']).strip()
295 parents = showval(child, b'%P')
296 commit_items = git.get_commit_items(child, git.cp())
297 WVPASSEQ(commit_items.parents, [commit])
300 restore_env_var(b'GIT_AUTHOR_NAME', orig_author_name)
301 restore_env_var(b'GIT_AUTHOR_EMAIL', orig_author_email)
302 restore_env_var(b'GIT_COMMITTER_NAME', orig_committer_name)
303 restore_env_var(b'GIT_COMMITTER_EMAIL', orig_committer_email)
306 def test_new_commit(tmpdir):
307 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
308 git.init_repo(bupdir)
312 tree = os.urandom(20)
313 parent = os.urandom(20)
314 author_name = b'Author'
315 author_mail = b'author@somewhere'
316 adate_sec = 1439657836
317 cdate_sec = adate_sec + 1
318 committer_name = b'Committer'
319 committer_mail = b'committer@somewhere'
320 adate_tz_sec = cdate_tz_sec = None
321 commit = w.new_commit(tree, parent,
322 b'%s <%s>' % (author_name, author_mail),
323 adate_sec, adate_tz_sec,
324 b'%s <%s>' % (committer_name, committer_mail),
325 cdate_sec, cdate_tz_sec,
326 b'There is a small mailbox here')
327 adate_tz_sec = -60 * 60
328 cdate_tz_sec = 120 * 60
329 commit_off = w.new_commit(tree, parent,
330 b'%s <%s>' % (author_name, author_mail),
331 adate_sec, adate_tz_sec,
332 b'%s <%s>' % (committer_name, committer_mail),
333 cdate_sec, cdate_tz_sec,
334 b'There is a small mailbox here')
337 commit_items = git.get_commit_items(hexlify(commit), git.cp())
338 local_author_offset = localtime(adate_sec).tm_gmtoff
339 local_committer_offset = localtime(cdate_sec).tm_gmtoff
340 WVPASSEQ(tree, unhexlify(commit_items.tree))
341 WVPASSEQ(1, len(commit_items.parents))
342 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
343 WVPASSEQ(author_name, commit_items.author_name)
344 WVPASSEQ(author_mail, commit_items.author_mail)
345 WVPASSEQ(adate_sec, commit_items.author_sec)
346 WVPASSEQ(local_author_offset, commit_items.author_offset)
347 WVPASSEQ(committer_name, commit_items.committer_name)
348 WVPASSEQ(committer_mail, commit_items.committer_mail)
349 WVPASSEQ(cdate_sec, commit_items.committer_sec)
350 WVPASSEQ(local_committer_offset, commit_items.committer_offset)
352 commit_items = git.get_commit_items(hexlify(commit_off), git.cp())
353 WVPASSEQ(tree, unhexlify(commit_items.tree))
354 WVPASSEQ(1, len(commit_items.parents))
355 WVPASSEQ(parent, unhexlify(commit_items.parents[0]))
356 WVPASSEQ(author_name, commit_items.author_name)
357 WVPASSEQ(author_mail, commit_items.author_mail)
358 WVPASSEQ(adate_sec, commit_items.author_sec)
359 WVPASSEQ(adate_tz_sec, commit_items.author_offset)
360 WVPASSEQ(committer_name, commit_items.committer_name)
361 WVPASSEQ(committer_mail, commit_items.committer_mail)
362 WVPASSEQ(cdate_sec, commit_items.committer_sec)
363 WVPASSEQ(cdate_tz_sec, commit_items.committer_offset)
366 def test_list_refs(tmpdir):
367 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
368 src = tmpdir + b'/src'
370 with open(src + b'/1', 'wb+') as f:
371 f.write(b'something\n')
372 with open(src + b'/2', 'wb+') as f:
373 f.write(b'something else\n')
374 git.init_repo(bupdir)
375 emptyset = frozenset()
376 WVPASSEQ(frozenset(git.list_refs()), emptyset)
377 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
378 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), emptyset)
379 exc(bup_exe, b'index', src)
380 exc(bup_exe, b'save', b'-n', b'src', b'--strip', src)
381 src_hash = exo(b'git', b'--git-dir', bupdir,
382 b'rev-parse', b'src').strip().split(b'\n')
383 assert(len(src_hash) == 1)
384 src_hash = unhexlify(src_hash[0])
385 tree_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
387 b'src:').strip().split(b'\n')[0])
388 blob_hash = unhexlify(exo(b'git', b'--git-dir', bupdir,
390 b'src:1').strip().split(b'\n')[0])
391 WVPASSEQ(frozenset(git.list_refs()),
392 frozenset([(b'refs/heads/src', src_hash)]))
393 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), emptyset)
394 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
395 frozenset([(b'refs/heads/src', src_hash)]))
396 exc(b'git', b'--git-dir', bupdir, b'tag', b'commit-tag', b'src')
397 WVPASSEQ(frozenset(git.list_refs()),
398 frozenset([(b'refs/heads/src', src_hash),
399 (b'refs/tags/commit-tag', src_hash)]))
400 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)),
401 frozenset([(b'refs/tags/commit-tag', src_hash)]))
402 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)),
403 frozenset([(b'refs/heads/src', src_hash)]))
404 exc(b'git', b'--git-dir', bupdir, b'tag', b'tree-tag', b'src:')
405 exc(b'git', b'--git-dir', bupdir, b'tag', b'blob-tag', b'src:1')
406 os.unlink(bupdir + b'/refs/heads/src')
407 expected_tags = frozenset([(b'refs/tags/commit-tag', src_hash),
408 (b'refs/tags/tree-tag', tree_hash),
409 (b'refs/tags/blob-tag', blob_hash)])
410 WVPASSEQ(frozenset(git.list_refs()), expected_tags)
411 WVPASSEQ(frozenset(git.list_refs(limit_to_heads=True)), frozenset([]))
412 WVPASSEQ(frozenset(git.list_refs(limit_to_tags=True)), expected_tags)
415 def test_git_date_str():
416 WVPASSEQ(b'0 +0000', git._git_date_str(0, 0))
417 WVPASSEQ(b'0 -0130', git._git_date_str(0, -90 * 60))
418 WVPASSEQ(b'0 +0130', git._git_date_str(0, 90 * 60))
421 def test_cat_pipe(tmpdir):
422 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
423 src = tmpdir + b'/src'
425 with open(src + b'/1', 'wb+') as f:
426 f.write(b'something\n')
427 with open(src + b'/2', 'wb+') as f:
428 f.write(b'something else\n')
429 git.init_repo(bupdir)
430 exc(bup_exe, b'index', src)
431 oidx = exo(bup_exe, b'save', b'-cn', b'src', b'--strip',
433 typ = exo(b'git', b'--git-dir', bupdir,
434 b'cat-file', b'-t', b'src').strip()
435 size = int(exo(b'git', b'--git-dir', bupdir,
436 b'cat-file', b'-s', b'src'))
437 it = git.cp().get(b'src')
441 WVPASSEQ((oidx, typ, size), get_info)
443 def _create_idx(d, i):
444 idx = git.PackIdxV2Writer()
445 # add 255 vaguely reasonable entries
447 idx.add(struct.pack('18xBB', i, s), s, 100 * s)
448 packbin = struct.pack('B19x', i)
449 packname = os.path.join(d, b'pack-%s.idx' % hexlify(packbin))
450 idx.write(packname, packbin)
452 def test_midx_close(tmpdir):
453 fddir = b'/proc/self/fd'
457 # not supported, not Linux, I guess
461 for fd in os.listdir(fddir):
463 yield os.readlink(os.path.join(fddir, fd))
467 def force_midx(objdir):
468 args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
471 environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
472 git.init_repo(bupdir)
473 # create a few dummy idxes
475 _create_idx(tmpdir, i)
476 git.auto_midx(tmpdir)
477 l = git.PackIdxList(tmpdir)
478 # this doesn't exist (yet)
479 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
480 for i in range(10, 15):
481 _create_idx(tmpdir, i)
482 # delete the midx ...
483 # TODO: why do we need to? git.auto_midx() below doesn't?!
484 for fn in os.listdir(tmpdir):
485 if fn.endswith(b'.midx'):
486 os.unlink(os.path.join(tmpdir, fn))
488 git.auto_midx(tmpdir)
489 # check it still doesn't exist - we haven't refreshed
490 WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
491 # check that we still have the midx open, this really
492 # just checks more for the kernel API ('deleted' string)
493 for fn in openfiles():
494 if not b'midx-' in fn:
496 WVPASSEQ(True, b'deleted' in fn)
497 # refresh the PackIdxList
499 # and check that an object in pack 10 exists now
500 WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
501 for fn in openfiles():
502 if not b'midx-' in fn:
504 # check that we don't have it open anymore
505 WVPASSEQ(False, b'deleted' in fn)