lib/bup/t/tbloom.py

   1
   2 from __future__ import absolute_import
   3 import errno, platform, tempfile
   4
   5 from wvtest import *
   6
   7 from bup import bloom
   8 from bup.helpers import mkdirp
   9 from buptest import no_lingering_errors, test_tempdir
  10
  11
  12 @wvtest
  13 def test_bloom():
  14     with no_lingering_errors():
  15         with test_tempdir('bup-tbloom-') as tmpdir:
  16             hashes = [os.urandom(20) for i in range(100)]
  17             class Idx:
  18                 pass
  19             ix = Idx()
  20             ix.name='dummy.idx'
  21             ix.shatable = ''.join(hashes)
  22             for k in (4, 5):
  23                 b = bloom.create(tmpdir + '/pybuptest.bloom', expected=100, k=k)
  24                 b.add_idx(ix)
  25                 WVPASSLT(b.pfalse_positive(), .1)
  26                 b.close()
  27                 b = bloom.ShaBloom(tmpdir + '/pybuptest.bloom')
  28                 all_present = True
  29                 for h in hashes:
  30                     all_present &= b.exists(h)
  31                 WVPASS(all_present)
  32                 false_positives = 0
  33                 for h in [os.urandom(20) for i in range(1000)]:
  34                     if b.exists(h):
  35                         false_positives += 1
  36                 WVPASSLT(false_positives, 5)
  37                 os.unlink(tmpdir + '/pybuptest.bloom')
  38
  39             tf = tempfile.TemporaryFile(dir=tmpdir)
  40             b = bloom.create('bup.bloom', f=tf, expected=100)
  41             WVPASSEQ(b.rwfile, tf)
  42             WVPASSEQ(b.k, 5)
  43
  44             # Test large (~1GiB) filter.  This may fail on s390 (31-bit
  45             # architecture), and anywhere else where the address space is
  46             # sufficiently limited.
  47             tf = tempfile.TemporaryFile(dir=tmpdir)
  48             skip_test = False
  49             try:
  50                 b = bloom.create('bup.bloom', f=tf, expected=2**28,
  51                                  delaywrite=False)
  52             except EnvironmentError as ex:
  53                 (ptr_width, linkage) = platform.architecture()
  54                 if ptr_width == '32bit' and ex.errno == errno.ENOMEM:
  55                     WVMSG('skipping large bloom filter test (mmap probably failed) '
  56                           + str(ex))
  57                     skip_test = True
  58                 else:
  59                     raise
  60             if not skip_test:
  61                 WVPASSEQ(b.k, 4)