lib/bup/t/tbloom.py

   1
   2 import errno, platform, tempfile
   3
   4 from wvtest import *
   5
   6 from bup import bloom
   7 from bup.helpers import mkdirp
   8 from buptest import no_lingering_errors, test_tempdir
   9
  10
  11 @wvtest
  12 def test_bloom():
  13     with no_lingering_errors():
  14         with test_tempdir('bup-tbloom-') as tmpdir:
  15             hashes = [os.urandom(20) for i in range(100)]
  16             class Idx:
  17                 pass
  18             ix = Idx()
  19             ix.name='dummy.idx'
  20             ix.shatable = ''.join(hashes)
  21             for k in (4, 5):
  22                 b = bloom.create(tmpdir + '/pybuptest.bloom', expected=100, k=k)
  23                 b.add_idx(ix)
  24                 WVPASSLT(b.pfalse_positive(), .1)
  25                 b.close()
  26                 b = bloom.ShaBloom(tmpdir + '/pybuptest.bloom')
  27                 all_present = True
  28                 for h in hashes:
  29                     all_present &= b.exists(h)
  30                 WVPASS(all_present)
  31                 false_positives = 0
  32                 for h in [os.urandom(20) for i in range(1000)]:
  33                     if b.exists(h):
  34                         false_positives += 1
  35                 WVPASSLT(false_positives, 5)
  36                 os.unlink(tmpdir + '/pybuptest.bloom')
  37
  38             tf = tempfile.TemporaryFile(dir=tmpdir)
  39             b = bloom.create('bup.bloom', f=tf, expected=100)
  40             WVPASSEQ(b.rwfile, tf)
  41             WVPASSEQ(b.k, 5)
  42
  43             # Test large (~1GiB) filter.  This may fail on s390 (31-bit
  44             # architecture), and anywhere else where the address space is
  45             # sufficiently limited.
  46             tf = tempfile.TemporaryFile(dir=tmpdir)
  47             skip_test = False
  48             try:
  49                 b = bloom.create('bup.bloom', f=tf, expected=2**28,
  50                                  delaywrite=False)
  51             except EnvironmentError as ex:
  52                 (ptr_width, linkage) = platform.architecture()
  53                 if ptr_width == '32bit' and ex.errno == errno.ENOMEM:
  54                     WVMSG('skipping large bloom filter test (mmap probably failed) '
  55                           + str(ex))
  56                     skip_test = True
  57                 else:
  58                     raise
  59             if not skip_test:
  60                 WVPASSEQ(b.k, 4)