memtest.py: a standalone program for testing memory usage in PackIndex.

The majority of the memory usage in bup split/save is now caused by
searching pack indexes for sha1 hashes.  The problem with this is that, in
the common case for a first full backup, *none* of the object hashes will be
found, so we'll *always* have to search *all* the packfiles.  With just 45
packfiles of 200k objects each, that makes about (18-8)*45 = 450 binary
search steps, or 100+ 4k pages that need to be loaded from disk, to check
*each* object hash.  memtest.py lets us see how fast RSS creeps up under
various conditions, and how different optimizations affect the result.

diff --git a/.gitignore b/.gitignore
index 0cb59087dd67851422f212bc3df7d65d1e167157..6c135e3e852993a3b145cf92d5bc427d3ebca356 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 bup
 bup-*
 randomgen
+memtest
 *.o
 *.so
 *.exe

diff --git a/Makefile b/Makefile
index 2249d0e7f376d36fb4aa139d0026872ea7d99c34..b2651548169a5b25ad1bd64926773b5f92abd1d8 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ endif
 default: all
 
 all: bup-split bup-join bup-save bup-init bup-server bup-index bup-tick \
-       bup randomgen$(EXT) chashsplit$(SOEXT)
+       bup memtest randomgen$(EXT) chashsplit$(SOEXT)
 
 randomgen$(EXT): randomgen.o
        $(CC) $(CFLAGS) -o $@ $<
@@ -55,6 +55,10 @@ bup-%: cmd-%.py
        rm -f $@
        ln -s $^ $@
        
+%: %.py
+       rm -f $@
+       ln -s $^ $@
+       
 bup-%: cmd-%.sh
        rm -f $@
        ln -s $^ $@

diff --git a/memtest.py b/memtest.py
new file mode 100755 (executable)
index 0000000..e14aa19
--- /dev/null
+++ b/memtest.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python2.5
+import sys, re, struct, mmap
+import git, options
+from helpers import *
+
+
+def s_from_bytes(bytes):
+    clist = [chr(b) for b in bytes]
+    return ''.join(clist)
+
+
+def report(count):
+    fields = ['VmSize', 'VmRSS', 'VmData', 'VmStk']
+    d = {}
+    for line in open('/proc/self/status').readlines():
+        l = re.split(r':\s*', line.strip(), 1)
+        d[l[0]] = l[1]
+    if count >= 0:
+        e1 = count
+        fields = [d[k] for k in fields]
+    else:
+        e1 = ''
+    print ('%9s  ' + ('%10s ' * len(fields))) % tuple([e1] + fields)
+
+
+optspec = """
+memtest [-n elements] [-c cycles]
+--
+n,number=  number of objects per cycle
+c,cycles=  number of cycles to run
+"""
+o = options.Options(sys.argv[0], optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+    o.usage()
+
+git.check_repo_or_die()
+m = git.MultiPackIndex(git.repo('objects/pack'))
+
+cycles = opt.cycles or 100
+number = opt.number or 10000
+
+report(-1)
+f = open('/dev/urandom')
+a = mmap.mmap(-1, 20)
+report(0)
+for c in xrange(cycles):
+    for n in xrange(number):
+        b = f.read(3)
+        if 0:
+            bytes = list(struct.unpack('!BBB', b)) + [0]*17
+            bytes[2] &= 0xf0
+            bin = struct.pack('!20s', s_from_bytes(bytes))
+        else:
+            a[0:2] = b[0:2]
+            a[2] = chr(ord(b[2]) & 0xf0)
+            bin = str(a[0:20])
+        #print bin.encode('hex')
+        m.exists(bin)
+    report((c+1)*number)