Adjust sparse restore tests for test fs block size

Change test-sparse-files.sh to detect the test fs block size (when
possible) and adjust its behavior accordingly.  If the block size can't
be determined, use a block size of 3MB, which is hoped to be larger than
any block sizes we'll encounter anytime soon.

Previously the tests might fail on filesystems with relatively large
block sizes, like those on the current Debian powerpc and ppc64el build
daemons (64k).

Thanks to Goswin Brederlow for mentioning that the Lucene block size is
1MB, to Robert Edmonds for running a build through the Debian buildds,
which revealed the problem, and to Julien Cristau for reporting the
block size on the failing buildds.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/t/test-sparse-files.sh b/t/test-sparse-files.sh
index c4f7d8acdba37626bda06467a5b3e635f91166f7..82275ff8bc045f82e1f85b46f0f5b624740b2f59 100755 (executable)
--- a/t/test-sparse-files.sh
+++ b/t/test-sparse-files.sh
@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 . ./wvtest-bup.sh || exit $?
+. t/lib.sh || exit $?
 
 set -o pipefail
 
-readonly mb=1048576
-readonly top="$(WVPASS pwd)" || exit $?
-readonly tmpdir="$(WVPASS wvmktempdir)" || exit $?
+mb=1048576
+top="$(WVPASS pwd)" || exit $?
+tmpdir="$(WVPASS wvmktempdir)" || exit $?
+readonly mb top tmpdir
 
 export BUP_DIR="$tmpdir/bup"
 export GIT_DIR="$tmpdir/bup"
@@ -14,49 +16,59 @@ bup() { "$top/bup" "$@"; }
 
 WVPASS cd "$tmpdir"
 
-WVPASS dd if=/dev/zero of=test-sparse seek=$((1024 * 256)) bs=1 count=1
-restore_size=$(WVPASS du -k -s test-sparse | WVPASS cut -f1) || exit $?
-if ! [ "$restore_size" -lt 256 ]; then
+# The 3MB guess is semi-arbitrary, but we've been informed that
+# Lustre, for example, uses 1MB, so guess higher than that, at least.
+block_size=$(python -c \
+  "import os; print getattr(os.stat('.'), 'st_blksize', 0) or $mb * 3") \
+    || exit $?
+data_size=$((block_size * 10))
+readonly block_size data_size
+
+WVPASS dd if=/dev/zero of=test-sparse-probe seek="$data_size" bs=1 count=1
+probe_size=$(WVPASS du -k -s test-sparse-probe | WVPASS cut -f1) || exit $?
+if [ "$probe_size" -ge "$((data_size / 1024))" ]; then
     WVSTART "no sparse support detected -- skipping tests"
     exit 0
 fi
 
+WVSTART "sparse restore on $(current-filesystem), assuming ${block_size}B blocks"
+
 WVPASS bup init
 WVPASS mkdir src
 
-WVPASS dd if=/dev/zero of=src/foo seek=$mb bs=1 count=1
+WVPASS dd if=/dev/zero of=src/foo seek="$data_size" bs=1 count=1
 WVPASS bup index src
 WVPASS bup save -n src src
 
 WVSTART "sparse file restore (all sparse)"
 WVPASS bup restore -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -gt 1000 ]
+WVPASS [ "$restore_size" -ge "$((data_size / 1024))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --no-sparse (all sparse)"
 WVPASS rm -r restore
 WVPASS bup restore --no-sparse -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -gt 1000 ]
+WVPASS [ "$restore_size" -ge "$((data_size / 1024))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --sparse (all sparse)"
 WVPASS rm -r restore
 WVPASS bup restore --sparse -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -lt 100 ]
+WVPASS [ "$restore_size" -le "$((3 * (block_size / 1024)))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --sparse (sparse end)"
 WVPASS echo "start" > src/foo
-WVPASS dd if=/dev/zero of=src/foo seek=$mb bs=1 count=1 conv=notrunc
+WVPASS dd if=/dev/zero of=src/foo seek="$data_size" bs=1 count=1 conv=notrunc
 WVPASS bup index src
 WVPASS bup save -n src src
 WVPASS rm -r restore
 WVPASS bup restore --sparse -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -lt 100 ]
+WVPASS [ "$restore_size" -le "$((3 * (block_size / 1024)))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --sparse (sparse middle)"
@@ -66,7 +78,7 @@ WVPASS bup save -n src src
 WVPASS rm -r restore
 WVPASS bup restore --sparse -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -lt 100 ]
+WVPASS [ "$restore_size" -le "$((5 * (block_size / 1024)))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --sparse (bracketed zero run in buf)"
@@ -80,29 +92,34 @@ WVPASS bup restore --sparse -C restore "src/latest/$(pwd)/"
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --sparse (sparse start)"
-WVPASS dd if=/dev/zero of=src/foo seek=$mb bs=1 count=1
+WVPASS dd if=/dev/zero of=src/foo seek="$data_size" bs=1 count=1
 WVPASS echo "end" >> src/foo
 WVPASS bup index src
 WVPASS bup save -n src src
 WVPASS rm -r restore
 WVPASS bup restore --sparse -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -lt 100 ]
+WVPASS [ "$restore_size" -le "$((5 * (block_size / 1024)))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
 WVSTART "sparse file restore --sparse (sparse start and end)"
-WVPASS dd if=/dev/zero of=src/foo seek=$mb bs=1 count=1
+WVPASS dd if=/dev/zero of=src/foo seek="$data_size" bs=1 count=1
 WVPASS echo "middle" >> src/foo
-WVPASS dd if=/dev/zero of=src/foo seek=$((2 * mb)) bs=1 count=1 conv=notrunc
+WVPASS dd if=/dev/zero of=src/foo seek=$((2 * data_size)) bs=1 count=1 conv=notrunc
 WVPASS bup index src
 WVPASS bup save -n src src
 WVPASS rm -r restore
 WVPASS bup restore --sparse -C restore "src/latest/$(pwd)/"
 restore_size=$(WVPASS du -k -s restore | WVPASS cut -f1) || exit $?
-WVPASS [ "$restore_size" -lt 100 ]
+WVPASS [ "$restore_size" -le "$((5 * (block_size / 1024)))" ]
 WVPASS "$top/t/compare-trees" -c src/ restore/src/
 
-WVSTART "sparse file restore --sparse (random)"
+if test "$block_size" -gt $mb; then
+    random_size="$block_size"
+else
+    random_size=1M
+fi
+WVSTART "sparse file restore --sparse (random $random_size)"
 WVPASS bup random 1M > src/foo
 WVPASS bup index src
 WVPASS bup save -n src src