Testing
=======
-You can run the test suite much more quickly via "make -j test" (as
-compared to "make test"), at the expense of slightly more confusing
-output (interleaved parallel test output), and inaccurate intermediate
-success/failure counts, but the final counts displayed should be
-correct.
+Individual tests can be run via
-Individual non-Python tests can be run via
+ ./pytest TEST
- ./wvtest run test/ext/TEST
+For example:
-and if you'd like to see all of the test output, you can omit the
-wvtest run wrapper: `test/ext/TEST`. Individual Python tests can be
-run via
-
- ./pytest test/int/test_something.py
+ ./pytest test/int/test_git.py
+ ./pytest test/ext/test-ftp
Internal tests that test bup's code directly are located in test/int,
and external tests that test bup from the outside, typically by
running the executable, are located in test/ext.
+Currently, all pytests must be located in either test/ext or test/int.
+Internal test filenames must match test_*.py, and external tests must
+be located in text/ext and their filenames must match test-* (see
+test/ext/conftest.py for the handling of the latter). Any paths
+matching those criteria will be automatically collected by pytest.
Some aspects of the environment are automatically restored after each
test via fixtures in conftest.py, including the state of the
test/tmp:
mkdir test/tmp
-runtests: runtests-python runtests-cmdline
-
-runtests-python: all test/tmp
+test: all test/tmp
./pytest
-cmdline_tests := \
- test/ext/test-cat-file.sh \
- test/ext/test-command-without-init-fails.sh \
- test/ext/test-compression.sh \
- test/ext/test-drecurse.sh \
- test/ext/test-fsck.sh \
- test/ext/test-fuse.sh \
- test/ext/test-help \
- test/ext/test-web.sh \
- test/ext/test-gc.sh \
- test/ext/test-import-duplicity.sh \
- test/ext/test-import-rdiff-backup.sh \
- test/ext/test-index.sh \
- test/ext/test-index-check-device.sh \
- test/ext/test-index-clear.sh \
- test/ext/test-list-idx.sh \
- test/ext/test-ls \
- test/ext/test-ls-remote \
- test/ext/test-main.sh \
- test/ext/test-meta.sh \
- test/ext/test-misc \
- test/ext/test-on.sh \
- test/ext/test-packsizelimit \
- test/ext/test-redundant-saves.sh \
- test/ext/test-restore-map-owner.sh \
- test/ext/test-restore-single-file.sh \
- test/ext/test-rm.sh \
- test/ext/test-rm-between-index-and-save.sh \
- test/ext/test-save-creates-no-unrefs.sh \
- test/ext/test-save-restore \
- test/ext/test-save-errors \
- test/ext/test-save-restore-excludes.sh \
- test/ext/test-save-strip-graft.sh \
- test/ext/test-save-with-valid-parent.sh \
- test/ext/test-sparse-files.sh \
- test/ext/test-split-join.sh \
- test/ext/test-tz.sh \
- test/ext/test-xdev.sh
-
-# For parallel runs.
-# The "pwd -P" here may not be appropriate in the long run, but we
-# need it until we settle the relevant drecurse/exclusion questions:
-# https://groups.google.com/forum/#!topic/bup-list/9ke-Mbp10Q0
-tmp-target-run-test%: all test/tmp
- $(pf); cd $$(pwd -P); TMPDIR="$(test_tmp)" \
- test/ext/test$* 2>&1 | tee -a test/tmp/test-log/$$$$.log
-
-runtests-cmdline: $(subst test/ext/test,tmp-target-run-test,$(cmdline_tests))
-
stupid:
PATH=/bin:/usr/bin $(MAKE) test
-test: all
- if test -e test/tmp/test-log; then rm -r test/tmp/test-log; fi
- mkdir -p test/tmp/test-log
- $(MAKE) runtests-python
- ./wvtest watch --no-counts \
- $(MAKE) runtests-cmdline 2>test/tmp/test-log/$$$$.log
- ./wvtest report test/tmp/test-log/*.log
-
check: test
distcheck: all
- ./wvtest run test/ext/test-release-archive.sh
+ ./pytest -m release
long-test: export BUP_TEST_LEVEL=11
long-test: test
+++ /dev/null
-#!/usr/bin/env perl
-#
-# WvTest:
-# Copyright (C) 2007-2009 Versabanq Innovations Inc. and contributors.
-# Copyright (C) 2015 Rob Browning <rlb@defaultvalue.org>
-# Licensed under the GNU Library General Public License, version 2.
-# See the included file named LICENSE for license information.
-#
-use strict;
-use warnings;
-use Getopt::Long qw(GetOptionsFromArray :config no_ignore_case bundling);
-use Pod::Usage;
-use Time::HiRes qw(time);
-
-my $per_test_warn_time = 100000; # upstream was 500
-my $per_test_bad_time = 100000; # upstream was 1000
-my $overall_test_warn_time = 100000; # upstream was 2000
-my $overall_test_bad_time = 100000; # upstream was 5000
-
-my $pid;
-my $istty = -t STDOUT;
-my @log = ();
-
-sub bigkill($)
-{
- my $pid = shift;
-
- if (@log) {
- print "\n" . join("\n", @log) . "\n";
- }
-
- print STDERR "\n! Killed by signal FAILED\n";
-
- ($pid > 0) || die("pid is '$pid'?!\n");
-
- local $SIG{CHLD} = sub { }; # this will wake us from sleep() faster
- kill 15, $pid;
- sleep(2);
-
- if ($pid > 1) {
- kill 9, -$pid;
- }
- kill 9, $pid;
-
- exit(125);
-}
-
-sub colourize($)
-{
- my $result = shift;
- my $pass = ($result eq "ok");
-
- if ($istty) {
- my $colour = $pass ? "\e[32;1m" : "\e[31;1m";
- return "$colour$result\e[0m";
- } else {
- return $result;
- }
-}
-
-sub mstime($$$)
-{
- my ($floatsec, $warntime, $badtime) = @_;
- my $ms = int($floatsec * 1000);
- my $str = sprintf("%d.%03ds", $ms/1000, $ms % 1000);
-
- if ($istty && $ms > $badtime) {
- return "\e[31;1m$str\e[0m";
- } elsif ($istty && $ms > $warntime) {
- return "\e[33;1m$str\e[0m";
- } else {
- return "$str";
- }
-}
-
-sub resultline($$)
-{
- my ($name, $result) = @_;
- return sprintf("! %-65s %s", $name, colourize($result));
-}
-
-my ($start, $stop);
-
-sub endsect()
-{
- $stop = time();
- if ($start) {
- printf " %s %s\n",
- mstime($stop - $start, $per_test_warn_time, $per_test_bad_time),
- colourize("ok");
- }
-}
-
-sub run
-{
- # dup_msgs should be true when "watching". In that case all top
- # level wvtest protocol messages should be duplicated to stderr so
- # that they can be safely captured for report to process later.
- my ($dup_msgs) = @_;
- my $show_counts = 1;
- GetOptionsFromArray(\@ARGV, 'counts!', \$show_counts)
- or pod2usage();
- pod2usage('$0: no command specified') if (@ARGV < 1);
-
- # always flush
- $| = 1;
-
- {
- my $msg = "Testing \"all\" in @ARGV:\n";
- print $msg;
- print STDERR $msg if $dup_msgs;
- }
-
- $pid = open(my $fh, "-|");
- if (!$pid) {
- # child
- setpgrp();
- open STDERR, '>&STDOUT' or die("Can't dup stdout: $!\n");
- exec(@ARGV);
- exit 126; # just in case
- }
-
- # parent
- my $allstart = time();
- local $SIG{INT} = sub { bigkill($pid); };
- local $SIG{TERM} = sub { bigkill($pid); };
- local $SIG{ALRM} = sub {
- print STDERR resultline('Alarm timed out! No test results for too long.\n',
- 'FAILED');
- bigkill($pid);
- };
-
- my ($gpasses, $gfails) = (0,0);
- while (<$fh>)
- {
- chomp;
- s/\r//g;
-
- if (/^\s*Testing "(.*)" in (.*):\s*$/)
- {
- alarm(300);
- my ($sect, $file) = ($1, $2);
-
- endsect();
-
- printf("! %s %s: ", $file, $sect);
- @log = ();
- $start = $stop;
- }
- elsif (/^!\s*(.*?)\s+(\S+)\s*$/)
- {
- alarm(300);
-
- my ($name, $result) = ($1, $2);
- my $pass = ($result eq "ok");
-
- if (!$start) {
- printf("\n! Startup: ");
- $start = time();
- }
-
- push @log, resultline($name, $result);
-
- if (!$pass) {
- $gfails++;
- if (@log) {
- print "\n" . join("\n", @log) . "\n";
- @log = ();
- }
- } else {
- $gpasses++;
- print ".";
- }
- }
- else
- {
- push @log, $_;
- }
- }
-
- endsect();
-
- my $newpid = waitpid($pid, 0);
- if ($newpid != $pid) {
- die("waitpid returned '$newpid', expected '$pid'\n");
- }
-
- my $code = $?;
- my $ret = ($code >> 8);
-
- # return death-from-signal exits as >128. This is what bash does if you ran
- # the program directly.
- if ($code && !$ret) { $ret = $code | 128; }
-
- if ($ret && @log) {
- print "\n" . join("\n", @log) . "\n";
- }
-
- if ($code != 0) {
- my $msg = resultline("Program returned non-zero exit code ($ret)",
- 'FAILED');
- print $msg;
- print STDERR "$msg\n" if $dup_msgs;
- }
-
- print "\n";
- if ($show_counts) {
- my $gtotal = $gpasses + $gfails;
- my $msg = sprintf("WvTest: %d test%s, %d failure%s\n",
- $gtotal, $gtotal == 1 ? "" : "s", $gfails,
- $gfails == 1 ? "" : "s");
- print $msg;
- print STDERR $msg if $dup_msgs;
- }
- {
- my $msg = sprintf("WvTest: result code $ret, total time %s\n",
- mstime(time() - $allstart,
- $overall_test_warn_time,
- $overall_test_bad_time));
- print $msg;
- print STDERR $msg if $dup_msgs;
- }
- return ($ret ? $ret : ($gfails ? 125 : 0));
-}
-
-sub report()
-{
- my ($gpasses, $gfails) = (0,0);
- for my $f (@ARGV)
- {
- my $fh;
- open($fh, '<:crlf', $f) or die "Unable to open $f: $!";
- while (<$fh>)
- {
- chomp;
- s/\r//g;
-
- if (/^\s*Testing "(.*)" in (.*):\s*$/) {
- @log = ();
- }
- elsif (/^!\s*(.*?)\s+(\S+)\s*$/) {
- my ($name, $result) = ($1, $2);
- my $pass = ($result eq "ok");
- push @log, resultline($name, $result);
- if (!$pass) {
- $gfails++;
- if (@log) {
- print "\n" . join("\n", @log) . "\n";
- @log = ();
- }
- } else {
- $gpasses++;
- }
- }
- else
- {
- push @log, $_;
- }
- }
- }
- my $gtotal = $gpasses + $gfails;
- printf("\nWvTest: %d test%s, %d failure%s\n",
- $gtotal, $gtotal == 1 ? "" : "s",
- $gfails, $gfails == 1 ? "" : "s");
- return ($gfails ? 125 : 0);
-}
-
-my ($show_help, $show_manual);
-Getopt::Long::Configure('no_permute');
-GetOptionsFromArray(\@ARGV,
- 'help|?' => \$show_help,
- 'man' => \$show_manual) or pod2usage();
-Getopt::Long::Configure('permute');
-pod2usage(-verbose => 1, -exitval => 0) if $show_help;
-pod2usage(-verbose => 2, -exitval => 0) if $show_manual;
-pod2usage(-msg => "$0: no action specified", -verbose => 1) if (@ARGV < 1);
-
-my $action = $ARGV[0];
-shift @ARGV;
-if ($action eq 'run') { exit run(0); }
-elsif ($action eq 'watch') { run(1); }
-elsif ($action eq 'report') { exit report(); }
-else { pod2usage(-msg => "$0: invalid action $action", -verbose => 1); }
-
-__END__
-
-=head1 NAME
-
-wvtest - the dumbest cross-platform test framework that could possibly work
-
-=head1 SYNOPSIS
-
- wvtest [GLOBAL...] run [RUN_OPT...] [--] command [arg...]
- wvtest [GLOBAL...] watch [RUN_OPT...] [--] command [arg...]
- wvtest [GLOBAL...] report [logfile...]
-
- GLOBAL:
- --help, -? display brief help message and exit
- --man display full documentation
- RUN_OPT:
- --[no-]counts [don't] show success/failure counts
-
-=head1 DESCRIPTION
-
-B<wvtest run some-tests> will run some-tests and report on the result.
-This should work fine as long as some-tests doesn't run any sub-tests
-in parallel.
-
-If you'd like to run your tests in parallel, use B<watch> and
-B<report> as described in the EXAMPLES below.
-
-=head1 EXAMPLES
-
- # Fine if ./tests doesn't produce any output in parallel.
- wvtest run ./tests
-
- # Use watch and report for parallel tests. Note that watch's stderr will
- # include copies of any top level messages - reporting non-zero
- # test command exits, etc., and so must be included in the report arguments.
- wvtest watch --no-counts \
- "sh -c '(test-1 2>&1 | tee test-1.log)& (test-2 2>&1 | tee test-2.log)&'" \
- 2>test-3.log \
- wvtest report test-1.log test-2.log test-3.log
-
-=cut
projects / bup.git / commitdiff