From 0144a07edc1a00b065bda208c36986b8201fb15a Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Thu, 28 May 2020 23:14:53 +0200 Subject: [PATCH] Migrate the remaining (standalone) test/ext tests to pytest Enable the test/ext/conftest.py collector to allow pytest to handle all of the test/ext standalone tests too. Signed-off-by: Johannes Berg Reviewed-by: Rob Browning [rlb@defaultvalue.org: show the pytest invocations; switch to use test/ext/conftest.py collector; use the release mark in check and distcheck; augment commit message; augment the information in HACKING] Signed-off-by: Rob Browning Tested-by: Rob Browning --- HACKING | 22 ++- Makefile | 63 +-------- pytest | 2 +- test/ext/conftest.py | 2 - wvtest | 325 ------------------------------------------- wvtest.sh | 0 6 files changed, 13 insertions(+), 401 deletions(-) delete mode 100755 wvtest mode change 100755 => 100644 wvtest.sh diff --git a/HACKING b/HACKING index c784573..31f48a5 100644 --- a/HACKING +++ b/HACKING @@ -60,26 +60,24 @@ We also love a good "Tested-by:" -- the more the merrier. Testing ======= -You can run the test suite much more quickly via "make -j test" (as -compared to "make test"), at the expense of slightly more confusing -output (interleaved parallel test output), and inaccurate intermediate -success/failure counts, but the final counts displayed should be -correct. +Individual tests can be run via -Individual non-Python tests can be run via + ./pytest TEST - ./wvtest run test/ext/TEST +For example: -and if you'd like to see all of the test output, you can omit the -wvtest run wrapper: `test/ext/TEST`. Individual Python tests can be -run via - - ./pytest test/int/test_something.py + ./pytest test/int/test_git.py + ./pytest test/ext/test-ftp Internal tests that test bup's code directly are located in test/int, and external tests that test bup from the outside, typically by running the executable, are located in test/ext. +Currently, all pytests must be located in either test/ext or test/int. +Internal test filenames must match test_*.py, and external tests must +be located in text/ext and their filenames must match test-* (see +test/ext/conftest.py for the handling of the latter). Any paths +matching those criteria will be automatically collected by pytest. Some aspects of the environment are automatically restored after each test via fixtures in conftest.py, including the state of the diff --git a/Makefile b/Makefile index 127df50..266f7ea 100644 --- a/Makefile +++ b/Makefile @@ -159,75 +159,16 @@ lib/bup/_helpers$(SOEXT): \ test/tmp: mkdir test/tmp -runtests: runtests-python runtests-cmdline - -runtests-python: all test/tmp +test: all test/tmp ./pytest -cmdline_tests := \ - test/ext/test-cat-file.sh \ - test/ext/test-command-without-init-fails.sh \ - test/ext/test-compression.sh \ - test/ext/test-drecurse.sh \ - test/ext/test-fsck.sh \ - test/ext/test-fuse.sh \ - test/ext/test-help \ - test/ext/test-web.sh \ - test/ext/test-gc.sh \ - test/ext/test-import-duplicity.sh \ - test/ext/test-import-rdiff-backup.sh \ - test/ext/test-index.sh \ - test/ext/test-index-check-device.sh \ - test/ext/test-index-clear.sh \ - test/ext/test-list-idx.sh \ - test/ext/test-ls \ - test/ext/test-ls-remote \ - test/ext/test-main.sh \ - test/ext/test-meta.sh \ - test/ext/test-misc \ - test/ext/test-on.sh \ - test/ext/test-packsizelimit \ - test/ext/test-redundant-saves.sh \ - test/ext/test-restore-map-owner.sh \ - test/ext/test-restore-single-file.sh \ - test/ext/test-rm.sh \ - test/ext/test-rm-between-index-and-save.sh \ - test/ext/test-save-creates-no-unrefs.sh \ - test/ext/test-save-restore \ - test/ext/test-save-errors \ - test/ext/test-save-restore-excludes.sh \ - test/ext/test-save-strip-graft.sh \ - test/ext/test-save-with-valid-parent.sh \ - test/ext/test-sparse-files.sh \ - test/ext/test-split-join.sh \ - test/ext/test-tz.sh \ - test/ext/test-xdev.sh - -# For parallel runs. -# The "pwd -P" here may not be appropriate in the long run, but we -# need it until we settle the relevant drecurse/exclusion questions: -# https://groups.google.com/forum/#!topic/bup-list/9ke-Mbp10Q0 -tmp-target-run-test%: all test/tmp - $(pf); cd $$(pwd -P); TMPDIR="$(test_tmp)" \ - test/ext/test$* 2>&1 | tee -a test/tmp/test-log/$$$$.log - -runtests-cmdline: $(subst test/ext/test,tmp-target-run-test,$(cmdline_tests)) - stupid: PATH=/bin:/usr/bin $(MAKE) test -test: all - if test -e test/tmp/test-log; then rm -r test/tmp/test-log; fi - mkdir -p test/tmp/test-log - $(MAKE) runtests-python - ./wvtest watch --no-counts \ - $(MAKE) runtests-cmdline 2>test/tmp/test-log/$$$$.log - ./wvtest report test/tmp/test-log/*.log - check: test distcheck: all - ./wvtest run test/ext/test-release-archive.sh + ./pytest -m release long-test: export BUP_TEST_LEVEL=11 long-test: test diff --git a/pytest b/pytest index a62ee1f..f35f626 100755 --- a/pytest +++ b/pytest @@ -6,4 +6,4 @@ script_home="$(cd "$(dirname "$0")" && pwd -P)" testlibdir="$script_home/test/lib" export PYTHONPATH="$testlibdir${PYTHONPATH:+:$PYTHONPATH}" -exec dev/bup-python -m pytest -v "$@" +exec dev/bup-python -m pytest -v -m 'not release' "$@" diff --git a/test/ext/conftest.py b/test/ext/conftest.py index e93d6b0..be1b682 100644 --- a/test/ext/conftest.py +++ b/test/ext/conftest.py @@ -65,8 +65,6 @@ class BupSubprocTestFile(pytest.File): yield BupSubprocTestRunner('', self) def pytest_collect_file(parent, path): - # Disabled until the upcoming changes that rely on it are in place. - return None base = path.basename if base.startswith('test-') and not base.endswith('~'): try: diff --git a/wvtest b/wvtest deleted file mode 100755 index 7d494f3..0000000 --- a/wvtest +++ /dev/null @@ -1,325 +0,0 @@ -#!/usr/bin/env perl -# -# WvTest: -# Copyright (C) 2007-2009 Versabanq Innovations Inc. and contributors. -# Copyright (C) 2015 Rob Browning -# Licensed under the GNU Library General Public License, version 2. -# See the included file named LICENSE for license information. -# -use strict; -use warnings; -use Getopt::Long qw(GetOptionsFromArray :config no_ignore_case bundling); -use Pod::Usage; -use Time::HiRes qw(time); - -my $per_test_warn_time = 100000; # upstream was 500 -my $per_test_bad_time = 100000; # upstream was 1000 -my $overall_test_warn_time = 100000; # upstream was 2000 -my $overall_test_bad_time = 100000; # upstream was 5000 - -my $pid; -my $istty = -t STDOUT; -my @log = (); - -sub bigkill($) -{ - my $pid = shift; - - if (@log) { - print "\n" . join("\n", @log) . "\n"; - } - - print STDERR "\n! Killed by signal FAILED\n"; - - ($pid > 0) || die("pid is '$pid'?!\n"); - - local $SIG{CHLD} = sub { }; # this will wake us from sleep() faster - kill 15, $pid; - sleep(2); - - if ($pid > 1) { - kill 9, -$pid; - } - kill 9, $pid; - - exit(125); -} - -sub colourize($) -{ - my $result = shift; - my $pass = ($result eq "ok"); - - if ($istty) { - my $colour = $pass ? "\e[32;1m" : "\e[31;1m"; - return "$colour$result\e[0m"; - } else { - return $result; - } -} - -sub mstime($$$) -{ - my ($floatsec, $warntime, $badtime) = @_; - my $ms = int($floatsec * 1000); - my $str = sprintf("%d.%03ds", $ms/1000, $ms % 1000); - - if ($istty && $ms > $badtime) { - return "\e[31;1m$str\e[0m"; - } elsif ($istty && $ms > $warntime) { - return "\e[33;1m$str\e[0m"; - } else { - return "$str"; - } -} - -sub resultline($$) -{ - my ($name, $result) = @_; - return sprintf("! %-65s %s", $name, colourize($result)); -} - -my ($start, $stop); - -sub endsect() -{ - $stop = time(); - if ($start) { - printf " %s %s\n", - mstime($stop - $start, $per_test_warn_time, $per_test_bad_time), - colourize("ok"); - } -} - -sub run -{ - # dup_msgs should be true when "watching". In that case all top - # level wvtest protocol messages should be duplicated to stderr so - # that they can be safely captured for report to process later. - my ($dup_msgs) = @_; - my $show_counts = 1; - GetOptionsFromArray(\@ARGV, 'counts!', \$show_counts) - or pod2usage(); - pod2usage('$0: no command specified') if (@ARGV < 1); - - # always flush - $| = 1; - - { - my $msg = "Testing \"all\" in @ARGV:\n"; - print $msg; - print STDERR $msg if $dup_msgs; - } - - $pid = open(my $fh, "-|"); - if (!$pid) { - # child - setpgrp(); - open STDERR, '>&STDOUT' or die("Can't dup stdout: $!\n"); - exec(@ARGV); - exit 126; # just in case - } - - # parent - my $allstart = time(); - local $SIG{INT} = sub { bigkill($pid); }; - local $SIG{TERM} = sub { bigkill($pid); }; - local $SIG{ALRM} = sub { - print STDERR resultline('Alarm timed out! No test results for too long.\n', - 'FAILED'); - bigkill($pid); - }; - - my ($gpasses, $gfails) = (0,0); - while (<$fh>) - { - chomp; - s/\r//g; - - if (/^\s*Testing "(.*)" in (.*):\s*$/) - { - alarm(300); - my ($sect, $file) = ($1, $2); - - endsect(); - - printf("! %s %s: ", $file, $sect); - @log = (); - $start = $stop; - } - elsif (/^!\s*(.*?)\s+(\S+)\s*$/) - { - alarm(300); - - my ($name, $result) = ($1, $2); - my $pass = ($result eq "ok"); - - if (!$start) { - printf("\n! Startup: "); - $start = time(); - } - - push @log, resultline($name, $result); - - if (!$pass) { - $gfails++; - if (@log) { - print "\n" . join("\n", @log) . "\n"; - @log = (); - } - } else { - $gpasses++; - print "."; - } - } - else - { - push @log, $_; - } - } - - endsect(); - - my $newpid = waitpid($pid, 0); - if ($newpid != $pid) { - die("waitpid returned '$newpid', expected '$pid'\n"); - } - - my $code = $?; - my $ret = ($code >> 8); - - # return death-from-signal exits as >128. This is what bash does if you ran - # the program directly. - if ($code && !$ret) { $ret = $code | 128; } - - if ($ret && @log) { - print "\n" . join("\n", @log) . "\n"; - } - - if ($code != 0) { - my $msg = resultline("Program returned non-zero exit code ($ret)", - 'FAILED'); - print $msg; - print STDERR "$msg\n" if $dup_msgs; - } - - print "\n"; - if ($show_counts) { - my $gtotal = $gpasses + $gfails; - my $msg = sprintf("WvTest: %d test%s, %d failure%s\n", - $gtotal, $gtotal == 1 ? "" : "s", $gfails, - $gfails == 1 ? "" : "s"); - print $msg; - print STDERR $msg if $dup_msgs; - } - { - my $msg = sprintf("WvTest: result code $ret, total time %s\n", - mstime(time() - $allstart, - $overall_test_warn_time, - $overall_test_bad_time)); - print $msg; - print STDERR $msg if $dup_msgs; - } - return ($ret ? $ret : ($gfails ? 125 : 0)); -} - -sub report() -{ - my ($gpasses, $gfails) = (0,0); - for my $f (@ARGV) - { - my $fh; - open($fh, '<:crlf', $f) or die "Unable to open $f: $!"; - while (<$fh>) - { - chomp; - s/\r//g; - - if (/^\s*Testing "(.*)" in (.*):\s*$/) { - @log = (); - } - elsif (/^!\s*(.*?)\s+(\S+)\s*$/) { - my ($name, $result) = ($1, $2); - my $pass = ($result eq "ok"); - push @log, resultline($name, $result); - if (!$pass) { - $gfails++; - if (@log) { - print "\n" . join("\n", @log) . "\n"; - @log = (); - } - } else { - $gpasses++; - } - } - else - { - push @log, $_; - } - } - } - my $gtotal = $gpasses + $gfails; - printf("\nWvTest: %d test%s, %d failure%s\n", - $gtotal, $gtotal == 1 ? "" : "s", - $gfails, $gfails == 1 ? "" : "s"); - return ($gfails ? 125 : 0); -} - -my ($show_help, $show_manual); -Getopt::Long::Configure('no_permute'); -GetOptionsFromArray(\@ARGV, - 'help|?' => \$show_help, - 'man' => \$show_manual) or pod2usage(); -Getopt::Long::Configure('permute'); -pod2usage(-verbose => 1, -exitval => 0) if $show_help; -pod2usage(-verbose => 2, -exitval => 0) if $show_manual; -pod2usage(-msg => "$0: no action specified", -verbose => 1) if (@ARGV < 1); - -my $action = $ARGV[0]; -shift @ARGV; -if ($action eq 'run') { exit run(0); } -elsif ($action eq 'watch') { run(1); } -elsif ($action eq 'report') { exit report(); } -else { pod2usage(-msg => "$0: invalid action $action", -verbose => 1); } - -__END__ - -=head1 NAME - -wvtest - the dumbest cross-platform test framework that could possibly work - -=head1 SYNOPSIS - - wvtest [GLOBAL...] run [RUN_OPT...] [--] command [arg...] - wvtest [GLOBAL...] watch [RUN_OPT...] [--] command [arg...] - wvtest [GLOBAL...] report [logfile...] - - GLOBAL: - --help, -? display brief help message and exit - --man display full documentation - RUN_OPT: - --[no-]counts [don't] show success/failure counts - -=head1 DESCRIPTION - -B will run some-tests and report on the result. -This should work fine as long as some-tests doesn't run any sub-tests -in parallel. - -If you'd like to run your tests in parallel, use B and -B as described in the EXAMPLES below. - -=head1 EXAMPLES - - # Fine if ./tests doesn't produce any output in parallel. - wvtest run ./tests - - # Use watch and report for parallel tests. Note that watch's stderr will - # include copies of any top level messages - reporting non-zero - # test command exits, etc., and so must be included in the report arguments. - wvtest watch --no-counts \ - "sh -c '(test-1 2>&1 | tee test-1.log)& (test-2 2>&1 | tee test-2.log)&'" \ - 2>test-3.log \ - wvtest report test-1.log test-2.log test-3.log - -=cut diff --git a/wvtest.sh b/wvtest.sh old mode 100755 new mode 100644 -- 2.39.2