libevent/test/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2010 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include <stdio.h>
  27 #include <stdlib.h>
  28 #include <string.h>
  29 #include <assert.h>
  30
  31 #ifdef TINYTEST_LOCAL
  32 #include "tinytest_local.h"
  33 #endif
  34
  35 #ifdef WIN32
  36 #include <windows.h>
  37 #else
  38 #include <sys/types.h>
  39 #include <sys/wait.h>
  40 #include <unistd.h>
  41 #endif
  42
  43 #ifndef __GNUC__
  44 #define __attribute__(x)
  45 #endif
  46
  47 #include "tinytest.h"
  48 #include "tinytest_macros.h"
  49
  50 #define LONGEST_TEST_NAME 16384
  51
  52 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  53 static int n_ok = 0; /**< Number of tests that have passed */
  54 static int n_bad = 0; /**< Number of tests that have failed. */
  55 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  56
  57 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  58 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  59 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  60 const char *verbosity_flag = "";
  61
  62 enum outcome { SKIP=2, OK=1, FAIL=0 };
  63 static enum outcome cur_test_outcome = 0;
  64 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  65 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  66 const char *cur_test_name = NULL;
  67
  68 #ifdef WIN32
  69 /** Pointer to argv[0] for win32. */
  70 static const char *commandname = NULL;
  71 #endif
  72
  73 static void usage(struct testgroup_t *groups, int list_groups)
  74   __attribute__((noreturn));
  75
  76 static enum outcome
  77 _testcase_run_bare(const struct testcase_t *testcase)
  78 {
  79         void *env = NULL;
  80         int outcome;
  81         if (testcase->setup) {
  82                 env = testcase->setup->setup_fn(testcase);
  83                 if (!env)
  84                         return FAIL;
  85                 else if (env == (void*)TT_SKIP)
  86                         return SKIP;
  87         }
  88
  89         cur_test_outcome = OK;
  90         testcase->fn(env);
  91         outcome = cur_test_outcome;
  92
  93         if (testcase->setup) {
  94                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
  95                         outcome = FAIL;
  96         }
  97
  98         return outcome;
  99 }
 100
 101 #define MAGIC_EXITCODE 42
 102
 103 static enum outcome
 104 _testcase_run_forked(const struct testgroup_t *group,
 105                      const struct testcase_t *testcase)
 106 {
 107 #ifdef WIN32
 108         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 109            we'll invoke our own exe (whose name we recall from the command
 110            line) with a command line that tells it to run just the test we
 111            want, and this time without forking.
 112
 113            (No, threads aren't an option.  The whole point of forking is to
 114            share no state between tests.)
 115          */
 116         int ok;
 117         char buffer[LONGEST_TEST_NAME+256];
 118         STARTUPINFOA si;
 119         PROCESS_INFORMATION info;
 120         DWORD exitcode;
 121
 122         if (!in_tinytest_main) {
 123                 printf("\nERROR.  On Windows, _testcase_run_forked must be"
 124                        " called from within tinytest_main.\n");
 125                 abort();
 126         }
 127         if (opt_verbosity>0)
 128                 printf("[forking] ");
 129
 130         snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
 131                  commandname, verbosity_flag, group->prefix, testcase->name);
 132
 133         memset(&si, 0, sizeof(si));
 134         memset(&info, 0, sizeof(info));
 135         si.cb = sizeof(si);
 136
 137         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 138                            0, NULL, NULL, &si, &info);
 139         if (!ok) {
 140                 printf("CreateProcess failed!\n");
 141                 return 0;
 142         }
 143         WaitForSingleObject(info.hProcess, INFINITE);
 144         GetExitCodeProcess(info.hProcess, &exitcode);
 145         CloseHandle(info.hProcess);
 146         CloseHandle(info.hThread);
 147         if (exitcode == 0)
 148                 return OK;
 149         else if (exitcode == MAGIC_EXITCODE)
 150                 return SKIP;
 151         else
 152                 return FAIL;
 153 #else
 154         int outcome_pipe[2];
 155         pid_t pid;
 156         (void)group;
 157
 158         if (pipe(outcome_pipe))
 159                 perror("opening pipe");
 160
 161         if (opt_verbosity>0)
 162                 printf("[forking] ");
 163         pid = fork();
 164         if (!pid) {
 165                 /* child. */
 166                 int test_r, write_r;
 167                 char b[1];
 168                 close(outcome_pipe[0]);
 169                 test_r = _testcase_run_bare(testcase);
 170                 assert(0<=(int)test_r && (int)test_r<=2);
 171                 b[0] = "NYS"[test_r];
 172                 write_r = (int)write(outcome_pipe[1], b, 1);
 173                 if (write_r != 1) {
 174                         perror("write outcome to pipe");
 175                         exit(1);
 176                 }
 177                 exit(0);
 178                 return FAIL; /* unreachable */
 179         } else {
 180                 /* parent */
 181                 int status, r;
 182                 char b[1];
 183                 /* Close this now, so that if the other side closes it,
 184                  * our read fails. */
 185                 close(outcome_pipe[1]);
 186                 r = (int)read(outcome_pipe[0], b, 1);
 187                 if (r == 0) {
 188                         printf("[Lost connection!] ");
 189                         return 0;
 190                 } else if (r != 1) {
 191                         perror("read outcome from pipe");
 192                 }
 193                 waitpid(pid, &status, 0);
 194                 close(outcome_pipe[0]);
 195                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 196         }
 197 #endif
 198 }
 199
 200 int
 201 testcase_run_one(const struct testgroup_t *group,
 202                  const struct testcase_t *testcase)
 203 {
 204         enum outcome outcome;
 205
 206         if (testcase->flags & TT_SKIP) {
 207                 if (opt_verbosity>0)
 208                         printf("%s%s: SKIPPED\n",
 209                             group->prefix, testcase->name);
 210                 ++n_skipped;
 211                 return SKIP;
 212         }
 213
 214         if (opt_verbosity>0 && !opt_forked) {
 215                 printf("%s%s: ", group->prefix, testcase->name);
 216         } else {
 217                 if (opt_verbosity==0) printf(".");
 218                 cur_test_prefix = group->prefix;
 219                 cur_test_name = testcase->name;
 220         }
 221
 222         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 223                 outcome = _testcase_run_forked(group, testcase);
 224         } else {
 225                 outcome = _testcase_run_bare(testcase);
 226         }
 227
 228         if (outcome == OK) {
 229                 ++n_ok;
 230                 if (opt_verbosity>0 && !opt_forked)
 231                         puts(opt_verbosity==1?"OK":"");
 232         } else if (outcome == SKIP) {
 233                 ++n_skipped;
 234                 if (opt_verbosity>0 && !opt_forked)
 235                         puts("SKIPPED");
 236         } else {
 237                 ++n_bad;
 238                 if (!opt_forked)
 239                         printf("\n  [%s FAILED]\n", testcase->name);
 240         }
 241
 242         if (opt_forked) {
 243                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 244                 return 1; /* unreachable */
 245         } else {
 246                 return (int)outcome;
 247         }
 248 }
 249
 250 int
 251 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag)
 252 {
 253         int i, j;
 254         size_t length = LONGEST_TEST_NAME;
 255         char fullname[LONGEST_TEST_NAME];
 256         int found=0;
 257         if (strstr(arg, ".."))
 258                 length = strstr(arg,"..")-arg;
 259         for (i=0; groups[i].prefix; ++i) {
 260                 for (j=0; groups[i].cases[j].name; ++j) {
 261                         snprintf(fullname, sizeof(fullname), "%s%s",
 262                                  groups[i].prefix, groups[i].cases[j].name);
 263                         if (!flag) /* Hack! */
 264                                 printf("    %s\n", fullname);
 265                         if (!strncmp(fullname, arg, length)) {
 266                                 groups[i].cases[j].flags |= flag;
 267                                 ++found;
 268                         }
 269                 }
 270         }
 271         return found;
 272 }
 273
 274 static void
 275 usage(struct testgroup_t *groups, int list_groups)
 276 {
 277         puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
 278         puts("  Specify tests by name, or using a prefix ending with '..'");
 279         puts("  To skip a test, list give its name prefixed with a colon.");
 280         puts("  Use --list-tests for a list of tests.");
 281         if (list_groups) {
 282                 puts("Known tests are:");
 283                 _tinytest_set_flag(groups, "..", 0);
 284         }
 285         exit(0);
 286 }
 287
 288 int
 289 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 290 {
 291         int i, j, n=0;
 292
 293 #ifdef WIN32
 294         commandname = v[0];
 295 #endif
 296         for (i=1; i<c; ++i) {
 297                 if (v[i][0] == '-') {
 298                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 299                                 opt_forked = 1;
 300                         } else if (!strcmp(v[i], "--no-fork")) {
 301                                 opt_nofork = 1;
 302                         } else if (!strcmp(v[i], "--quiet")) {
 303                                 opt_verbosity = -1;
 304                                 verbosity_flag = "--quiet";
 305                         } else if (!strcmp(v[i], "--verbose")) {
 306                                 opt_verbosity = 2;
 307                                 verbosity_flag = "--verbose";
 308                         } else if (!strcmp(v[i], "--terse")) {
 309                                 opt_verbosity = 0;
 310                                 verbosity_flag = "--terse";
 311                         } else if (!strcmp(v[i], "--help")) {
 312                                 usage(groups, 0);
 313                         } else if (!strcmp(v[i], "--list-tests")) {
 314                                 usage(groups, 1);
 315                         } else {
 316                                 printf("Unknown option %s.  Try --help\n",v[i]);
 317                                 return -1;
 318                         }
 319                 } else {
 320                         const char *test = v[i];
 321                         int flag = _TT_ENABLED;
 322                         if (test[0] == ':') {
 323                                 ++test;
 324                                 flag = TT_SKIP;
 325                         } else {
 326                                 ++n;
 327                         }
 328                         if (!_tinytest_set_flag(groups, test, flag)) {
 329                                 printf("No such test as %s!\n", v[i]);
 330                                 return -1;
 331                         }
 332                 }
 333         }
 334         if (!n)
 335                 _tinytest_set_flag(groups, "..", _TT_ENABLED);
 336
 337         setvbuf(stdout, NULL, _IONBF, 0);
 338
 339         ++in_tinytest_main;
 340         for (i=0; groups[i].prefix; ++i)
 341                 for (j=0; groups[i].cases[j].name; ++j)
 342                         if (groups[i].cases[j].flags & _TT_ENABLED)
 343                                 testcase_run_one(&groups[i],
 344                                                  &groups[i].cases[j]);
 345
 346         --in_tinytest_main;
 347
 348         if (opt_verbosity==0)
 349                 puts("");
 350
 351         if (n_bad)
 352                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 353                        n_bad+n_ok,n_skipped);
 354         else if (opt_verbosity >= 1)
 355                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 356
 357         return (n_bad == 0) ? 0 : 1;
 358 }
 359
 360 int
 361 _tinytest_get_verbosity(void)
 362 {
 363         return opt_verbosity;
 364 }
 365
 366 void
 367 _tinytest_set_test_failed(void)
 368 {
 369         if (opt_verbosity <= 0 && cur_test_name) {
 370                 if (opt_verbosity==0) puts("");
 371                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 372                 cur_test_name = NULL;
 373         }
 374         cur_test_outcome = 0;
 375 }
 376
 377 void
 378 _tinytest_set_test_skipped(void)
 379 {
 380         if (cur_test_outcome==OK)
 381                 cur_test_outcome = SKIP;
 382 }
 383