1 /* $NetBSD: tinytest.c,v 1.7 2024/08/18 20:47:24 christos Exp $ */
2
3 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 #ifdef TINYTEST_LOCAL
28 #include "tinytest_local.h"
29 #endif
30
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <assert.h>
35
36 #ifndef NO_FORKING
37
38 #ifdef _WIN32
39 #include <windows.h>
40 #else
41 #include <sys/types.h>
42 #include <sys/wait.h>
43 #include <unistd.h>
44 #endif
45
46 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
47 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
48 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
49 /* Workaround for a stupid bug in OSX 10.6 */
50 #define FORK_BREAKS_GCOV
51 #include <vproc.h>
52 #endif
53 #endif
54
55 #endif /* !NO_FORKING */
56
57 #ifndef __GNUC__
58 #define __attribute__(x)
59 #endif
60
61 #include "tinytest.h"
62 #include "tinytest_macros.h"
63
64 #define LONGEST_TEST_NAME 16384
65 #define DEFAULT_TESTCASE_TIMEOUT 30U
66 #define MAGIC_EXITCODE 42
67
68 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
69 static int n_ok = 0; /**< Number of tests that have passed */
70 static int n_bad = 0; /**< Number of tests that have failed. */
71 static int n_skipped = 0; /**< Number of tests that have been skipped. */
72
73 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
74 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
75 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
76 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
77 const char *verbosity_flag = "";
78
79 const struct testlist_alias_t *cfg_aliases=NULL;
80
81 enum outcome { SKIP=2, OK=1, FAIL=0 };
82 static enum outcome cur_test_outcome = 0;
83 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
84 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
85 const char *cur_test_name = NULL;
86
87 static void usage(struct testgroup_t *groups, int list_groups)
88 __attribute__((noreturn));
89 static int process_test_option(struct testgroup_t *groups, const char *test);
90
91 #ifdef _WIN32
92 /* Copy of argv[0] for win32. */
93 static char commandname[MAX_PATH+1];
94
95 struct timeout_thread_args {
96 const testcase_fn *fn;
97 void *env;
98 };
99
100 static DWORD WINAPI
timeout_thread_proc_(LPVOID arg)101 timeout_thread_proc_(LPVOID arg)
102 {
103 struct timeout_thread_args *args = arg;
104 (*(args->fn))(args->env);
105 ExitThread(cur_test_outcome == FAIL ? 1 : 0);
106 }
107
108 static enum outcome
testcase_run_in_thread_(const struct testcase_t * testcase,void * env)109 testcase_run_in_thread_(const struct testcase_t *testcase, void *env)
110 {
111 /* We will never run testcase in a new thread when the
112 timeout is set to zero */
113 assert(opt_timeout);
114 DWORD ret, tid;
115 HANDLE handle;
116 struct timeout_thread_args args = {
117 &(testcase->fn),
118 env
119 };
120
121 handle =CreateThread(NULL, 0, timeout_thread_proc_,
122 (LPVOID)&args, 0, &tid);
123 ret = WaitForSingleObject(handle, opt_timeout * 1000U);
124 if (ret == WAIT_OBJECT_0) {
125 ret = 0;
126 if (!GetExitCodeThread(handle, &ret)) {
127 printf("GetExitCodeThread failed\n");
128 ret = 1;
129 }
130 } else if (ret == WAIT_TIMEOUT) {
131 printf("timeout\n");
132 } else {
133 printf("Wait failed\n");
134 }
135 CloseHandle(handle);
136 if (ret == 0)
137 return OK;
138 else if (ret == MAGIC_EXITCODE)
139 return SKIP;
140 else
141 return FAIL;
142 }
143 #else
testcase_set_timeout_(void)144 static unsigned int testcase_set_timeout_(void)
145 {
146 return alarm(opt_timeout);
147 }
148
testcase_reset_timeout_(void)149 static unsigned int testcase_reset_timeout_(void)
150 {
151 return alarm(0);
152 }
153 #endif
154
155 static enum outcome
testcase_run_bare_(const struct testcase_t * testcase)156 testcase_run_bare_(const struct testcase_t *testcase)
157 {
158 void *env = NULL;
159 int outcome;
160 if (testcase->setup) {
161 env = testcase->setup->setup_fn(testcase);
162 if (!env)
163 return FAIL;
164 else if (env == (void*)TT_SKIP)
165 return SKIP;
166 }
167
168 cur_test_outcome = OK;
169 {
170 if (opt_timeout) {
171 #ifdef _WIN32
172 cur_test_outcome = testcase_run_in_thread_(testcase, env);
173 #else
174 testcase_set_timeout_();
175 testcase->fn(env);
176 testcase_reset_timeout_();
177 #endif
178 } else {
179 testcase->fn(env);
180 }
181 }
182 outcome = cur_test_outcome;
183
184 if (testcase->setup) {
185 if (testcase->setup->cleanup_fn(testcase, env) == 0)
186 outcome = FAIL;
187 }
188
189 return outcome;
190 }
191
192
193 #ifndef NO_FORKING
194
195 static enum outcome
testcase_run_forked_(const struct testgroup_t * group,const struct testcase_t * testcase)196 testcase_run_forked_(const struct testgroup_t *group,
197 const struct testcase_t *testcase)
198 {
199 #ifdef _WIN32
200 /* Fork? On Win32? How primitive! We'll do what the smart kids do:
201 we'll invoke our own exe (whose name we recall from the command
202 line) with a command line that tells it to run just the test we
203 want, and this time without forking.
204
205 (No, threads aren't an option. The whole point of forking is to
206 share no state between tests.)
207 */
208 int ok;
209 char buffer[LONGEST_TEST_NAME+256];
210 STARTUPINFOA si;
211 PROCESS_INFORMATION info;
212 DWORD ret;
213
214 if (!in_tinytest_main) {
215 printf("\nERROR. On Windows, testcase_run_forked_ must be"
216 " called from within tinytest_main.\n");
217 abort();
218 }
219 if (opt_verbosity>0)
220 printf("[forking] ");
221
222 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s --timeout 0 %s%s",
223 commandname, verbosity_flag, group->prefix, testcase->name);
224
225 memset(&si, 0, sizeof(si));
226 memset(&info, 0, sizeof(info));
227 si.cb = sizeof(si);
228
229 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
230 0, NULL, NULL, &si, &info);
231 if (!ok) {
232 printf("CreateProcess failed!\n");
233 return FAIL;
234 }
235 ret = WaitForSingleObject(info.hProcess,
236 (opt_timeout ? opt_timeout * 1000U : INFINITE));
237
238 if (ret == WAIT_OBJECT_0) {
239 GetExitCodeProcess(info.hProcess, &ret);
240 } else if (ret == WAIT_TIMEOUT) {
241 printf("timeout\n");
242 } else {
243 printf("Wait failed\n");
244 }
245 CloseHandle(info.hProcess);
246 CloseHandle(info.hThread);
247 if (ret == 0)
248 return OK;
249 else if (ret == MAGIC_EXITCODE)
250 return SKIP;
251 else
252 return FAIL;
253 #else
254 int outcome_pipe[2];
255 pid_t pid;
256 (void)group;
257
258 if (pipe(outcome_pipe))
259 perror("opening pipe");
260
261 if (opt_verbosity>0)
262 printf("[forking] ");
263 pid = fork();
264 #ifdef FORK_BREAKS_GCOV
265 vproc_transaction_begin(0);
266 #endif
267 if (!pid) {
268 /* child. */
269 int test_r, write_r;
270 char b[1];
271 close(outcome_pipe[0]);
272 test_r = testcase_run_bare_(testcase);
273 assert(0<=(int)test_r && (int)test_r<=2);
274 b[0] = "NYS"[test_r];
275 write_r = (int)write(outcome_pipe[1], b, 1);
276 if (write_r != 1) {
277 perror("write outcome to pipe");
278 exit(1);
279 }
280 exit(0);
281 return FAIL; /* unreachable */
282 } else {
283 /* parent */
284 int status, r, exitcode;
285 char b[1];
286 /* Close this now, so that if the other side closes it,
287 * our read fails. */
288 close(outcome_pipe[1]);
289 r = (int)read(outcome_pipe[0], b, 1);
290 if (r == 0) {
291 printf("[Lost connection!] ");
292 return FAIL;
293 } else if (r != 1) {
294 perror("read outcome from pipe");
295 }
296 waitpid(pid, &status, 0);
297 exitcode = WEXITSTATUS(status);
298 close(outcome_pipe[0]);
299 if (opt_verbosity>1)
300 printf("%s%s: exited with %i (%i)\n", group->prefix, testcase->name, exitcode, status);
301 if (exitcode != 0)
302 {
303 printf("[atexit failure!] ");
304 return FAIL;
305 }
306 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
307 }
308 #endif
309 }
310
311 #endif /* !NO_FORKING */
312
313 int
testcase_run_one(const struct testgroup_t * group,const struct testcase_t * testcase)314 testcase_run_one(const struct testgroup_t *group,
315 const struct testcase_t *testcase)
316 {
317 enum outcome outcome;
318
319 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
320 if (opt_verbosity>0)
321 printf("%s%s: %s\n",
322 group->prefix, testcase->name,
323 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
324 ++n_skipped;
325 return SKIP;
326 }
327
328 if (opt_verbosity>0 && !opt_forked) {
329 printf("%s%s: ", group->prefix, testcase->name);
330 } else {
331 if (opt_verbosity==0) printf(".");
332 cur_test_prefix = group->prefix;
333 cur_test_name = testcase->name;
334 }
335
336 #ifndef NO_FORKING
337 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
338 outcome = testcase_run_forked_(group, testcase);
339 } else {
340 #else
341 {
342 #endif
343 outcome = testcase_run_bare_(testcase);
344 }
345
346 if (outcome == OK) {
347 if (opt_verbosity>0 && !opt_forked)
348 puts(opt_verbosity==1?"OK":"");
349 } else if (outcome == SKIP) {
350 if (opt_verbosity>0 && !opt_forked)
351 puts("SKIPPED");
352 } else {
353 if (!opt_forked)
354 printf("\n [%s FAILED]\n", testcase->name);
355 }
356
357 if (opt_forked) {
358 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
359 return 1; /* unreachable */
360 } else {
361 return (int)outcome;
362 }
363 }
364
365 int
366 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
367 {
368 int i, j;
369 size_t length = LONGEST_TEST_NAME;
370 char fullname[LONGEST_TEST_NAME];
371 int found=0;
372 if (strstr(arg, ".."))
373 length = strstr(arg,"..")-arg;
374 for (i=0; groups[i].prefix; ++i) {
375 for (j=0; groups[i].cases[j].name; ++j) {
376 struct testcase_t *testcase = &groups[i].cases[j];
377 snprintf(fullname, sizeof(fullname), "%s%s",
378 groups[i].prefix, testcase->name);
379 if (!flag) { /* Hack! */
380 printf(" %s", fullname);
381 if (testcase->flags & TT_OFF_BY_DEFAULT)
382 puts(" (Off by default)");
383 else if (testcase->flags & TT_SKIP)
384 puts(" (DISABLED)");
385 else
386 puts("");
387 }
388 if (!strncmp(fullname, arg, length)) {
389 if (set)
390 testcase->flags |= flag;
391 else
392 testcase->flags &= ~flag;
393 ++found;
394 }
395 }
396 }
397 return found;
398 }
399
400 static void
401 usage(struct testgroup_t *groups, int list_groups)
402 {
403 puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
404 puts(" Specify tests by name, or using a prefix ending with '..'");
405 puts(" To skip a test, prefix its name with a colon.");
406 puts(" To enable a disabled test, prefix its name with a plus.");
407 puts(" Use --list-tests for a list of tests.");
408 if (list_groups) {
409 puts("Known tests are:");
410 tinytest_set_flag_(groups, "..", 1, 0);
411 }
412 exit(0);
413 }
414
415 static int
416 process_test_alias(struct testgroup_t *groups, const char *test)
417 {
418 int i, j, n, r;
419 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
420 if (!strcmp(cfg_aliases[i].name, test)) {
421 n = 0;
422 for (j = 0; cfg_aliases[i].tests[j]; ++j) {
423 r = process_test_option(groups, cfg_aliases[i].tests[j]);
424 if (r<0)
425 return -1;
426 n += r;
427 }
428 return n;
429 }
430 }
431 printf("No such test alias as @%s!",test);
432 return -1;
433 }
434
435 static int
436 process_test_option(struct testgroup_t *groups, const char *test)
437 {
438 int flag = TT_ENABLED_;
439 int n = 0;
440 if (test[0] == '@') {
441 return process_test_alias(groups, test + 1);
442 } else if (test[0] == ':') {
443 ++test;
444 flag = TT_SKIP;
445 } else if (test[0] == '+') {
446 ++test;
447 ++n;
448 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
449 printf("No such test as %s!\n", test);
450 return -1;
451 }
452 } else {
453 ++n;
454 }
455 if (!tinytest_set_flag_(groups, test, 1, flag)) {
456 printf("No such test as %s!\n", test);
457 return -1;
458 }
459 return n;
460 }
461
462 void
463 tinytest_set_aliases(const struct testlist_alias_t *aliases)
464 {
465 cfg_aliases = aliases;
466 }
467
468 int
469 tinytest_main(int c, const char **v, struct testgroup_t *groups)
470 {
471 int i, j, n=0;
472
473 #ifdef _WIN32
474 const char *sp = strrchr(v[0], '.');
475 const char *extension = "";
476 if (!sp || stricmp(sp, ".exe"))
477 extension = ".exe"; /* Add an exe so CreateProcess will work */
478 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
479 commandname[MAX_PATH]='\0';
480 #endif
481 for (i=1; i<c; ++i) {
482 if (v[i][0] == '-') {
483 if (!strcmp(v[i], "--RUNNING-FORKED")) {
484 opt_forked = 1;
485 } else if (!strcmp(v[i], "--no-fork")) {
486 opt_nofork = 1;
487 } else if (!strcmp(v[i], "--quiet")) {
488 opt_verbosity = -1;
489 verbosity_flag = "--quiet";
490 } else if (!strcmp(v[i], "--verbose")) {
491 opt_verbosity = 2;
492 verbosity_flag = "--verbose";
493 } else if (!strcmp(v[i], "--terse")) {
494 opt_verbosity = 0;
495 verbosity_flag = "--terse";
496 } else if (!strcmp(v[i], "--help")) {
497 usage(groups, 0);
498 } else if (!strcmp(v[i], "--list-tests")) {
499 usage(groups, 1);
500 } else if (!strcmp(v[i], "--timeout")) {
501 ++i;
502 if (i >= c) {
503 fprintf(stderr, "--timeout requires argument\n");
504 return -1;
505 }
506 opt_timeout = (unsigned)atoi(v[i]);
507 } else {
508 fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
509 return -1;
510 }
511 } else {
512 int r = process_test_option(groups, v[i]);
513 if (r<0)
514 return -1;
515 n += r;
516 }
517 }
518 if (!n)
519 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
520
521 #ifdef _IONBF
522 setvbuf(stdout, NULL, _IONBF, 0);
523 #endif
524
525 ++in_tinytest_main;
526 for (i = 0; groups[i].prefix; ++i) {
527 struct testgroup_t *group = &groups[i];
528 for (j = 0; group->cases[j].name; ++j) {
529 struct testcase_t *testcase = &group->cases[j];
530 int test_attempts = 3;
531 int test_ret_err;
532
533 if (!(testcase->flags & TT_ENABLED_))
534 continue;
535
536 for (;;) {
537 test_ret_err = testcase_run_one(group, testcase);
538
539 if (test_ret_err == OK)
540 break;
541 if (!(testcase->flags & TT_RETRIABLE))
542 break;
543 printf("\n [RETRYING %s (%i)]\n", testcase->name, test_attempts);
544 if (!test_attempts--)
545 break;
546 }
547
548 switch (test_ret_err) {
549 case OK: ++n_ok; break;
550 case SKIP: ++n_skipped; break;
551 default: ++n_bad; break;
552 }
553 }
554 }
555
556 --in_tinytest_main;
557
558 if (opt_verbosity==0)
559 puts("");
560
561 if (n_bad)
562 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
563 n_bad+n_ok,n_skipped);
564 else if (opt_verbosity >= 1)
565 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
566
567 return (n_bad == 0) ? 0 : 1;
568 }
569
570 int
571 tinytest_get_verbosity_(void)
572 {
573 return opt_verbosity;
574 }
575
576 void
577 tinytest_set_test_failed_(void)
578 {
579 if (opt_verbosity <= 0 && cur_test_name) {
580 if (opt_verbosity==0) puts("");
581 printf("%s%s: ", cur_test_prefix, cur_test_name);
582 cur_test_name = NULL;
583 }
584 cur_test_outcome = FAIL;
585 }
586
587 void
588 tinytest_set_test_skipped_(void)
589 {
590 if (cur_test_outcome==OK)
591 cur_test_outcome = SKIP;
592 }
593
594 char *
595 tinytest_format_hex_(const void *val_, unsigned long len)
596 {
597 const unsigned char *val = val_;
598 char *result, *cp;
599 size_t i;
600
601 if (!val)
602 return strdup("null");
603 if (!(result = malloc(len*2+1)))
604 return strdup("<allocation failure>");
605 cp = result;
606 for (i=0;i<len;++i) {
607 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
608 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
609 }
610 *cp = 0;
611 return result;
612 }
613