michael@0: /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
michael@0:  *
michael@0:  * Redistribution and use in source and binary forms, with or without
michael@0:  * modification, are permitted provided that the following conditions
michael@0:  * are met:
michael@0:  * 1. Redistributions of source code must retain the above copyright
michael@0:  *    notice, this list of conditions and the following disclaimer.
michael@0:  * 2. Redistributions in binary form must reproduce the above copyright
michael@0:  *    notice, this list of conditions and the following disclaimer in the
michael@0:  *    documentation and/or other materials provided with the distribution.
michael@0:  * 3. The name of the author may not be used to endorse or promote products
michael@0:  *    derived from this software without specific prior written permission.
michael@0:  *
michael@0:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
michael@0:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
michael@0:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
michael@0:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
michael@0:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
michael@0:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
michael@0:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
michael@0:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
michael@0:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
michael@0:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
michael@0:  */
michael@0: 
michael@0: #include <stdio.h>
michael@0: #include <stdlib.h>
michael@0: #include <string.h>
michael@0: #include <assert.h>
michael@0: 
michael@0: #ifdef TINYTEST_LOCAL
michael@0: #include "tinytest_local.h"
michael@0: #endif
michael@0: 
michael@0: #ifdef WIN32
michael@0: #include <windows.h>
michael@0: #else
michael@0: #include <sys/types.h>
michael@0: #include <sys/wait.h>
michael@0: #include <unistd.h>
michael@0: #endif
michael@0: 
michael@0: #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
michael@0: #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
michael@0:     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
michael@0: /* Workaround for a stupid bug in OSX 10.6 */
michael@0: #define FORK_BREAKS_GCOV
michael@0: #include <vproc.h>
michael@0: #endif
michael@0: #endif
michael@0: 
michael@0: #ifndef __GNUC__
michael@0: #define __attribute__(x)
michael@0: #endif
michael@0: 
michael@0: #include "tinytest.h"
michael@0: #include "tinytest_macros.h"
michael@0: 
michael@0: #define LONGEST_TEST_NAME 16384
michael@0: 
michael@0: static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
michael@0: static int n_ok = 0; /**< Number of tests that have passed */
michael@0: static int n_bad = 0; /**< Number of tests that have failed. */
michael@0: static int n_skipped = 0; /**< Number of tests that have been skipped. */
michael@0: 
michael@0: static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
michael@0: static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
michael@0: static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
michael@0: const char *verbosity_flag = "";
michael@0: 
michael@0: enum outcome { SKIP=2, OK=1, FAIL=0 };
michael@0: static enum outcome cur_test_outcome = 0;
michael@0: const char *cur_test_prefix = NULL; /**< prefix of the current test group */
michael@0: /** Name of the current test, if we haven't logged is yet. Used for --quiet */
michael@0: const char *cur_test_name = NULL;
michael@0: 
michael@0: #ifdef WIN32
michael@0: /* Copy of argv[0] for win32. */
michael@0: static char commandname[MAX_PATH+1];
michael@0: #endif
michael@0: 
michael@0: static void usage(struct testgroup_t *groups, int list_groups)
michael@0:   __attribute__((noreturn));
michael@0: 
michael@0: static enum outcome
michael@0: _testcase_run_bare(const struct testcase_t *testcase)
michael@0: {
michael@0: 	void *env = NULL;
michael@0: 	int outcome;
michael@0: 	if (testcase->setup) {
michael@0: 		env = testcase->setup->setup_fn(testcase);
michael@0: 		if (!env)
michael@0: 			return FAIL;
michael@0: 		else if (env == (void*)TT_SKIP)
michael@0: 			return SKIP;
michael@0: 	}
michael@0: 
michael@0: 	cur_test_outcome = OK;
michael@0: 	testcase->fn(env);
michael@0: 	outcome = cur_test_outcome;
michael@0: 
michael@0: 	if (testcase->setup) {
michael@0: 		if (testcase->setup->cleanup_fn(testcase, env) == 0)
michael@0: 			outcome = FAIL;
michael@0: 	}
michael@0: 
michael@0: 	return outcome;
michael@0: }
michael@0: 
michael@0: #define MAGIC_EXITCODE 42
michael@0: 
michael@0: static enum outcome
michael@0: _testcase_run_forked(const struct testgroup_t *group,
michael@0: 		     const struct testcase_t *testcase)
michael@0: {
michael@0: #ifdef WIN32
michael@0: 	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
michael@0: 	   we'll invoke our own exe (whose name we recall from the command
michael@0: 	   line) with a command line that tells it to run just the test we
michael@0: 	   want, and this time without forking.
michael@0: 
michael@0: 	   (No, threads aren't an option.  The whole point of forking is to
michael@0: 	   share no state between tests.)
michael@0: 	 */
michael@0: 	int ok;
michael@0: 	char buffer[LONGEST_TEST_NAME+256];
michael@0: 	STARTUPINFOA si;
michael@0: 	PROCESS_INFORMATION info;
michael@0: 	DWORD exitcode;
michael@0: 
michael@0: 	if (!in_tinytest_main) {
michael@0: 		printf("\nERROR.  On Windows, _testcase_run_forked must be"
michael@0: 		       " called from within tinytest_main.\n");
michael@0: 		abort();
michael@0: 	}
michael@0: 	if (opt_verbosity>0)
michael@0: 		printf("[forking] ");
michael@0: 
michael@0: 	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
michael@0: 		 commandname, verbosity_flag, group->prefix, testcase->name);
michael@0: 
michael@0: 	memset(&si, 0, sizeof(si));
michael@0: 	memset(&info, 0, sizeof(info));
michael@0: 	si.cb = sizeof(si);
michael@0: 
michael@0: 	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
michael@0: 			   0, NULL, NULL, &si, &info);
michael@0: 	if (!ok) {
michael@0: 		printf("CreateProcess failed!\n");
michael@0: 		return 0;
michael@0: 	}
michael@0: 	WaitForSingleObject(info.hProcess, INFINITE);
michael@0: 	GetExitCodeProcess(info.hProcess, &exitcode);
michael@0: 	CloseHandle(info.hProcess);
michael@0: 	CloseHandle(info.hThread);
michael@0: 	if (exitcode == 0)
michael@0: 		return OK;
michael@0: 	else if (exitcode == MAGIC_EXITCODE)
michael@0: 		return SKIP;
michael@0: 	else
michael@0: 		return FAIL;
michael@0: #else
michael@0: 	int outcome_pipe[2];
michael@0: 	pid_t pid;
michael@0: 	(void)group;
michael@0: 
michael@0: 	if (pipe(outcome_pipe))
michael@0: 		perror("opening pipe");
michael@0: 
michael@0: 	if (opt_verbosity>0)
michael@0: 		printf("[forking] ");
michael@0: 	pid = fork();
michael@0: #ifdef FORK_BREAKS_GCOV
michael@0: 	vproc_transaction_begin(0);
michael@0: #endif
michael@0: 	if (!pid) {
michael@0: 		/* child. */
michael@0: 		int test_r, write_r;
michael@0: 		char b[1];
michael@0: 		close(outcome_pipe[0]);
michael@0: 		test_r = _testcase_run_bare(testcase);
michael@0: 		assert(0<=(int)test_r && (int)test_r<=2);
michael@0: 		b[0] = "NYS"[test_r];
michael@0: 		write_r = (int)write(outcome_pipe[1], b, 1);
michael@0: 		if (write_r != 1) {
michael@0: 			perror("write outcome to pipe");
michael@0: 			exit(1);
michael@0: 		}
michael@0: 		exit(0);
michael@0: 		return FAIL; /* unreachable */
michael@0: 	} else {
michael@0: 		/* parent */
michael@0: 		int status, r;
michael@0: 		char b[1];
michael@0: 		/* Close this now, so that if the other side closes it,
michael@0: 		 * our read fails. */
michael@0: 		close(outcome_pipe[1]);
michael@0: 		r = (int)read(outcome_pipe[0], b, 1);
michael@0: 		if (r == 0) {
michael@0: 			printf("[Lost connection!] ");
michael@0: 			return 0;
michael@0: 		} else if (r != 1) {
michael@0: 			perror("read outcome from pipe");
michael@0: 		}
michael@0: 		waitpid(pid, &status, 0);
michael@0: 		close(outcome_pipe[0]);
michael@0: 		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
michael@0: 	}
michael@0: #endif
michael@0: }
michael@0: 
michael@0: int
michael@0: testcase_run_one(const struct testgroup_t *group,
michael@0: 		 const struct testcase_t *testcase)
michael@0: {
michael@0: 	enum outcome outcome;
michael@0: 
michael@0: 	if (testcase->flags & TT_SKIP) {
michael@0: 		if (opt_verbosity>0)
michael@0: 			printf("%s%s: SKIPPED\n",
michael@0: 			    group->prefix, testcase->name);
michael@0: 		++n_skipped;
michael@0: 		return SKIP;
michael@0: 	}
michael@0: 
michael@0: 	if (opt_verbosity>0 && !opt_forked) {
michael@0: 		printf("%s%s: ", group->prefix, testcase->name);
michael@0: 	} else {
michael@0: 		if (opt_verbosity==0) printf(".");
michael@0: 		cur_test_prefix = group->prefix;
michael@0: 		cur_test_name = testcase->name;
michael@0: 	}
michael@0: 
michael@0: 	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
michael@0: 		outcome = _testcase_run_forked(group, testcase);
michael@0: 	} else {
michael@0: 		outcome = _testcase_run_bare(testcase);
michael@0: 	}
michael@0: 
michael@0: 	if (outcome == OK) {
michael@0: 		++n_ok;
michael@0: 		if (opt_verbosity>0 && !opt_forked)
michael@0: 			puts(opt_verbosity==1?"OK":"");
michael@0: 	} else if (outcome == SKIP) {
michael@0: 		++n_skipped;
michael@0: 		if (opt_verbosity>0 && !opt_forked)
michael@0: 			puts("SKIPPED");
michael@0: 	} else {
michael@0: 		++n_bad;
michael@0: 		if (!opt_forked)
michael@0: 			printf("\n  [%s FAILED]\n", testcase->name);
michael@0: 	}
michael@0: 
michael@0: 	if (opt_forked) {
michael@0: 		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
michael@0: 		return 1; /* unreachable */
michael@0: 	} else {
michael@0: 		return (int)outcome;
michael@0: 	}
michael@0: }
michael@0: 
michael@0: int
michael@0: _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag)
michael@0: {
michael@0: 	int i, j;
michael@0: 	size_t length = LONGEST_TEST_NAME;
michael@0: 	char fullname[LONGEST_TEST_NAME];
michael@0: 	int found=0;
michael@0: 	if (strstr(arg, ".."))
michael@0: 		length = strstr(arg,"..")-arg;
michael@0: 	for (i=0; groups[i].prefix; ++i) {
michael@0: 		for (j=0; groups[i].cases[j].name; ++j) {
michael@0: 			snprintf(fullname, sizeof(fullname), "%s%s",
michael@0: 				 groups[i].prefix, groups[i].cases[j].name);
michael@0: 			if (!flag) /* Hack! */
michael@0: 				printf("    %s\n", fullname);
michael@0: 			if (!strncmp(fullname, arg, length)) {
michael@0: 				groups[i].cases[j].flags |= flag;
michael@0: 				++found;
michael@0: 			}
michael@0: 		}
michael@0: 	}
michael@0: 	return found;
michael@0: }
michael@0: 
michael@0: static void
michael@0: usage(struct testgroup_t *groups, int list_groups)
michael@0: {
michael@0: 	puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
michael@0: 	puts("  Specify tests by name, or using a prefix ending with '..'");
michael@0: 	puts("  To skip a test, list give its name prefixed with a colon.");
michael@0: 	puts("  Use --list-tests for a list of tests.");
michael@0: 	if (list_groups) {
michael@0: 		puts("Known tests are:");
michael@0: 		_tinytest_set_flag(groups, "..", 0);
michael@0: 	}
michael@0: 	exit(0);
michael@0: }
michael@0: 
michael@0: int
michael@0: tinytest_main(int c, const char **v, struct testgroup_t *groups)
michael@0: {
michael@0: 	int i, j, n=0;
michael@0: 
michael@0: #ifdef WIN32
michael@0: 	const char *sp = strrchr(v[0], '.');
michael@0: 	const char *extension = "";
michael@0: 	if (!sp || stricmp(sp, ".exe"))
michael@0: 		extension = ".exe"; /* Add an exe so CreateProcess will work */
michael@0: 	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
michael@0: 	commandname[MAX_PATH]='\0';
michael@0: #endif
michael@0: 	for (i=1; i<c; ++i) {
michael@0: 		if (v[i][0] == '-') {
michael@0: 			if (!strcmp(v[i], "--RUNNING-FORKED")) {
michael@0: 				opt_forked = 1;
michael@0: 			} else if (!strcmp(v[i], "--no-fork")) {
michael@0: 				opt_nofork = 1;
michael@0: 			} else if (!strcmp(v[i], "--quiet")) {
michael@0: 				opt_verbosity = -1;
michael@0: 				verbosity_flag = "--quiet";
michael@0: 			} else if (!strcmp(v[i], "--verbose")) {
michael@0: 				opt_verbosity = 2;
michael@0: 				verbosity_flag = "--verbose";
michael@0: 			} else if (!strcmp(v[i], "--terse")) {
michael@0: 				opt_verbosity = 0;
michael@0: 				verbosity_flag = "--terse";
michael@0: 			} else if (!strcmp(v[i], "--help")) {
michael@0: 				usage(groups, 0);
michael@0: 			} else if (!strcmp(v[i], "--list-tests")) {
michael@0: 				usage(groups, 1);
michael@0: 			} else {
michael@0: 				printf("Unknown option %s.  Try --help\n",v[i]);
michael@0: 				return -1;
michael@0: 			}
michael@0: 		} else {
michael@0: 			const char *test = v[i];
michael@0: 			int flag = _TT_ENABLED;
michael@0: 			if (test[0] == ':') {
michael@0: 				++test;
michael@0: 				flag = TT_SKIP;
michael@0: 			} else {
michael@0: 				++n;
michael@0: 			}
michael@0: 			if (!_tinytest_set_flag(groups, test, flag)) {
michael@0: 				printf("No such test as %s!\n", v[i]);
michael@0: 				return -1;
michael@0: 			}
michael@0: 		}
michael@0: 	}
michael@0: 	if (!n)
michael@0: 		_tinytest_set_flag(groups, "..", _TT_ENABLED);
michael@0: 
michael@0: 	setvbuf(stdout, NULL, _IONBF, 0);
michael@0: 
michael@0: 	++in_tinytest_main;
michael@0: 	for (i=0; groups[i].prefix; ++i)
michael@0: 		for (j=0; groups[i].cases[j].name; ++j)
michael@0: 			if (groups[i].cases[j].flags & _TT_ENABLED)
michael@0: 				testcase_run_one(&groups[i],
michael@0: 						 &groups[i].cases[j]);
michael@0: 
michael@0: 	--in_tinytest_main;
michael@0: 
michael@0: 	if (opt_verbosity==0)
michael@0: 		puts("");
michael@0: 
michael@0: 	if (n_bad)
michael@0: 		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
michael@0: 		       n_bad+n_ok,n_skipped);
michael@0: 	else if (opt_verbosity >= 1)
michael@0: 		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
michael@0: 
michael@0: 	return (n_bad == 0) ? 0 : 1;
michael@0: }
michael@0: 
michael@0: int
michael@0: _tinytest_get_verbosity(void)
michael@0: {
michael@0: 	return opt_verbosity;
michael@0: }
michael@0: 
michael@0: void
michael@0: _tinytest_set_test_failed(void)
michael@0: {
michael@0: 	if (opt_verbosity <= 0 && cur_test_name) {
michael@0: 		if (opt_verbosity==0) puts("");
michael@0: 		printf("%s%s: ", cur_test_prefix, cur_test_name);
michael@0: 		cur_test_name = NULL;
michael@0: 	}
michael@0: 	cur_test_outcome = 0;
michael@0: }
michael@0: 
michael@0: void
michael@0: _tinytest_set_test_skipped(void)
michael@0: {
michael@0: 	if (cur_test_outcome==OK)
michael@0: 		cur_test_outcome = SKIP;
michael@0: }
michael@0: