1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/tests/jstests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,333 @@
1.4 +#!/usr/bin/env python
1.5 +"""
1.6 +The JS Shell Test Harness.
1.7 +
1.8 +See the adjacent README.txt for more details.
1.9 +"""
1.10 +
1.11 +import os, sys, textwrap
1.12 +from os.path import abspath, dirname, realpath
1.13 +from copy import copy
1.14 +from subprocess import list2cmdline, call
1.15 +
1.16 +from lib.results import NullTestOutput
1.17 +from lib.tests import TestCase, TBPL_FLAGS
1.18 +from lib.results import ResultsSink
1.19 +from lib.progressbar import ProgressBar
1.20 +
1.21 +if (sys.platform.startswith('linux') or
1.22 + sys.platform.startswith('darwin')
1.23 + ):
1.24 + from lib.tasks_unix import run_all_tests
1.25 +else:
1.26 + from lib.tasks_win import run_all_tests
1.27 +
1.28 +def run_tests(options, tests, results):
1.29 + """Run the given tests, sending raw results to the given results accumulator."""
1.30 + try:
1.31 + completed = run_all_tests(tests, results, options)
1.32 + except KeyboardInterrupt:
1.33 + completed = False
1.34 +
1.35 + results.finish(completed)
1.36 +
1.37 +def get_cpu_count():
1.38 + """
1.39 + Guess at a reasonable parallelism count to set as the default for the
1.40 + current machine and run.
1.41 + """
1.42 + # Python 2.6+
1.43 + try:
1.44 + import multiprocessing
1.45 + return multiprocessing.cpu_count()
1.46 + except (ImportError,NotImplementedError):
1.47 + pass
1.48 +
1.49 + # POSIX
1.50 + try:
1.51 + res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
1.52 + if res > 0:
1.53 + return res
1.54 + except (AttributeError,ValueError):
1.55 + pass
1.56 +
1.57 + # Windows
1.58 + try:
1.59 + res = int(os.environ['NUMBER_OF_PROCESSORS'])
1.60 + if res > 0:
1.61 + return res
1.62 + except (KeyError, ValueError):
1.63 + pass
1.64 +
1.65 + return 1
1.66 +
1.67 +def parse_args():
1.68 + """
1.69 + Parse command line arguments.
1.70 + Returns a tuple of: (options, js_shell, requested_paths, excluded_paths)
1.71 + options :object: The raw OptionParser output.
1.72 + js_shell :str: The absolute location of the shell to test with.
1.73 + requested_paths :set<str>: Test paths specially requested on the CLI.
1.74 + excluded_paths :set<str>: Test paths specifically excluded by the CLI.
1.75 + """
1.76 + from optparse import OptionParser, OptionGroup
1.77 + op = OptionParser(usage=textwrap.dedent("""
1.78 + %prog [OPTIONS] JS_SHELL [TESTS]
1.79 +
1.80 + Shell output format: [ pass | fail | timeout | skip ] progress | time
1.81 + """).strip())
1.82 + op.add_option('--xul-info', dest='xul_info_src',
1.83 + help='config data for xulRuntime (avoids search for config/autoconf.mk)')
1.84 +
1.85 + harness_og = OptionGroup(op, "Harness Controls", "Control how tests are run.")
1.86 + harness_og.add_option('-j', '--worker-count', type=int, default=max(1, get_cpu_count()),
1.87 + help='Number of tests to run in parallel (default %default)')
1.88 + harness_og.add_option('-t', '--timeout', type=float, default=150.0,
1.89 + help='Set maximum time a test is allows to run (in seconds).')
1.90 + harness_og.add_option('-a', '--args', dest='shell_args', default='',
1.91 + help='Extra args to pass to the JS shell.')
1.92 + harness_og.add_option('--jitflags', default='', help="Obsolete. Does nothing.")
1.93 + harness_og.add_option('--tbpl', action='store_true',
1.94 + help='Runs each test in all configurations tbpl tests.')
1.95 + harness_og.add_option('-g', '--debug', action='store_true', help='Run a test in debugger.')
1.96 + harness_og.add_option('--debugger', default='gdb -q --args', help='Debugger command.')
1.97 + harness_og.add_option('-J', '--jorendb', action='store_true', help='Run under JS debugger.')
1.98 + harness_og.add_option('--passthrough', action='store_true', help='Run tests with stdin/stdout attached to caller.')
1.99 + harness_og.add_option('--valgrind', action='store_true', help='Run tests in valgrind.')
1.100 + harness_og.add_option('--valgrind-args', default='', help='Extra args to pass to valgrind.')
1.101 + op.add_option_group(harness_og)
1.102 +
1.103 + input_og = OptionGroup(op, "Inputs", "Change what tests are run.")
1.104 + input_og.add_option('-f', '--file', dest='test_file', action='append',
1.105 + help='Get tests from the given file.')
1.106 + input_og.add_option('-x', '--exclude-file', action='append',
1.107 + help='Exclude tests from the given file.')
1.108 + input_og.add_option('-d', '--exclude-random', dest='random', action='store_false',
1.109 + help='Exclude tests marked as "random."')
1.110 + input_og.add_option('--run-skipped', action='store_true', help='Run tests marked as "skip."')
1.111 + input_og.add_option('--run-only-skipped', action='store_true', help='Run only tests marked as "skip."')
1.112 + input_og.add_option('--run-slow-tests', action='store_true',
1.113 + help='Do not skip tests marked as "slow."')
1.114 + input_og.add_option('--no-extensions', action='store_true',
1.115 + help='Run only tests conforming to the ECMAScript 5 standard.')
1.116 + op.add_option_group(input_og)
1.117 +
1.118 + output_og = OptionGroup(op, "Output", "Modify the harness and tests output.")
1.119 + output_og.add_option('-s', '--show-cmd', action='store_true',
1.120 + help='Show exact commandline used to run each test.')
1.121 + output_og.add_option('-o', '--show-output', action='store_true',
1.122 + help="Print each test's output to the file given by --output-file.")
1.123 + output_og.add_option('-F', '--failed-only', action='store_true',
1.124 + help="If a --show-* option is given, only print output for failed tests.")
1.125 + output_og.add_option('-O', '--output-file',
1.126 + help='Write all output to the given file (default: stdout).')
1.127 + output_og.add_option('--failure-file',
1.128 + help='Write all not-passed tests to the given file.')
1.129 + output_og.add_option('--no-progress', dest='hide_progress', action='store_true',
1.130 + help='Do not show the progress bar.')
1.131 + output_og.add_option('--tinderbox', action='store_true',
1.132 + help='Use tinderbox-parseable output format.')
1.133 + op.add_option_group(output_og)
1.134 +
1.135 + special_og = OptionGroup(op, "Special", "Special modes that do not run tests.")
1.136 + special_og.add_option('--make-manifests', metavar='BASE_TEST_PATH',
1.137 + help='Generate reftest manifest files.')
1.138 + op.add_option_group(special_og)
1.139 + options, args = op.parse_args()
1.140 +
1.141 + # Acquire the JS shell given on the command line.
1.142 + options.js_shell = None
1.143 + requested_paths = set()
1.144 + if len(args) > 0:
1.145 + options.js_shell = abspath(args[0])
1.146 + requested_paths |= set(args[1:])
1.147 +
1.148 + # If we do not have a shell, we must be in a special mode.
1.149 + if options.js_shell is None and not options.make_manifests:
1.150 + op.error('missing JS_SHELL argument')
1.151 +
1.152 + # Valgrind and gdb are mutually exclusive.
1.153 + if options.valgrind and options.debug:
1.154 + op.error("--valgrind and --debug are mutually exclusive.")
1.155 +
1.156 + # Fill the debugger field, as needed.
1.157 + prefix = options.debugger.split() if options.debug else []
1.158 + if options.valgrind:
1.159 + prefix = ['valgrind'] + options.valgrind_args.split()
1.160 + if os.uname()[0] == 'Darwin':
1.161 + prefix.append('--dsymutil=yes')
1.162 + options.show_output = True
1.163 +
1.164 + js_cmd_args = options.shell_args.split()
1.165 + if options.jorendb:
1.166 + options.passthrough = True
1.167 + options.hide_progress = True
1.168 + options.worker_count = 1
1.169 + debugger_path = realpath(os.path.join(abspath(dirname(abspath(__file__))), '..', '..', 'examples', 'jorendb.js'))
1.170 + js_cmd_args.extend([ '-d', '-f', debugger_path, '--' ])
1.171 + TestCase.set_js_cmd_prefix(options.js_shell, js_cmd_args, prefix)
1.172 +
1.173 + # If files with lists of tests to run were specified, add them to the
1.174 + # requested tests set.
1.175 + if options.test_file:
1.176 + for test_file in options.test_file:
1.177 + requested_paths |= set([line.strip() for line in open(test_file).readlines()])
1.178 +
1.179 + # If files with lists of tests to exclude were specified, add them to the
1.180 + # excluded tests set.
1.181 + excluded_paths = set()
1.182 + if options.exclude_file:
1.183 + for filename in options.exclude_file:
1.184 + try:
1.185 + fp = open(filename, 'r')
1.186 + for line in fp:
1.187 + if line.startswith('#'): continue
1.188 + line = line.strip()
1.189 + if not line: continue
1.190 + excluded_paths |= set((line,))
1.191 + finally:
1.192 + fp.close()
1.193 +
1.194 + # Handle output redirection, if requested and relevant.
1.195 + options.output_fp = sys.stdout
1.196 + if options.output_file:
1.197 + if not options.show_cmd:
1.198 + options.show_output = True
1.199 + try:
1.200 + options.output_fp = open(options.output_file, 'w')
1.201 + except IOError, ex:
1.202 + raise SystemExit("Failed to open output file: " + str(ex))
1.203 +
1.204 + options.show = options.show_cmd or options.show_output
1.205 +
1.206 + # Hide the progress bar if it will get in the way of other output.
1.207 + options.hide_progress = (options.tinderbox or
1.208 + not ProgressBar.conservative_isatty() or
1.209 + options.hide_progress)
1.210 +
1.211 + return (options, requested_paths, excluded_paths)
1.212 +
1.213 +def load_tests(options, requested_paths, excluded_paths):
1.214 + """
1.215 + Returns a tuple: (skipped_tests, test_list)
1.216 + skip_list: [iterable<Test>] Tests found but skipped.
1.217 + test_list: [iterable<Test>] Tests found that should be run.
1.218 + """
1.219 + import lib.manifest as manifest
1.220 +
1.221 + if options.js_shell is None:
1.222 + xul_tester = manifest.NullXULInfoTester()
1.223 + else:
1.224 + if options.xul_info_src is None:
1.225 + xul_info = manifest.XULInfo.create(options.js_shell)
1.226 + else:
1.227 + xul_abi, xul_os, xul_debug = options.xul_info_src.split(r':')
1.228 + xul_debug = xul_debug.lower() is 'true'
1.229 + xul_info = manifest.XULInfo(xul_abi, xul_os, xul_debug)
1.230 + xul_tester = manifest.XULInfoTester(xul_info, options.js_shell)
1.231 +
1.232 + test_dir = dirname(abspath(__file__))
1.233 + test_list = manifest.load(test_dir, xul_tester)
1.234 + skip_list = []
1.235 +
1.236 + if options.make_manifests:
1.237 + manifest.make_manifests(options.make_manifests, test_list)
1.238 + sys.exit()
1.239 +
1.240 + # Create a new test list. Apply each TBPL configuration to every test.
1.241 + if options.tbpl:
1.242 + new_test_list = []
1.243 + flags_list = TBPL_FLAGS
1.244 + for test in test_list:
1.245 + for jitflags in flags_list:
1.246 + tmp_test = copy(test)
1.247 + tmp_test.options = copy(test.options)
1.248 + tmp_test.options.extend(jitflags)
1.249 + new_test_list.append(tmp_test)
1.250 + test_list = new_test_list
1.251 +
1.252 + if options.jitflags:
1.253 + print("Warning: the --jitflags option is obsolete and does nothing now.")
1.254 +
1.255 + if options.test_file:
1.256 + paths = set()
1.257 + for test_file in options.test_file:
1.258 + paths |= set([ line.strip() for line in open(test_file).readlines()])
1.259 + test_list = [ _ for _ in test_list if _.path in paths ]
1.260 +
1.261 + if requested_paths:
1.262 + def p(path):
1.263 + for arg in requested_paths:
1.264 + if path.find(arg) != -1:
1.265 + return True
1.266 + return False
1.267 + test_list = [ _ for _ in test_list if p(_.path) ]
1.268 +
1.269 + if options.exclude_file:
1.270 + test_list = [_ for _ in test_list if _.path not in excluded_paths]
1.271 +
1.272 + if options.no_extensions:
1.273 + pattern = os.sep + 'extensions' + os.sep
1.274 + test_list = [_ for _ in test_list if pattern not in _.path]
1.275 +
1.276 + if not options.random:
1.277 + test_list = [ _ for _ in test_list if not _.random ]
1.278 +
1.279 + if options.run_only_skipped:
1.280 + options.run_skipped = True
1.281 + test_list = [ _ for _ in test_list if not _.enable ]
1.282 +
1.283 + if not options.run_slow_tests:
1.284 + test_list = [ _ for _ in test_list if not _.slow ]
1.285 +
1.286 + if not options.run_skipped:
1.287 + skip_list = [ _ for _ in test_list if not _.enable ]
1.288 + test_list = [ _ for _ in test_list if _.enable ]
1.289 +
1.290 + return skip_list, test_list
1.291 +
1.292 +def main():
1.293 + options, requested_paths, excluded_paths = parse_args()
1.294 + skip_list, test_list = load_tests(options, requested_paths, excluded_paths)
1.295 +
1.296 + if not test_list:
1.297 + print 'no tests selected'
1.298 + return 1
1.299 +
1.300 + test_dir = dirname(abspath(__file__))
1.301 +
1.302 + if options.debug:
1.303 + if len(test_list) > 1:
1.304 + print('Multiple tests match command line arguments, debugger can only run one')
1.305 + for tc in test_list:
1.306 + print(' %s'%tc.path)
1.307 + return 2
1.308 +
1.309 + cmd = test_list[0].get_command(TestCase.js_cmd_prefix)
1.310 + if options.show_cmd:
1.311 + print list2cmdline(cmd)
1.312 + if test_dir not in ('', '.'):
1.313 + os.chdir(test_dir)
1.314 + call(cmd)
1.315 + return 0
1.316 +
1.317 + curdir = os.getcwd()
1.318 + if test_dir not in ('', '.'):
1.319 + os.chdir(test_dir)
1.320 +
1.321 + results = None
1.322 + try:
1.323 + results = ResultsSink(options, len(skip_list) + len(test_list))
1.324 + for t in skip_list:
1.325 + results.push(NullTestOutput(t))
1.326 + run_tests(options, test_list, results)
1.327 + finally:
1.328 + os.chdir(curdir)
1.329 +
1.330 + if results is None or not results.all_passed():
1.331 + return 1
1.332 +
1.333 + return 0
1.334 +
1.335 +if __name__ == '__main__':
1.336 + sys.exit(main())
