michael@0: #!/usr/bin/env python
michael@0: """
michael@0: The JS Shell Test Harness.
michael@0: 
michael@0: See the adjacent README.txt for more details.
michael@0: """
michael@0: 
michael@0: import os, sys, textwrap
michael@0: from os.path import abspath, dirname, realpath
michael@0: from copy import copy
michael@0: from subprocess import list2cmdline, call
michael@0: 
michael@0: from lib.results import NullTestOutput
michael@0: from lib.tests import TestCase, TBPL_FLAGS
michael@0: from lib.results import ResultsSink
michael@0: from lib.progressbar import ProgressBar
michael@0: 
michael@0: if (sys.platform.startswith('linux') or
michael@0:     sys.platform.startswith('darwin')
michael@0:    ):
michael@0:     from lib.tasks_unix import run_all_tests
michael@0: else:
michael@0:     from lib.tasks_win import run_all_tests
michael@0: 
michael@0: def run_tests(options, tests, results):
michael@0:     """Run the given tests, sending raw results to the given results accumulator."""
michael@0:     try:
michael@0:         completed = run_all_tests(tests, results, options)
michael@0:     except KeyboardInterrupt:
michael@0:         completed = False
michael@0: 
michael@0:     results.finish(completed)
michael@0: 
michael@0: def get_cpu_count():
michael@0:     """
michael@0:     Guess at a reasonable parallelism count to set as the default for the
michael@0:     current machine and run.
michael@0:     """
michael@0:     # Python 2.6+
michael@0:     try:
michael@0:         import multiprocessing
michael@0:         return multiprocessing.cpu_count()
michael@0:     except (ImportError,NotImplementedError):
michael@0:         pass
michael@0: 
michael@0:     # POSIX
michael@0:     try:
michael@0:         res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
michael@0:         if res > 0:
michael@0:             return res
michael@0:     except (AttributeError,ValueError):
michael@0:         pass
michael@0: 
michael@0:     # Windows
michael@0:     try:
michael@0:         res = int(os.environ['NUMBER_OF_PROCESSORS'])
michael@0:         if res > 0:
michael@0:             return res
michael@0:     except (KeyError, ValueError):
michael@0:         pass
michael@0: 
michael@0:     return 1
michael@0: 
michael@0: def parse_args():
michael@0:     """
michael@0:     Parse command line arguments.
michael@0:     Returns a tuple of: (options, js_shell, requested_paths, excluded_paths)
michael@0:         options :object: The raw OptionParser output.
michael@0:         js_shell :str: The absolute location of the shell to test with.
michael@0:         requested_paths :set<str>: Test paths specially requested on the CLI.
michael@0:         excluded_paths :set<str>: Test paths specifically excluded by the CLI.
michael@0:     """
michael@0:     from optparse import OptionParser, OptionGroup
michael@0:     op = OptionParser(usage=textwrap.dedent("""
michael@0:         %prog [OPTIONS] JS_SHELL [TESTS]
michael@0: 
michael@0:         Shell output format: [ pass | fail | timeout | skip ] progress | time
michael@0:         """).strip())
michael@0:     op.add_option('--xul-info', dest='xul_info_src',
michael@0:                   help='config data for xulRuntime (avoids search for config/autoconf.mk)')
michael@0: 
michael@0:     harness_og = OptionGroup(op, "Harness Controls", "Control how tests are run.")
michael@0:     harness_og.add_option('-j', '--worker-count', type=int, default=max(1, get_cpu_count()),
michael@0:                           help='Number of tests to run in parallel (default %default)')
michael@0:     harness_og.add_option('-t', '--timeout', type=float, default=150.0,
michael@0:                           help='Set maximum time a test is allows to run (in seconds).')
michael@0:     harness_og.add_option('-a', '--args', dest='shell_args', default='',
michael@0:                           help='Extra args to pass to the JS shell.')
michael@0:     harness_og.add_option('--jitflags', default='', help="Obsolete. Does nothing.")
michael@0:     harness_og.add_option('--tbpl', action='store_true',
michael@0:                           help='Runs each test in all configurations tbpl tests.')
michael@0:     harness_og.add_option('-g', '--debug', action='store_true', help='Run a test in debugger.')
michael@0:     harness_og.add_option('--debugger', default='gdb -q --args', help='Debugger command.')
michael@0:     harness_og.add_option('-J', '--jorendb', action='store_true', help='Run under JS debugger.')
michael@0:     harness_og.add_option('--passthrough', action='store_true', help='Run tests with stdin/stdout attached to caller.')
michael@0:     harness_og.add_option('--valgrind', action='store_true', help='Run tests in valgrind.')
michael@0:     harness_og.add_option('--valgrind-args', default='', help='Extra args to pass to valgrind.')
michael@0:     op.add_option_group(harness_og)
michael@0: 
michael@0:     input_og = OptionGroup(op, "Inputs", "Change what tests are run.")
michael@0:     input_og.add_option('-f', '--file', dest='test_file', action='append',
michael@0:                         help='Get tests from the given file.')
michael@0:     input_og.add_option('-x', '--exclude-file', action='append',
michael@0:                         help='Exclude tests from the given file.')
michael@0:     input_og.add_option('-d', '--exclude-random', dest='random', action='store_false',
michael@0:                         help='Exclude tests marked as "random."')
michael@0:     input_og.add_option('--run-skipped', action='store_true', help='Run tests marked as "skip."')
michael@0:     input_og.add_option('--run-only-skipped', action='store_true', help='Run only tests marked as "skip."')
michael@0:     input_og.add_option('--run-slow-tests', action='store_true',
michael@0:                         help='Do not skip tests marked as "slow."')
michael@0:     input_og.add_option('--no-extensions', action='store_true',
michael@0:                         help='Run only tests conforming to the ECMAScript 5 standard.')
michael@0:     op.add_option_group(input_og)
michael@0: 
michael@0:     output_og = OptionGroup(op, "Output", "Modify the harness and tests output.")
michael@0:     output_og.add_option('-s', '--show-cmd', action='store_true',
michael@0:                          help='Show exact commandline used to run each test.')
michael@0:     output_og.add_option('-o', '--show-output', action='store_true',
michael@0:                          help="Print each test's output to the file given by --output-file.")
michael@0:     output_og.add_option('-F', '--failed-only', action='store_true',
michael@0:                          help="If a --show-* option is given, only print output for failed tests.")
michael@0:     output_og.add_option('-O', '--output-file',
michael@0:                          help='Write all output to the given file (default: stdout).')
michael@0:     output_og.add_option('--failure-file',
michael@0:                          help='Write all not-passed tests to the given file.')
michael@0:     output_og.add_option('--no-progress', dest='hide_progress', action='store_true',
michael@0:                          help='Do not show the progress bar.')
michael@0:     output_og.add_option('--tinderbox', action='store_true',
michael@0:                          help='Use tinderbox-parseable output format.')
michael@0:     op.add_option_group(output_og)
michael@0: 
michael@0:     special_og = OptionGroup(op, "Special", "Special modes that do not run tests.")
michael@0:     special_og.add_option('--make-manifests', metavar='BASE_TEST_PATH',
michael@0:                           help='Generate reftest manifest files.')
michael@0:     op.add_option_group(special_og)
michael@0:     options, args = op.parse_args()
michael@0: 
michael@0:     # Acquire the JS shell given on the command line.
michael@0:     options.js_shell = None
michael@0:     requested_paths = set()
michael@0:     if len(args) > 0:
michael@0:         options.js_shell = abspath(args[0])
michael@0:         requested_paths |= set(args[1:])
michael@0: 
michael@0:     # If we do not have a shell, we must be in a special mode.
michael@0:     if options.js_shell is None and not options.make_manifests:
michael@0:         op.error('missing JS_SHELL argument')
michael@0: 
michael@0:     # Valgrind and gdb are mutually exclusive.
michael@0:     if options.valgrind and options.debug:
michael@0:         op.error("--valgrind and --debug are mutually exclusive.")
michael@0: 
michael@0:     # Fill the debugger field, as needed.
michael@0:     prefix = options.debugger.split() if options.debug else []
michael@0:     if options.valgrind:
michael@0:         prefix = ['valgrind'] + options.valgrind_args.split()
michael@0:         if os.uname()[0] == 'Darwin':
michael@0:             prefix.append('--dsymutil=yes')
michael@0:         options.show_output = True
michael@0: 
michael@0:     js_cmd_args = options.shell_args.split()
michael@0:     if options.jorendb:
michael@0:         options.passthrough = True
michael@0:         options.hide_progress = True
michael@0:         options.worker_count = 1
michael@0:         debugger_path = realpath(os.path.join(abspath(dirname(abspath(__file__))), '..', '..', 'examples', 'jorendb.js'))
michael@0:         js_cmd_args.extend([ '-d', '-f', debugger_path, '--' ])
michael@0:     TestCase.set_js_cmd_prefix(options.js_shell, js_cmd_args, prefix)
michael@0: 
michael@0:     # If files with lists of tests to run were specified, add them to the
michael@0:     # requested tests set.
michael@0:     if options.test_file:
michael@0:         for test_file in options.test_file:
michael@0:             requested_paths |= set([line.strip() for line in open(test_file).readlines()])
michael@0: 
michael@0:     # If files with lists of tests to exclude were specified, add them to the
michael@0:     # excluded tests set.
michael@0:     excluded_paths = set()
michael@0:     if options.exclude_file:
michael@0:         for filename in options.exclude_file:
michael@0:             try:
michael@0:                 fp = open(filename, 'r')
michael@0:                 for line in fp:
michael@0:                     if line.startswith('#'): continue
michael@0:                     line = line.strip()
michael@0:                     if not line: continue
michael@0:                     excluded_paths |= set((line,))
michael@0:             finally:
michael@0:                 fp.close()
michael@0: 
michael@0:     # Handle output redirection, if requested and relevant.
michael@0:     options.output_fp = sys.stdout
michael@0:     if options.output_file:
michael@0:         if not options.show_cmd:
michael@0:             options.show_output = True
michael@0:         try:
michael@0:             options.output_fp = open(options.output_file, 'w')
michael@0:         except IOError, ex:
michael@0:             raise SystemExit("Failed to open output file: " + str(ex))
michael@0: 
michael@0:     options.show = options.show_cmd or options.show_output
michael@0: 
michael@0:     # Hide the progress bar if it will get in the way of other output.
michael@0:     options.hide_progress = (options.tinderbox or
michael@0:                              not ProgressBar.conservative_isatty() or
michael@0:                              options.hide_progress)
michael@0: 
michael@0:     return (options, requested_paths, excluded_paths)
michael@0: 
michael@0: def load_tests(options, requested_paths, excluded_paths):
michael@0:     """
michael@0:     Returns a tuple: (skipped_tests, test_list)
michael@0:         skip_list: [iterable<Test>] Tests found but skipped.
michael@0:         test_list: [iterable<Test>] Tests found that should be run.
michael@0:     """
michael@0:     import lib.manifest as manifest
michael@0: 
michael@0:     if options.js_shell is None:
michael@0:         xul_tester = manifest.NullXULInfoTester()
michael@0:     else:
michael@0:         if options.xul_info_src is None:
michael@0:             xul_info = manifest.XULInfo.create(options.js_shell)
michael@0:         else:
michael@0:             xul_abi, xul_os, xul_debug = options.xul_info_src.split(r':')
michael@0:             xul_debug = xul_debug.lower() is 'true'
michael@0:             xul_info = manifest.XULInfo(xul_abi, xul_os, xul_debug)
michael@0:         xul_tester = manifest.XULInfoTester(xul_info, options.js_shell)
michael@0: 
michael@0:     test_dir = dirname(abspath(__file__))
michael@0:     test_list = manifest.load(test_dir, xul_tester)
michael@0:     skip_list = []
michael@0: 
michael@0:     if options.make_manifests:
michael@0:         manifest.make_manifests(options.make_manifests, test_list)
michael@0:         sys.exit()
michael@0: 
michael@0:     # Create a new test list. Apply each TBPL configuration to every test.
michael@0:     if options.tbpl:
michael@0:         new_test_list = []
michael@0:         flags_list = TBPL_FLAGS
michael@0:         for test in test_list:
michael@0:             for jitflags in flags_list:
michael@0:                 tmp_test = copy(test)
michael@0:                 tmp_test.options = copy(test.options)
michael@0:                 tmp_test.options.extend(jitflags)
michael@0:                 new_test_list.append(tmp_test)
michael@0:         test_list = new_test_list
michael@0: 
michael@0:     if options.jitflags:
michael@0:         print("Warning: the --jitflags option is obsolete and does nothing now.")
michael@0: 
michael@0:     if options.test_file:
michael@0:         paths = set()
michael@0:         for test_file in options.test_file:
michael@0:             paths |= set([ line.strip() for line in open(test_file).readlines()])
michael@0:         test_list = [ _ for _ in test_list if _.path in paths ]
michael@0: 
michael@0:     if requested_paths:
michael@0:         def p(path):
michael@0:             for arg in requested_paths:
michael@0:                 if path.find(arg) != -1:
michael@0:                     return True
michael@0:             return False
michael@0:         test_list = [ _ for _ in test_list if p(_.path) ]
michael@0: 
michael@0:     if options.exclude_file:
michael@0:         test_list = [_ for _ in test_list if _.path not in excluded_paths]
michael@0: 
michael@0:     if options.no_extensions:
michael@0:         pattern = os.sep + 'extensions' + os.sep
michael@0:         test_list = [_ for _ in test_list if pattern not in _.path]
michael@0: 
michael@0:     if not options.random:
michael@0:         test_list = [ _ for _ in test_list if not _.random ]
michael@0: 
michael@0:     if options.run_only_skipped:
michael@0:         options.run_skipped = True
michael@0:         test_list = [ _ for _ in test_list if not _.enable ]
michael@0: 
michael@0:     if not options.run_slow_tests:
michael@0:         test_list = [ _ for _ in test_list if not _.slow ]
michael@0: 
michael@0:     if not options.run_skipped:
michael@0:         skip_list = [ _ for _ in test_list if not _.enable ]
michael@0:         test_list = [ _ for _ in test_list if _.enable ]
michael@0: 
michael@0:     return skip_list, test_list
michael@0: 
michael@0: def main():
michael@0:     options, requested_paths, excluded_paths = parse_args()
michael@0:     skip_list, test_list = load_tests(options, requested_paths, excluded_paths)
michael@0: 
michael@0:     if not test_list:
michael@0:         print 'no tests selected'
michael@0:         return 1
michael@0: 
michael@0:     test_dir = dirname(abspath(__file__))
michael@0: 
michael@0:     if options.debug:
michael@0:         if len(test_list) > 1:
michael@0:             print('Multiple tests match command line arguments, debugger can only run one')
michael@0:             for tc in test_list:
michael@0:                 print('    %s'%tc.path)
michael@0:             return 2
michael@0: 
michael@0:         cmd = test_list[0].get_command(TestCase.js_cmd_prefix)
michael@0:         if options.show_cmd:
michael@0:             print list2cmdline(cmd)
michael@0:         if test_dir not in ('', '.'):
michael@0:             os.chdir(test_dir)
michael@0:         call(cmd)
michael@0:         return 0
michael@0: 
michael@0:     curdir = os.getcwd()
michael@0:     if test_dir not in ('', '.'):
michael@0:         os.chdir(test_dir)
michael@0: 
michael@0:     results = None
michael@0:     try:
michael@0:         results = ResultsSink(options, len(skip_list) + len(test_list))
michael@0:         for t in skip_list:
michael@0:             results.push(NullTestOutput(t))
michael@0:         run_tests(options, test_list, results)
michael@0:     finally:
michael@0:         os.chdir(curdir)
michael@0: 
michael@0:     if results is None or not results.all_passed():
michael@0:         return 1
michael@0: 
michael@0:     return 0
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     sys.exit(main())