michael@0: #!/usr/bin/env python
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: 
michael@0: # jit_test.py -- Python harness for JavaScript trace tests.
michael@0: 
michael@0: from __future__ import print_function
michael@0: import os, posixpath, sys, tempfile, traceback, time
michael@0: import subprocess
michael@0: from subprocess import Popen, PIPE
michael@0: from threading import Thread
michael@0: import signal
michael@0: import StringIO
michael@0: 
michael@0: try:
michael@0:     from multiprocessing import Process, Manager, cpu_count
michael@0:     HAVE_MULTIPROCESSING = True
michael@0: except ImportError:
michael@0:     HAVE_MULTIPROCESSING = False
michael@0: 
michael@0: from progressbar import ProgressBar, NullProgressBar
michael@0: from results import TestOutput
michael@0: 
michael@0: TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__))
michael@0: JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR))
michael@0: TOP_SRC_DIR = os.path.dirname(os.path.dirname(JS_DIR))
michael@0: TEST_DIR = os.path.join(JS_DIR, 'jit-test', 'tests')
michael@0: LIB_DIR = os.path.join(JS_DIR, 'jit-test', 'lib') + os.path.sep
michael@0: JS_CACHE_DIR = os.path.join(JS_DIR, 'jit-test', '.js-cache')
michael@0: JS_TESTS_DIR = posixpath.join(JS_DIR, 'tests')
michael@0: 
michael@0: # Backported from Python 3.1 posixpath.py
michael@0: def _relpath(path, start=None):
michael@0:     """Return a relative version of a path"""
michael@0: 
michael@0:     if not path:
michael@0:         raise ValueError("no path specified")
michael@0: 
michael@0:     if start is None:
michael@0:         start = os.curdir
michael@0: 
michael@0:     start_list = os.path.abspath(start).split(os.sep)
michael@0:     path_list = os.path.abspath(path).split(os.sep)
michael@0: 
michael@0:     # Work out how much of the filepath is shared by start and path.
michael@0:     i = len(os.path.commonprefix([start_list, path_list]))
michael@0: 
michael@0:     rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
michael@0:     if not rel_list:
michael@0:         return os.curdir
michael@0:     return os.path.join(*rel_list)
michael@0: 
michael@0: os.path.relpath = _relpath
michael@0: 
michael@0: class Test:
michael@0: 
michael@0:     VALGRIND_CMD = []
michael@0:     paths = (d for d in os.environ['PATH'].split(os.pathsep))
michael@0:     valgrinds = (os.path.join(d, 'valgrind') for d in paths)
michael@0:     if any(os.path.exists(p) for p in valgrinds):
michael@0:         VALGRIND_CMD = [
michael@0:             'valgrind', '-q', '--smc-check=all-non-file',
michael@0:             '--error-exitcode=1', '--gen-suppressions=all',
michael@0:             '--show-possibly-lost=no', '--leak-check=full',
michael@0:         ]
michael@0:         if os.uname()[0] == 'Darwin':
michael@0:             VALGRIND_CMD.append('--dsymutil=yes')
michael@0: 
michael@0:     del paths
michael@0:     del valgrinds
michael@0: 
michael@0:     def __init__(self, path):
michael@0:         # Absolute path of the test file.
michael@0:         self.path = path
michael@0: 
michael@0:         # Path relative to the top mozilla/ directory.
michael@0:         self.relpath_top = os.path.relpath(path, TOP_SRC_DIR)
michael@0: 
michael@0:         # Path relative to mozilla/js/src/jit-test/tests/.
michael@0:         self.relpath_tests = os.path.relpath(path, TEST_DIR)
michael@0: 
michael@0:         self.jitflags = []     # jit flags to enable
michael@0:         self.slow = False      # True means the test is slow-running
michael@0:         self.allow_oom = False # True means that OOM is not considered a failure
michael@0:         self.valgrind = False  # True means run under valgrind
michael@0:         self.tz_pacific = False # True means force Pacific time for the test
michael@0:         self.expect_error = '' # Errors to expect and consider passing
michael@0:         self.expect_status = 0 # Exit status to expect from shell
michael@0: 
michael@0:     def copy(self):
michael@0:         t = Test(self.path)
michael@0:         t.jitflags = self.jitflags[:]
michael@0:         t.slow = self.slow
michael@0:         t.allow_oom = self.allow_oom
michael@0:         t.valgrind = self.valgrind
michael@0:         t.tz_pacific = self.tz_pacific
michael@0:         t.expect_error = self.expect_error
michael@0:         t.expect_status = self.expect_status
michael@0:         return t
michael@0: 
michael@0:     COOKIE = '|jit-test|'
michael@0:     CacheDir = JS_CACHE_DIR
michael@0: 
michael@0:     @classmethod
michael@0:     def from_file(cls, path, options):
michael@0:         test = cls(path)
michael@0: 
michael@0:         line = open(path).readline()
michael@0:         i = line.find(cls.COOKIE)
michael@0:         if i != -1:
michael@0:             meta = line[i + len(cls.COOKIE):].strip('\n')
michael@0:             parts = meta.split(';')
michael@0:             for part in parts:
michael@0:                 part = part.strip()
michael@0:                 if not part:
michael@0:                     continue
michael@0:                 name, _, value = part.partition(':')
michael@0:                 if value:
michael@0:                     value = value.strip()
michael@0:                     if name == 'error':
michael@0:                         test.expect_error = value
michael@0:                     elif name == 'exitstatus':
michael@0:                         try:
michael@0:                             test.expect_status = int(value, 0);
michael@0:                         except ValueError:
michael@0:                             print("warning: couldn't parse exit status %s" % value)
michael@0:                     elif name == 'thread-count':
michael@0:                         try:
michael@0:                             test.jitflags.append('--thread-count=' + int(value, 0));
michael@0:                         except ValueError:
michael@0:                             print("warning: couldn't parse thread-count %s" % value)
michael@0:                     else:
michael@0:                         print('warning: unrecognized |jit-test| attribute %s' % part)
michael@0:                 else:
michael@0:                     if name == 'slow':
michael@0:                         test.slow = True
michael@0:                     elif name == 'allow-oom':
michael@0:                         test.allow_oom = True
michael@0:                     elif name == 'valgrind':
michael@0:                         test.valgrind = options.valgrind
michael@0:                     elif name == 'tz-pacific':
michael@0:                         test.tz_pacific = True
michael@0:                     elif name == 'debug':
michael@0:                         test.jitflags.append('--debugjit')
michael@0:                     elif name == 'ion-eager':
michael@0:                         test.jitflags.append('--ion-eager')
michael@0:                     elif name == 'no-ion':
michael@0:                         test.jitflags.append('--no-ion')
michael@0:                     elif name == 'dump-bytecode':
michael@0:                         test.jitflags.append('--dump-bytecode')
michael@0:                     else:
michael@0:                         print('warning: unrecognized |jit-test| attribute %s' % part)
michael@0: 
michael@0:         if options.valgrind_all:
michael@0:             test.valgrind = True
michael@0: 
michael@0:         return test
michael@0: 
michael@0:     def command(self, prefix, libdir, remote_prefix=None):
michael@0:         path = self.path
michael@0:         if remote_prefix:
michael@0:             path = self.path.replace(TEST_DIR, remote_prefix)
michael@0: 
michael@0:         scriptdir_var = os.path.dirname(path);
michael@0:         if not scriptdir_var.endswith('/'):
michael@0:             scriptdir_var += '/'
michael@0: 
michael@0:         # Platforms where subprocess immediately invokes exec do not care
michael@0:         # whether we use double or single quotes. On windows and when using
michael@0:         # a remote device, however, we have to be careful to use the quote
michael@0:         # style that is the opposite of what the exec wrapper uses.
michael@0:         # This uses %r to get single quotes on windows and special cases
michael@0:         # the remote device.
michael@0:         fmt = 'const platform=%r; const libdir=%r; const scriptdir=%r'
michael@0:         if remote_prefix:
michael@0:             fmt = 'const platform="%s"; const libdir="%s"; const scriptdir="%s"'
michael@0:         expr = fmt % (sys.platform, libdir, scriptdir_var)
michael@0: 
michael@0:         # We may have specified '-a' or '-d' twice: once via --jitflags, once
michael@0:         # via the "|jit-test|" line.  Remove dups because they are toggles.
michael@0:         cmd = prefix + ['--js-cache', Test.CacheDir]
michael@0:         cmd += list(set(self.jitflags)) + ['-e', expr, '-f', path]
michael@0:         if self.valgrind:
michael@0:             cmd = self.VALGRIND_CMD + cmd
michael@0:         return cmd
michael@0: 
michael@0: def find_tests(substring=None):
michael@0:     ans = []
michael@0:     for dirpath, dirnames, filenames in os.walk(TEST_DIR):
michael@0:         dirnames.sort()
michael@0:         filenames.sort()
michael@0:         if dirpath == '.':
michael@0:             continue
michael@0:         for filename in filenames:
michael@0:             if not filename.endswith('.js'):
michael@0:                 continue
michael@0:             if filename in ('shell.js', 'browser.js', 'jsref.js'):
michael@0:                 continue
michael@0:             test = os.path.join(dirpath, filename)
michael@0:             if substring is None or substring in os.path.relpath(test, TEST_DIR):
michael@0:                 ans.append(test)
michael@0:     return ans
michael@0: 
michael@0: def tmppath(token):
michael@0:     fd, path = tempfile.mkstemp(prefix=token)
michael@0:     os.close(fd)
michael@0:     return path
michael@0: 
michael@0: def read_and_unlink(path):
michael@0:     f = open(path)
michael@0:     d = f.read()
michael@0:     f.close()
michael@0:     os.unlink(path)
michael@0:     return d
michael@0: 
michael@0: def th_run_cmd(cmdline, options, l):
michael@0:     # close_fds is not supported on Windows and will cause a ValueError.
michael@0:     if sys.platform != 'win32':
michael@0:         options["close_fds"] = True
michael@0:     p = Popen(cmdline, stdin=PIPE, stdout=PIPE, stderr=PIPE, **options)
michael@0: 
michael@0:     l[0] = p
michael@0:     out, err = p.communicate()
michael@0:     l[1] = (out, err, p.returncode)
michael@0: 
michael@0: def run_timeout_cmd(cmdline, options, timeout=60.0):
michael@0:     l = [ None, None ]
michael@0:     timed_out = False
michael@0:     th = Thread(target=th_run_cmd, args=(cmdline, options, l))
michael@0: 
michael@0:     # If our SIGINT handler is set to SIG_IGN (ignore)
michael@0:     # then we are running as a child process for parallel
michael@0:     # execution and we must ensure to kill our child
michael@0:     # when we are signaled to exit.
michael@0:     sigint_handler = signal.getsignal(signal.SIGINT)
michael@0:     sigterm_handler = signal.getsignal(signal.SIGTERM)
michael@0:     if (sigint_handler == signal.SIG_IGN):
michael@0:         def handleChildSignal(sig, frame):
michael@0:             try:
michael@0:                 if sys.platform != 'win32':
michael@0:                     os.kill(l[0].pid, signal.SIGKILL)
michael@0:                 else:
michael@0:                     import ctypes
michael@0:                     ctypes.windll.kernel32.TerminateProcess(int(l[0]._handle), -1)
michael@0:             except OSError:
michael@0:                 pass
michael@0:             if (sig == signal.SIGTERM):
michael@0:                 sys.exit(0)
michael@0:         signal.signal(signal.SIGINT, handleChildSignal)
michael@0:         signal.signal(signal.SIGTERM, handleChildSignal)
michael@0: 
michael@0:     th.start()
michael@0:     th.join(timeout)
michael@0:     while th.isAlive():
michael@0:         if l[0] is not None:
michael@0:             try:
michael@0:                 # In Python 3, we could just do l[0].kill().
michael@0:                 if sys.platform != 'win32':
michael@0:                     os.kill(l[0].pid, signal.SIGKILL)
michael@0:                 else:
michael@0:                     import ctypes
michael@0:                     ctypes.windll.kernel32.TerminateProcess(int(l[0]._handle), -1)
michael@0:                 time.sleep(.1)
michael@0:                 timed_out = True
michael@0:             except OSError:
michael@0:                 # Expecting a "No such process" error
michael@0:                 pass
michael@0:     th.join()
michael@0: 
michael@0:     # Restore old signal handlers
michael@0:     if (sigint_handler == signal.SIG_IGN):
michael@0:         signal.signal(signal.SIGINT, signal.SIG_IGN)
michael@0:         signal.signal(signal.SIGTERM, sigterm_handler)
michael@0: 
michael@0:     (out, err, code) = l[1]
michael@0: 
michael@0:     return (out, err, code, timed_out)
michael@0: 
michael@0: def run_cmd(cmdline, env, timeout):
michael@0:     return run_timeout_cmd(cmdline, { 'env': env }, timeout)
michael@0: 
michael@0: def run_cmd_avoid_stdio(cmdline, env, timeout):
michael@0:     stdoutPath, stderrPath = tmppath('jsstdout'), tmppath('jsstderr')
michael@0:     env['JS_STDOUT'] = stdoutPath
michael@0:     env['JS_STDERR'] = stderrPath
michael@0:     _, __, code = run_timeout_cmd(cmdline, { 'env': env }, timeout)
michael@0:     return read_and_unlink(stdoutPath), read_and_unlink(stderrPath), code
michael@0: 
michael@0: def run_test(test, prefix, options):
michael@0:     cmd = test.command(prefix, LIB_DIR)
michael@0:     if options.show_cmd:
michael@0:         print(subprocess.list2cmdline(cmd))
michael@0: 
michael@0:     if options.avoid_stdio:
michael@0:         run = run_cmd_avoid_stdio
michael@0:     else:
michael@0:         run = run_cmd
michael@0: 
michael@0:     env = os.environ.copy()
michael@0:     if test.tz_pacific:
michael@0:         env['TZ'] = 'PST8PDT'
michael@0: 
michael@0:     # Ensure interpreter directory is in shared library path.
michael@0:     pathvar = ''
michael@0:     if sys.platform.startswith('linux'):
michael@0:         pathvar = 'LD_LIBRARY_PATH'
michael@0:     elif sys.platform.startswith('darwin'):
michael@0:         pathvar = 'DYLD_LIBRARY_PATH'
michael@0:     elif sys.platform.startswith('win'):
michael@0:         pathvar = 'PATH'
michael@0:     if pathvar:
michael@0:         bin_dir = os.path.dirname(cmd[0])
michael@0:         if pathvar in env:
michael@0:             env[pathvar] = '%s%s%s' % (bin_dir, os.pathsep, env[pathvar])
michael@0:         else:
michael@0:             env[pathvar] = bin_dir
michael@0: 
michael@0:     out, err, code, timed_out = run(cmd, env, options.timeout)
michael@0:     return TestOutput(test, cmd, out, err, code, None, timed_out)
michael@0: 
michael@0: def run_test_remote(test, device, prefix, options):
michael@0:     cmd = test.command(prefix, posixpath.join(options.remote_test_root, 'lib/'), posixpath.join(options.remote_test_root, 'tests'))
michael@0:     if options.show_cmd:
michael@0:         print(subprocess.list2cmdline(cmd))
michael@0: 
michael@0:     env = {}
michael@0:     if test.tz_pacific:
michael@0:         env['TZ'] = 'PST8PDT'
michael@0: 
michael@0:     env['LD_LIBRARY_PATH'] = options.remote_test_root
michael@0: 
michael@0:     buf = StringIO.StringIO()
michael@0:     returncode = device.shell(cmd, buf, env=env, cwd=options.remote_test_root,
michael@0:                               timeout=int(options.timeout))
michael@0: 
michael@0:     out = buf.getvalue()
michael@0:     # We can't distinguish between stdout and stderr so we pass
michael@0:     # the same buffer to both.
michael@0:     return TestOutput(test, cmd, out, out, returncode, None, False)
michael@0: 
michael@0: def check_output(out, err, rc, timed_out, test):
michael@0:     if timed_out:
michael@0:         # The shell sometimes hangs on shutdown on Windows 7 and Windows
michael@0:         # Server 2008. See bug 970063 comment 7 for a description of the
michael@0:         # problem. Until bug 956899 is fixed, ignore timeouts on these
michael@0:         # platforms (versions 6.0 and 6.1).
michael@0:         if sys.platform == 'win32':
michael@0:             ver = sys.getwindowsversion()
michael@0:             if ver.major == 6 and ver.minor <= 1:
michael@0:                 return True
michael@0:         return False
michael@0: 
michael@0:     if test.expect_error:
michael@0:         # The shell exits with code 3 on uncaught exceptions.
michael@0:         # Sometimes 0 is returned on Windows for unknown reasons.
michael@0:         # See bug 899697.
michael@0:         if sys.platform in ['win32', 'cygwin']:
michael@0:             if rc != 3 and rc != 0:
michael@0:                 return False
michael@0:         else:
michael@0:             if rc != 3:
michael@0:                 return False
michael@0: 
michael@0:         return test.expect_error in err
michael@0: 
michael@0:     for line in out.split('\n'):
michael@0:         if line.startswith('Trace stats check failed'):
michael@0:             return False
michael@0: 
michael@0:     for line in err.split('\n'):
michael@0:         if 'Assertion failed:' in line:
michael@0:             return False
michael@0: 
michael@0:     if rc != test.expect_status:
michael@0:         # Tests which expect a timeout check for exit code 6.
michael@0:         # Sometimes 0 is returned on Windows for unknown reasons.
michael@0:         # See bug 899697.
michael@0:         if sys.platform in ['win32', 'cygwin'] and rc == 0:
michael@0:             return True
michael@0: 
michael@0:         # Allow a non-zero exit code if we want to allow OOM, but only if we
michael@0:         # actually got OOM.
michael@0:         return test.allow_oom and 'out of memory' in err and 'Assertion failure' not in err
michael@0: 
michael@0:     return True
michael@0: 
michael@0: def print_tinderbox(ok, res):
michael@0:     # Output test failures in a TBPL parsable format, eg:
michael@0:     # TEST-RESULT | filename.js | Failure description (code N, args "--foobar")
michael@0:     #
michael@0:     # Example:
michael@0:     # TEST-PASS | foo/bar/baz.js | (code 0, args "--ion-eager")
michael@0:     # TEST-UNEXPECTED-FAIL | foo/bar/baz.js | TypeError: or something (code -9, args "--no-ion")
michael@0:     # INFO exit-status     : 3
michael@0:     # INFO timed-out       : False
michael@0:     # INFO stdout          > foo
michael@0:     # INFO stdout          > bar
michael@0:     # INFO stdout          > baz
michael@0:     # INFO stderr         2> TypeError: or something
michael@0:     # TEST-UNEXPECTED-FAIL | jit_test.py: Test execution interrupted by user
michael@0:     result = "TEST-PASS" if ok else "TEST-UNEXPECTED-FAIL"
michael@0:     message = "Success" if ok else res.describe_failure()
michael@0:     jitflags = " ".join(res.test.jitflags)
michael@0:     print("{} | {} | {} (code {}, args \"{}\")".format(
michael@0:           result, res.test.relpath_top, message, res.rc, jitflags))
michael@0: 
michael@0:     # For failed tests, print as much information as we have, to aid debugging.
michael@0:     if ok:
michael@0:         return
michael@0:     print("INFO exit-status     : {}".format(res.rc))
michael@0:     print("INFO timed-out       : {}".format(res.timed_out))
michael@0:     for line in res.out.splitlines():
michael@0:         print("INFO stdout          > " + line.strip())
michael@0:     for line in res.err.splitlines():
michael@0:         print("INFO stderr         2> " + line.strip())
michael@0: 
michael@0: def wrap_parallel_run_test(test, prefix, resultQueue, options):
michael@0:     # Ignore SIGINT in the child
michael@0:     signal.signal(signal.SIGINT, signal.SIG_IGN)
michael@0:     result = run_test(test, prefix, options)
michael@0:     resultQueue.put(result)
michael@0:     return result
michael@0: 
michael@0: def run_tests_parallel(tests, prefix, options):
michael@0:     # This queue will contain the results of the various tests run.
michael@0:     # We could make this queue a global variable instead of using
michael@0:     # a manager to share, but this will not work on Windows.
michael@0:     queue_manager = Manager()
michael@0:     async_test_result_queue = queue_manager.Queue()
michael@0: 
michael@0:     # This queue will be used by the result process to indicate
michael@0:     # that it has received a result and we can start a new process
michael@0:     # on our end. The advantage is that we don't have to sleep and
michael@0:     # check for worker completion ourselves regularly.
michael@0:     notify_queue = queue_manager.Queue()
michael@0: 
michael@0:     # This queue will contain the return value of the function
michael@0:     # processing the test results.
michael@0:     total_tests = len(tests) * options.repeat
michael@0:     result_process_return_queue = queue_manager.Queue()
michael@0:     result_process = Process(target=process_test_results_parallel,
michael@0:                              args=(async_test_result_queue, result_process_return_queue,
michael@0:                                    notify_queue, total_tests, options))
michael@0:     result_process.start()
michael@0: 
michael@0:     # Ensure that a SIGTERM is handled the same way as SIGINT
michael@0:     # to terminate all child processes.
michael@0:     sigint_handler = signal.getsignal(signal.SIGINT)
michael@0:     signal.signal(signal.SIGTERM, sigint_handler)
michael@0: 
michael@0:     worker_processes = []
michael@0: 
michael@0:     def remove_completed_workers(workers):
michael@0:         new_workers = []
michael@0:         for worker in workers:
michael@0:             if worker.is_alive():
michael@0:                 new_workers.append(worker)
michael@0:             else:
michael@0:                 worker.join()
michael@0:         return new_workers
michael@0: 
michael@0:     try:
michael@0:         testcnt = 0
michael@0:         # Initially start as many jobs as allowed to run parallel
michael@0:         for i in range(min(options.max_jobs,total_tests)):
michael@0:             notify_queue.put(True)
michael@0: 
michael@0:         # For every item in the notify queue, start one new worker.
michael@0:         # Every completed worker adds a new item to this queue.
michael@0:         while notify_queue.get():
michael@0:             if (testcnt < total_tests):
michael@0:                 # Start one new worker
michael@0:                 test = tests[testcnt % len(tests)]
michael@0:                 worker_process = Process(target=wrap_parallel_run_test, args=(test, prefix, async_test_result_queue, options))
michael@0:                 worker_processes.append(worker_process)
michael@0:                 worker_process.start()
michael@0:                 testcnt += 1
michael@0: 
michael@0:                 # Collect completed workers
michael@0:                 worker_processes = remove_completed_workers(worker_processes)
michael@0:             else:
michael@0:                 break
michael@0: 
michael@0:         # Wait for all processes to terminate
michael@0:         while len(worker_processes) > 0:
michael@0:             worker_processes = remove_completed_workers(worker_processes)
michael@0: 
michael@0:         # Signal completion to result processor, then wait for it to complete on its own
michael@0:         async_test_result_queue.put(None)
michael@0:         result_process.join()
michael@0: 
michael@0:         # Return what the result process has returned to us
michael@0:         return result_process_return_queue.get()
michael@0:     except (Exception, KeyboardInterrupt) as e:
michael@0:         # Print the exception if it's not an interrupt,
michael@0:         # might point to a bug or other faulty condition
michael@0:         if not isinstance(e,KeyboardInterrupt):
michael@0:             traceback.print_exc()
michael@0: 
michael@0:         for worker in worker_processes:
michael@0:             try:
michael@0:                 worker.terminate()
michael@0:             except:
michael@0:                 pass
michael@0: 
michael@0:         result_process.terminate()
michael@0: 
michael@0:     return False
michael@0: 
michael@0: def get_parallel_results(async_test_result_queue, notify_queue):
michael@0:     while True:
michael@0:         async_test_result = async_test_result_queue.get()
michael@0: 
michael@0:         # Check if we are supposed to terminate
michael@0:         if (async_test_result == None):
michael@0:             return
michael@0: 
michael@0:         # Notify parent that we got a result
michael@0:         notify_queue.put(True)
michael@0: 
michael@0:         yield async_test_result
michael@0: 
michael@0: def process_test_results_parallel(async_test_result_queue, return_queue, notify_queue, num_tests, options):
michael@0:     gen = get_parallel_results(async_test_result_queue, notify_queue)
michael@0:     ok = process_test_results(gen, num_tests, options)
michael@0:     return_queue.put(ok)
michael@0: 
michael@0: def print_test_summary(num_tests, failures, complete, doing, options):
michael@0:     if failures:
michael@0:         if options.write_failures:
michael@0:             try:
michael@0:                 out = open(options.write_failures, 'w')
michael@0:                 # Don't write duplicate entries when we are doing multiple failures per job.
michael@0:                 written = set()
michael@0:                 for res in failures:
michael@0:                     if res.test.path not in written:
michael@0:                         out.write(os.path.relpath(res.test.path, TEST_DIR) + '\n')
michael@0:                         if options.write_failure_output:
michael@0:                             out.write(res.out)
michael@0:                             out.write(res.err)
michael@0:                             out.write('Exit code: ' + str(res.rc) + "\n")
michael@0:                         written.add(res.test.path)
michael@0:                 out.close()
michael@0:             except IOError:
michael@0:                 sys.stderr.write("Exception thrown trying to write failure file '%s'\n"%
michael@0:                                  options.write_failures)
michael@0:                 traceback.print_exc()
michael@0:                 sys.stderr.write('---\n')
michael@0: 
michael@0:         def show_test(res):
michael@0:             if options.show_failed:
michael@0:                 print('    ' + subprocess.list2cmdline(res.cmd))
michael@0:             else:
michael@0:                 print('    ' + ' '.join(res.test.jitflags + [res.test.path]))
michael@0: 
michael@0:         print('FAILURES:')
michael@0:         for res in failures:
michael@0:             if not res.timed_out:
michael@0:                 show_test(res)
michael@0: 
michael@0:         print('TIMEOUTS:')
michael@0:         for res in failures:
michael@0:             if res.timed_out:
michael@0:                 show_test(res)
michael@0:     else:
michael@0:         print('PASSED ALL' + ('' if complete else ' (partial run -- interrupted by user %s)' % doing))
michael@0: 
michael@0:     if options.tinderbox:
michael@0:         num_failures = len(failures) if failures else 0
michael@0:         print('Result summary:')
michael@0:         print('Passed: %d' % (num_tests - num_failures))
michael@0:         print('Failed: %d' % num_failures)
michael@0: 
michael@0:     return not failures
michael@0: 
michael@0: def process_test_results(results, num_tests, options):
michael@0:     pb = NullProgressBar()
michael@0:     if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
michael@0:         fmt = [
michael@0:             {'value': 'PASS',    'color': 'green'},
michael@0:             {'value': 'FAIL',    'color': 'red'},
michael@0:             {'value': 'TIMEOUT', 'color': 'blue'},
michael@0:             {'value': 'SKIP',    'color': 'brightgray'},
michael@0:         ]
michael@0:         pb = ProgressBar(num_tests, fmt)
michael@0: 
michael@0:     failures = []
michael@0:     timeouts = 0
michael@0:     complete = False
michael@0:     doing = 'before starting'
michael@0:     try:
michael@0:         for i, res in enumerate(results):
michael@0:             if options.show_output:
michael@0:                 sys.stdout.write(res.out)
michael@0:                 sys.stdout.write(res.err)
michael@0:                 sys.stdout.write('Exit code: %s\n' % res.rc)
michael@0:             if res.test.valgrind:
michael@0:                 sys.stdout.write(res.err)
michael@0: 
michael@0:             ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test)
michael@0:             doing = 'after %s' % res.test.relpath_tests
michael@0:             if not ok:
michael@0:                 failures.append(res)
michael@0:                 if res.timed_out:
michael@0:                     pb.message("TIMEOUT - %s" % res.test.relpath_tests)
michael@0:                     timeouts += 1
michael@0:                 else:
michael@0:                     pb.message("FAIL - %s" % res.test.relpath_tests)
michael@0: 
michael@0:             if options.tinderbox:
michael@0:                 print_tinderbox(ok, res)
michael@0: 
michael@0:             n = i + 1
michael@0:             pb.update(n, {
michael@0:                 'PASS': n - len(failures),
michael@0:                 'FAIL': len(failures),
michael@0:                 'TIMEOUT': timeouts,
michael@0:                 'SKIP': 0}
michael@0:             )
michael@0:         complete = True
michael@0:     except KeyboardInterrupt:
michael@0:         print("TEST-UNEXPECTED-FAIL | jit_test.py" +
michael@0:               " : Test execution interrupted by user")
michael@0: 
michael@0:     pb.finish(True)
michael@0:     return print_test_summary(num_tests, failures, complete, doing, options)
michael@0: 
michael@0: def get_serial_results(tests, prefix, options):
michael@0:     for i in xrange(0, options.repeat):
michael@0:         for test in tests:
michael@0:             yield run_test(test, prefix, options)
michael@0: 
michael@0: def run_tests(tests, prefix, options):
michael@0:     gen = get_serial_results(tests, prefix, options)
michael@0:     ok = process_test_results(gen, len(tests) * options.repeat, options)
michael@0:     return ok
michael@0: 
michael@0: def get_remote_results(tests, device, prefix, options):
michael@0:     for i in xrange(0, options.repeat):
michael@0:         for test in tests:
michael@0:             yield run_test_remote(test, device, prefix, options)
michael@0: 
michael@0: def push_libs(options, device):
michael@0:     # This saves considerable time in pushing unnecessary libraries
michael@0:     # to the device but needs to be updated if the dependencies change.
michael@0:     required_libs = ['libnss3.so', 'libmozglue.so']
michael@0: 
michael@0:     for file in os.listdir(options.local_lib):
michael@0:         if file in required_libs:
michael@0:             remote_file = posixpath.join(options.remote_test_root, file)
michael@0:             device.pushFile(os.path.join(options.local_lib, file), remote_file)
michael@0: 
michael@0: def push_progs(options, device, progs):
michael@0:     for local_file in progs:
michael@0:         remote_file = posixpath.join(options.remote_test_root, os.path.basename(local_file))
michael@0:         device.pushFile(local_file, remote_file)
michael@0: 
michael@0: def run_tests_remote(tests, prefix, options):
michael@0:     # Setup device with everything needed to run our tests.
michael@0:     from mozdevice import devicemanager, devicemanagerADB, devicemanagerSUT
michael@0: 
michael@0:     if options.device_transport == 'adb':
michael@0:         if options.device_ip:
michael@0:             dm = devicemanagerADB.DeviceManagerADB(options.device_ip, options.device_port, deviceSerial=options.device_serial, packageName=None, deviceRoot=options.remote_test_root)
michael@0:         else:
michael@0:             dm = devicemanagerADB.DeviceManagerADB(deviceSerial=options.device_serial, packageName=None, deviceRoot=options.remote_test_root)
michael@0:     else:
michael@0:         dm = devicemanagerSUT.DeviceManagerSUT(options.device_ip, options.device_port, deviceRoot=options.remote_test_root)
michael@0:         if options.device_ip == None:
michael@0:             print('Error: you must provide a device IP to connect to via the --device option')
michael@0:             sys.exit(1)
michael@0: 
michael@0:     # Update the test root to point to our test directory.
michael@0:     jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests')
michael@0:     options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests')
michael@0: 
michael@0:     # Push js shell and libraries.
michael@0:     if dm.dirExists(jit_tests_dir):
michael@0:         dm.removeDir(jit_tests_dir)
michael@0:     dm.mkDirs(options.remote_test_root)
michael@0:     push_libs(options, dm)
michael@0:     push_progs(options, dm, [prefix[0]])
michael@0:     dm.chmodDir(options.remote_test_root)
michael@0: 
michael@0:     Test.CacheDir = posixpath.join(options.remote_test_root, '.js-cache')
michael@0:     dm.mkDir(Test.CacheDir)
michael@0: 
michael@0:     dm.pushDir(JS_TESTS_DIR, posixpath.join(jit_tests_dir, 'tests'), timeout=600)
michael@0: 
michael@0:     dm.pushDir(os.path.dirname(TEST_DIR), options.remote_test_root, timeout=600)
michael@0:     prefix[0] = os.path.join(options.remote_test_root, 'js')
michael@0: 
michael@0:     # Run all tests.
michael@0:     gen = get_remote_results(tests, dm, prefix, options)
michael@0:     ok = process_test_results(gen, len(tests) * options.repeat, options)
michael@0:     return ok
michael@0: 
michael@0: def parse_jitflags(options):
michael@0:     jitflags = [ [ '-' + flag for flag in flags ]
michael@0:                  for flags in options.jitflags.split(',') ]
michael@0:     for flags in jitflags:
michael@0:         for flag in flags:
michael@0:             if flag not in ('-m', '-a', '-p', '-d', '-n'):
michael@0:                 print('Invalid jit flag: "%s"' % flag)
michael@0:                 sys.exit(1)
michael@0:     return jitflags
michael@0: 
michael@0: def platform_might_be_android():
michael@0:     try:
michael@0:         # The python package for SL4A provides an |android| module.
michael@0:         # If that module is present, we're likely in SL4A-python on
michael@0:         # device.  False positives and negatives are possible,
michael@0:         # however.
michael@0:         import android
michael@0:         return True
michael@0:     except ImportError:
michael@0:         return False
michael@0: 
michael@0: def stdio_might_be_broken():
michael@0:     return platform_might_be_android()
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     print('Use ../jit-test/jit_test.py to run these tests.')