michael@0: #!/usr/bin/env python
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: # run-tests.py -- Python harness for GDB SpiderMonkey support
michael@0: 
michael@0: import os, re, subprocess, sys, traceback
michael@0: from threading import Thread
michael@0: 
michael@0: # From this directory:
michael@0: import progressbar
michael@0: from taskpool import TaskPool, get_cpu_count
michael@0: 
michael@0: # Backported from Python 3.1 posixpath.py
michael@0: def _relpath(path, start=None):
michael@0:     """Return a relative version of a path"""
michael@0: 
michael@0:     if not path:
michael@0:         raise ValueError("no path specified")
michael@0: 
michael@0:     if start is None:
michael@0:         start = os.curdir
michael@0: 
michael@0:     start_list = os.path.abspath(start).split(os.sep)
michael@0:     path_list = os.path.abspath(path).split(os.sep)
michael@0: 
michael@0:     # Work out how much of the filepath is shared by start and path.
michael@0:     i = len(os.path.commonprefix([start_list, path_list]))
michael@0: 
michael@0:     rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
michael@0:     if not rel_list:
michael@0:         return os.curdir
michael@0:     return os.path.join(*rel_list)
michael@0: 
michael@0: os.path.relpath = _relpath
michael@0: 
michael@0: # Characters that need to be escaped when used in shell words.
michael@0: shell_need_escapes = re.compile('[^\w\d%+,-./:=@\'"]', re.DOTALL)
michael@0: # Characters that need to be escaped within double-quoted strings.
michael@0: shell_dquote_escapes = re.compile('[^\w\d%+,-./:=@"]', re.DOTALL)
michael@0: def make_shell_cmd(l):
michael@0:     def quote(s):
michael@0:         if shell_need_escapes.search(s):
michael@0:             if s.find("'") < 0:
michael@0:                 return "'" + s + "'"
michael@0:             return '"' + shell_dquote_escapes.sub('\\g<0>', s) + '"'
michael@0:         return s
michael@0: 
michael@0:     return ' '.join([quote(_) for _ in l])
michael@0: 
michael@0: # An instance of this class collects the lists of passing, failing, and
michael@0: # timing-out tests, runs the progress bar, and prints a summary at the end.
michael@0: class Summary(object):
michael@0: 
michael@0:     class SummaryBar(progressbar.ProgressBar):
michael@0:         def __init__(self, limit):
michael@0:             super(Summary.SummaryBar, self).__init__('', limit, 24)
michael@0:         def start(self):
michael@0:             self.label = '[starting           ]'
michael@0:             self.update(0)
michael@0:         def counts(self, run, failures, timeouts):
michael@0:             self.label = '[%4d|%4d|%4d|%4d]' % (run - failures, failures, timeouts, run)
michael@0:             self.update(run)
michael@0: 
michael@0:     def __init__(self, num_tests):
michael@0:         self.run = 0
michael@0:         self.failures = []              # kind of judgemental; "unexpecteds"?
michael@0:         self.timeouts = []
michael@0:         if not OPTIONS.hide_progress:
michael@0:             self.bar = Summary.SummaryBar(num_tests)
michael@0: 
michael@0:     # Progress bar control.
michael@0:     def start(self):
michael@0:         if not OPTIONS.hide_progress:
michael@0:             self.bar.start()
michael@0:     def update(self):
michael@0:         if not OPTIONS.hide_progress:
michael@0:             self.bar.counts(self.run, len(self.failures), len(self.timeouts))
michael@0:     # Call 'thunk' to show some output, while getting the progress bar out of the way.
michael@0:     def interleave_output(self, thunk):
michael@0:         if not OPTIONS.hide_progress:
michael@0:             self.bar.clear()
michael@0:         thunk()
michael@0:         self.update()
michael@0: 
michael@0:     def passed(self, test):
michael@0:         self.run += 1
michael@0:         self.update()
michael@0: 
michael@0:     def failed(self, test):
michael@0:         self.run += 1
michael@0:         self.failures.append(test)
michael@0:         self.update()
michael@0: 
michael@0:     def timeout(self, test):
michael@0:         self.run += 1
michael@0:         self.timeouts.append(test)
michael@0:         self.update()
michael@0: 
michael@0:     def finish(self):
michael@0:         if not OPTIONS.hide_progress:
michael@0:             self.bar.finish()
michael@0: 
michael@0:         if self.failures:
michael@0: 
michael@0:             print "tests failed:"
michael@0:             for test in self.failures:
michael@0:                 test.show(sys.stdout)
michael@0: 
michael@0:             if OPTIONS.worklist:
michael@0:                 try:
michael@0:                     with open(OPTIONS.worklist) as out:
michael@0:                         for test in self.failures:
michael@0:                             out.write(test.name + '\n')
michael@0:                 except IOError as err:
michael@0:                     sys.stderr.write("Error writing worklist file '%s': %s"
michael@0:                                      % (OPTIONS.worklist, err))
michael@0:                     sys.exit(1)
michael@0: 
michael@0:             if OPTIONS.write_failures:
michael@0:                 try:
michael@0:                     with open(OPTIONS.write_failures) as out:
michael@0:                         for test in self.failures:
michael@0:                             test.show(out)
michael@0:                 except IOError as err:
michael@0:                     sys.stderr.write("Error writing worklist file '%s': %s"
michael@0:                                      % (OPTIONS.write_failures, err))
michael@0:                     sys.exit(1)
michael@0: 
michael@0:         if self.timeouts:
michael@0:             print "tests timed out:"
michael@0:             for test in self.timeouts:
michael@0:                 test.show(sys.stdout)
michael@0: 
michael@0:         if self.failures or self.timeouts:
michael@0:             sys.exit(2)
michael@0: 
michael@0: class Test(TaskPool.Task):
michael@0:     def __init__(self, path, summary):
michael@0:         super(Test, self).__init__()
michael@0:         self.test_path = path           # path to .py test file
michael@0:         self.summary = summary
michael@0: 
michael@0:         # test.name is the name of the test relative to the top of the test
michael@0:         # directory. This is what we use to report failures and timeouts,
michael@0:         # and when writing test lists.
michael@0:         self.name = os.path.relpath(self.test_path, OPTIONS.testdir)
michael@0: 
michael@0:         self.stdout = ''
michael@0:         self.stderr = ''
michael@0:         self.returncode = None
michael@0: 
michael@0:     def cmd(self):
michael@0:         testlibdir = os.path.normpath(os.path.join(OPTIONS.testdir, '..', 'lib-for-tests'))
michael@0:         return [OPTIONS.gdb_executable,
michael@0:                 '-nw',          # Don't create a window (unnecessary?)
michael@0:                 '-nx',          # Don't read .gdbinit.
michael@0:                 '--ex', 'add-auto-load-safe-path %s' % (OPTIONS.builddir,),
michael@0:                 '--ex', 'set env LD_LIBRARY_PATH %s' % (OPTIONS.libdir,),
michael@0:                 '--ex', 'file %s' % (os.path.join(OPTIONS.builddir, 'gdb-tests'),),
michael@0:                 '--eval-command', 'python testlibdir=%r' % (testlibdir,),
michael@0:                 '--eval-command', 'python testscript=%r' % (self.test_path,),
michael@0:                 '--eval-command', 'python execfile(%r)' % os.path.join(testlibdir, 'catcher.py')]
michael@0: 
michael@0:     def start(self, pipe, deadline):
michael@0:         super(Test, self).start(pipe, deadline)
michael@0:         if OPTIONS.show_cmd:
michael@0:             self.summary.interleave_output(lambda: self.show_cmd(sys.stdout))
michael@0: 
michael@0:     def onStdout(self, text):
michael@0:         self.stdout += text
michael@0: 
michael@0:     def onStderr(self, text):
michael@0:         self.stderr += text
michael@0: 
michael@0:     def onFinished(self, returncode):
michael@0:         self.returncode = returncode
michael@0:         if OPTIONS.show_output:
michael@0:             self.summary.interleave_output(lambda: self.show_output(sys.stdout))
michael@0:         if returncode != 0:
michael@0:             self.summary.failed(self)
michael@0:         else:
michael@0:             self.summary.passed(self)
michael@0: 
michael@0:     def onTimeout(self):
michael@0:         self.summary.timeout(self)
michael@0: 
michael@0:     def show_cmd(self, out):
michael@0:         print "Command: ", make_shell_cmd(self.cmd())
michael@0: 
michael@0:     def show_output(self, out):
michael@0:         if self.stdout:
michael@0:             out.write('Standard output:')
michael@0:             out.write('\n' + self.stdout + '\n')
michael@0:         if self.stderr:
michael@0:             out.write('Standard error:')
michael@0:             out.write('\n' + self.stderr + '\n')
michael@0: 
michael@0:     def show(self, out):
michael@0:         out.write(self.name + '\n')
michael@0:         if OPTIONS.write_failure_output:
michael@0:             out.write('Command: %s\n' % (make_shell_cmd(self.cmd()),))
michael@0:             self.show_output(out)
michael@0:             out.write('GDB exit code: %r\n' % (self.returncode,))
michael@0: 
michael@0: def find_tests(dir, substring = None):
michael@0:     ans = []
michael@0:     for dirpath, dirnames, filenames in os.walk(dir):
michael@0:         if dirpath == '.':
michael@0:             continue
michael@0:         for filename in filenames:
michael@0:             if not filename.endswith('.py'):
michael@0:                 continue
michael@0:             test = os.path.join(dirpath, filename)
michael@0:             if substring is None or substring in os.path.relpath(test, dir):
michael@0:                 ans.append(test)
michael@0:     return ans
michael@0: 
michael@0: def build_test_exec(builddir):
michael@0:     p = subprocess.check_call(['make', 'gdb-tests'], cwd=builddir)
michael@0: 
michael@0: def run_tests(tests, summary):
michael@0:     pool = TaskPool(tests, job_limit=OPTIONS.workercount, timeout=OPTIONS.timeout)
michael@0:     pool.run_all()
michael@0: 
michael@0: OPTIONS = None
michael@0: def main(argv):
michael@0:     global OPTIONS
michael@0:     script_path = os.path.abspath(__file__)
michael@0:     script_dir = os.path.dirname(script_path)
michael@0: 
michael@0:     # LIBDIR is the directory in which we find the SpiderMonkey shared
michael@0:     # library, to link against.
michael@0:     #
michael@0:     # The [TESTS] optional arguments are paths of test files relative
michael@0:     # to the jit-test/tests directory.
michael@0:     from optparse import OptionParser
michael@0:     op = OptionParser(usage='%prog [options] LIBDIR [TESTS...]')
michael@0:     op.add_option('-s', '--show-cmd', dest='show_cmd', action='store_true',
michael@0:                   help='show GDB shell command run')
michael@0:     op.add_option('-o', '--show-output', dest='show_output', action='store_true',
michael@0:                   help='show output from GDB')
michael@0:     op.add_option('-x', '--exclude', dest='exclude', action='append',
michael@0:                   help='exclude given test dir or path')
michael@0:     op.add_option('-t', '--timeout', dest='timeout',  type=float, default=150.0,
michael@0:                   help='set test timeout in seconds')
michael@0:     op.add_option('-j', '--worker-count', dest='workercount', type=int,
michael@0:                   help='Run [WORKERCOUNT] tests at a time')
michael@0:     op.add_option('--no-progress', dest='hide_progress', action='store_true',
michael@0:                   help='hide progress bar')
michael@0:     op.add_option('--worklist', dest='worklist', metavar='FILE',
michael@0:                   help='Read tests to run from [FILE] (or run all if [FILE] not found);\n'
michael@0:                        'write failures back to [FILE]')
michael@0:     op.add_option('-r', '--read-tests', dest='read_tests', metavar='FILE',
michael@0:                   help='Run test files listed in [FILE]')
michael@0:     op.add_option('-w', '--write-failures', dest='write_failures', metavar='FILE',
michael@0:                   help='Write failing tests to [FILE]')
michael@0:     op.add_option('--write-failure-output', dest='write_failure_output', action='store_true',
michael@0:                   help='With --write-failures=FILE, additionally write the output of failed tests to [FILE]')
michael@0:     op.add_option('--gdb', dest='gdb_executable', metavar='EXECUTABLE', default='gdb',
michael@0:                   help='Run tests with [EXECUTABLE], rather than plain \'gdb\'.')
michael@0:     op.add_option('--srcdir', dest='srcdir',
michael@0:                   default=os.path.abspath(os.path.join(script_dir, '..')),
michael@0:                   help='Use SpiderMonkey sources in [SRCDIR].')
michael@0:     op.add_option('--testdir', dest='testdir', default=os.path.join(script_dir, 'tests'),
michael@0:                   help='Find tests in [TESTDIR].')
michael@0:     op.add_option('--builddir', dest='builddir',
michael@0:                   help='Build test executable in [BUILDDIR].')
michael@0:     (OPTIONS, args) = op.parse_args(argv)
michael@0:     if len(args) < 1:
michael@0:         op.error('missing LIBDIR argument')
michael@0:     OPTIONS.libdir = os.path.abspath(args[0])
michael@0:     test_args = args[1:]
michael@0: 
michael@0:     if not OPTIONS.workercount:
michael@0:         OPTIONS.workercount = get_cpu_count()
michael@0: 
michael@0:     # Compute default for OPTIONS.builddir now, since we've computed OPTIONS.libdir.
michael@0:     if not OPTIONS.builddir:
michael@0:         OPTIONS.builddir = os.path.join(OPTIONS.libdir, 'gdb')
michael@0: 
michael@0:     test_set = set()
michael@0: 
michael@0:     # All the various sources of test names accumulate.
michael@0:     if test_args:
michael@0:         for arg in test_args:
michael@0:             test_set.update(find_tests(OPTIONS.testdir, arg))
michael@0:     if OPTIONS.worklist:
michael@0:         try:
michael@0:             with open(OPTIONS.worklist) as f:
michael@0:                 for line in f:
michael@0:                     test_set.update(os.path.join(test_dir, line.strip('\n')))
michael@0:         except IOError:
michael@0:             # With worklist, a missing file means to start the process with
michael@0:             # the complete list of tests.
michael@0:             sys.stderr.write("Couldn't read worklist file '%s'; running all tests\n"
michael@0:                              % (OPTIONS.worklist,))
michael@0:             test_set = set(find_tests(OPTIONS.testdir))
michael@0:     if OPTIONS.read_tests:
michael@0:         try:
michael@0:             with open(OPTIONS.read_tests) as f:
michael@0:                 for line in f:
michael@0:                     test_set.update(os.path.join(test_dir, line.strip('\n')))
michael@0:         except IOError as err:
michael@0:             sys.stderr.write("Error trying to read test file '%s': %s\n"
michael@0:                              % (OPTIONS.read_tests, err))
michael@0:             sys.exit(1)
michael@0: 
michael@0:     # If none of the above options were passed, and no tests were listed
michael@0:     # explicitly, use the complete set.
michael@0:     if not test_args and not OPTIONS.worklist and not OPTIONS.read_tests:
michael@0:         test_set = set(find_tests(OPTIONS.testdir))
michael@0: 
michael@0:     if OPTIONS.exclude:
michael@0:         exclude_set = set()
michael@0:         for exclude in OPTIONS.exclude:
michael@0:             exclude_set.update(find_tests(test_dir, exclude))
michael@0:         test_set -= exclude_set
michael@0: 
michael@0:     if not test_set:
michael@0:         sys.stderr.write("No tests found matching command line arguments.\n")
michael@0:         sys.exit(1)
michael@0: 
michael@0:     summary = Summary(len(test_set))
michael@0:     test_list = [ Test(_, summary) for _ in sorted(test_set) ]
michael@0: 
michael@0:     # Build the test executable from all the .cpp files found in the test
michael@0:     # directory tree.
michael@0:     try:
michael@0:         build_test_exec(OPTIONS.builddir)
michael@0:     except subprocess.CalledProcessError as err:
michael@0:         sys.stderr.write("Error building test executable: %s\n" % (err,))
michael@0:         sys.exit(1)
michael@0: 
michael@0:     # Run the tests.
michael@0:     try:
michael@0:         summary.start()
michael@0:         run_tests(test_list, summary)
michael@0:         summary.finish()
michael@0:     except OSError as err:
michael@0:         sys.stderr.write("Error running tests: %s\n" % (err,))
michael@0:         sys.exit(1)
michael@0: 
michael@0:     sys.exit(0)
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     main(sys.argv[1:])