1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/gdb/run-tests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,348 @@
1.4 +#!/usr/bin/env python
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +
1.9 +# run-tests.py -- Python harness for GDB SpiderMonkey support
1.10 +
1.11 +import os, re, subprocess, sys, traceback
1.12 +from threading import Thread
1.13 +
1.14 +# From this directory:
1.15 +import progressbar
1.16 +from taskpool import TaskPool, get_cpu_count
1.17 +
1.18 +# Backported from Python 3.1 posixpath.py
1.19 +def _relpath(path, start=None):
1.20 + """Return a relative version of a path"""
1.21 +
1.22 + if not path:
1.23 + raise ValueError("no path specified")
1.24 +
1.25 + if start is None:
1.26 + start = os.curdir
1.27 +
1.28 + start_list = os.path.abspath(start).split(os.sep)
1.29 + path_list = os.path.abspath(path).split(os.sep)
1.30 +
1.31 + # Work out how much of the filepath is shared by start and path.
1.32 + i = len(os.path.commonprefix([start_list, path_list]))
1.33 +
1.34 + rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
1.35 + if not rel_list:
1.36 + return os.curdir
1.37 + return os.path.join(*rel_list)
1.38 +
1.39 +os.path.relpath = _relpath
1.40 +
1.41 +# Characters that need to be escaped when used in shell words.
1.42 +shell_need_escapes = re.compile('[^\w\d%+,-./:=@\'"]', re.DOTALL)
1.43 +# Characters that need to be escaped within double-quoted strings.
1.44 +shell_dquote_escapes = re.compile('[^\w\d%+,-./:=@"]', re.DOTALL)
1.45 +def make_shell_cmd(l):
1.46 + def quote(s):
1.47 + if shell_need_escapes.search(s):
1.48 + if s.find("'") < 0:
1.49 + return "'" + s + "'"
1.50 + return '"' + shell_dquote_escapes.sub('\\g<0>', s) + '"'
1.51 + return s
1.52 +
1.53 + return ' '.join([quote(_) for _ in l])
1.54 +
1.55 +# An instance of this class collects the lists of passing, failing, and
1.56 +# timing-out tests, runs the progress bar, and prints a summary at the end.
1.57 +class Summary(object):
1.58 +
1.59 + class SummaryBar(progressbar.ProgressBar):
1.60 + def __init__(self, limit):
1.61 + super(Summary.SummaryBar, self).__init__('', limit, 24)
1.62 + def start(self):
1.63 + self.label = '[starting ]'
1.64 + self.update(0)
1.65 + def counts(self, run, failures, timeouts):
1.66 + self.label = '[%4d|%4d|%4d|%4d]' % (run - failures, failures, timeouts, run)
1.67 + self.update(run)
1.68 +
1.69 + def __init__(self, num_tests):
1.70 + self.run = 0
1.71 + self.failures = [] # kind of judgemental; "unexpecteds"?
1.72 + self.timeouts = []
1.73 + if not OPTIONS.hide_progress:
1.74 + self.bar = Summary.SummaryBar(num_tests)
1.75 +
1.76 + # Progress bar control.
1.77 + def start(self):
1.78 + if not OPTIONS.hide_progress:
1.79 + self.bar.start()
1.80 + def update(self):
1.81 + if not OPTIONS.hide_progress:
1.82 + self.bar.counts(self.run, len(self.failures), len(self.timeouts))
1.83 + # Call 'thunk' to show some output, while getting the progress bar out of the way.
1.84 + def interleave_output(self, thunk):
1.85 + if not OPTIONS.hide_progress:
1.86 + self.bar.clear()
1.87 + thunk()
1.88 + self.update()
1.89 +
1.90 + def passed(self, test):
1.91 + self.run += 1
1.92 + self.update()
1.93 +
1.94 + def failed(self, test):
1.95 + self.run += 1
1.96 + self.failures.append(test)
1.97 + self.update()
1.98 +
1.99 + def timeout(self, test):
1.100 + self.run += 1
1.101 + self.timeouts.append(test)
1.102 + self.update()
1.103 +
1.104 + def finish(self):
1.105 + if not OPTIONS.hide_progress:
1.106 + self.bar.finish()
1.107 +
1.108 + if self.failures:
1.109 +
1.110 + print "tests failed:"
1.111 + for test in self.failures:
1.112 + test.show(sys.stdout)
1.113 +
1.114 + if OPTIONS.worklist:
1.115 + try:
1.116 + with open(OPTIONS.worklist) as out:
1.117 + for test in self.failures:
1.118 + out.write(test.name + '\n')
1.119 + except IOError as err:
1.120 + sys.stderr.write("Error writing worklist file '%s': %s"
1.121 + % (OPTIONS.worklist, err))
1.122 + sys.exit(1)
1.123 +
1.124 + if OPTIONS.write_failures:
1.125 + try:
1.126 + with open(OPTIONS.write_failures) as out:
1.127 + for test in self.failures:
1.128 + test.show(out)
1.129 + except IOError as err:
1.130 + sys.stderr.write("Error writing worklist file '%s': %s"
1.131 + % (OPTIONS.write_failures, err))
1.132 + sys.exit(1)
1.133 +
1.134 + if self.timeouts:
1.135 + print "tests timed out:"
1.136 + for test in self.timeouts:
1.137 + test.show(sys.stdout)
1.138 +
1.139 + if self.failures or self.timeouts:
1.140 + sys.exit(2)
1.141 +
1.142 +class Test(TaskPool.Task):
1.143 + def __init__(self, path, summary):
1.144 + super(Test, self).__init__()
1.145 + self.test_path = path # path to .py test file
1.146 + self.summary = summary
1.147 +
1.148 + # test.name is the name of the test relative to the top of the test
1.149 + # directory. This is what we use to report failures and timeouts,
1.150 + # and when writing test lists.
1.151 + self.name = os.path.relpath(self.test_path, OPTIONS.testdir)
1.152 +
1.153 + self.stdout = ''
1.154 + self.stderr = ''
1.155 + self.returncode = None
1.156 +
1.157 + def cmd(self):
1.158 + testlibdir = os.path.normpath(os.path.join(OPTIONS.testdir, '..', 'lib-for-tests'))
1.159 + return [OPTIONS.gdb_executable,
1.160 + '-nw', # Don't create a window (unnecessary?)
1.161 + '-nx', # Don't read .gdbinit.
1.162 + '--ex', 'add-auto-load-safe-path %s' % (OPTIONS.builddir,),
1.163 + '--ex', 'set env LD_LIBRARY_PATH %s' % (OPTIONS.libdir,),
1.164 + '--ex', 'file %s' % (os.path.join(OPTIONS.builddir, 'gdb-tests'),),
1.165 + '--eval-command', 'python testlibdir=%r' % (testlibdir,),
1.166 + '--eval-command', 'python testscript=%r' % (self.test_path,),
1.167 + '--eval-command', 'python execfile(%r)' % os.path.join(testlibdir, 'catcher.py')]
1.168 +
1.169 + def start(self, pipe, deadline):
1.170 + super(Test, self).start(pipe, deadline)
1.171 + if OPTIONS.show_cmd:
1.172 + self.summary.interleave_output(lambda: self.show_cmd(sys.stdout))
1.173 +
1.174 + def onStdout(self, text):
1.175 + self.stdout += text
1.176 +
1.177 + def onStderr(self, text):
1.178 + self.stderr += text
1.179 +
1.180 + def onFinished(self, returncode):
1.181 + self.returncode = returncode
1.182 + if OPTIONS.show_output:
1.183 + self.summary.interleave_output(lambda: self.show_output(sys.stdout))
1.184 + if returncode != 0:
1.185 + self.summary.failed(self)
1.186 + else:
1.187 + self.summary.passed(self)
1.188 +
1.189 + def onTimeout(self):
1.190 + self.summary.timeout(self)
1.191 +
1.192 + def show_cmd(self, out):
1.193 + print "Command: ", make_shell_cmd(self.cmd())
1.194 +
1.195 + def show_output(self, out):
1.196 + if self.stdout:
1.197 + out.write('Standard output:')
1.198 + out.write('\n' + self.stdout + '\n')
1.199 + if self.stderr:
1.200 + out.write('Standard error:')
1.201 + out.write('\n' + self.stderr + '\n')
1.202 +
1.203 + def show(self, out):
1.204 + out.write(self.name + '\n')
1.205 + if OPTIONS.write_failure_output:
1.206 + out.write('Command: %s\n' % (make_shell_cmd(self.cmd()),))
1.207 + self.show_output(out)
1.208 + out.write('GDB exit code: %r\n' % (self.returncode,))
1.209 +
1.210 +def find_tests(dir, substring = None):
1.211 + ans = []
1.212 + for dirpath, dirnames, filenames in os.walk(dir):
1.213 + if dirpath == '.':
1.214 + continue
1.215 + for filename in filenames:
1.216 + if not filename.endswith('.py'):
1.217 + continue
1.218 + test = os.path.join(dirpath, filename)
1.219 + if substring is None or substring in os.path.relpath(test, dir):
1.220 + ans.append(test)
1.221 + return ans
1.222 +
1.223 +def build_test_exec(builddir):
1.224 + p = subprocess.check_call(['make', 'gdb-tests'], cwd=builddir)
1.225 +
1.226 +def run_tests(tests, summary):
1.227 + pool = TaskPool(tests, job_limit=OPTIONS.workercount, timeout=OPTIONS.timeout)
1.228 + pool.run_all()
1.229 +
1.230 +OPTIONS = None
1.231 +def main(argv):
1.232 + global OPTIONS
1.233 + script_path = os.path.abspath(__file__)
1.234 + script_dir = os.path.dirname(script_path)
1.235 +
1.236 + # LIBDIR is the directory in which we find the SpiderMonkey shared
1.237 + # library, to link against.
1.238 + #
1.239 + # The [TESTS] optional arguments are paths of test files relative
1.240 + # to the jit-test/tests directory.
1.241 + from optparse import OptionParser
1.242 + op = OptionParser(usage='%prog [options] LIBDIR [TESTS...]')
1.243 + op.add_option('-s', '--show-cmd', dest='show_cmd', action='store_true',
1.244 + help='show GDB shell command run')
1.245 + op.add_option('-o', '--show-output', dest='show_output', action='store_true',
1.246 + help='show output from GDB')
1.247 + op.add_option('-x', '--exclude', dest='exclude', action='append',
1.248 + help='exclude given test dir or path')
1.249 + op.add_option('-t', '--timeout', dest='timeout', type=float, default=150.0,
1.250 + help='set test timeout in seconds')
1.251 + op.add_option('-j', '--worker-count', dest='workercount', type=int,
1.252 + help='Run [WORKERCOUNT] tests at a time')
1.253 + op.add_option('--no-progress', dest='hide_progress', action='store_true',
1.254 + help='hide progress bar')
1.255 + op.add_option('--worklist', dest='worklist', metavar='FILE',
1.256 + help='Read tests to run from [FILE] (or run all if [FILE] not found);\n'
1.257 + 'write failures back to [FILE]')
1.258 + op.add_option('-r', '--read-tests', dest='read_tests', metavar='FILE',
1.259 + help='Run test files listed in [FILE]')
1.260 + op.add_option('-w', '--write-failures', dest='write_failures', metavar='FILE',
1.261 + help='Write failing tests to [FILE]')
1.262 + op.add_option('--write-failure-output', dest='write_failure_output', action='store_true',
1.263 + help='With --write-failures=FILE, additionally write the output of failed tests to [FILE]')
1.264 + op.add_option('--gdb', dest='gdb_executable', metavar='EXECUTABLE', default='gdb',
1.265 + help='Run tests with [EXECUTABLE], rather than plain \'gdb\'.')
1.266 + op.add_option('--srcdir', dest='srcdir',
1.267 + default=os.path.abspath(os.path.join(script_dir, '..')),
1.268 + help='Use SpiderMonkey sources in [SRCDIR].')
1.269 + op.add_option('--testdir', dest='testdir', default=os.path.join(script_dir, 'tests'),
1.270 + help='Find tests in [TESTDIR].')
1.271 + op.add_option('--builddir', dest='builddir',
1.272 + help='Build test executable in [BUILDDIR].')
1.273 + (OPTIONS, args) = op.parse_args(argv)
1.274 + if len(args) < 1:
1.275 + op.error('missing LIBDIR argument')
1.276 + OPTIONS.libdir = os.path.abspath(args[0])
1.277 + test_args = args[1:]
1.278 +
1.279 + if not OPTIONS.workercount:
1.280 + OPTIONS.workercount = get_cpu_count()
1.281 +
1.282 + # Compute default for OPTIONS.builddir now, since we've computed OPTIONS.libdir.
1.283 + if not OPTIONS.builddir:
1.284 + OPTIONS.builddir = os.path.join(OPTIONS.libdir, 'gdb')
1.285 +
1.286 + test_set = set()
1.287 +
1.288 + # All the various sources of test names accumulate.
1.289 + if test_args:
1.290 + for arg in test_args:
1.291 + test_set.update(find_tests(OPTIONS.testdir, arg))
1.292 + if OPTIONS.worklist:
1.293 + try:
1.294 + with open(OPTIONS.worklist) as f:
1.295 + for line in f:
1.296 + test_set.update(os.path.join(test_dir, line.strip('\n')))
1.297 + except IOError:
1.298 + # With worklist, a missing file means to start the process with
1.299 + # the complete list of tests.
1.300 + sys.stderr.write("Couldn't read worklist file '%s'; running all tests\n"
1.301 + % (OPTIONS.worklist,))
1.302 + test_set = set(find_tests(OPTIONS.testdir))
1.303 + if OPTIONS.read_tests:
1.304 + try:
1.305 + with open(OPTIONS.read_tests) as f:
1.306 + for line in f:
1.307 + test_set.update(os.path.join(test_dir, line.strip('\n')))
1.308 + except IOError as err:
1.309 + sys.stderr.write("Error trying to read test file '%s': %s\n"
1.310 + % (OPTIONS.read_tests, err))
1.311 + sys.exit(1)
1.312 +
1.313 + # If none of the above options were passed, and no tests were listed
1.314 + # explicitly, use the complete set.
1.315 + if not test_args and not OPTIONS.worklist and not OPTIONS.read_tests:
1.316 + test_set = set(find_tests(OPTIONS.testdir))
1.317 +
1.318 + if OPTIONS.exclude:
1.319 + exclude_set = set()
1.320 + for exclude in OPTIONS.exclude:
1.321 + exclude_set.update(find_tests(test_dir, exclude))
1.322 + test_set -= exclude_set
1.323 +
1.324 + if not test_set:
1.325 + sys.stderr.write("No tests found matching command line arguments.\n")
1.326 + sys.exit(1)
1.327 +
1.328 + summary = Summary(len(test_set))
1.329 + test_list = [ Test(_, summary) for _ in sorted(test_set) ]
1.330 +
1.331 + # Build the test executable from all the .cpp files found in the test
1.332 + # directory tree.
1.333 + try:
1.334 + build_test_exec(OPTIONS.builddir)
1.335 + except subprocess.CalledProcessError as err:
1.336 + sys.stderr.write("Error building test executable: %s\n" % (err,))
1.337 + sys.exit(1)
1.338 +
1.339 + # Run the tests.
1.340 + try:
1.341 + summary.start()
1.342 + run_tests(test_list, summary)
1.343 + summary.finish()
1.344 + except OSError as err:
1.345 + sys.stderr.write("Error running tests: %s\n" % (err,))
1.346 + sys.exit(1)
1.347 +
1.348 + sys.exit(0)
1.349 +
1.350 +if __name__ == '__main__':
1.351 + main(sys.argv[1:])
