1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/xpcshell/runxpcshelltests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1619 @@
1.4 +#!/usr/bin/env python
1.5 +#
1.6 +# This Source Code Form is subject to the terms of the Mozilla Public
1.7 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.9 +
1.10 +import copy
1.11 +import json
1.12 +import math
1.13 +import os
1.14 +import os.path
1.15 +import random
1.16 +import re
1.17 +import shutil
1.18 +import signal
1.19 +import socket
1.20 +import sys
1.21 +import time
1.22 +import traceback
1.23 +import xml.dom.minidom
1.24 +from collections import deque
1.25 +from distutils import dir_util
1.26 +from multiprocessing import cpu_count
1.27 +from optparse import OptionParser
1.28 +from subprocess import Popen, PIPE, STDOUT
1.29 +from tempfile import mkdtemp, gettempdir
1.30 +from threading import Timer, Thread, Event, RLock
1.31 +
1.32 +try:
1.33 + import psutil
1.34 + HAVE_PSUTIL = True
1.35 +except ImportError:
1.36 + HAVE_PSUTIL = False
1.37 +
1.38 +from automation import Automation, getGlobalLog, resetGlobalLog
1.39 +from automationutils import *
1.40 +
1.41 +# Printing buffered output in case of a failure or verbose mode will result
1.42 +# in buffered output interleaved with other threads' output.
1.43 +# To prevent his, each call to the logger as well as any blocks of output that
1.44 +# are intended to be continuous are protected by the same lock.
1.45 +LOG_MUTEX = RLock()
1.46 +
1.47 +HARNESS_TIMEOUT = 5 * 60
1.48 +
1.49 +# benchmarking on tbpl revealed that this works best for now
1.50 +NUM_THREADS = int(cpu_count() * 4)
1.51 +
1.52 +FAILURE_ACTIONS = set(['test_unexpected_fail',
1.53 + 'test_unexpected_pass',
1.54 + 'javascript_error'])
1.55 +ACTION_STRINGS = {
1.56 + "test_unexpected_fail": "TEST-UNEXPECTED-FAIL",
1.57 + "test_known_fail": "TEST-KNOWN-FAIL",
1.58 + "test_unexpected_pass": "TEST-UNEXPECTED-PASS",
1.59 + "javascript_error": "TEST-UNEXPECTED-FAIL",
1.60 + "test_pass": "TEST-PASS",
1.61 + "test_info": "TEST-INFO"
1.62 +}
1.63 +
1.64 +# --------------------------------------------------------------
1.65 +# TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
1.66 +#
1.67 +here = os.path.dirname(__file__)
1.68 +mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
1.69 +
1.70 +if os.path.isdir(mozbase):
1.71 + for package in os.listdir(mozbase):
1.72 + sys.path.append(os.path.join(mozbase, package))
1.73 +
1.74 +import manifestparser
1.75 +import mozcrash
1.76 +import mozinfo
1.77 +
1.78 +# --------------------------------------------------------------
1.79 +
1.80 +# TODO: perhaps this should be in a more generally shared location?
1.81 +# This regex matches all of the C0 and C1 control characters
1.82 +# (U+0000 through U+001F; U+007F; U+0080 through U+009F),
1.83 +# except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C).
1.84 +# A raw string is deliberately not used.
1.85 +_cleanup_encoding_re = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]')
1.86 +def _cleanup_encoding_repl(m):
1.87 + c = m.group(0)
1.88 + return '\\\\' if c == '\\' else '\\x{0:02X}'.format(ord(c))
1.89 +def cleanup_encoding(s):
1.90 + """S is either a byte or unicode string. Either way it may
1.91 + contain control characters, unpaired surrogates, reserved code
1.92 + points, etc. If it is a byte string, it is assumed to be
1.93 + UTF-8, but it may not be *correct* UTF-8. Produce a byte
1.94 + string that can safely be dumped into a (generally UTF-8-coded)
1.95 + logfile."""
1.96 + if not isinstance(s, unicode):
1.97 + s = s.decode('utf-8', 'replace')
1.98 + if s.endswith('\n'):
1.99 + # A new line is always added by head.js to delimit messages,
1.100 + # however consumers will want to supply their own.
1.101 + s = s[:-1]
1.102 + # Replace all C0 and C1 control characters with \xNN escapes.
1.103 + s = _cleanup_encoding_re.sub(_cleanup_encoding_repl, s)
1.104 + return s.encode('utf-8', 'backslashreplace')
1.105 +
1.106 +""" Control-C handling """
1.107 +gotSIGINT = False
1.108 +def markGotSIGINT(signum, stackFrame):
1.109 + global gotSIGINT
1.110 + gotSIGINT = True
1.111 +
1.112 +class XPCShellTestThread(Thread):
1.113 + def __init__(self, test_object, event, cleanup_dir_list, retry=True,
1.114 + tests_root_dir=None, app_dir_key=None, interactive=False,
1.115 + verbose=False, pStdout=None, pStderr=None, keep_going=False,
1.116 + log=None, **kwargs):
1.117 + Thread.__init__(self)
1.118 + self.daemon = True
1.119 +
1.120 + self.test_object = test_object
1.121 + self.cleanup_dir_list = cleanup_dir_list
1.122 + self.retry = retry
1.123 +
1.124 + self.appPath = kwargs.get('appPath')
1.125 + self.xrePath = kwargs.get('xrePath')
1.126 + self.testingModulesDir = kwargs.get('testingModulesDir')
1.127 + self.debuggerInfo = kwargs.get('debuggerInfo')
1.128 + self.pluginsPath = kwargs.get('pluginsPath')
1.129 + self.httpdManifest = kwargs.get('httpdManifest')
1.130 + self.httpdJSPath = kwargs.get('httpdJSPath')
1.131 + self.headJSPath = kwargs.get('headJSPath')
1.132 + self.testharnessdir = kwargs.get('testharnessdir')
1.133 + self.profileName = kwargs.get('profileName')
1.134 + self.singleFile = kwargs.get('singleFile')
1.135 + self.env = copy.deepcopy(kwargs.get('env'))
1.136 + self.symbolsPath = kwargs.get('symbolsPath')
1.137 + self.logfiles = kwargs.get('logfiles')
1.138 + self.xpcshell = kwargs.get('xpcshell')
1.139 + self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
1.140 + self.failureManifest = kwargs.get('failureManifest')
1.141 + self.on_message = kwargs.get('on_message')
1.142 +
1.143 + self.tests_root_dir = tests_root_dir
1.144 + self.app_dir_key = app_dir_key
1.145 + self.interactive = interactive
1.146 + self.verbose = verbose
1.147 + self.pStdout = pStdout
1.148 + self.pStderr = pStderr
1.149 + self.keep_going = keep_going
1.150 + self.log = log
1.151 +
1.152 + # only one of these will be set to 1. adding them to the totals in
1.153 + # the harness
1.154 + self.passCount = 0
1.155 + self.todoCount = 0
1.156 + self.failCount = 0
1.157 +
1.158 + self.output_lines = []
1.159 + self.has_failure_output = False
1.160 + self.saw_proc_start = False
1.161 + self.saw_proc_end = False
1.162 +
1.163 + # event from main thread to signal work done
1.164 + self.event = event
1.165 + self.done = False # explicitly set flag so we don't rely on thread.isAlive
1.166 +
1.167 + def run(self):
1.168 + try:
1.169 + self.run_test()
1.170 + except Exception as e:
1.171 + self.exception = e
1.172 + self.traceback = traceback.format_exc()
1.173 + else:
1.174 + self.exception = None
1.175 + self.traceback = None
1.176 + if self.retry:
1.177 + self.log.info("TEST-INFO | %s | Test failed or timed out, will retry."
1.178 + % self.test_object['name'])
1.179 + self.done = True
1.180 + self.event.set()
1.181 +
1.182 + def kill(self, proc):
1.183 + """
1.184 + Simple wrapper to kill a process.
1.185 + On a remote system, this is overloaded to handle remote process communication.
1.186 + """
1.187 + return proc.kill()
1.188 +
1.189 + def removeDir(self, dirname):
1.190 + """
1.191 + Simple wrapper to remove (recursively) a given directory.
1.192 + On a remote system, we need to overload this to work on the remote filesystem.
1.193 + """
1.194 + shutil.rmtree(dirname)
1.195 +
1.196 + def poll(self, proc):
1.197 + """
1.198 + Simple wrapper to check if a process has terminated.
1.199 + On a remote system, this is overloaded to handle remote process communication.
1.200 + """
1.201 + return proc.poll()
1.202 +
1.203 + def createLogFile(self, test_file, stdout):
1.204 + """
1.205 + For a given test file and stdout buffer, create a log file.
1.206 + On a remote system we have to fix the test name since it can contain directories.
1.207 + """
1.208 + with open(test_file + ".log", "w") as f:
1.209 + f.write(stdout)
1.210 +
1.211 + def getReturnCode(self, proc):
1.212 + """
1.213 + Simple wrapper to get the return code for a given process.
1.214 + On a remote system we overload this to work with the remote process management.
1.215 + """
1.216 + return proc.returncode
1.217 +
1.218 + def communicate(self, proc):
1.219 + """
1.220 + Simple wrapper to communicate with a process.
1.221 + On a remote system, this is overloaded to handle remote process communication.
1.222 + """
1.223 + # Processing of incremental output put here to
1.224 + # sidestep issues on remote platforms, where what we know
1.225 + # as proc is a file pulled off of a device.
1.226 + if proc.stdout:
1.227 + while True:
1.228 + line = proc.stdout.readline()
1.229 + if not line:
1.230 + break
1.231 + self.process_line(line)
1.232 +
1.233 + if self.saw_proc_start and not self.saw_proc_end:
1.234 + self.has_failure_output = True
1.235 +
1.236 + return proc.communicate()
1.237 +
1.238 + def launchProcess(self, cmd, stdout, stderr, env, cwd):
1.239 + """
1.240 + Simple wrapper to launch a process.
1.241 + On a remote system, this is more complex and we need to overload this function.
1.242 + """
1.243 + if HAVE_PSUTIL:
1.244 + popen_func = psutil.Popen
1.245 + else:
1.246 + popen_func = Popen
1.247 + proc = popen_func(cmd, stdout=stdout, stderr=stderr,
1.248 + env=env, cwd=cwd)
1.249 + return proc
1.250 +
1.251 + def checkForCrashes(self,
1.252 + dump_directory,
1.253 + symbols_path,
1.254 + test_name=None):
1.255 + """
1.256 + Simple wrapper to check for crashes.
1.257 + On a remote system, this is more complex and we need to overload this function.
1.258 + """
1.259 + return mozcrash.check_for_crashes(dump_directory, symbols_path, test_name=test_name)
1.260 +
1.261 + def logCommand(self, name, completeCmd, testdir):
1.262 + self.log.info("TEST-INFO | %s | full command: %r" % (name, completeCmd))
1.263 + self.log.info("TEST-INFO | %s | current directory: %r" % (name, testdir))
1.264 + # Show only those environment variables that are changed from
1.265 + # the ambient environment.
1.266 + changedEnv = (set("%s=%s" % i for i in self.env.iteritems())
1.267 + - set("%s=%s" % i for i in os.environ.iteritems()))
1.268 + self.log.info("TEST-INFO | %s | environment: %s" % (name, list(changedEnv)))
1.269 +
1.270 + def testTimeout(self, test_file, proc):
1.271 + if not self.retry:
1.272 + self.log.error("TEST-UNEXPECTED-FAIL | %s | Test timed out" % test_file)
1.273 + self.done = True
1.274 + Automation().killAndGetStackNoScreenshot(proc.pid, self.appPath, self.debuggerInfo)
1.275 +
1.276 + def buildCmdTestFile(self, name):
1.277 + """
1.278 + Build the command line arguments for the test file.
1.279 + On a remote system, this may be overloaded to use a remote path structure.
1.280 + """
1.281 + return ['-e', 'const _TEST_FILE = ["%s"];' %
1.282 + replaceBackSlashes(name)]
1.283 +
1.284 + def setupTempDir(self):
1.285 + tempDir = mkdtemp()
1.286 + self.env["XPCSHELL_TEST_TEMP_DIR"] = tempDir
1.287 + if self.interactive:
1.288 + self.log.info("TEST-INFO | temp dir is %s" % tempDir)
1.289 + return tempDir
1.290 +
1.291 + def setupPluginsDir(self):
1.292 + if not os.path.isdir(self.pluginsPath):
1.293 + return None
1.294 +
1.295 + pluginsDir = mkdtemp()
1.296 + # shutil.copytree requires dst to not exist. Deleting the tempdir
1.297 + # would make a race condition possible in a concurrent environment,
1.298 + # so we are using dir_utils.copy_tree which accepts an existing dst
1.299 + dir_util.copy_tree(self.pluginsPath, pluginsDir)
1.300 + if self.interactive:
1.301 + self.log.info("TEST-INFO | plugins dir is %s" % pluginsDir)
1.302 + return pluginsDir
1.303 +
1.304 + def setupProfileDir(self):
1.305 + """
1.306 + Create a temporary folder for the profile and set appropriate environment variables.
1.307 + When running check-interactive and check-one, the directory is well-defined and
1.308 + retained for inspection once the tests complete.
1.309 +
1.310 + On a remote system, this may be overloaded to use a remote path structure.
1.311 + """
1.312 + if self.interactive or self.singleFile:
1.313 + profileDir = os.path.join(gettempdir(), self.profileName, "xpcshellprofile")
1.314 + try:
1.315 + # This could be left over from previous runs
1.316 + self.removeDir(profileDir)
1.317 + except:
1.318 + pass
1.319 + os.makedirs(profileDir)
1.320 + else:
1.321 + profileDir = mkdtemp()
1.322 + self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
1.323 + if self.interactive or self.singleFile:
1.324 + self.log.info("TEST-INFO | profile dir is %s" % profileDir)
1.325 + return profileDir
1.326 +
1.327 + def buildCmdHead(self, headfiles, tailfiles, xpcscmd):
1.328 + """
1.329 + Build the command line arguments for the head and tail files,
1.330 + along with the address of the webserver which some tests require.
1.331 +
1.332 + On a remote system, this is overloaded to resolve quoting issues over a secondary command line.
1.333 + """
1.334 + cmdH = ", ".join(['"' + replaceBackSlashes(f) + '"'
1.335 + for f in headfiles])
1.336 + cmdT = ", ".join(['"' + replaceBackSlashes(f) + '"'
1.337 + for f in tailfiles])
1.338 + return xpcscmd + \
1.339 + ['-e', 'const _SERVER_ADDR = "localhost"',
1.340 + '-e', 'const _HEAD_FILES = [%s];' % cmdH,
1.341 + '-e', 'const _TAIL_FILES = [%s];' % cmdT]
1.342 +
1.343 + def getHeadAndTailFiles(self, test_object):
1.344 + """Obtain the list of head and tail files.
1.345 +
1.346 + Returns a 2-tuple. The first element is a list of head files. The second
1.347 + is a list of tail files.
1.348 + """
1.349 + def sanitize_list(s, kind):
1.350 + for f in s.strip().split(' '):
1.351 + f = f.strip()
1.352 + if len(f) < 1:
1.353 + continue
1.354 +
1.355 + path = os.path.normpath(os.path.join(test_object['here'], f))
1.356 + if not os.path.exists(path):
1.357 + raise Exception('%s file does not exist: %s' % (kind, path))
1.358 +
1.359 + if not os.path.isfile(path):
1.360 + raise Exception('%s file is not a file: %s' % (kind, path))
1.361 +
1.362 + yield path
1.363 +
1.364 + return (list(sanitize_list(test_object['head'], 'head')),
1.365 + list(sanitize_list(test_object['tail'], 'tail')))
1.366 +
1.367 + def buildXpcsCmd(self, testdir):
1.368 + """
1.369 + Load the root head.js file as the first file in our test path, before other head, test, and tail files.
1.370 + On a remote system, we overload this to add additional command line arguments, so this gets overloaded.
1.371 + """
1.372 + # - NOTE: if you rename/add any of the constants set here, update
1.373 + # do_load_child_test_harness() in head.js
1.374 + if not self.appPath:
1.375 + self.appPath = self.xrePath
1.376 +
1.377 + self.xpcsCmd = [
1.378 + self.xpcshell,
1.379 + '-g', self.xrePath,
1.380 + '-a', self.appPath,
1.381 + '-r', self.httpdManifest,
1.382 + '-m',
1.383 + '-s',
1.384 + '-e', 'const _HTTPD_JS_PATH = "%s";' % self.httpdJSPath,
1.385 + '-e', 'const _HEAD_JS_PATH = "%s";' % self.headJSPath
1.386 + ]
1.387 +
1.388 + if self.testingModulesDir:
1.389 + # Escape backslashes in string literal.
1.390 + sanitized = self.testingModulesDir.replace('\\', '\\\\')
1.391 + self.xpcsCmd.extend([
1.392 + '-e',
1.393 + 'const _TESTING_MODULES_DIR = "%s";' % sanitized
1.394 + ])
1.395 +
1.396 + self.xpcsCmd.extend(['-f', os.path.join(self.testharnessdir, 'head.js')])
1.397 +
1.398 + if self.debuggerInfo:
1.399 + self.xpcsCmd = [self.debuggerInfo["path"]] + self.debuggerInfo["args"] + self.xpcsCmd
1.400 +
1.401 + # Automation doesn't specify a pluginsPath and xpcshell defaults to
1.402 + # $APPDIR/plugins. We do the same here so we can carry on with
1.403 + # setting up every test with its own plugins directory.
1.404 + if not self.pluginsPath:
1.405 + self.pluginsPath = os.path.join(self.appPath, 'plugins')
1.406 +
1.407 + self.pluginsDir = self.setupPluginsDir()
1.408 + if self.pluginsDir:
1.409 + self.xpcsCmd.extend(['-p', self.pluginsDir])
1.410 +
1.411 + def cleanupDir(self, directory, name, xunit_result):
1.412 + if not os.path.exists(directory):
1.413 + return
1.414 +
1.415 + TRY_LIMIT = 25 # up to TRY_LIMIT attempts (one every second), because
1.416 + # the Windows filesystem is slow to react to the changes
1.417 + try_count = 0
1.418 + while try_count < TRY_LIMIT:
1.419 + try:
1.420 + self.removeDir(directory)
1.421 + except OSError:
1.422 + self.log.info("TEST-INFO | Failed to remove directory: %s. Waiting." % directory)
1.423 + # We suspect the filesystem may still be making changes. Wait a
1.424 + # little bit and try again.
1.425 + time.sleep(1)
1.426 + try_count += 1
1.427 + else:
1.428 + # removed fine
1.429 + return
1.430 +
1.431 + # we try cleaning up again later at the end of the run
1.432 + self.cleanup_dir_list.append(directory)
1.433 +
1.434 + def clean_temp_dirs(self, name, stdout):
1.435 + # We don't want to delete the profile when running check-interactive
1.436 + # or check-one.
1.437 + if self.profileDir and not self.interactive and not self.singleFile:
1.438 + self.cleanupDir(self.profileDir, name, self.xunit_result)
1.439 +
1.440 + self.cleanupDir(self.tempDir, name, self.xunit_result)
1.441 +
1.442 + if self.pluginsDir:
1.443 + self.cleanupDir(self.pluginsDir, name, self.xunit_result)
1.444 +
1.445 + def message_from_line(self, line):
1.446 + """ Given a line of raw output, convert to a string message. """
1.447 + if isinstance(line, basestring):
1.448 + # This function has received unstructured output.
1.449 + if line:
1.450 + if 'TEST-UNEXPECTED-' in line:
1.451 + self.has_failure_output = True
1.452 + return line
1.453 +
1.454 + msg = ['%s: ' % line['process'] if 'process' in line else '']
1.455 +
1.456 + # Each call to the logger in head.js either specified '_message'
1.457 + # or both 'source_file' and 'diagnostic'. If either of these are
1.458 + # missing, they ended up being undefined as a result of the way
1.459 + # the test was run.
1.460 + if '_message' in line:
1.461 + msg.append(line['_message'])
1.462 + if 'diagnostic' in line:
1.463 + msg.append('\nDiagnostic: %s' % line['diagnostic'])
1.464 + else:
1.465 + msg.append('%s | %s | %s' % (ACTION_STRINGS[line['action']],
1.466 + line.get('source_file', 'undefined'),
1.467 + line.get('diagnostic', 'undefined')))
1.468 +
1.469 + msg.append('\n%s' % line['stack'] if 'stack' in line else '')
1.470 + return ''.join(msg)
1.471 +
1.472 + def parse_output(self, output):
1.473 + """Parses process output for structured messages and saves output as it is
1.474 + read. Sets self.has_failure_output in case of evidence of a failure"""
1.475 + for line_string in output.splitlines():
1.476 + self.process_line(line_string)
1.477 +
1.478 + if self.saw_proc_start and not self.saw_proc_end:
1.479 + self.has_failure_output = True
1.480 +
1.481 + def report_message(self, line):
1.482 + """ Reports a message to a consumer, both as a strucutured and
1.483 + human-readable log message. """
1.484 +
1.485 + message = cleanup_encoding(self.message_from_line(line))
1.486 + if message.endswith('\n'):
1.487 + # A new line is always added by head.js to delimit messages,
1.488 + # however consumers will want to supply their own.
1.489 + message = message[:-1]
1.490 +
1.491 + if self.on_message:
1.492 + self.on_message(line, message)
1.493 + else:
1.494 + self.output_lines.append(message)
1.495 +
1.496 + def process_line(self, line_string):
1.497 + """ Parses a single line of output, determining its significance and
1.498 + reporting a message.
1.499 + """
1.500 + try:
1.501 + line_object = json.loads(line_string)
1.502 + if not isinstance(line_object, dict):
1.503 + self.report_message(line_string)
1.504 + return
1.505 + except ValueError:
1.506 + self.report_message(line_string)
1.507 + return
1.508 +
1.509 + if 'action' not in line_object:
1.510 + # In case a test outputs something that happens to be valid
1.511 + # JSON.
1.512 + self.report_message(line_string)
1.513 + return
1.514 +
1.515 + action = line_object['action']
1.516 + self.report_message(line_object)
1.517 +
1.518 + if action in FAILURE_ACTIONS:
1.519 + self.has_failure_output = True
1.520 + elif action == 'child_test_start':
1.521 + self.saw_proc_start = True
1.522 + elif action == 'child_test_end':
1.523 + self.saw_proc_end = True
1.524 +
1.525 + def log_output(self, output):
1.526 + """Prints given output line-by-line to avoid overflowing buffers."""
1.527 + self.log.info(">>>>>>>")
1.528 + if output:
1.529 + if isinstance(output, basestring):
1.530 + output = output.splitlines()
1.531 + for part in output:
1.532 + # For multi-line output, such as a stack trace
1.533 + for line in part.splitlines():
1.534 + try:
1.535 + line = line.decode('utf-8')
1.536 + except UnicodeDecodeError:
1.537 + self.log.info("TEST-INFO | %s | Detected non UTF-8 output."\
1.538 + " Please modify the test to only print UTF-8." %
1.539 + self.test_object['name'])
1.540 + # add '?' instead of funky bytes
1.541 + line = line.decode('utf-8', 'replace')
1.542 + self.log.info(line)
1.543 + self.log.info("<<<<<<<")
1.544 +
1.545 + def run_test(self):
1.546 + """Run an individual xpcshell test."""
1.547 + global gotSIGINT
1.548 +
1.549 + name = self.test_object['path']
1.550 +
1.551 + self.xunit_result = {'name': self.test_object['name'], 'classname': 'xpcshell'}
1.552 +
1.553 + # The xUnit package is defined as the path component between the root
1.554 + # dir and the test with path characters replaced with '.' (using Java
1.555 + # class notation).
1.556 + if self.tests_root_dir is not None:
1.557 + self.tests_root_dir = os.path.normpath(self.tests_root_dir)
1.558 + if os.path.normpath(self.test_object['here']).find(self.tests_root_dir) != 0:
1.559 + raise Exception('tests_root_dir is not a parent path of %s' %
1.560 + self.test_object['here'])
1.561 + relpath = self.test_object['here'][len(self.tests_root_dir):].lstrip('/\\')
1.562 + self.xunit_result['classname'] = relpath.replace('/', '.').replace('\\', '.')
1.563 +
1.564 + # Check for skipped tests
1.565 + if 'disabled' in self.test_object:
1.566 + self.log.info('TEST-INFO | skipping %s | %s' %
1.567 + (name, self.test_object['disabled']))
1.568 +
1.569 + self.xunit_result['skipped'] = True
1.570 + self.retry = False
1.571 +
1.572 + self.keep_going = True
1.573 + return
1.574 +
1.575 + # Check for known-fail tests
1.576 + expected = self.test_object['expected'] == 'pass'
1.577 +
1.578 + # By default self.appPath will equal the gre dir. If specified in the
1.579 + # xpcshell.ini file, set a different app dir for this test.
1.580 + if self.app_dir_key and self.app_dir_key in self.test_object:
1.581 + rel_app_dir = self.test_object[self.app_dir_key]
1.582 + rel_app_dir = os.path.join(self.xrePath, rel_app_dir)
1.583 + self.appPath = os.path.abspath(rel_app_dir)
1.584 + else:
1.585 + self.appPath = None
1.586 +
1.587 + test_dir = os.path.dirname(name)
1.588 + self.buildXpcsCmd(test_dir)
1.589 + head_files, tail_files = self.getHeadAndTailFiles(self.test_object)
1.590 + cmdH = self.buildCmdHead(head_files, tail_files, self.xpcsCmd)
1.591 +
1.592 + # Create a profile and a temp dir that the JS harness can stick
1.593 + # a profile and temporary data in
1.594 + self.profileDir = self.setupProfileDir()
1.595 + self.tempDir = self.setupTempDir()
1.596 +
1.597 + # The test file will have to be loaded after the head files.
1.598 + cmdT = self.buildCmdTestFile(name)
1.599 +
1.600 + args = self.xpcsRunArgs[:]
1.601 + if 'debug' in self.test_object:
1.602 + args.insert(0, '-d')
1.603 +
1.604 + completeCmd = cmdH + cmdT + args
1.605 +
1.606 + testTimeoutInterval = HARNESS_TIMEOUT
1.607 + # Allow a test to request a multiple of the timeout if it is expected to take long
1.608 + if 'requesttimeoutfactor' in self.test_object:
1.609 + testTimeoutInterval *= int(self.test_object['requesttimeoutfactor'])
1.610 +
1.611 + testTimer = None
1.612 + if not self.interactive and not self.debuggerInfo:
1.613 + testTimer = Timer(testTimeoutInterval, lambda: self.testTimeout(name, proc))
1.614 + testTimer.start()
1.615 +
1.616 + proc = None
1.617 + stdout = None
1.618 + stderr = None
1.619 +
1.620 + try:
1.621 + self.log.info("TEST-INFO | %s | running test ..." % name)
1.622 + if self.verbose:
1.623 + self.logCommand(name, completeCmd, test_dir)
1.624 +
1.625 + startTime = time.time()
1.626 + proc = self.launchProcess(completeCmd,
1.627 + stdout=self.pStdout, stderr=self.pStderr, env=self.env, cwd=test_dir)
1.628 +
1.629 + if self.interactive:
1.630 + self.log.info("TEST-INFO | %s | Process ID: %d" % (name, proc.pid))
1.631 +
1.632 + stdout, stderr = self.communicate(proc)
1.633 +
1.634 + if self.interactive:
1.635 + # Not sure what else to do here...
1.636 + self.keep_going = True
1.637 + return
1.638 +
1.639 + if testTimer:
1.640 + testTimer.cancel()
1.641 +
1.642 + if stdout:
1.643 + self.parse_output(stdout)
1.644 + result = not (self.has_failure_output or
1.645 + (self.getReturnCode(proc) != 0))
1.646 +
1.647 + if result != expected:
1.648 + if self.retry:
1.649 + self.clean_temp_dirs(name, stdout)
1.650 + return
1.651 +
1.652 + failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS")
1.653 + message = "%s | %s | test failed (with xpcshell return code: %d)" % (
1.654 + failureType, name, self.getReturnCode(proc))
1.655 + if self.output_lines:
1.656 + message += ", see following log:"
1.657 +
1.658 + with LOG_MUTEX:
1.659 + self.log.error(message)
1.660 + self.log_output(self.output_lines)
1.661 +
1.662 + self.failCount += 1
1.663 + self.xunit_result["passed"] = False
1.664 +
1.665 + self.xunit_result["failure"] = {
1.666 + "type": failureType,
1.667 + "message": message,
1.668 + "text": stdout
1.669 + }
1.670 +
1.671 + if self.failureManifest:
1.672 + with open(self.failureManifest, 'a') as f:
1.673 + f.write('[%s]\n' % self.test_object['path'])
1.674 + for k, v in self.test_object.items():
1.675 + f.write('%s = %s\n' % (k, v))
1.676 +
1.677 + else:
1.678 + now = time.time()
1.679 + timeTaken = (now - startTime) * 1000
1.680 + self.xunit_result["time"] = now - startTime
1.681 +
1.682 + with LOG_MUTEX:
1.683 + self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken))
1.684 + if self.verbose:
1.685 + self.log_output(self.output_lines)
1.686 +
1.687 + self.xunit_result["passed"] = True
1.688 + self.retry = False
1.689 +
1.690 + if expected:
1.691 + self.passCount = 1
1.692 + else:
1.693 + self.todoCount = 1
1.694 + self.xunit_result["todo"] = True
1.695 +
1.696 + if self.checkForCrashes(self.tempDir, self.symbolsPath, test_name=name):
1.697 + if self.retry:
1.698 + self.clean_temp_dirs(name, stdout)
1.699 + return
1.700 +
1.701 + message = "PROCESS-CRASH | %s | application crashed" % name
1.702 + self.failCount = 1
1.703 + self.xunit_result["passed"] = False
1.704 + self.xunit_result["failure"] = {
1.705 + "type": "PROCESS-CRASH",
1.706 + "message": message,
1.707 + "text": stdout
1.708 + }
1.709 +
1.710 + if self.logfiles and stdout:
1.711 + self.createLogFile(name, stdout)
1.712 +
1.713 + finally:
1.714 + # We can sometimes get here before the process has terminated, which would
1.715 + # cause removeDir() to fail - so check for the process & kill it it needed.
1.716 + if proc and self.poll(proc) is None:
1.717 + self.kill(proc)
1.718 +
1.719 + if self.retry:
1.720 + self.clean_temp_dirs(name, stdout)
1.721 + return
1.722 +
1.723 + with LOG_MUTEX:
1.724 + message = "TEST-UNEXPECTED-FAIL | %s | Process still running after test!" % name
1.725 + self.log.error(message)
1.726 + self.log_output(self.output_lines)
1.727 +
1.728 + self.failCount = 1
1.729 + self.xunit_result["passed"] = False
1.730 + self.xunit_result["failure"] = {
1.731 + "type": "TEST-UNEXPECTED-FAIL",
1.732 + "message": message,
1.733 + "text": stdout
1.734 + }
1.735 +
1.736 + self.clean_temp_dirs(name, stdout)
1.737 +
1.738 + if gotSIGINT:
1.739 + self.xunit_result["passed"] = False
1.740 + self.xunit_result["time"] = "0.0"
1.741 + self.xunit_result["failure"] = {
1.742 + "type": "SIGINT",
1.743 + "message": "Received SIGINT",
1.744 + "text": "Received SIGINT (control-C) during test execution."
1.745 + }
1.746 +
1.747 + self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution")
1.748 + if self.keep_going:
1.749 + gotSIGINT = False
1.750 + else:
1.751 + self.keep_going = False
1.752 + return
1.753 +
1.754 + self.keep_going = True
1.755 +
1.756 +class XPCShellTests(object):
1.757 +
1.758 + log = getGlobalLog()
1.759 + oldcwd = os.getcwd()
1.760 +
1.761 + def __init__(self, log=None):
1.762 + """ Init logging and node status """
1.763 + if log:
1.764 + resetGlobalLog(log)
1.765 +
1.766 + # Each method of the underlying logger must acquire the log
1.767 + # mutex before writing to stdout.
1.768 + log_funs = ['debug', 'info', 'warning', 'error', 'critical', 'log']
1.769 + for fun_name in log_funs:
1.770 + unwrapped = getattr(self.log, fun_name, None)
1.771 + if unwrapped:
1.772 + def wrap(fn):
1.773 + def wrapped(*args, **kwargs):
1.774 + with LOG_MUTEX:
1.775 + fn(*args, **kwargs)
1.776 + return wrapped
1.777 + setattr(self.log, fun_name, wrap(unwrapped))
1.778 +
1.779 + self.nodeProc = {}
1.780 +
1.781 + def buildTestList(self):
1.782 + """
1.783 + read the xpcshell.ini manifest and set self.alltests to be
1.784 + an array of test objects.
1.785 +
1.786 + if we are chunking tests, it will be done here as well
1.787 + """
1.788 + if isinstance(self.manifest, manifestparser.TestManifest):
1.789 + mp = self.manifest
1.790 + else:
1.791 + mp = manifestparser.TestManifest(strict=False)
1.792 + if self.manifest is None:
1.793 + for testdir in self.testdirs:
1.794 + if testdir:
1.795 + mp.read(os.path.join(testdir, 'xpcshell.ini'))
1.796 + else:
1.797 + mp.read(self.manifest)
1.798 +
1.799 + self.buildTestPath()
1.800 +
1.801 + try:
1.802 + self.alltests = mp.active_tests(**mozinfo.info)
1.803 + except TypeError:
1.804 + sys.stderr.write("*** offending mozinfo.info: %s\n" % repr(mozinfo.info))
1.805 + raise
1.806 +
1.807 + if self.singleFile is None and self.totalChunks > 1:
1.808 + self.chunkTests()
1.809 +
1.810 + def chunkTests(self):
1.811 + """
1.812 + Split the list of tests up into [totalChunks] pieces and filter the
1.813 + self.alltests based on thisChunk, so we only run a subset.
1.814 + """
1.815 + totalTests = len(self.alltests)
1.816 + testsPerChunk = math.ceil(totalTests / float(self.totalChunks))
1.817 + start = int(round((self.thisChunk-1) * testsPerChunk))
1.818 + end = int(start + testsPerChunk)
1.819 + if end > totalTests:
1.820 + end = totalTests
1.821 + self.log.info("Running tests %d-%d/%d", start+1, end, totalTests)
1.822 + self.alltests = self.alltests[start:end]
1.823 +
1.824 + def setAbsPath(self):
1.825 + """
1.826 + Set the absolute path for xpcshell, httpdjspath and xrepath.
1.827 + These 3 variables depend on input from the command line and we need to allow for absolute paths.
1.828 + This function is overloaded for a remote solution as os.path* won't work remotely.
1.829 + """
1.830 + self.testharnessdir = os.path.dirname(os.path.abspath(__file__))
1.831 + self.headJSPath = self.testharnessdir.replace("\\", "/") + "/head.js"
1.832 + self.xpcshell = os.path.abspath(self.xpcshell)
1.833 +
1.834 + # we assume that httpd.js lives in components/ relative to xpcshell
1.835 + self.httpdJSPath = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.js')
1.836 + self.httpdJSPath = replaceBackSlashes(self.httpdJSPath)
1.837 +
1.838 + self.httpdManifest = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.manifest')
1.839 + self.httpdManifest = replaceBackSlashes(self.httpdManifest)
1.840 +
1.841 + if self.xrePath is None:
1.842 + self.xrePath = os.path.dirname(self.xpcshell)
1.843 + else:
1.844 + self.xrePath = os.path.abspath(self.xrePath)
1.845 +
1.846 + if self.mozInfo is None:
1.847 + self.mozInfo = os.path.join(self.testharnessdir, "mozinfo.json")
1.848 +
1.849 + def buildCoreEnvironment(self):
1.850 + """
1.851 + Add environment variables likely to be used across all platforms, including remote systems.
1.852 + """
1.853 + # Make assertions fatal
1.854 + self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
1.855 + # Crash reporting interferes with debugging
1.856 + if not self.debuggerInfo:
1.857 + self.env["MOZ_CRASHREPORTER"] = "1"
1.858 + # Don't launch the crash reporter client
1.859 + self.env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
1.860 + # Capturing backtraces is very slow on some platforms, and it's
1.861 + # disabled by automation.py too
1.862 + self.env["NS_TRACE_MALLOC_DISABLE_STACKS"] = "1"
1.863 + # Don't permit remote connections.
1.864 + self.env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
1.865 +
1.866 + def buildEnvironment(self):
1.867 + """
1.868 + Create and returns a dictionary of self.env to include all the appropriate env variables and values.
1.869 + On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
1.870 + """
1.871 + self.env = dict(os.environ)
1.872 + self.buildCoreEnvironment()
1.873 + if sys.platform == 'win32':
1.874 + self.env["PATH"] = self.env["PATH"] + ";" + self.xrePath
1.875 + elif sys.platform in ('os2emx', 'os2knix'):
1.876 + os.environ["BEGINLIBPATH"] = self.xrePath + ";" + self.env["BEGINLIBPATH"]
1.877 + os.environ["LIBPATHSTRICT"] = "T"
1.878 + elif sys.platform == 'osx' or sys.platform == "darwin":
1.879 + self.env["DYLD_LIBRARY_PATH"] = self.xrePath
1.880 + else: # unix or linux?
1.881 + if not "LD_LIBRARY_PATH" in self.env or self.env["LD_LIBRARY_PATH"] is None:
1.882 + self.env["LD_LIBRARY_PATH"] = self.xrePath
1.883 + else:
1.884 + self.env["LD_LIBRARY_PATH"] = ":".join([self.xrePath, self.env["LD_LIBRARY_PATH"]])
1.885 +
1.886 + if "asan" in self.mozInfo and self.mozInfo["asan"]:
1.887 + # ASan symbolizer support
1.888 + llvmsym = os.path.join(self.xrePath, "llvm-symbolizer")
1.889 + if os.path.isfile(llvmsym):
1.890 + self.env["ASAN_SYMBOLIZER_PATH"] = llvmsym
1.891 + self.log.info("INFO | runxpcshelltests.py | ASan using symbolizer at %s", llvmsym)
1.892 + else:
1.893 + self.log.info("INFO | runxpcshelltests.py | ASan symbolizer binary not found: %s", llvmsym)
1.894 +
1.895 + return self.env
1.896 +
1.897 + def getPipes(self):
1.898 + """
1.899 + Determine the value of the stdout and stderr for the test.
1.900 + Return value is a list (pStdout, pStderr).
1.901 + """
1.902 + if self.interactive:
1.903 + pStdout = None
1.904 + pStderr = None
1.905 + else:
1.906 + if (self.debuggerInfo and self.debuggerInfo["interactive"]):
1.907 + pStdout = None
1.908 + pStderr = None
1.909 + else:
1.910 + if sys.platform == 'os2emx':
1.911 + pStdout = None
1.912 + else:
1.913 + pStdout = PIPE
1.914 + pStderr = STDOUT
1.915 + return pStdout, pStderr
1.916 +
1.917 + def buildTestPath(self):
1.918 + """
1.919 + If we specifiy a testpath, set the self.testPath variable to be the given directory or file.
1.920 +
1.921 + |testPath| will be the optional path only, or |None|.
1.922 + |singleFile| will be the optional test only, or |None|.
1.923 + """
1.924 + self.singleFile = None
1.925 + if self.testPath is not None:
1.926 + if self.testPath.endswith('.js'):
1.927 + # Split into path and file.
1.928 + if self.testPath.find('/') == -1:
1.929 + # Test only.
1.930 + self.singleFile = self.testPath
1.931 + else:
1.932 + # Both path and test.
1.933 + # Reuse |testPath| temporarily.
1.934 + self.testPath = self.testPath.rsplit('/', 1)
1.935 + self.singleFile = self.testPath[1]
1.936 + self.testPath = self.testPath[0]
1.937 + else:
1.938 + # Path only.
1.939 + # Simply remove optional ending separator.
1.940 + self.testPath = self.testPath.rstrip("/")
1.941 +
1.942 + def verifyDirPath(self, dirname):
1.943 + """
1.944 + Simple wrapper to get the absolute path for a given directory name.
1.945 + On a remote system, we need to overload this to work on the remote filesystem.
1.946 + """
1.947 + return os.path.abspath(dirname)
1.948 +
1.949 + def trySetupNode(self):
1.950 + """
1.951 + Run node for SPDY tests, if available, and updates mozinfo as appropriate.
1.952 + """
1.953 + nodeMozInfo = {'hasNode': False} # Assume the worst
1.954 + nodeBin = None
1.955 +
1.956 + # We try to find the node executable in the path given to us by the user in
1.957 + # the MOZ_NODE_PATH environment variable
1.958 + localPath = os.getenv('MOZ_NODE_PATH', None)
1.959 + if localPath and os.path.exists(localPath) and os.path.isfile(localPath):
1.960 + nodeBin = localPath
1.961 +
1.962 + if nodeBin:
1.963 + self.log.info('Found node at %s' % (nodeBin,))
1.964 +
1.965 + def startServer(name, serverJs):
1.966 + if os.path.exists(serverJs):
1.967 + # OK, we found our SPDY server, let's try to get it running
1.968 + self.log.info('Found %s at %s' % (name, serverJs))
1.969 + try:
1.970 + # We pipe stdin to node because the spdy server will exit when its
1.971 + # stdin reaches EOF
1.972 + process = Popen([nodeBin, serverJs], stdin=PIPE, stdout=PIPE,
1.973 + stderr=STDOUT, env=self.env, cwd=os.getcwd())
1.974 + self.nodeProc[name] = process
1.975 +
1.976 + # Check to make sure the server starts properly by waiting for it to
1.977 + # tell us it's started
1.978 + msg = process.stdout.readline()
1.979 + if 'server listening' in msg:
1.980 + nodeMozInfo['hasNode'] = True
1.981 + except OSError, e:
1.982 + # This occurs if the subprocess couldn't be started
1.983 + self.log.error('Could not run %s server: %s' % (name, str(e)))
1.984 +
1.985 + myDir = os.path.split(os.path.abspath(__file__))[0]
1.986 + startServer('moz-spdy', os.path.join(myDir, 'moz-spdy', 'moz-spdy.js'))
1.987 + startServer('moz-http2', os.path.join(myDir, 'moz-http2', 'moz-http2.js'))
1.988 +
1.989 + mozinfo.update(nodeMozInfo)
1.990 +
1.991 + def shutdownNode(self):
1.992 + """
1.993 + Shut down our node process, if it exists
1.994 + """
1.995 + for name, proc in self.nodeProc.iteritems():
1.996 + self.log.info('Node %s server shutting down ...' % name)
1.997 + proc.terminate()
1.998 +
1.999 + def writeXunitResults(self, results, name=None, filename=None, fh=None):
1.1000 + """
1.1001 + Write Xunit XML from results.
1.1002 +
1.1003 + The function receives an iterable of results dicts. Each dict must have
1.1004 + the following keys:
1.1005 +
1.1006 + classname - The "class" name of the test.
1.1007 + name - The simple name of the test.
1.1008 +
1.1009 + In addition, it must have one of the following saying how the test
1.1010 + executed:
1.1011 +
1.1012 + passed - Boolean indicating whether the test passed. False if it
1.1013 + failed.
1.1014 + skipped - True if the test was skipped.
1.1015 +
1.1016 + The following keys are optional:
1.1017 +
1.1018 + time - Execution time of the test in decimal seconds.
1.1019 + failure - Dict describing test failure. Requires keys:
1.1020 + type - String type of failure.
1.1021 + message - String describing basic failure.
1.1022 + text - Verbose string describing failure.
1.1023 +
1.1024 + Arguments:
1.1025 +
1.1026 + |name|, Name of the test suite. Many tools expect Java class dot notation
1.1027 + e.g. dom.simple.foo. A directory with '/' converted to '.' is a good
1.1028 + choice.
1.1029 + |fh|, File handle to write XML to.
1.1030 + |filename|, File name to write XML to.
1.1031 + |results|, Iterable of tuples describing the results.
1.1032 + """
1.1033 + if filename is None and fh is None:
1.1034 + raise Exception("One of filename or fh must be defined.")
1.1035 +
1.1036 + if name is None:
1.1037 + name = "xpcshell"
1.1038 + else:
1.1039 + assert isinstance(name, basestring)
1.1040 +
1.1041 + if filename is not None:
1.1042 + fh = open(filename, 'wb')
1.1043 +
1.1044 + doc = xml.dom.minidom.Document()
1.1045 + testsuite = doc.createElement("testsuite")
1.1046 + testsuite.setAttribute("name", name)
1.1047 + doc.appendChild(testsuite)
1.1048 +
1.1049 + total = 0
1.1050 + passed = 0
1.1051 + failed = 0
1.1052 + skipped = 0
1.1053 +
1.1054 + for result in results:
1.1055 + total += 1
1.1056 +
1.1057 + if result.get("skipped", None):
1.1058 + skipped += 1
1.1059 + elif result["passed"]:
1.1060 + passed += 1
1.1061 + else:
1.1062 + failed += 1
1.1063 +
1.1064 + testcase = doc.createElement("testcase")
1.1065 + testcase.setAttribute("classname", result["classname"])
1.1066 + testcase.setAttribute("name", result["name"])
1.1067 +
1.1068 + if "time" in result:
1.1069 + testcase.setAttribute("time", str(result["time"]))
1.1070 + else:
1.1071 + # It appears most tools expect the time attribute to be present.
1.1072 + testcase.setAttribute("time", "0")
1.1073 +
1.1074 + if "failure" in result:
1.1075 + failure = doc.createElement("failure")
1.1076 + failure.setAttribute("type", str(result["failure"]["type"]))
1.1077 + failure.setAttribute("message", result["failure"]["message"])
1.1078 +
1.1079 + # Lossy translation but required to not break CDATA. Also, text could
1.1080 + # be None and Python 2.5's minidom doesn't accept None. Later versions
1.1081 + # do, however.
1.1082 + cdata = result["failure"]["text"]
1.1083 + if not isinstance(cdata, str):
1.1084 + cdata = ""
1.1085 +
1.1086 + cdata = cdata.replace("]]>", "]] >")
1.1087 + text = doc.createCDATASection(cdata)
1.1088 + failure.appendChild(text)
1.1089 + testcase.appendChild(failure)
1.1090 +
1.1091 + if result.get("skipped", None):
1.1092 + e = doc.createElement("skipped")
1.1093 + testcase.appendChild(e)
1.1094 +
1.1095 + testsuite.appendChild(testcase)
1.1096 +
1.1097 + testsuite.setAttribute("tests", str(total))
1.1098 + testsuite.setAttribute("failures", str(failed))
1.1099 + testsuite.setAttribute("skip", str(skipped))
1.1100 +
1.1101 + doc.writexml(fh, addindent=" ", newl="\n", encoding="utf-8")
1.1102 +
1.1103 + def post_to_autolog(self, results, name):
1.1104 + from moztest.results import TestContext, TestResult, TestResultCollection
1.1105 + from moztest.output.autolog import AutologOutput
1.1106 +
1.1107 + context = TestContext(
1.1108 + testgroup='b2g xpcshell testsuite',
1.1109 + operating_system='android',
1.1110 + arch='emulator',
1.1111 + harness='xpcshell',
1.1112 + hostname=socket.gethostname(),
1.1113 + tree='b2g',
1.1114 + buildtype='opt',
1.1115 + )
1.1116 +
1.1117 + collection = TestResultCollection('b2g emulator testsuite')
1.1118 +
1.1119 + for result in results:
1.1120 + duration = result.get('time', 0)
1.1121 +
1.1122 + if 'skipped' in result:
1.1123 + outcome = 'SKIPPED'
1.1124 + elif 'todo' in result:
1.1125 + outcome = 'KNOWN-FAIL'
1.1126 + elif result['passed']:
1.1127 + outcome = 'PASS'
1.1128 + else:
1.1129 + outcome = 'UNEXPECTED-FAIL'
1.1130 +
1.1131 + output = None
1.1132 + if 'failure' in result:
1.1133 + output = result['failure']['text']
1.1134 +
1.1135 + t = TestResult(name=result['name'], test_class=name,
1.1136 + time_start=0, context=context)
1.1137 + t.finish(result=outcome, time_end=duration, output=output)
1.1138 +
1.1139 + collection.append(t)
1.1140 + collection.time_taken += duration
1.1141 +
1.1142 + out = AutologOutput()
1.1143 + out.post(out.make_testgroups(collection))
1.1144 +
1.1145 + def buildXpcsRunArgs(self):
1.1146 + """
1.1147 + Add arguments to run the test or make it interactive.
1.1148 + """
1.1149 + if self.interactive:
1.1150 + self.xpcsRunArgs = [
1.1151 + '-e', 'print("To start the test, type |_execute_test();|.");',
1.1152 + '-i']
1.1153 + else:
1.1154 + self.xpcsRunArgs = ['-e', '_execute_test(); quit(0);']
1.1155 +
1.1156 + def addTestResults(self, test):
1.1157 + self.passCount += test.passCount
1.1158 + self.failCount += test.failCount
1.1159 + self.todoCount += test.todoCount
1.1160 + self.xunitResults.append(test.xunit_result)
1.1161 +
1.1162 + def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
1.1163 + manifest=None, testdirs=None, testPath=None, mobileArgs=None,
1.1164 + interactive=False, verbose=False, keepGoing=False, logfiles=True,
1.1165 + thisChunk=1, totalChunks=1, debugger=None,
1.1166 + debuggerArgs=None, debuggerInteractive=False,
1.1167 + profileName=None, mozInfo=None, sequential=False, shuffle=False,
1.1168 + testsRootDir=None, xunitFilename=None, xunitName=None,
1.1169 + testingModulesDir=None, autolog=False, pluginsPath=None,
1.1170 + testClass=XPCShellTestThread, failureManifest=None,
1.1171 + on_message=None, **otherOptions):
1.1172 + """Run xpcshell tests.
1.1173 +
1.1174 + |xpcshell|, is the xpcshell executable to use to run the tests.
1.1175 + |xrePath|, if provided, is the path to the XRE to use.
1.1176 + |appPath|, if provided, is the path to an application directory.
1.1177 + |symbolsPath|, if provided is the path to a directory containing
1.1178 + breakpad symbols for processing crashes in tests.
1.1179 + |manifest|, if provided, is a file containing a list of
1.1180 + test directories to run.
1.1181 + |testdirs|, if provided, is a list of absolute paths of test directories.
1.1182 + No-manifest only option.
1.1183 + |testPath|, if provided, indicates a single path and/or test to run.
1.1184 + |pluginsPath|, if provided, custom plugins directory to be returned from
1.1185 + the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST.
1.1186 + |interactive|, if set to True, indicates to provide an xpcshell prompt
1.1187 + instead of automatically executing the test.
1.1188 + |verbose|, if set to True, will cause stdout/stderr from tests to
1.1189 + be printed always
1.1190 + |logfiles|, if set to False, indicates not to save output to log files.
1.1191 + Non-interactive only option.
1.1192 + |debuggerInfo|, if set, specifies the debugger and debugger arguments
1.1193 + that will be used to launch xpcshell.
1.1194 + |profileName|, if set, specifies the name of the application for the profile
1.1195 + directory if running only a subset of tests.
1.1196 + |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
1.1197 + |shuffle|, if True, execute tests in random order.
1.1198 + |testsRootDir|, absolute path to root directory of all tests. This is used
1.1199 + by xUnit generation to determine the package name of the tests.
1.1200 + |xunitFilename|, if set, specifies the filename to which to write xUnit XML
1.1201 + results.
1.1202 + |xunitName|, if outputting an xUnit XML file, the str value to use for the
1.1203 + testsuite name.
1.1204 + |testingModulesDir|, if provided, specifies where JS modules reside.
1.1205 + xpcshell will register a resource handler mapping this path.
1.1206 + |otherOptions| may be present for the convenience of subclasses
1.1207 + """
1.1208 +
1.1209 + global gotSIGINT
1.1210 +
1.1211 + if testdirs is None:
1.1212 + testdirs = []
1.1213 +
1.1214 + if xunitFilename is not None or xunitName is not None:
1.1215 + if not isinstance(testsRootDir, basestring):
1.1216 + raise Exception("testsRootDir must be a str when outputting xUnit.")
1.1217 +
1.1218 + if not os.path.isabs(testsRootDir):
1.1219 + testsRootDir = os.path.abspath(testsRootDir)
1.1220 +
1.1221 + if not os.path.exists(testsRootDir):
1.1222 + raise Exception("testsRootDir path does not exists: %s" %
1.1223 + testsRootDir)
1.1224 +
1.1225 + # Try to guess modules directory.
1.1226 + # This somewhat grotesque hack allows the buildbot machines to find the
1.1227 + # modules directory without having to configure the buildbot hosts. This
1.1228 + # code path should never be executed in local runs because the build system
1.1229 + # should always set this argument.
1.1230 + if not testingModulesDir:
1.1231 + ourDir = os.path.dirname(__file__)
1.1232 + possible = os.path.join(ourDir, os.path.pardir, 'modules')
1.1233 +
1.1234 + if os.path.isdir(possible):
1.1235 + testingModulesDir = possible
1.1236 +
1.1237 + if testingModulesDir:
1.1238 + # The resource loader expects native paths. Depending on how we were
1.1239 + # invoked, a UNIX style path may sneak in on Windows. We try to
1.1240 + # normalize that.
1.1241 + testingModulesDir = os.path.normpath(testingModulesDir)
1.1242 +
1.1243 + if not os.path.isabs(testingModulesDir):
1.1244 + testingModulesDir = os.path.abspath(testingModulesDir)
1.1245 +
1.1246 + if not testingModulesDir.endswith(os.path.sep):
1.1247 + testingModulesDir += os.path.sep
1.1248 +
1.1249 + self.xpcshell = xpcshell
1.1250 + self.xrePath = xrePath
1.1251 + self.appPath = appPath
1.1252 + self.symbolsPath = symbolsPath
1.1253 + self.manifest = manifest
1.1254 + self.testdirs = testdirs
1.1255 + self.testPath = testPath
1.1256 + self.interactive = interactive
1.1257 + self.verbose = verbose
1.1258 + self.keepGoing = keepGoing
1.1259 + self.logfiles = logfiles
1.1260 + self.on_message = on_message
1.1261 + self.totalChunks = totalChunks
1.1262 + self.thisChunk = thisChunk
1.1263 + self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive)
1.1264 + self.profileName = profileName or "xpcshell"
1.1265 + self.mozInfo = mozInfo
1.1266 + self.testingModulesDir = testingModulesDir
1.1267 + self.pluginsPath = pluginsPath
1.1268 + self.sequential = sequential
1.1269 +
1.1270 + if not testdirs and not manifest:
1.1271 + # nothing to test!
1.1272 + self.log.error("Error: No test dirs or test manifest specified!")
1.1273 + return False
1.1274 +
1.1275 + self.testCount = 0
1.1276 + self.passCount = 0
1.1277 + self.failCount = 0
1.1278 + self.todoCount = 0
1.1279 +
1.1280 + self.setAbsPath()
1.1281 + self.buildXpcsRunArgs()
1.1282 +
1.1283 + self.event = Event()
1.1284 +
1.1285 + # Handle filenames in mozInfo
1.1286 + if not isinstance(self.mozInfo, dict):
1.1287 + mozInfoFile = self.mozInfo
1.1288 + if not os.path.isfile(mozInfoFile):
1.1289 + self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile)
1.1290 + return False
1.1291 + self.mozInfo = json.load(open(mozInfoFile))
1.1292 +
1.1293 + # mozinfo.info is used as kwargs. Some builds are done with
1.1294 + # an older Python that can't handle Unicode keys in kwargs.
1.1295 + # All of the keys in question should be ASCII.
1.1296 + fixedInfo = {}
1.1297 + for k, v in self.mozInfo.items():
1.1298 + if isinstance(k, unicode):
1.1299 + k = k.encode('ascii')
1.1300 + fixedInfo[k] = v
1.1301 + self.mozInfo = fixedInfo
1.1302 +
1.1303 + mozinfo.update(self.mozInfo)
1.1304 +
1.1305 + # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized.
1.1306 + self.buildEnvironment()
1.1307 +
1.1308 + # The appDirKey is a optional entry in either the default or individual test
1.1309 + # sections that defines a relative application directory for test runs. If
1.1310 + # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell
1.1311 + # test harness.
1.1312 + appDirKey = None
1.1313 + if "appname" in self.mozInfo:
1.1314 + appDirKey = self.mozInfo["appname"] + "-appdir"
1.1315 +
1.1316 + # We have to do this before we build the test list so we know whether or
1.1317 + # not to run tests that depend on having the node spdy server
1.1318 + self.trySetupNode()
1.1319 +
1.1320 + pStdout, pStderr = self.getPipes()
1.1321 +
1.1322 + self.buildTestList()
1.1323 + if self.singleFile:
1.1324 + self.sequential = True
1.1325 +
1.1326 + if shuffle:
1.1327 + random.shuffle(self.alltests)
1.1328 +
1.1329 + self.xunitResults = []
1.1330 + self.cleanup_dir_list = []
1.1331 + self.try_again_list = []
1.1332 +
1.1333 + kwargs = {
1.1334 + 'appPath': self.appPath,
1.1335 + 'xrePath': self.xrePath,
1.1336 + 'testingModulesDir': self.testingModulesDir,
1.1337 + 'debuggerInfo': self.debuggerInfo,
1.1338 + 'pluginsPath': self.pluginsPath,
1.1339 + 'httpdManifest': self.httpdManifest,
1.1340 + 'httpdJSPath': self.httpdJSPath,
1.1341 + 'headJSPath': self.headJSPath,
1.1342 + 'testharnessdir': self.testharnessdir,
1.1343 + 'profileName': self.profileName,
1.1344 + 'singleFile': self.singleFile,
1.1345 + 'env': self.env, # making a copy of this in the testthreads
1.1346 + 'symbolsPath': self.symbolsPath,
1.1347 + 'logfiles': self.logfiles,
1.1348 + 'xpcshell': self.xpcshell,
1.1349 + 'xpcsRunArgs': self.xpcsRunArgs,
1.1350 + 'failureManifest': failureManifest,
1.1351 + 'on_message': self.on_message,
1.1352 + }
1.1353 +
1.1354 + if self.sequential:
1.1355 + # Allow user to kill hung xpcshell subprocess with SIGINT
1.1356 + # when we are only running tests sequentially.
1.1357 + signal.signal(signal.SIGINT, markGotSIGINT)
1.1358 +
1.1359 + if self.debuggerInfo:
1.1360 + # Force a sequential run
1.1361 + self.sequential = True
1.1362 +
1.1363 + # If we have an interactive debugger, disable SIGINT entirely.
1.1364 + if self.debuggerInfo["interactive"]:
1.1365 + signal.signal(signal.SIGINT, lambda signum, frame: None)
1.1366 +
1.1367 + # create a queue of all tests that will run
1.1368 + tests_queue = deque()
1.1369 + # also a list for the tests that need to be run sequentially
1.1370 + sequential_tests = []
1.1371 + for test_object in self.alltests:
1.1372 + name = test_object['path']
1.1373 + if self.singleFile and not name.endswith(self.singleFile):
1.1374 + continue
1.1375 +
1.1376 + if self.testPath and name.find(self.testPath) == -1:
1.1377 + continue
1.1378 +
1.1379 + self.testCount += 1
1.1380 +
1.1381 + test = testClass(test_object, self.event, self.cleanup_dir_list,
1.1382 + tests_root_dir=testsRootDir, app_dir_key=appDirKey,
1.1383 + interactive=interactive, verbose=verbose, pStdout=pStdout,
1.1384 + pStderr=pStderr, keep_going=keepGoing, log=self.log,
1.1385 + mobileArgs=mobileArgs, **kwargs)
1.1386 + if 'run-sequentially' in test_object or self.sequential:
1.1387 + sequential_tests.append(test)
1.1388 + else:
1.1389 + tests_queue.append(test)
1.1390 +
1.1391 + if self.sequential:
1.1392 + self.log.info("INFO | Running tests sequentially.")
1.1393 + else:
1.1394 + self.log.info("INFO | Using at most %d threads." % NUM_THREADS)
1.1395 +
1.1396 + # keep a set of NUM_THREADS running tests and start running the
1.1397 + # tests in the queue at most NUM_THREADS at a time
1.1398 + running_tests = set()
1.1399 + keep_going = True
1.1400 + exceptions = []
1.1401 + tracebacks = []
1.1402 + while tests_queue or running_tests:
1.1403 + # if we're not supposed to continue and all of the running tests
1.1404 + # are done, stop
1.1405 + if not keep_going and not running_tests:
1.1406 + break
1.1407 +
1.1408 + # if there's room to run more tests, start running them
1.1409 + while keep_going and tests_queue and (len(running_tests) < NUM_THREADS):
1.1410 + test = tests_queue.popleft()
1.1411 + running_tests.add(test)
1.1412 + test.start()
1.1413 +
1.1414 + # queue is full (for now) or no more new tests,
1.1415 + # process the finished tests so far
1.1416 +
1.1417 + # wait for at least one of the tests to finish
1.1418 + self.event.wait(1)
1.1419 + self.event.clear()
1.1420 +
1.1421 + # find what tests are done (might be more than 1)
1.1422 + done_tests = set()
1.1423 + for test in running_tests:
1.1424 + if test.done:
1.1425 + done_tests.add(test)
1.1426 + test.join(1) # join with timeout so we don't hang on blocked threads
1.1427 + # if the test had trouble, we will try running it again
1.1428 + # at the end of the run
1.1429 + if test.retry or test.is_alive():
1.1430 + # if the join call timed out, test.is_alive => True
1.1431 + self.try_again_list.append(test.test_object)
1.1432 + continue
1.1433 + # did the test encounter any exception?
1.1434 + if test.exception:
1.1435 + exceptions.append(test.exception)
1.1436 + tracebacks.append(test.traceback)
1.1437 + # we won't add any more tests, will just wait for
1.1438 + # the currently running ones to finish
1.1439 + keep_going = False
1.1440 + keep_going = keep_going and test.keep_going
1.1441 + self.addTestResults(test)
1.1442 +
1.1443 + # make room for new tests to run
1.1444 + running_tests.difference_update(done_tests)
1.1445 +
1.1446 + if keep_going:
1.1447 + # run the other tests sequentially
1.1448 + for test in sequential_tests:
1.1449 + if not keep_going:
1.1450 + self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
1.1451 + "(Use --keep-going to keep running tests after killing one with SIGINT)")
1.1452 + break
1.1453 + # we don't want to retry these tests
1.1454 + test.retry = False
1.1455 + test.start()
1.1456 + test.join()
1.1457 + self.addTestResults(test)
1.1458 + # did the test encounter any exception?
1.1459 + if test.exception:
1.1460 + exceptions.append(test.exception)
1.1461 + tracebacks.append(test.traceback)
1.1462 + break
1.1463 + keep_going = test.keep_going
1.1464 +
1.1465 + # retry tests that failed when run in parallel
1.1466 + if self.try_again_list:
1.1467 + self.log.info("Retrying tests that failed when run in parallel.")
1.1468 + for test_object in self.try_again_list:
1.1469 + test = testClass(test_object, self.event, self.cleanup_dir_list,
1.1470 + retry=False, tests_root_dir=testsRootDir,
1.1471 + app_dir_key=appDirKey, interactive=interactive,
1.1472 + verbose=verbose, pStdout=pStdout, pStderr=pStderr,
1.1473 + keep_going=keepGoing, log=self.log, mobileArgs=mobileArgs,
1.1474 + **kwargs)
1.1475 + test.start()
1.1476 + test.join()
1.1477 + self.addTestResults(test)
1.1478 + # did the test encounter any exception?
1.1479 + if test.exception:
1.1480 + exceptions.append(test.exception)
1.1481 + tracebacks.append(test.traceback)
1.1482 + break
1.1483 + keep_going = test.keep_going
1.1484 +
1.1485 + # restore default SIGINT behaviour
1.1486 + signal.signal(signal.SIGINT, signal.SIG_DFL)
1.1487 +
1.1488 + self.shutdownNode()
1.1489 + # Clean up any slacker directories that might be lying around
1.1490 + # Some might fail because of windows taking too long to unlock them.
1.1491 + # We don't do anything if this fails because the test slaves will have
1.1492 + # their $TEMP dirs cleaned up on reboot anyway.
1.1493 + for directory in self.cleanup_dir_list:
1.1494 + try:
1.1495 + shutil.rmtree(directory)
1.1496 + except:
1.1497 + self.log.info("INFO | %s could not be cleaned up." % directory)
1.1498 +
1.1499 + if exceptions:
1.1500 + self.log.info("INFO | Following exceptions were raised:")
1.1501 + for t in tracebacks:
1.1502 + self.log.error(t)
1.1503 + raise exceptions[0]
1.1504 +
1.1505 + if self.testCount == 0:
1.1506 + self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?")
1.1507 + self.failCount = 1
1.1508 +
1.1509 + self.log.info("INFO | Result summary:")
1.1510 + self.log.info("INFO | Passed: %d" % self.passCount)
1.1511 + self.log.info("INFO | Failed: %d" % self.failCount)
1.1512 + self.log.info("INFO | Todo: %d" % self.todoCount)
1.1513 + self.log.info("INFO | Retried: %d" % len(self.try_again_list))
1.1514 +
1.1515 + if autolog:
1.1516 + self.post_to_autolog(self.xunitResults, xunitName)
1.1517 +
1.1518 + if xunitFilename is not None:
1.1519 + self.writeXunitResults(filename=xunitFilename, results=self.xunitResults,
1.1520 + name=xunitName)
1.1521 +
1.1522 + if gotSIGINT and not keepGoing:
1.1523 + self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
1.1524 + "(Use --keep-going to keep running tests after killing one with SIGINT)")
1.1525 + return False
1.1526 +
1.1527 + return self.failCount == 0
1.1528 +
1.1529 +class XPCShellOptions(OptionParser):
1.1530 + def __init__(self):
1.1531 + """Process command line arguments and call runTests() to do the real work."""
1.1532 + OptionParser.__init__(self)
1.1533 +
1.1534 + addCommonOptions(self)
1.1535 + self.add_option("--app-path",
1.1536 + type="string", dest="appPath", default=None,
1.1537 + help="application directory (as opposed to XRE directory)")
1.1538 + self.add_option("--autolog",
1.1539 + action="store_true", dest="autolog", default=False,
1.1540 + help="post to autolog")
1.1541 + self.add_option("--interactive",
1.1542 + action="store_true", dest="interactive", default=False,
1.1543 + help="don't automatically run tests, drop to an xpcshell prompt")
1.1544 + self.add_option("--verbose",
1.1545 + action="store_true", dest="verbose", default=False,
1.1546 + help="always print stdout and stderr from tests")
1.1547 + self.add_option("--keep-going",
1.1548 + action="store_true", dest="keepGoing", default=False,
1.1549 + help="continue running tests after test killed with control-C (SIGINT)")
1.1550 + self.add_option("--logfiles",
1.1551 + action="store_true", dest="logfiles", default=True,
1.1552 + help="create log files (default, only used to override --no-logfiles)")
1.1553 + self.add_option("--manifest",
1.1554 + type="string", dest="manifest", default=None,
1.1555 + help="Manifest of test directories to use")
1.1556 + self.add_option("--no-logfiles",
1.1557 + action="store_false", dest="logfiles",
1.1558 + help="don't create log files")
1.1559 + self.add_option("--sequential",
1.1560 + action="store_true", dest="sequential", default=False,
1.1561 + help="Run all tests sequentially")
1.1562 + self.add_option("--test-path",
1.1563 + type="string", dest="testPath", default=None,
1.1564 + help="single path and/or test filename to test")
1.1565 + self.add_option("--tests-root-dir",
1.1566 + type="string", dest="testsRootDir", default=None,
1.1567 + help="absolute path to directory where all tests are located. this is typically $(objdir)/_tests")
1.1568 + self.add_option("--testing-modules-dir",
1.1569 + dest="testingModulesDir", default=None,
1.1570 + help="Directory where testing modules are located.")
1.1571 + self.add_option("--test-plugin-path",
1.1572 + type="string", dest="pluginsPath", default=None,
1.1573 + help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
1.1574 + "By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
1.1575 + "to return for NS_APP_PLUGINS_DIR_LIST when queried.")
1.1576 + self.add_option("--total-chunks",
1.1577 + type = "int", dest = "totalChunks", default=1,
1.1578 + help = "how many chunks to split the tests up into")
1.1579 + self.add_option("--this-chunk",
1.1580 + type = "int", dest = "thisChunk", default=1,
1.1581 + help = "which chunk to run between 1 and --total-chunks")
1.1582 + self.add_option("--profile-name",
1.1583 + type = "string", dest="profileName", default=None,
1.1584 + help="name of application profile being tested")
1.1585 + self.add_option("--build-info-json",
1.1586 + type = "string", dest="mozInfo", default=None,
1.1587 + help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
1.1588 + self.add_option("--shuffle",
1.1589 + action="store_true", dest="shuffle", default=False,
1.1590 + help="Execute tests in random order")
1.1591 + self.add_option("--xunit-file", dest="xunitFilename",
1.1592 + help="path to file where xUnit results will be written.")
1.1593 + self.add_option("--xunit-suite-name", dest="xunitName",
1.1594 + help="name to record for this xUnit test suite. Many "
1.1595 + "tools expect Java class notation, e.g. "
1.1596 + "dom.basic.foo")
1.1597 + self.add_option("--failure-manifest", dest="failureManifest",
1.1598 + action="store",
1.1599 + help="path to file where failure manifest will be written.")
1.1600 +
1.1601 +def main():
1.1602 + parser = XPCShellOptions()
1.1603 + options, args = parser.parse_args()
1.1604 +
1.1605 + if len(args) < 2 and options.manifest is None or \
1.1606 + (len(args) < 1 and options.manifest is not None):
1.1607 + print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
1.1608 + or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
1.1609 + sys.argv[0])
1.1610 + sys.exit(1)
1.1611 +
1.1612 + xpcsh = XPCShellTests()
1.1613 +
1.1614 + if options.interactive and not options.testPath:
1.1615 + print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
1.1616 + sys.exit(1)
1.1617 +
1.1618 + if not xpcsh.runTests(args[0], testdirs=args[1:], **options.__dict__):
1.1619 + sys.exit(1)
1.1620 +
1.1621 +if __name__ == '__main__':
1.1622 + main()
