michael@0: #!/usr/bin/env python michael@0: # michael@0: # This Source Code Form is subject to the terms of the Mozilla Public michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/. michael@0: michael@0: import copy michael@0: import json michael@0: import math michael@0: import os michael@0: import os.path michael@0: import random michael@0: import re michael@0: import shutil michael@0: import signal michael@0: import socket michael@0: import sys michael@0: import time michael@0: import traceback michael@0: import xml.dom.minidom michael@0: from collections import deque michael@0: from distutils import dir_util michael@0: from multiprocessing import cpu_count michael@0: from optparse import OptionParser michael@0: from subprocess import Popen, PIPE, STDOUT michael@0: from tempfile import mkdtemp, gettempdir michael@0: from threading import Timer, Thread, Event, RLock michael@0: michael@0: try: michael@0: import psutil michael@0: HAVE_PSUTIL = True michael@0: except ImportError: michael@0: HAVE_PSUTIL = False michael@0: michael@0: from automation import Automation, getGlobalLog, resetGlobalLog michael@0: from automationutils import * michael@0: michael@0: # Printing buffered output in case of a failure or verbose mode will result michael@0: # in buffered output interleaved with other threads' output. michael@0: # To prevent his, each call to the logger as well as any blocks of output that michael@0: # are intended to be continuous are protected by the same lock. michael@0: LOG_MUTEX = RLock() michael@0: michael@0: HARNESS_TIMEOUT = 5 * 60 michael@0: michael@0: # benchmarking on tbpl revealed that this works best for now michael@0: NUM_THREADS = int(cpu_count() * 4) michael@0: michael@0: FAILURE_ACTIONS = set(['test_unexpected_fail', michael@0: 'test_unexpected_pass', michael@0: 'javascript_error']) michael@0: ACTION_STRINGS = { michael@0: "test_unexpected_fail": "TEST-UNEXPECTED-FAIL", michael@0: "test_known_fail": "TEST-KNOWN-FAIL", michael@0: "test_unexpected_pass": "TEST-UNEXPECTED-PASS", michael@0: "javascript_error": "TEST-UNEXPECTED-FAIL", michael@0: "test_pass": "TEST-PASS", michael@0: "test_info": "TEST-INFO" michael@0: } michael@0: michael@0: # -------------------------------------------------------------- michael@0: # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900 michael@0: # michael@0: here = os.path.dirname(__file__) michael@0: mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase')) michael@0: michael@0: if os.path.isdir(mozbase): michael@0: for package in os.listdir(mozbase): michael@0: sys.path.append(os.path.join(mozbase, package)) michael@0: michael@0: import manifestparser michael@0: import mozcrash michael@0: import mozinfo michael@0: michael@0: # -------------------------------------------------------------- michael@0: michael@0: # TODO: perhaps this should be in a more generally shared location? michael@0: # This regex matches all of the C0 and C1 control characters michael@0: # (U+0000 through U+001F; U+007F; U+0080 through U+009F), michael@0: # except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C). michael@0: # A raw string is deliberately not used. michael@0: _cleanup_encoding_re = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]') michael@0: def _cleanup_encoding_repl(m): michael@0: c = m.group(0) michael@0: return '\\\\' if c == '\\' else '\\x{0:02X}'.format(ord(c)) michael@0: def cleanup_encoding(s): michael@0: """S is either a byte or unicode string. Either way it may michael@0: contain control characters, unpaired surrogates, reserved code michael@0: points, etc. If it is a byte string, it is assumed to be michael@0: UTF-8, but it may not be *correct* UTF-8. Produce a byte michael@0: string that can safely be dumped into a (generally UTF-8-coded) michael@0: logfile.""" michael@0: if not isinstance(s, unicode): michael@0: s = s.decode('utf-8', 'replace') michael@0: if s.endswith('\n'): michael@0: # A new line is always added by head.js to delimit messages, michael@0: # however consumers will want to supply their own. michael@0: s = s[:-1] michael@0: # Replace all C0 and C1 control characters with \xNN escapes. michael@0: s = _cleanup_encoding_re.sub(_cleanup_encoding_repl, s) michael@0: return s.encode('utf-8', 'backslashreplace') michael@0: michael@0: """ Control-C handling """ michael@0: gotSIGINT = False michael@0: def markGotSIGINT(signum, stackFrame): michael@0: global gotSIGINT michael@0: gotSIGINT = True michael@0: michael@0: class XPCShellTestThread(Thread): michael@0: def __init__(self, test_object, event, cleanup_dir_list, retry=True, michael@0: tests_root_dir=None, app_dir_key=None, interactive=False, michael@0: verbose=False, pStdout=None, pStderr=None, keep_going=False, michael@0: log=None, **kwargs): michael@0: Thread.__init__(self) michael@0: self.daemon = True michael@0: michael@0: self.test_object = test_object michael@0: self.cleanup_dir_list = cleanup_dir_list michael@0: self.retry = retry michael@0: michael@0: self.appPath = kwargs.get('appPath') michael@0: self.xrePath = kwargs.get('xrePath') michael@0: self.testingModulesDir = kwargs.get('testingModulesDir') michael@0: self.debuggerInfo = kwargs.get('debuggerInfo') michael@0: self.pluginsPath = kwargs.get('pluginsPath') michael@0: self.httpdManifest = kwargs.get('httpdManifest') michael@0: self.httpdJSPath = kwargs.get('httpdJSPath') michael@0: self.headJSPath = kwargs.get('headJSPath') michael@0: self.testharnessdir = kwargs.get('testharnessdir') michael@0: self.profileName = kwargs.get('profileName') michael@0: self.singleFile = kwargs.get('singleFile') michael@0: self.env = copy.deepcopy(kwargs.get('env')) michael@0: self.symbolsPath = kwargs.get('symbolsPath') michael@0: self.logfiles = kwargs.get('logfiles') michael@0: self.xpcshell = kwargs.get('xpcshell') michael@0: self.xpcsRunArgs = kwargs.get('xpcsRunArgs') michael@0: self.failureManifest = kwargs.get('failureManifest') michael@0: self.on_message = kwargs.get('on_message') michael@0: michael@0: self.tests_root_dir = tests_root_dir michael@0: self.app_dir_key = app_dir_key michael@0: self.interactive = interactive michael@0: self.verbose = verbose michael@0: self.pStdout = pStdout michael@0: self.pStderr = pStderr michael@0: self.keep_going = keep_going michael@0: self.log = log michael@0: michael@0: # only one of these will be set to 1. adding them to the totals in michael@0: # the harness michael@0: self.passCount = 0 michael@0: self.todoCount = 0 michael@0: self.failCount = 0 michael@0: michael@0: self.output_lines = [] michael@0: self.has_failure_output = False michael@0: self.saw_proc_start = False michael@0: self.saw_proc_end = False michael@0: michael@0: # event from main thread to signal work done michael@0: self.event = event michael@0: self.done = False # explicitly set flag so we don't rely on thread.isAlive michael@0: michael@0: def run(self): michael@0: try: michael@0: self.run_test() michael@0: except Exception as e: michael@0: self.exception = e michael@0: self.traceback = traceback.format_exc() michael@0: else: michael@0: self.exception = None michael@0: self.traceback = None michael@0: if self.retry: michael@0: self.log.info("TEST-INFO | %s | Test failed or timed out, will retry." michael@0: % self.test_object['name']) michael@0: self.done = True michael@0: self.event.set() michael@0: michael@0: def kill(self, proc): michael@0: """ michael@0: Simple wrapper to kill a process. michael@0: On a remote system, this is overloaded to handle remote process communication. michael@0: """ michael@0: return proc.kill() michael@0: michael@0: def removeDir(self, dirname): michael@0: """ michael@0: Simple wrapper to remove (recursively) a given directory. michael@0: On a remote system, we need to overload this to work on the remote filesystem. michael@0: """ michael@0: shutil.rmtree(dirname) michael@0: michael@0: def poll(self, proc): michael@0: """ michael@0: Simple wrapper to check if a process has terminated. michael@0: On a remote system, this is overloaded to handle remote process communication. michael@0: """ michael@0: return proc.poll() michael@0: michael@0: def createLogFile(self, test_file, stdout): michael@0: """ michael@0: For a given test file and stdout buffer, create a log file. michael@0: On a remote system we have to fix the test name since it can contain directories. michael@0: """ michael@0: with open(test_file + ".log", "w") as f: michael@0: f.write(stdout) michael@0: michael@0: def getReturnCode(self, proc): michael@0: """ michael@0: Simple wrapper to get the return code for a given process. michael@0: On a remote system we overload this to work with the remote process management. michael@0: """ michael@0: return proc.returncode michael@0: michael@0: def communicate(self, proc): michael@0: """ michael@0: Simple wrapper to communicate with a process. michael@0: On a remote system, this is overloaded to handle remote process communication. michael@0: """ michael@0: # Processing of incremental output put here to michael@0: # sidestep issues on remote platforms, where what we know michael@0: # as proc is a file pulled off of a device. michael@0: if proc.stdout: michael@0: while True: michael@0: line = proc.stdout.readline() michael@0: if not line: michael@0: break michael@0: self.process_line(line) michael@0: michael@0: if self.saw_proc_start and not self.saw_proc_end: michael@0: self.has_failure_output = True michael@0: michael@0: return proc.communicate() michael@0: michael@0: def launchProcess(self, cmd, stdout, stderr, env, cwd): michael@0: """ michael@0: Simple wrapper to launch a process. michael@0: On a remote system, this is more complex and we need to overload this function. michael@0: """ michael@0: if HAVE_PSUTIL: michael@0: popen_func = psutil.Popen michael@0: else: michael@0: popen_func = Popen michael@0: proc = popen_func(cmd, stdout=stdout, stderr=stderr, michael@0: env=env, cwd=cwd) michael@0: return proc michael@0: michael@0: def checkForCrashes(self, michael@0: dump_directory, michael@0: symbols_path, michael@0: test_name=None): michael@0: """ michael@0: Simple wrapper to check for crashes. michael@0: On a remote system, this is more complex and we need to overload this function. michael@0: """ michael@0: return mozcrash.check_for_crashes(dump_directory, symbols_path, test_name=test_name) michael@0: michael@0: def logCommand(self, name, completeCmd, testdir): michael@0: self.log.info("TEST-INFO | %s | full command: %r" % (name, completeCmd)) michael@0: self.log.info("TEST-INFO | %s | current directory: %r" % (name, testdir)) michael@0: # Show only those environment variables that are changed from michael@0: # the ambient environment. michael@0: changedEnv = (set("%s=%s" % i for i in self.env.iteritems()) michael@0: - set("%s=%s" % i for i in os.environ.iteritems())) michael@0: self.log.info("TEST-INFO | %s | environment: %s" % (name, list(changedEnv))) michael@0: michael@0: def testTimeout(self, test_file, proc): michael@0: if not self.retry: michael@0: self.log.error("TEST-UNEXPECTED-FAIL | %s | Test timed out" % test_file) michael@0: self.done = True michael@0: Automation().killAndGetStackNoScreenshot(proc.pid, self.appPath, self.debuggerInfo) michael@0: michael@0: def buildCmdTestFile(self, name): michael@0: """ michael@0: Build the command line arguments for the test file. michael@0: On a remote system, this may be overloaded to use a remote path structure. michael@0: """ michael@0: return ['-e', 'const _TEST_FILE = ["%s"];' % michael@0: replaceBackSlashes(name)] michael@0: michael@0: def setupTempDir(self): michael@0: tempDir = mkdtemp() michael@0: self.env["XPCSHELL_TEST_TEMP_DIR"] = tempDir michael@0: if self.interactive: michael@0: self.log.info("TEST-INFO | temp dir is %s" % tempDir) michael@0: return tempDir michael@0: michael@0: def setupPluginsDir(self): michael@0: if not os.path.isdir(self.pluginsPath): michael@0: return None michael@0: michael@0: pluginsDir = mkdtemp() michael@0: # shutil.copytree requires dst to not exist. Deleting the tempdir michael@0: # would make a race condition possible in a concurrent environment, michael@0: # so we are using dir_utils.copy_tree which accepts an existing dst michael@0: dir_util.copy_tree(self.pluginsPath, pluginsDir) michael@0: if self.interactive: michael@0: self.log.info("TEST-INFO | plugins dir is %s" % pluginsDir) michael@0: return pluginsDir michael@0: michael@0: def setupProfileDir(self): michael@0: """ michael@0: Create a temporary folder for the profile and set appropriate environment variables. michael@0: When running check-interactive and check-one, the directory is well-defined and michael@0: retained for inspection once the tests complete. michael@0: michael@0: On a remote system, this may be overloaded to use a remote path structure. michael@0: """ michael@0: if self.interactive or self.singleFile: michael@0: profileDir = os.path.join(gettempdir(), self.profileName, "xpcshellprofile") michael@0: try: michael@0: # This could be left over from previous runs michael@0: self.removeDir(profileDir) michael@0: except: michael@0: pass michael@0: os.makedirs(profileDir) michael@0: else: michael@0: profileDir = mkdtemp() michael@0: self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir michael@0: if self.interactive or self.singleFile: michael@0: self.log.info("TEST-INFO | profile dir is %s" % profileDir) michael@0: return profileDir michael@0: michael@0: def buildCmdHead(self, headfiles, tailfiles, xpcscmd): michael@0: """ michael@0: Build the command line arguments for the head and tail files, michael@0: along with the address of the webserver which some tests require. michael@0: michael@0: On a remote system, this is overloaded to resolve quoting issues over a secondary command line. michael@0: """ michael@0: cmdH = ", ".join(['"' + replaceBackSlashes(f) + '"' michael@0: for f in headfiles]) michael@0: cmdT = ", ".join(['"' + replaceBackSlashes(f) + '"' michael@0: for f in tailfiles]) michael@0: return xpcscmd + \ michael@0: ['-e', 'const _SERVER_ADDR = "localhost"', michael@0: '-e', 'const _HEAD_FILES = [%s];' % cmdH, michael@0: '-e', 'const _TAIL_FILES = [%s];' % cmdT] michael@0: michael@0: def getHeadAndTailFiles(self, test_object): michael@0: """Obtain the list of head and tail files. michael@0: michael@0: Returns a 2-tuple. The first element is a list of head files. The second michael@0: is a list of tail files. michael@0: """ michael@0: def sanitize_list(s, kind): michael@0: for f in s.strip().split(' '): michael@0: f = f.strip() michael@0: if len(f) < 1: michael@0: continue michael@0: michael@0: path = os.path.normpath(os.path.join(test_object['here'], f)) michael@0: if not os.path.exists(path): michael@0: raise Exception('%s file does not exist: %s' % (kind, path)) michael@0: michael@0: if not os.path.isfile(path): michael@0: raise Exception('%s file is not a file: %s' % (kind, path)) michael@0: michael@0: yield path michael@0: michael@0: return (list(sanitize_list(test_object['head'], 'head')), michael@0: list(sanitize_list(test_object['tail'], 'tail'))) michael@0: michael@0: def buildXpcsCmd(self, testdir): michael@0: """ michael@0: Load the root head.js file as the first file in our test path, before other head, test, and tail files. michael@0: On a remote system, we overload this to add additional command line arguments, so this gets overloaded. michael@0: """ michael@0: # - NOTE: if you rename/add any of the constants set here, update michael@0: # do_load_child_test_harness() in head.js michael@0: if not self.appPath: michael@0: self.appPath = self.xrePath michael@0: michael@0: self.xpcsCmd = [ michael@0: self.xpcshell, michael@0: '-g', self.xrePath, michael@0: '-a', self.appPath, michael@0: '-r', self.httpdManifest, michael@0: '-m', michael@0: '-s', michael@0: '-e', 'const _HTTPD_JS_PATH = "%s";' % self.httpdJSPath, michael@0: '-e', 'const _HEAD_JS_PATH = "%s";' % self.headJSPath michael@0: ] michael@0: michael@0: if self.testingModulesDir: michael@0: # Escape backslashes in string literal. michael@0: sanitized = self.testingModulesDir.replace('\\', '\\\\') michael@0: self.xpcsCmd.extend([ michael@0: '-e', michael@0: 'const _TESTING_MODULES_DIR = "%s";' % sanitized michael@0: ]) michael@0: michael@0: self.xpcsCmd.extend(['-f', os.path.join(self.testharnessdir, 'head.js')]) michael@0: michael@0: if self.debuggerInfo: michael@0: self.xpcsCmd = [self.debuggerInfo["path"]] + self.debuggerInfo["args"] + self.xpcsCmd michael@0: michael@0: # Automation doesn't specify a pluginsPath and xpcshell defaults to michael@0: # $APPDIR/plugins. We do the same here so we can carry on with michael@0: # setting up every test with its own plugins directory. michael@0: if not self.pluginsPath: michael@0: self.pluginsPath = os.path.join(self.appPath, 'plugins') michael@0: michael@0: self.pluginsDir = self.setupPluginsDir() michael@0: if self.pluginsDir: michael@0: self.xpcsCmd.extend(['-p', self.pluginsDir]) michael@0: michael@0: def cleanupDir(self, directory, name, xunit_result): michael@0: if not os.path.exists(directory): michael@0: return michael@0: michael@0: TRY_LIMIT = 25 # up to TRY_LIMIT attempts (one every second), because michael@0: # the Windows filesystem is slow to react to the changes michael@0: try_count = 0 michael@0: while try_count < TRY_LIMIT: michael@0: try: michael@0: self.removeDir(directory) michael@0: except OSError: michael@0: self.log.info("TEST-INFO | Failed to remove directory: %s. Waiting." % directory) michael@0: # We suspect the filesystem may still be making changes. Wait a michael@0: # little bit and try again. michael@0: time.sleep(1) michael@0: try_count += 1 michael@0: else: michael@0: # removed fine michael@0: return michael@0: michael@0: # we try cleaning up again later at the end of the run michael@0: self.cleanup_dir_list.append(directory) michael@0: michael@0: def clean_temp_dirs(self, name, stdout): michael@0: # We don't want to delete the profile when running check-interactive michael@0: # or check-one. michael@0: if self.profileDir and not self.interactive and not self.singleFile: michael@0: self.cleanupDir(self.profileDir, name, self.xunit_result) michael@0: michael@0: self.cleanupDir(self.tempDir, name, self.xunit_result) michael@0: michael@0: if self.pluginsDir: michael@0: self.cleanupDir(self.pluginsDir, name, self.xunit_result) michael@0: michael@0: def message_from_line(self, line): michael@0: """ Given a line of raw output, convert to a string message. """ michael@0: if isinstance(line, basestring): michael@0: # This function has received unstructured output. michael@0: if line: michael@0: if 'TEST-UNEXPECTED-' in line: michael@0: self.has_failure_output = True michael@0: return line michael@0: michael@0: msg = ['%s: ' % line['process'] if 'process' in line else ''] michael@0: michael@0: # Each call to the logger in head.js either specified '_message' michael@0: # or both 'source_file' and 'diagnostic'. If either of these are michael@0: # missing, they ended up being undefined as a result of the way michael@0: # the test was run. michael@0: if '_message' in line: michael@0: msg.append(line['_message']) michael@0: if 'diagnostic' in line: michael@0: msg.append('\nDiagnostic: %s' % line['diagnostic']) michael@0: else: michael@0: msg.append('%s | %s | %s' % (ACTION_STRINGS[line['action']], michael@0: line.get('source_file', 'undefined'), michael@0: line.get('diagnostic', 'undefined'))) michael@0: michael@0: msg.append('\n%s' % line['stack'] if 'stack' in line else '') michael@0: return ''.join(msg) michael@0: michael@0: def parse_output(self, output): michael@0: """Parses process output for structured messages and saves output as it is michael@0: read. Sets self.has_failure_output in case of evidence of a failure""" michael@0: for line_string in output.splitlines(): michael@0: self.process_line(line_string) michael@0: michael@0: if self.saw_proc_start and not self.saw_proc_end: michael@0: self.has_failure_output = True michael@0: michael@0: def report_message(self, line): michael@0: """ Reports a message to a consumer, both as a strucutured and michael@0: human-readable log message. """ michael@0: michael@0: message = cleanup_encoding(self.message_from_line(line)) michael@0: if message.endswith('\n'): michael@0: # A new line is always added by head.js to delimit messages, michael@0: # however consumers will want to supply their own. michael@0: message = message[:-1] michael@0: michael@0: if self.on_message: michael@0: self.on_message(line, message) michael@0: else: michael@0: self.output_lines.append(message) michael@0: michael@0: def process_line(self, line_string): michael@0: """ Parses a single line of output, determining its significance and michael@0: reporting a message. michael@0: """ michael@0: try: michael@0: line_object = json.loads(line_string) michael@0: if not isinstance(line_object, dict): michael@0: self.report_message(line_string) michael@0: return michael@0: except ValueError: michael@0: self.report_message(line_string) michael@0: return michael@0: michael@0: if 'action' not in line_object: michael@0: # In case a test outputs something that happens to be valid michael@0: # JSON. michael@0: self.report_message(line_string) michael@0: return michael@0: michael@0: action = line_object['action'] michael@0: self.report_message(line_object) michael@0: michael@0: if action in FAILURE_ACTIONS: michael@0: self.has_failure_output = True michael@0: elif action == 'child_test_start': michael@0: self.saw_proc_start = True michael@0: elif action == 'child_test_end': michael@0: self.saw_proc_end = True michael@0: michael@0: def log_output(self, output): michael@0: """Prints given output line-by-line to avoid overflowing buffers.""" michael@0: self.log.info(">>>>>>>") michael@0: if output: michael@0: if isinstance(output, basestring): michael@0: output = output.splitlines() michael@0: for part in output: michael@0: # For multi-line output, such as a stack trace michael@0: for line in part.splitlines(): michael@0: try: michael@0: line = line.decode('utf-8') michael@0: except UnicodeDecodeError: michael@0: self.log.info("TEST-INFO | %s | Detected non UTF-8 output."\ michael@0: " Please modify the test to only print UTF-8." % michael@0: self.test_object['name']) michael@0: # add '?' instead of funky bytes michael@0: line = line.decode('utf-8', 'replace') michael@0: self.log.info(line) michael@0: self.log.info("<<<<<<<") michael@0: michael@0: def run_test(self): michael@0: """Run an individual xpcshell test.""" michael@0: global gotSIGINT michael@0: michael@0: name = self.test_object['path'] michael@0: michael@0: self.xunit_result = {'name': self.test_object['name'], 'classname': 'xpcshell'} michael@0: michael@0: # The xUnit package is defined as the path component between the root michael@0: # dir and the test with path characters replaced with '.' (using Java michael@0: # class notation). michael@0: if self.tests_root_dir is not None: michael@0: self.tests_root_dir = os.path.normpath(self.tests_root_dir) michael@0: if os.path.normpath(self.test_object['here']).find(self.tests_root_dir) != 0: michael@0: raise Exception('tests_root_dir is not a parent path of %s' % michael@0: self.test_object['here']) michael@0: relpath = self.test_object['here'][len(self.tests_root_dir):].lstrip('/\\') michael@0: self.xunit_result['classname'] = relpath.replace('/', '.').replace('\\', '.') michael@0: michael@0: # Check for skipped tests michael@0: if 'disabled' in self.test_object: michael@0: self.log.info('TEST-INFO | skipping %s | %s' % michael@0: (name, self.test_object['disabled'])) michael@0: michael@0: self.xunit_result['skipped'] = True michael@0: self.retry = False michael@0: michael@0: self.keep_going = True michael@0: return michael@0: michael@0: # Check for known-fail tests michael@0: expected = self.test_object['expected'] == 'pass' michael@0: michael@0: # By default self.appPath will equal the gre dir. If specified in the michael@0: # xpcshell.ini file, set a different app dir for this test. michael@0: if self.app_dir_key and self.app_dir_key in self.test_object: michael@0: rel_app_dir = self.test_object[self.app_dir_key] michael@0: rel_app_dir = os.path.join(self.xrePath, rel_app_dir) michael@0: self.appPath = os.path.abspath(rel_app_dir) michael@0: else: michael@0: self.appPath = None michael@0: michael@0: test_dir = os.path.dirname(name) michael@0: self.buildXpcsCmd(test_dir) michael@0: head_files, tail_files = self.getHeadAndTailFiles(self.test_object) michael@0: cmdH = self.buildCmdHead(head_files, tail_files, self.xpcsCmd) michael@0: michael@0: # Create a profile and a temp dir that the JS harness can stick michael@0: # a profile and temporary data in michael@0: self.profileDir = self.setupProfileDir() michael@0: self.tempDir = self.setupTempDir() michael@0: michael@0: # The test file will have to be loaded after the head files. michael@0: cmdT = self.buildCmdTestFile(name) michael@0: michael@0: args = self.xpcsRunArgs[:] michael@0: if 'debug' in self.test_object: michael@0: args.insert(0, '-d') michael@0: michael@0: completeCmd = cmdH + cmdT + args michael@0: michael@0: testTimeoutInterval = HARNESS_TIMEOUT michael@0: # Allow a test to request a multiple of the timeout if it is expected to take long michael@0: if 'requesttimeoutfactor' in self.test_object: michael@0: testTimeoutInterval *= int(self.test_object['requesttimeoutfactor']) michael@0: michael@0: testTimer = None michael@0: if not self.interactive and not self.debuggerInfo: michael@0: testTimer = Timer(testTimeoutInterval, lambda: self.testTimeout(name, proc)) michael@0: testTimer.start() michael@0: michael@0: proc = None michael@0: stdout = None michael@0: stderr = None michael@0: michael@0: try: michael@0: self.log.info("TEST-INFO | %s | running test ..." % name) michael@0: if self.verbose: michael@0: self.logCommand(name, completeCmd, test_dir) michael@0: michael@0: startTime = time.time() michael@0: proc = self.launchProcess(completeCmd, michael@0: stdout=self.pStdout, stderr=self.pStderr, env=self.env, cwd=test_dir) michael@0: michael@0: if self.interactive: michael@0: self.log.info("TEST-INFO | %s | Process ID: %d" % (name, proc.pid)) michael@0: michael@0: stdout, stderr = self.communicate(proc) michael@0: michael@0: if self.interactive: michael@0: # Not sure what else to do here... michael@0: self.keep_going = True michael@0: return michael@0: michael@0: if testTimer: michael@0: testTimer.cancel() michael@0: michael@0: if stdout: michael@0: self.parse_output(stdout) michael@0: result = not (self.has_failure_output or michael@0: (self.getReturnCode(proc) != 0)) michael@0: michael@0: if result != expected: michael@0: if self.retry: michael@0: self.clean_temp_dirs(name, stdout) michael@0: return michael@0: michael@0: failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS") michael@0: message = "%s | %s | test failed (with xpcshell return code: %d)" % ( michael@0: failureType, name, self.getReturnCode(proc)) michael@0: if self.output_lines: michael@0: message += ", see following log:" michael@0: michael@0: with LOG_MUTEX: michael@0: self.log.error(message) michael@0: self.log_output(self.output_lines) michael@0: michael@0: self.failCount += 1 michael@0: self.xunit_result["passed"] = False michael@0: michael@0: self.xunit_result["failure"] = { michael@0: "type": failureType, michael@0: "message": message, michael@0: "text": stdout michael@0: } michael@0: michael@0: if self.failureManifest: michael@0: with open(self.failureManifest, 'a') as f: michael@0: f.write('[%s]\n' % self.test_object['path']) michael@0: for k, v in self.test_object.items(): michael@0: f.write('%s = %s\n' % (k, v)) michael@0: michael@0: else: michael@0: now = time.time() michael@0: timeTaken = (now - startTime) * 1000 michael@0: self.xunit_result["time"] = now - startTime michael@0: michael@0: with LOG_MUTEX: michael@0: self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken)) michael@0: if self.verbose: michael@0: self.log_output(self.output_lines) michael@0: michael@0: self.xunit_result["passed"] = True michael@0: self.retry = False michael@0: michael@0: if expected: michael@0: self.passCount = 1 michael@0: else: michael@0: self.todoCount = 1 michael@0: self.xunit_result["todo"] = True michael@0: michael@0: if self.checkForCrashes(self.tempDir, self.symbolsPath, test_name=name): michael@0: if self.retry: michael@0: self.clean_temp_dirs(name, stdout) michael@0: return michael@0: michael@0: message = "PROCESS-CRASH | %s | application crashed" % name michael@0: self.failCount = 1 michael@0: self.xunit_result["passed"] = False michael@0: self.xunit_result["failure"] = { michael@0: "type": "PROCESS-CRASH", michael@0: "message": message, michael@0: "text": stdout michael@0: } michael@0: michael@0: if self.logfiles and stdout: michael@0: self.createLogFile(name, stdout) michael@0: michael@0: finally: michael@0: # We can sometimes get here before the process has terminated, which would michael@0: # cause removeDir() to fail - so check for the process & kill it it needed. michael@0: if proc and self.poll(proc) is None: michael@0: self.kill(proc) michael@0: michael@0: if self.retry: michael@0: self.clean_temp_dirs(name, stdout) michael@0: return michael@0: michael@0: with LOG_MUTEX: michael@0: message = "TEST-UNEXPECTED-FAIL | %s | Process still running after test!" % name michael@0: self.log.error(message) michael@0: self.log_output(self.output_lines) michael@0: michael@0: self.failCount = 1 michael@0: self.xunit_result["passed"] = False michael@0: self.xunit_result["failure"] = { michael@0: "type": "TEST-UNEXPECTED-FAIL", michael@0: "message": message, michael@0: "text": stdout michael@0: } michael@0: michael@0: self.clean_temp_dirs(name, stdout) michael@0: michael@0: if gotSIGINT: michael@0: self.xunit_result["passed"] = False michael@0: self.xunit_result["time"] = "0.0" michael@0: self.xunit_result["failure"] = { michael@0: "type": "SIGINT", michael@0: "message": "Received SIGINT", michael@0: "text": "Received SIGINT (control-C) during test execution." michael@0: } michael@0: michael@0: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution") michael@0: if self.keep_going: michael@0: gotSIGINT = False michael@0: else: michael@0: self.keep_going = False michael@0: return michael@0: michael@0: self.keep_going = True michael@0: michael@0: class XPCShellTests(object): michael@0: michael@0: log = getGlobalLog() michael@0: oldcwd = os.getcwd() michael@0: michael@0: def __init__(self, log=None): michael@0: """ Init logging and node status """ michael@0: if log: michael@0: resetGlobalLog(log) michael@0: michael@0: # Each method of the underlying logger must acquire the log michael@0: # mutex before writing to stdout. michael@0: log_funs = ['debug', 'info', 'warning', 'error', 'critical', 'log'] michael@0: for fun_name in log_funs: michael@0: unwrapped = getattr(self.log, fun_name, None) michael@0: if unwrapped: michael@0: def wrap(fn): michael@0: def wrapped(*args, **kwargs): michael@0: with LOG_MUTEX: michael@0: fn(*args, **kwargs) michael@0: return wrapped michael@0: setattr(self.log, fun_name, wrap(unwrapped)) michael@0: michael@0: self.nodeProc = {} michael@0: michael@0: def buildTestList(self): michael@0: """ michael@0: read the xpcshell.ini manifest and set self.alltests to be michael@0: an array of test objects. michael@0: michael@0: if we are chunking tests, it will be done here as well michael@0: """ michael@0: if isinstance(self.manifest, manifestparser.TestManifest): michael@0: mp = self.manifest michael@0: else: michael@0: mp = manifestparser.TestManifest(strict=False) michael@0: if self.manifest is None: michael@0: for testdir in self.testdirs: michael@0: if testdir: michael@0: mp.read(os.path.join(testdir, 'xpcshell.ini')) michael@0: else: michael@0: mp.read(self.manifest) michael@0: michael@0: self.buildTestPath() michael@0: michael@0: try: michael@0: self.alltests = mp.active_tests(**mozinfo.info) michael@0: except TypeError: michael@0: sys.stderr.write("*** offending mozinfo.info: %s\n" % repr(mozinfo.info)) michael@0: raise michael@0: michael@0: if self.singleFile is None and self.totalChunks > 1: michael@0: self.chunkTests() michael@0: michael@0: def chunkTests(self): michael@0: """ michael@0: Split the list of tests up into [totalChunks] pieces and filter the michael@0: self.alltests based on thisChunk, so we only run a subset. michael@0: """ michael@0: totalTests = len(self.alltests) michael@0: testsPerChunk = math.ceil(totalTests / float(self.totalChunks)) michael@0: start = int(round((self.thisChunk-1) * testsPerChunk)) michael@0: end = int(start + testsPerChunk) michael@0: if end > totalTests: michael@0: end = totalTests michael@0: self.log.info("Running tests %d-%d/%d", start+1, end, totalTests) michael@0: self.alltests = self.alltests[start:end] michael@0: michael@0: def setAbsPath(self): michael@0: """ michael@0: Set the absolute path for xpcshell, httpdjspath and xrepath. michael@0: These 3 variables depend on input from the command line and we need to allow for absolute paths. michael@0: This function is overloaded for a remote solution as os.path* won't work remotely. michael@0: """ michael@0: self.testharnessdir = os.path.dirname(os.path.abspath(__file__)) michael@0: self.headJSPath = self.testharnessdir.replace("\\", "/") + "/head.js" michael@0: self.xpcshell = os.path.abspath(self.xpcshell) michael@0: michael@0: # we assume that httpd.js lives in components/ relative to xpcshell michael@0: self.httpdJSPath = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.js') michael@0: self.httpdJSPath = replaceBackSlashes(self.httpdJSPath) michael@0: michael@0: self.httpdManifest = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.manifest') michael@0: self.httpdManifest = replaceBackSlashes(self.httpdManifest) michael@0: michael@0: if self.xrePath is None: michael@0: self.xrePath = os.path.dirname(self.xpcshell) michael@0: else: michael@0: self.xrePath = os.path.abspath(self.xrePath) michael@0: michael@0: if self.mozInfo is None: michael@0: self.mozInfo = os.path.join(self.testharnessdir, "mozinfo.json") michael@0: michael@0: def buildCoreEnvironment(self): michael@0: """ michael@0: Add environment variables likely to be used across all platforms, including remote systems. michael@0: """ michael@0: # Make assertions fatal michael@0: self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" michael@0: # Crash reporting interferes with debugging michael@0: if not self.debuggerInfo: michael@0: self.env["MOZ_CRASHREPORTER"] = "1" michael@0: # Don't launch the crash reporter client michael@0: self.env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" michael@0: # Capturing backtraces is very slow on some platforms, and it's michael@0: # disabled by automation.py too michael@0: self.env["NS_TRACE_MALLOC_DISABLE_STACKS"] = "1" michael@0: # Don't permit remote connections. michael@0: self.env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" michael@0: michael@0: def buildEnvironment(self): michael@0: """ michael@0: Create and returns a dictionary of self.env to include all the appropriate env variables and values. michael@0: On a remote system, we overload this to set different values and are missing things like os.environ and PATH. michael@0: """ michael@0: self.env = dict(os.environ) michael@0: self.buildCoreEnvironment() michael@0: if sys.platform == 'win32': michael@0: self.env["PATH"] = self.env["PATH"] + ";" + self.xrePath michael@0: elif sys.platform in ('os2emx', 'os2knix'): michael@0: os.environ["BEGINLIBPATH"] = self.xrePath + ";" + self.env["BEGINLIBPATH"] michael@0: os.environ["LIBPATHSTRICT"] = "T" michael@0: elif sys.platform == 'osx' or sys.platform == "darwin": michael@0: self.env["DYLD_LIBRARY_PATH"] = self.xrePath michael@0: else: # unix or linux? michael@0: if not "LD_LIBRARY_PATH" in self.env or self.env["LD_LIBRARY_PATH"] is None: michael@0: self.env["LD_LIBRARY_PATH"] = self.xrePath michael@0: else: michael@0: self.env["LD_LIBRARY_PATH"] = ":".join([self.xrePath, self.env["LD_LIBRARY_PATH"]]) michael@0: michael@0: if "asan" in self.mozInfo and self.mozInfo["asan"]: michael@0: # ASan symbolizer support michael@0: llvmsym = os.path.join(self.xrePath, "llvm-symbolizer") michael@0: if os.path.isfile(llvmsym): michael@0: self.env["ASAN_SYMBOLIZER_PATH"] = llvmsym michael@0: self.log.info("INFO | runxpcshelltests.py | ASan using symbolizer at %s", llvmsym) michael@0: else: michael@0: self.log.info("INFO | runxpcshelltests.py | ASan symbolizer binary not found: %s", llvmsym) michael@0: michael@0: return self.env michael@0: michael@0: def getPipes(self): michael@0: """ michael@0: Determine the value of the stdout and stderr for the test. michael@0: Return value is a list (pStdout, pStderr). michael@0: """ michael@0: if self.interactive: michael@0: pStdout = None michael@0: pStderr = None michael@0: else: michael@0: if (self.debuggerInfo and self.debuggerInfo["interactive"]): michael@0: pStdout = None michael@0: pStderr = None michael@0: else: michael@0: if sys.platform == 'os2emx': michael@0: pStdout = None michael@0: else: michael@0: pStdout = PIPE michael@0: pStderr = STDOUT michael@0: return pStdout, pStderr michael@0: michael@0: def buildTestPath(self): michael@0: """ michael@0: If we specifiy a testpath, set the self.testPath variable to be the given directory or file. michael@0: michael@0: |testPath| will be the optional path only, or |None|. michael@0: |singleFile| will be the optional test only, or |None|. michael@0: """ michael@0: self.singleFile = None michael@0: if self.testPath is not None: michael@0: if self.testPath.endswith('.js'): michael@0: # Split into path and file. michael@0: if self.testPath.find('/') == -1: michael@0: # Test only. michael@0: self.singleFile = self.testPath michael@0: else: michael@0: # Both path and test. michael@0: # Reuse |testPath| temporarily. michael@0: self.testPath = self.testPath.rsplit('/', 1) michael@0: self.singleFile = self.testPath[1] michael@0: self.testPath = self.testPath[0] michael@0: else: michael@0: # Path only. michael@0: # Simply remove optional ending separator. michael@0: self.testPath = self.testPath.rstrip("/") michael@0: michael@0: def verifyDirPath(self, dirname): michael@0: """ michael@0: Simple wrapper to get the absolute path for a given directory name. michael@0: On a remote system, we need to overload this to work on the remote filesystem. michael@0: """ michael@0: return os.path.abspath(dirname) michael@0: michael@0: def trySetupNode(self): michael@0: """ michael@0: Run node for SPDY tests, if available, and updates mozinfo as appropriate. michael@0: """ michael@0: nodeMozInfo = {'hasNode': False} # Assume the worst michael@0: nodeBin = None michael@0: michael@0: # We try to find the node executable in the path given to us by the user in michael@0: # the MOZ_NODE_PATH environment variable michael@0: localPath = os.getenv('MOZ_NODE_PATH', None) michael@0: if localPath and os.path.exists(localPath) and os.path.isfile(localPath): michael@0: nodeBin = localPath michael@0: michael@0: if nodeBin: michael@0: self.log.info('Found node at %s' % (nodeBin,)) michael@0: michael@0: def startServer(name, serverJs): michael@0: if os.path.exists(serverJs): michael@0: # OK, we found our SPDY server, let's try to get it running michael@0: self.log.info('Found %s at %s' % (name, serverJs)) michael@0: try: michael@0: # We pipe stdin to node because the spdy server will exit when its michael@0: # stdin reaches EOF michael@0: process = Popen([nodeBin, serverJs], stdin=PIPE, stdout=PIPE, michael@0: stderr=STDOUT, env=self.env, cwd=os.getcwd()) michael@0: self.nodeProc[name] = process michael@0: michael@0: # Check to make sure the server starts properly by waiting for it to michael@0: # tell us it's started michael@0: msg = process.stdout.readline() michael@0: if 'server listening' in msg: michael@0: nodeMozInfo['hasNode'] = True michael@0: except OSError, e: michael@0: # This occurs if the subprocess couldn't be started michael@0: self.log.error('Could not run %s server: %s' % (name, str(e))) michael@0: michael@0: myDir = os.path.split(os.path.abspath(__file__))[0] michael@0: startServer('moz-spdy', os.path.join(myDir, 'moz-spdy', 'moz-spdy.js')) michael@0: startServer('moz-http2', os.path.join(myDir, 'moz-http2', 'moz-http2.js')) michael@0: michael@0: mozinfo.update(nodeMozInfo) michael@0: michael@0: def shutdownNode(self): michael@0: """ michael@0: Shut down our node process, if it exists michael@0: """ michael@0: for name, proc in self.nodeProc.iteritems(): michael@0: self.log.info('Node %s server shutting down ...' % name) michael@0: proc.terminate() michael@0: michael@0: def writeXunitResults(self, results, name=None, filename=None, fh=None): michael@0: """ michael@0: Write Xunit XML from results. michael@0: michael@0: The function receives an iterable of results dicts. Each dict must have michael@0: the following keys: michael@0: michael@0: classname - The "class" name of the test. michael@0: name - The simple name of the test. michael@0: michael@0: In addition, it must have one of the following saying how the test michael@0: executed: michael@0: michael@0: passed - Boolean indicating whether the test passed. False if it michael@0: failed. michael@0: skipped - True if the test was skipped. michael@0: michael@0: The following keys are optional: michael@0: michael@0: time - Execution time of the test in decimal seconds. michael@0: failure - Dict describing test failure. Requires keys: michael@0: type - String type of failure. michael@0: message - String describing basic failure. michael@0: text - Verbose string describing failure. michael@0: michael@0: Arguments: michael@0: michael@0: |name|, Name of the test suite. Many tools expect Java class dot notation michael@0: e.g. dom.simple.foo. A directory with '/' converted to '.' is a good michael@0: choice. michael@0: |fh|, File handle to write XML to. michael@0: |filename|, File name to write XML to. michael@0: |results|, Iterable of tuples describing the results. michael@0: """ michael@0: if filename is None and fh is None: michael@0: raise Exception("One of filename or fh must be defined.") michael@0: michael@0: if name is None: michael@0: name = "xpcshell" michael@0: else: michael@0: assert isinstance(name, basestring) michael@0: michael@0: if filename is not None: michael@0: fh = open(filename, 'wb') michael@0: michael@0: doc = xml.dom.minidom.Document() michael@0: testsuite = doc.createElement("testsuite") michael@0: testsuite.setAttribute("name", name) michael@0: doc.appendChild(testsuite) michael@0: michael@0: total = 0 michael@0: passed = 0 michael@0: failed = 0 michael@0: skipped = 0 michael@0: michael@0: for result in results: michael@0: total += 1 michael@0: michael@0: if result.get("skipped", None): michael@0: skipped += 1 michael@0: elif result["passed"]: michael@0: passed += 1 michael@0: else: michael@0: failed += 1 michael@0: michael@0: testcase = doc.createElement("testcase") michael@0: testcase.setAttribute("classname", result["classname"]) michael@0: testcase.setAttribute("name", result["name"]) michael@0: michael@0: if "time" in result: michael@0: testcase.setAttribute("time", str(result["time"])) michael@0: else: michael@0: # It appears most tools expect the time attribute to be present. michael@0: testcase.setAttribute("time", "0") michael@0: michael@0: if "failure" in result: michael@0: failure = doc.createElement("failure") michael@0: failure.setAttribute("type", str(result["failure"]["type"])) michael@0: failure.setAttribute("message", result["failure"]["message"]) michael@0: michael@0: # Lossy translation but required to not break CDATA. Also, text could michael@0: # be None and Python 2.5's minidom doesn't accept None. Later versions michael@0: # do, however. michael@0: cdata = result["failure"]["text"] michael@0: if not isinstance(cdata, str): michael@0: cdata = "" michael@0: michael@0: cdata = cdata.replace("]]>", "]] >") michael@0: text = doc.createCDATASection(cdata) michael@0: failure.appendChild(text) michael@0: testcase.appendChild(failure) michael@0: michael@0: if result.get("skipped", None): michael@0: e = doc.createElement("skipped") michael@0: testcase.appendChild(e) michael@0: michael@0: testsuite.appendChild(testcase) michael@0: michael@0: testsuite.setAttribute("tests", str(total)) michael@0: testsuite.setAttribute("failures", str(failed)) michael@0: testsuite.setAttribute("skip", str(skipped)) michael@0: michael@0: doc.writexml(fh, addindent=" ", newl="\n", encoding="utf-8") michael@0: michael@0: def post_to_autolog(self, results, name): michael@0: from moztest.results import TestContext, TestResult, TestResultCollection michael@0: from moztest.output.autolog import AutologOutput michael@0: michael@0: context = TestContext( michael@0: testgroup='b2g xpcshell testsuite', michael@0: operating_system='android', michael@0: arch='emulator', michael@0: harness='xpcshell', michael@0: hostname=socket.gethostname(), michael@0: tree='b2g', michael@0: buildtype='opt', michael@0: ) michael@0: michael@0: collection = TestResultCollection('b2g emulator testsuite') michael@0: michael@0: for result in results: michael@0: duration = result.get('time', 0) michael@0: michael@0: if 'skipped' in result: michael@0: outcome = 'SKIPPED' michael@0: elif 'todo' in result: michael@0: outcome = 'KNOWN-FAIL' michael@0: elif result['passed']: michael@0: outcome = 'PASS' michael@0: else: michael@0: outcome = 'UNEXPECTED-FAIL' michael@0: michael@0: output = None michael@0: if 'failure' in result: michael@0: output = result['failure']['text'] michael@0: michael@0: t = TestResult(name=result['name'], test_class=name, michael@0: time_start=0, context=context) michael@0: t.finish(result=outcome, time_end=duration, output=output) michael@0: michael@0: collection.append(t) michael@0: collection.time_taken += duration michael@0: michael@0: out = AutologOutput() michael@0: out.post(out.make_testgroups(collection)) michael@0: michael@0: def buildXpcsRunArgs(self): michael@0: """ michael@0: Add arguments to run the test or make it interactive. michael@0: """ michael@0: if self.interactive: michael@0: self.xpcsRunArgs = [ michael@0: '-e', 'print("To start the test, type |_execute_test();|.");', michael@0: '-i'] michael@0: else: michael@0: self.xpcsRunArgs = ['-e', '_execute_test(); quit(0);'] michael@0: michael@0: def addTestResults(self, test): michael@0: self.passCount += test.passCount michael@0: self.failCount += test.failCount michael@0: self.todoCount += test.todoCount michael@0: self.xunitResults.append(test.xunit_result) michael@0: michael@0: def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None, michael@0: manifest=None, testdirs=None, testPath=None, mobileArgs=None, michael@0: interactive=False, verbose=False, keepGoing=False, logfiles=True, michael@0: thisChunk=1, totalChunks=1, debugger=None, michael@0: debuggerArgs=None, debuggerInteractive=False, michael@0: profileName=None, mozInfo=None, sequential=False, shuffle=False, michael@0: testsRootDir=None, xunitFilename=None, xunitName=None, michael@0: testingModulesDir=None, autolog=False, pluginsPath=None, michael@0: testClass=XPCShellTestThread, failureManifest=None, michael@0: on_message=None, **otherOptions): michael@0: """Run xpcshell tests. michael@0: michael@0: |xpcshell|, is the xpcshell executable to use to run the tests. michael@0: |xrePath|, if provided, is the path to the XRE to use. michael@0: |appPath|, if provided, is the path to an application directory. michael@0: |symbolsPath|, if provided is the path to a directory containing michael@0: breakpad symbols for processing crashes in tests. michael@0: |manifest|, if provided, is a file containing a list of michael@0: test directories to run. michael@0: |testdirs|, if provided, is a list of absolute paths of test directories. michael@0: No-manifest only option. michael@0: |testPath|, if provided, indicates a single path and/or test to run. michael@0: |pluginsPath|, if provided, custom plugins directory to be returned from michael@0: the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST. michael@0: |interactive|, if set to True, indicates to provide an xpcshell prompt michael@0: instead of automatically executing the test. michael@0: |verbose|, if set to True, will cause stdout/stderr from tests to michael@0: be printed always michael@0: |logfiles|, if set to False, indicates not to save output to log files. michael@0: Non-interactive only option. michael@0: |debuggerInfo|, if set, specifies the debugger and debugger arguments michael@0: that will be used to launch xpcshell. michael@0: |profileName|, if set, specifies the name of the application for the profile michael@0: directory if running only a subset of tests. michael@0: |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict. michael@0: |shuffle|, if True, execute tests in random order. michael@0: |testsRootDir|, absolute path to root directory of all tests. This is used michael@0: by xUnit generation to determine the package name of the tests. michael@0: |xunitFilename|, if set, specifies the filename to which to write xUnit XML michael@0: results. michael@0: |xunitName|, if outputting an xUnit XML file, the str value to use for the michael@0: testsuite name. michael@0: |testingModulesDir|, if provided, specifies where JS modules reside. michael@0: xpcshell will register a resource handler mapping this path. michael@0: |otherOptions| may be present for the convenience of subclasses michael@0: """ michael@0: michael@0: global gotSIGINT michael@0: michael@0: if testdirs is None: michael@0: testdirs = [] michael@0: michael@0: if xunitFilename is not None or xunitName is not None: michael@0: if not isinstance(testsRootDir, basestring): michael@0: raise Exception("testsRootDir must be a str when outputting xUnit.") michael@0: michael@0: if not os.path.isabs(testsRootDir): michael@0: testsRootDir = os.path.abspath(testsRootDir) michael@0: michael@0: if not os.path.exists(testsRootDir): michael@0: raise Exception("testsRootDir path does not exists: %s" % michael@0: testsRootDir) michael@0: michael@0: # Try to guess modules directory. michael@0: # This somewhat grotesque hack allows the buildbot machines to find the michael@0: # modules directory without having to configure the buildbot hosts. This michael@0: # code path should never be executed in local runs because the build system michael@0: # should always set this argument. michael@0: if not testingModulesDir: michael@0: ourDir = os.path.dirname(__file__) michael@0: possible = os.path.join(ourDir, os.path.pardir, 'modules') michael@0: michael@0: if os.path.isdir(possible): michael@0: testingModulesDir = possible michael@0: michael@0: if testingModulesDir: michael@0: # The resource loader expects native paths. Depending on how we were michael@0: # invoked, a UNIX style path may sneak in on Windows. We try to michael@0: # normalize that. michael@0: testingModulesDir = os.path.normpath(testingModulesDir) michael@0: michael@0: if not os.path.isabs(testingModulesDir): michael@0: testingModulesDir = os.path.abspath(testingModulesDir) michael@0: michael@0: if not testingModulesDir.endswith(os.path.sep): michael@0: testingModulesDir += os.path.sep michael@0: michael@0: self.xpcshell = xpcshell michael@0: self.xrePath = xrePath michael@0: self.appPath = appPath michael@0: self.symbolsPath = symbolsPath michael@0: self.manifest = manifest michael@0: self.testdirs = testdirs michael@0: self.testPath = testPath michael@0: self.interactive = interactive michael@0: self.verbose = verbose michael@0: self.keepGoing = keepGoing michael@0: self.logfiles = logfiles michael@0: self.on_message = on_message michael@0: self.totalChunks = totalChunks michael@0: self.thisChunk = thisChunk michael@0: self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive) michael@0: self.profileName = profileName or "xpcshell" michael@0: self.mozInfo = mozInfo michael@0: self.testingModulesDir = testingModulesDir michael@0: self.pluginsPath = pluginsPath michael@0: self.sequential = sequential michael@0: michael@0: if not testdirs and not manifest: michael@0: # nothing to test! michael@0: self.log.error("Error: No test dirs or test manifest specified!") michael@0: return False michael@0: michael@0: self.testCount = 0 michael@0: self.passCount = 0 michael@0: self.failCount = 0 michael@0: self.todoCount = 0 michael@0: michael@0: self.setAbsPath() michael@0: self.buildXpcsRunArgs() michael@0: michael@0: self.event = Event() michael@0: michael@0: # Handle filenames in mozInfo michael@0: if not isinstance(self.mozInfo, dict): michael@0: mozInfoFile = self.mozInfo michael@0: if not os.path.isfile(mozInfoFile): michael@0: self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile) michael@0: return False michael@0: self.mozInfo = json.load(open(mozInfoFile)) michael@0: michael@0: # mozinfo.info is used as kwargs. Some builds are done with michael@0: # an older Python that can't handle Unicode keys in kwargs. michael@0: # All of the keys in question should be ASCII. michael@0: fixedInfo = {} michael@0: for k, v in self.mozInfo.items(): michael@0: if isinstance(k, unicode): michael@0: k = k.encode('ascii') michael@0: fixedInfo[k] = v michael@0: self.mozInfo = fixedInfo michael@0: michael@0: mozinfo.update(self.mozInfo) michael@0: michael@0: # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized. michael@0: self.buildEnvironment() michael@0: michael@0: # The appDirKey is a optional entry in either the default or individual test michael@0: # sections that defines a relative application directory for test runs. If michael@0: # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell michael@0: # test harness. michael@0: appDirKey = None michael@0: if "appname" in self.mozInfo: michael@0: appDirKey = self.mozInfo["appname"] + "-appdir" michael@0: michael@0: # We have to do this before we build the test list so we know whether or michael@0: # not to run tests that depend on having the node spdy server michael@0: self.trySetupNode() michael@0: michael@0: pStdout, pStderr = self.getPipes() michael@0: michael@0: self.buildTestList() michael@0: if self.singleFile: michael@0: self.sequential = True michael@0: michael@0: if shuffle: michael@0: random.shuffle(self.alltests) michael@0: michael@0: self.xunitResults = [] michael@0: self.cleanup_dir_list = [] michael@0: self.try_again_list = [] michael@0: michael@0: kwargs = { michael@0: 'appPath': self.appPath, michael@0: 'xrePath': self.xrePath, michael@0: 'testingModulesDir': self.testingModulesDir, michael@0: 'debuggerInfo': self.debuggerInfo, michael@0: 'pluginsPath': self.pluginsPath, michael@0: 'httpdManifest': self.httpdManifest, michael@0: 'httpdJSPath': self.httpdJSPath, michael@0: 'headJSPath': self.headJSPath, michael@0: 'testharnessdir': self.testharnessdir, michael@0: 'profileName': self.profileName, michael@0: 'singleFile': self.singleFile, michael@0: 'env': self.env, # making a copy of this in the testthreads michael@0: 'symbolsPath': self.symbolsPath, michael@0: 'logfiles': self.logfiles, michael@0: 'xpcshell': self.xpcshell, michael@0: 'xpcsRunArgs': self.xpcsRunArgs, michael@0: 'failureManifest': failureManifest, michael@0: 'on_message': self.on_message, michael@0: } michael@0: michael@0: if self.sequential: michael@0: # Allow user to kill hung xpcshell subprocess with SIGINT michael@0: # when we are only running tests sequentially. michael@0: signal.signal(signal.SIGINT, markGotSIGINT) michael@0: michael@0: if self.debuggerInfo: michael@0: # Force a sequential run michael@0: self.sequential = True michael@0: michael@0: # If we have an interactive debugger, disable SIGINT entirely. michael@0: if self.debuggerInfo["interactive"]: michael@0: signal.signal(signal.SIGINT, lambda signum, frame: None) michael@0: michael@0: # create a queue of all tests that will run michael@0: tests_queue = deque() michael@0: # also a list for the tests that need to be run sequentially michael@0: sequential_tests = [] michael@0: for test_object in self.alltests: michael@0: name = test_object['path'] michael@0: if self.singleFile and not name.endswith(self.singleFile): michael@0: continue michael@0: michael@0: if self.testPath and name.find(self.testPath) == -1: michael@0: continue michael@0: michael@0: self.testCount += 1 michael@0: michael@0: test = testClass(test_object, self.event, self.cleanup_dir_list, michael@0: tests_root_dir=testsRootDir, app_dir_key=appDirKey, michael@0: interactive=interactive, verbose=verbose, pStdout=pStdout, michael@0: pStderr=pStderr, keep_going=keepGoing, log=self.log, michael@0: mobileArgs=mobileArgs, **kwargs) michael@0: if 'run-sequentially' in test_object or self.sequential: michael@0: sequential_tests.append(test) michael@0: else: michael@0: tests_queue.append(test) michael@0: michael@0: if self.sequential: michael@0: self.log.info("INFO | Running tests sequentially.") michael@0: else: michael@0: self.log.info("INFO | Using at most %d threads." % NUM_THREADS) michael@0: michael@0: # keep a set of NUM_THREADS running tests and start running the michael@0: # tests in the queue at most NUM_THREADS at a time michael@0: running_tests = set() michael@0: keep_going = True michael@0: exceptions = [] michael@0: tracebacks = [] michael@0: while tests_queue or running_tests: michael@0: # if we're not supposed to continue and all of the running tests michael@0: # are done, stop michael@0: if not keep_going and not running_tests: michael@0: break michael@0: michael@0: # if there's room to run more tests, start running them michael@0: while keep_going and tests_queue and (len(running_tests) < NUM_THREADS): michael@0: test = tests_queue.popleft() michael@0: running_tests.add(test) michael@0: test.start() michael@0: michael@0: # queue is full (for now) or no more new tests, michael@0: # process the finished tests so far michael@0: michael@0: # wait for at least one of the tests to finish michael@0: self.event.wait(1) michael@0: self.event.clear() michael@0: michael@0: # find what tests are done (might be more than 1) michael@0: done_tests = set() michael@0: for test in running_tests: michael@0: if test.done: michael@0: done_tests.add(test) michael@0: test.join(1) # join with timeout so we don't hang on blocked threads michael@0: # if the test had trouble, we will try running it again michael@0: # at the end of the run michael@0: if test.retry or test.is_alive(): michael@0: # if the join call timed out, test.is_alive => True michael@0: self.try_again_list.append(test.test_object) michael@0: continue michael@0: # did the test encounter any exception? michael@0: if test.exception: michael@0: exceptions.append(test.exception) michael@0: tracebacks.append(test.traceback) michael@0: # we won't add any more tests, will just wait for michael@0: # the currently running ones to finish michael@0: keep_going = False michael@0: keep_going = keep_going and test.keep_going michael@0: self.addTestResults(test) michael@0: michael@0: # make room for new tests to run michael@0: running_tests.difference_update(done_tests) michael@0: michael@0: if keep_going: michael@0: # run the other tests sequentially michael@0: for test in sequential_tests: michael@0: if not keep_going: michael@0: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ michael@0: "(Use --keep-going to keep running tests after killing one with SIGINT)") michael@0: break michael@0: # we don't want to retry these tests michael@0: test.retry = False michael@0: test.start() michael@0: test.join() michael@0: self.addTestResults(test) michael@0: # did the test encounter any exception? michael@0: if test.exception: michael@0: exceptions.append(test.exception) michael@0: tracebacks.append(test.traceback) michael@0: break michael@0: keep_going = test.keep_going michael@0: michael@0: # retry tests that failed when run in parallel michael@0: if self.try_again_list: michael@0: self.log.info("Retrying tests that failed when run in parallel.") michael@0: for test_object in self.try_again_list: michael@0: test = testClass(test_object, self.event, self.cleanup_dir_list, michael@0: retry=False, tests_root_dir=testsRootDir, michael@0: app_dir_key=appDirKey, interactive=interactive, michael@0: verbose=verbose, pStdout=pStdout, pStderr=pStderr, michael@0: keep_going=keepGoing, log=self.log, mobileArgs=mobileArgs, michael@0: **kwargs) michael@0: test.start() michael@0: test.join() michael@0: self.addTestResults(test) michael@0: # did the test encounter any exception? michael@0: if test.exception: michael@0: exceptions.append(test.exception) michael@0: tracebacks.append(test.traceback) michael@0: break michael@0: keep_going = test.keep_going michael@0: michael@0: # restore default SIGINT behaviour michael@0: signal.signal(signal.SIGINT, signal.SIG_DFL) michael@0: michael@0: self.shutdownNode() michael@0: # Clean up any slacker directories that might be lying around michael@0: # Some might fail because of windows taking too long to unlock them. michael@0: # We don't do anything if this fails because the test slaves will have michael@0: # their $TEMP dirs cleaned up on reboot anyway. michael@0: for directory in self.cleanup_dir_list: michael@0: try: michael@0: shutil.rmtree(directory) michael@0: except: michael@0: self.log.info("INFO | %s could not be cleaned up." % directory) michael@0: michael@0: if exceptions: michael@0: self.log.info("INFO | Following exceptions were raised:") michael@0: for t in tracebacks: michael@0: self.log.error(t) michael@0: raise exceptions[0] michael@0: michael@0: if self.testCount == 0: michael@0: self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?") michael@0: self.failCount = 1 michael@0: michael@0: self.log.info("INFO | Result summary:") michael@0: self.log.info("INFO | Passed: %d" % self.passCount) michael@0: self.log.info("INFO | Failed: %d" % self.failCount) michael@0: self.log.info("INFO | Todo: %d" % self.todoCount) michael@0: self.log.info("INFO | Retried: %d" % len(self.try_again_list)) michael@0: michael@0: if autolog: michael@0: self.post_to_autolog(self.xunitResults, xunitName) michael@0: michael@0: if xunitFilename is not None: michael@0: self.writeXunitResults(filename=xunitFilename, results=self.xunitResults, michael@0: name=xunitName) michael@0: michael@0: if gotSIGINT and not keepGoing: michael@0: self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \ michael@0: "(Use --keep-going to keep running tests after killing one with SIGINT)") michael@0: return False michael@0: michael@0: return self.failCount == 0 michael@0: michael@0: class XPCShellOptions(OptionParser): michael@0: def __init__(self): michael@0: """Process command line arguments and call runTests() to do the real work.""" michael@0: OptionParser.__init__(self) michael@0: michael@0: addCommonOptions(self) michael@0: self.add_option("--app-path", michael@0: type="string", dest="appPath", default=None, michael@0: help="application directory (as opposed to XRE directory)") michael@0: self.add_option("--autolog", michael@0: action="store_true", dest="autolog", default=False, michael@0: help="post to autolog") michael@0: self.add_option("--interactive", michael@0: action="store_true", dest="interactive", default=False, michael@0: help="don't automatically run tests, drop to an xpcshell prompt") michael@0: self.add_option("--verbose", michael@0: action="store_true", dest="verbose", default=False, michael@0: help="always print stdout and stderr from tests") michael@0: self.add_option("--keep-going", michael@0: action="store_true", dest="keepGoing", default=False, michael@0: help="continue running tests after test killed with control-C (SIGINT)") michael@0: self.add_option("--logfiles", michael@0: action="store_true", dest="logfiles", default=True, michael@0: help="create log files (default, only used to override --no-logfiles)") michael@0: self.add_option("--manifest", michael@0: type="string", dest="manifest", default=None, michael@0: help="Manifest of test directories to use") michael@0: self.add_option("--no-logfiles", michael@0: action="store_false", dest="logfiles", michael@0: help="don't create log files") michael@0: self.add_option("--sequential", michael@0: action="store_true", dest="sequential", default=False, michael@0: help="Run all tests sequentially") michael@0: self.add_option("--test-path", michael@0: type="string", dest="testPath", default=None, michael@0: help="single path and/or test filename to test") michael@0: self.add_option("--tests-root-dir", michael@0: type="string", dest="testsRootDir", default=None, michael@0: help="absolute path to directory where all tests are located. this is typically $(objdir)/_tests") michael@0: self.add_option("--testing-modules-dir", michael@0: dest="testingModulesDir", default=None, michael@0: help="Directory where testing modules are located.") michael@0: self.add_option("--test-plugin-path", michael@0: type="string", dest="pluginsPath", default=None, michael@0: help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. " michael@0: "By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory " michael@0: "to return for NS_APP_PLUGINS_DIR_LIST when queried.") michael@0: self.add_option("--total-chunks", michael@0: type = "int", dest = "totalChunks", default=1, michael@0: help = "how many chunks to split the tests up into") michael@0: self.add_option("--this-chunk", michael@0: type = "int", dest = "thisChunk", default=1, michael@0: help = "which chunk to run between 1 and --total-chunks") michael@0: self.add_option("--profile-name", michael@0: type = "string", dest="profileName", default=None, michael@0: help="name of application profile being tested") michael@0: self.add_option("--build-info-json", michael@0: type = "string", dest="mozInfo", default=None, michael@0: help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.") michael@0: self.add_option("--shuffle", michael@0: action="store_true", dest="shuffle", default=False, michael@0: help="Execute tests in random order") michael@0: self.add_option("--xunit-file", dest="xunitFilename", michael@0: help="path to file where xUnit results will be written.") michael@0: self.add_option("--xunit-suite-name", dest="xunitName", michael@0: help="name to record for this xUnit test suite. Many " michael@0: "tools expect Java class notation, e.g. " michael@0: "dom.basic.foo") michael@0: self.add_option("--failure-manifest", dest="failureManifest", michael@0: action="store", michael@0: help="path to file where failure manifest will be written.") michael@0: michael@0: def main(): michael@0: parser = XPCShellOptions() michael@0: options, args = parser.parse_args() michael@0: michael@0: if len(args) < 2 and options.manifest is None or \ michael@0: (len(args) < 1 and options.manifest is not None): michael@0: print >>sys.stderr, """Usage: %s michael@0: or: %s --manifest=test.manifest """ % (sys.argv[0], michael@0: sys.argv[0]) michael@0: sys.exit(1) michael@0: michael@0: xpcsh = XPCShellTests() michael@0: michael@0: if options.interactive and not options.testPath: michael@0: print >>sys.stderr, "Error: You must specify a test filename in interactive mode!" michael@0: sys.exit(1) michael@0: michael@0: if not xpcsh.runTests(args[0], testdirs=args[1:], **options.__dict__): michael@0: sys.exit(1) michael@0: michael@0: if __name__ == '__main__': michael@0: main()