1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/mochitest/runtests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1561 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +"""
1.9 +Runs the Mochitest test harness.
1.10 +"""
1.11 +
1.12 +from __future__ import with_statement
1.13 +import os
1.14 +import sys
1.15 +SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
1.16 +sys.path.insert(0, SCRIPT_DIR);
1.17 +
1.18 +import glob
1.19 +import json
1.20 +import mozcrash
1.21 +import mozinfo
1.22 +import mozprocess
1.23 +import mozrunner
1.24 +import optparse
1.25 +import re
1.26 +import shutil
1.27 +import signal
1.28 +import subprocess
1.29 +import tempfile
1.30 +import time
1.31 +import traceback
1.32 +import urllib2
1.33 +import zipfile
1.34 +
1.35 +from automationutils import environment, getDebuggerInfo, isURL, KeyValueParseError, parseKeyValue, processLeakLog, systemMemory, dumpScreen, ShutdownLeaks, printstatus
1.36 +from datetime import datetime
1.37 +from manifestparser import TestManifest
1.38 +from mochitest_options import MochitestOptions
1.39 +from mozprofile import Profile, Preferences
1.40 +from mozprofile.permissions import ServerLocations
1.41 +from urllib import quote_plus as encodeURIComponent
1.42 +
1.43 +# This should use the `which` module already in tree, but it is
1.44 +# not yet present in the mozharness environment
1.45 +from mozrunner.utils import findInPath as which
1.46 +
1.47 +# set up logging handler a la automation.py.in for compatability
1.48 +import logging
1.49 +log = logging.getLogger()
1.50 +def resetGlobalLog():
1.51 + while log.handlers:
1.52 + log.removeHandler(log.handlers[0])
1.53 + handler = logging.StreamHandler(sys.stdout)
1.54 + log.setLevel(logging.INFO)
1.55 + log.addHandler(handler)
1.56 +resetGlobalLog()
1.57 +
1.58 +###########################
1.59 +# Option for NSPR logging #
1.60 +###########################
1.61 +
1.62 +# Set the desired log modules you want an NSPR log be produced by a try run for, or leave blank to disable the feature.
1.63 +# This will be passed to NSPR_LOG_MODULES environment variable. Try run will then put a download link for the log file
1.64 +# on tbpl.mozilla.org.
1.65 +
1.66 +NSPR_LOG_MODULES = ""
1.67 +
1.68 +####################
1.69 +# PROCESS HANDLING #
1.70 +####################
1.71 +
1.72 +def call(*args, **kwargs):
1.73 + """front-end function to mozprocess.ProcessHandler"""
1.74 + # TODO: upstream -> mozprocess
1.75 + # https://bugzilla.mozilla.org/show_bug.cgi?id=791383
1.76 + process = mozprocess.ProcessHandler(*args, **kwargs)
1.77 + process.run()
1.78 + return process.wait()
1.79 +
1.80 +def killPid(pid):
1.81 + # see also https://bugzilla.mozilla.org/show_bug.cgi?id=911249#c58
1.82 + try:
1.83 + os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
1.84 + except Exception, e:
1.85 + log.info("Failed to kill process %d: %s", pid, str(e))
1.86 +
1.87 +if mozinfo.isWin:
1.88 + import ctypes, ctypes.wintypes, time, msvcrt
1.89 +
1.90 + def isPidAlive(pid):
1.91 + STILL_ACTIVE = 259
1.92 + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
1.93 + pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
1.94 + if not pHandle:
1.95 + return False
1.96 + pExitCode = ctypes.wintypes.DWORD()
1.97 + ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode))
1.98 + ctypes.windll.kernel32.CloseHandle(pHandle)
1.99 + return pExitCode.value == STILL_ACTIVE
1.100 +
1.101 +else:
1.102 + import errno
1.103 +
1.104 + def isPidAlive(pid):
1.105 + try:
1.106 + # kill(pid, 0) checks for a valid PID without actually sending a signal
1.107 + # The method throws OSError if the PID is invalid, which we catch below.
1.108 + os.kill(pid, 0)
1.109 +
1.110 + # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
1.111 + # the process terminates before we get to this point.
1.112 + wpid, wstatus = os.waitpid(pid, os.WNOHANG)
1.113 + return wpid == 0
1.114 + except OSError, err:
1.115 + # Catch the errors we might expect from os.kill/os.waitpid,
1.116 + # and re-raise any others
1.117 + if err.errno == errno.ESRCH or err.errno == errno.ECHILD:
1.118 + return False
1.119 + raise
1.120 +# TODO: ^ upstream isPidAlive to mozprocess
1.121 +
1.122 +#######################
1.123 +# HTTP SERVER SUPPORT #
1.124 +#######################
1.125 +
1.126 +class MochitestServer(object):
1.127 + "Web server used to serve Mochitests, for closer fidelity to the real web."
1.128 +
1.129 + def __init__(self, options):
1.130 + if isinstance(options, optparse.Values):
1.131 + options = vars(options)
1.132 + self._closeWhenDone = options['closeWhenDone']
1.133 + self._utilityPath = options['utilityPath']
1.134 + self._xrePath = options['xrePath']
1.135 + self._profileDir = options['profilePath']
1.136 + self.webServer = options['webServer']
1.137 + self.httpPort = options['httpPort']
1.138 + self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort }
1.139 + self.testPrefix = "'webapprt_'" if options.get('webapprtContent') else "undefined"
1.140 +
1.141 + if options.get('httpdPath'):
1.142 + self._httpdPath = options['httpdPath']
1.143 + else:
1.144 + self._httpdPath = SCRIPT_DIR
1.145 + self._httpdPath = os.path.abspath(self._httpdPath)
1.146 +
1.147 + def start(self):
1.148 + "Run the Mochitest server, returning the process ID of the server."
1.149 +
1.150 + # get testing environment
1.151 + env = environment(xrePath=self._xrePath)
1.152 + env["XPCOM_DEBUG_BREAK"] = "warn"
1.153 +
1.154 + # When running with an ASan build, our xpcshell server will also be ASan-enabled,
1.155 + # thus consuming too much resources when running together with the browser on
1.156 + # the test slaves. Try to limit the amount of resources by disabling certain
1.157 + # features.
1.158 + env["ASAN_OPTIONS"] = "quarantine_size=1:redzone=32:malloc_context_size=5"
1.159 +
1.160 + if mozinfo.isWin:
1.161 + env["PATH"] = env["PATH"] + ";" + str(self._xrePath)
1.162 +
1.163 + args = ["-g", self._xrePath,
1.164 + "-v", "170",
1.165 + "-f", os.path.join(self._httpdPath, "httpd.js"),
1.166 + "-e", """const _PROFILE_PATH = '%(profile)s'; const _SERVER_PORT = '%(port)s'; const _SERVER_ADDR = '%(server)s'; const _TEST_PREFIX = %(testPrefix)s; const _DISPLAY_RESULTS = %(displayResults)s;""" %
1.167 + {"profile" : self._profileDir.replace('\\', '\\\\'), "port" : self.httpPort, "server" : self.webServer,
1.168 + "testPrefix" : self.testPrefix, "displayResults" : str(not self._closeWhenDone).lower() },
1.169 + "-f", os.path.join(SCRIPT_DIR, "server.js")]
1.170 +
1.171 + xpcshell = os.path.join(self._utilityPath,
1.172 + "xpcshell" + mozinfo.info['bin_suffix'])
1.173 + command = [xpcshell] + args
1.174 + self._process = mozprocess.ProcessHandler(command, cwd=SCRIPT_DIR, env=env)
1.175 + self._process.run()
1.176 + log.info("%s : launching %s", self.__class__.__name__, command)
1.177 + pid = self._process.pid
1.178 + log.info("runtests.py | Server pid: %d", pid)
1.179 +
1.180 + def ensureReady(self, timeout):
1.181 + assert timeout >= 0
1.182 +
1.183 + aliveFile = os.path.join(self._profileDir, "server_alive.txt")
1.184 + i = 0
1.185 + while i < timeout:
1.186 + if os.path.exists(aliveFile):
1.187 + break
1.188 + time.sleep(1)
1.189 + i += 1
1.190 + else:
1.191 + log.error("TEST-UNEXPECTED-FAIL | runtests.py | Timed out while waiting for server startup.")
1.192 + self.stop()
1.193 + sys.exit(1)
1.194 +
1.195 + def stop(self):
1.196 + try:
1.197 + with urllib2.urlopen(self.shutdownURL) as c:
1.198 + c.read()
1.199 +
1.200 + # TODO: need ProcessHandler.poll()
1.201 + # https://bugzilla.mozilla.org/show_bug.cgi?id=912285
1.202 + # rtncode = self._process.poll()
1.203 + rtncode = self._process.proc.poll()
1.204 + if rtncode is None:
1.205 + # TODO: need ProcessHandler.terminate() and/or .send_signal()
1.206 + # https://bugzilla.mozilla.org/show_bug.cgi?id=912285
1.207 + # self._process.terminate()
1.208 + self._process.proc.terminate()
1.209 + except:
1.210 + self._process.kill()
1.211 +
1.212 +class WebSocketServer(object):
1.213 + "Class which encapsulates the mod_pywebsocket server"
1.214 +
1.215 + def __init__(self, options, scriptdir, debuggerInfo=None):
1.216 + self.port = options.webSocketPort
1.217 + self._scriptdir = scriptdir
1.218 + self.debuggerInfo = debuggerInfo
1.219 +
1.220 + def start(self):
1.221 + # Invoke pywebsocket through a wrapper which adds special SIGINT handling.
1.222 + #
1.223 + # If we're in an interactive debugger, the wrapper causes the server to
1.224 + # ignore SIGINT so the server doesn't capture a ctrl+c meant for the
1.225 + # debugger.
1.226 + #
1.227 + # If we're not in an interactive debugger, the wrapper causes the server to
1.228 + # die silently upon receiving a SIGINT.
1.229 + scriptPath = 'pywebsocket_wrapper.py'
1.230 + script = os.path.join(self._scriptdir, scriptPath)
1.231 +
1.232 + cmd = [sys.executable, script]
1.233 + if self.debuggerInfo and self.debuggerInfo['interactive']:
1.234 + cmd += ['--interactive']
1.235 + cmd += ['-p', str(self.port), '-w', self._scriptdir, '-l', \
1.236 + os.path.join(self._scriptdir, "websock.log"), \
1.237 + '--log-level=debug', '--allow-handlers-outside-root-dir']
1.238 + # start the process
1.239 + self._process = mozprocess.ProcessHandler(cmd, cwd=SCRIPT_DIR)
1.240 + self._process.run()
1.241 + pid = self._process.pid
1.242 + log.info("runtests.py | Websocket server pid: %d", pid)
1.243 +
1.244 + def stop(self):
1.245 + self._process.kill()
1.246 +
1.247 +class MochitestUtilsMixin(object):
1.248 + """
1.249 + Class containing some utility functions common to both local and remote
1.250 + mochitest runners
1.251 + """
1.252 +
1.253 + # TODO Utility classes are a code smell. This class is temporary
1.254 + # and should be removed when desktop mochitests are refactored
1.255 + # on top of mozbase. Each of the functions in here should
1.256 + # probably live somewhere in mozbase
1.257 +
1.258 + oldcwd = os.getcwd()
1.259 + jarDir = 'mochijar'
1.260 +
1.261 + # Path to the test script on the server
1.262 + TEST_PATH = "tests"
1.263 + CHROME_PATH = "redirect.html"
1.264 + urlOpts = []
1.265 +
1.266 + def __init__(self):
1.267 + self.update_mozinfo()
1.268 + self.server = None
1.269 + self.wsserver = None
1.270 + self.sslTunnel = None
1.271 + self._locations = None
1.272 +
1.273 + def update_mozinfo(self):
1.274 + """walk up directories to find mozinfo.json update the info"""
1.275 + # TODO: This should go in a more generic place, e.g. mozinfo
1.276 +
1.277 + path = SCRIPT_DIR
1.278 + dirs = set()
1.279 + while path != os.path.expanduser('~'):
1.280 + if path in dirs:
1.281 + break
1.282 + dirs.add(path)
1.283 + path = os.path.split(path)[0]
1.284 +
1.285 + mozinfo.find_and_update_from_json(*dirs)
1.286 +
1.287 + def getFullPath(self, path):
1.288 + " Get an absolute path relative to self.oldcwd."
1.289 + return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
1.290 +
1.291 + def getLogFilePath(self, logFile):
1.292 + """ return the log file path relative to the device we are testing on, in most cases
1.293 + it will be the full path on the local system
1.294 + """
1.295 + return self.getFullPath(logFile)
1.296 +
1.297 + @property
1.298 + def locations(self):
1.299 + if self._locations is not None:
1.300 + return self._locations
1.301 + locations_file = os.path.join(SCRIPT_DIR, 'server-locations.txt')
1.302 + self._locations = ServerLocations(locations_file)
1.303 + return self._locations
1.304 +
1.305 + def buildURLOptions(self, options, env):
1.306 + """ Add test control options from the command line to the url
1.307 +
1.308 + URL parameters to test URL:
1.309 +
1.310 + autorun -- kick off tests automatically
1.311 + closeWhenDone -- closes the browser after the tests
1.312 + hideResultsTable -- hides the table of individual test results
1.313 + logFile -- logs test run to an absolute path
1.314 + totalChunks -- how many chunks to split tests into
1.315 + thisChunk -- which chunk to run
1.316 + startAt -- name of test to start at
1.317 + endAt -- name of test to end at
1.318 + timeout -- per-test timeout in seconds
1.319 + repeat -- How many times to repeat the test, ie: repeat=1 will run the test twice.
1.320 + """
1.321 +
1.322 + # allow relative paths for logFile
1.323 + if options.logFile:
1.324 + options.logFile = self.getLogFilePath(options.logFile)
1.325 +
1.326 + # Note that all tests under options.subsuite need to be browser chrome tests.
1.327 + if options.browserChrome or options.chrome or options.subsuite or \
1.328 + options.a11y or options.webapprtChrome:
1.329 + self.makeTestConfig(options)
1.330 + else:
1.331 + if options.autorun:
1.332 + self.urlOpts.append("autorun=1")
1.333 + if options.timeout:
1.334 + self.urlOpts.append("timeout=%d" % options.timeout)
1.335 + if options.closeWhenDone:
1.336 + self.urlOpts.append("closeWhenDone=1")
1.337 + if options.logFile:
1.338 + self.urlOpts.append("logFile=" + encodeURIComponent(options.logFile))
1.339 + self.urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel))
1.340 + if options.consoleLevel:
1.341 + self.urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel))
1.342 + if options.totalChunks:
1.343 + self.urlOpts.append("totalChunks=%d" % options.totalChunks)
1.344 + self.urlOpts.append("thisChunk=%d" % options.thisChunk)
1.345 + if options.chunkByDir:
1.346 + self.urlOpts.append("chunkByDir=%d" % options.chunkByDir)
1.347 + if options.startAt:
1.348 + self.urlOpts.append("startAt=%s" % options.startAt)
1.349 + if options.endAt:
1.350 + self.urlOpts.append("endAt=%s" % options.endAt)
1.351 + if options.shuffle:
1.352 + self.urlOpts.append("shuffle=1")
1.353 + if "MOZ_HIDE_RESULTS_TABLE" in env and env["MOZ_HIDE_RESULTS_TABLE"] == "1":
1.354 + self.urlOpts.append("hideResultsTable=1")
1.355 + if options.runUntilFailure:
1.356 + self.urlOpts.append("runUntilFailure=1")
1.357 + if options.repeat:
1.358 + self.urlOpts.append("repeat=%d" % options.repeat)
1.359 + if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0:
1.360 + self.urlOpts.append("testname=%s" % ("/").join([self.TEST_PATH, options.testPath]))
1.361 + if options.testManifest:
1.362 + self.urlOpts.append("testManifest=%s" % options.testManifest)
1.363 + if hasattr(options, 'runOnly') and options.runOnly:
1.364 + self.urlOpts.append("runOnly=true")
1.365 + else:
1.366 + self.urlOpts.append("runOnly=false")
1.367 + if options.manifestFile:
1.368 + self.urlOpts.append("manifestFile=%s" % options.manifestFile)
1.369 + if options.failureFile:
1.370 + self.urlOpts.append("failureFile=%s" % self.getFullPath(options.failureFile))
1.371 + if options.runSlower:
1.372 + self.urlOpts.append("runSlower=true")
1.373 + if options.debugOnFailure:
1.374 + self.urlOpts.append("debugOnFailure=true")
1.375 + if options.dumpOutputDirectory:
1.376 + self.urlOpts.append("dumpOutputDirectory=%s" % encodeURIComponent(options.dumpOutputDirectory))
1.377 + if options.dumpAboutMemoryAfterTest:
1.378 + self.urlOpts.append("dumpAboutMemoryAfterTest=true")
1.379 + if options.dumpDMDAfterTest:
1.380 + self.urlOpts.append("dumpDMDAfterTest=true")
1.381 + if options.quiet:
1.382 + self.urlOpts.append("quiet=true")
1.383 +
1.384 + def getTestFlavor(self, options):
1.385 + if options.browserChrome:
1.386 + return "browser-chrome"
1.387 + elif options.chrome:
1.388 + return "chrome"
1.389 + elif options.a11y:
1.390 + return "a11y"
1.391 + elif options.webapprtChrome:
1.392 + return "webapprt-chrome"
1.393 + else:
1.394 + return "mochitest"
1.395 +
1.396 + # This check can be removed when bug 983867 is fixed.
1.397 + def isTest(self, options, filename):
1.398 + allow_js_css = False
1.399 + if options.browserChrome:
1.400 + allow_js_css = True
1.401 + testPattern = re.compile(r"browser_.+\.js")
1.402 + elif options.chrome or options.a11y:
1.403 + testPattern = re.compile(r"(browser|test)_.+\.(xul|html|js|xhtml)")
1.404 + elif options.webapprtContent:
1.405 + testPattern = re.compile(r"webapprt_")
1.406 + elif options.webapprtChrome:
1.407 + allow_js_css = True
1.408 + testPattern = re.compile(r"browser_")
1.409 + else:
1.410 + testPattern = re.compile(r"test_")
1.411 +
1.412 + if not allow_js_css and (".js" in filename or ".css" in filename):
1.413 + return False
1.414 +
1.415 + pathPieces = filename.split("/")
1.416 +
1.417 + return (testPattern.match(pathPieces[-1]) and
1.418 + not re.search(r'\^headers\^$', filename))
1.419 +
1.420 + def getTestPath(self, options):
1.421 + if options.ipcplugins:
1.422 + return "dom/plugins/test"
1.423 + else:
1.424 + return options.testPath
1.425 +
1.426 + def getTestRoot(self, options):
1.427 + if options.browserChrome:
1.428 + if options.immersiveMode:
1.429 + return 'metro'
1.430 + return 'browser'
1.431 + elif options.a11y:
1.432 + return 'a11y'
1.433 + elif options.webapprtChrome:
1.434 + return 'webapprtChrome'
1.435 + elif options.chrome:
1.436 + return 'chrome'
1.437 + return self.TEST_PATH
1.438 +
1.439 + def buildTestURL(self, options):
1.440 + testHost = "http://mochi.test:8888"
1.441 + testPath = self.getTestPath(options)
1.442 + testURL = "/".join([testHost, self.TEST_PATH, testPath])
1.443 + if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, testPath)) and options.repeat > 0:
1.444 + testURL = "/".join([testHost, self.TEST_PATH, os.path.dirname(testPath)])
1.445 + if options.chrome or options.a11y:
1.446 + testURL = "/".join([testHost, self.CHROME_PATH])
1.447 + elif options.browserChrome:
1.448 + testURL = "about:blank"
1.449 + return testURL
1.450 +
1.451 + def buildTestPath(self, options):
1.452 + """ Build the url path to the specific test harness and test file or directory
1.453 + Build a manifest of tests to run and write out a json file for the harness to read
1.454 + """
1.455 + manifest = None
1.456 +
1.457 + testRoot = self.getTestRoot(options)
1.458 + # testdir refers to 'mochitest' here.
1.459 + testdir = SCRIPT_DIR.split(os.getcwd())[-1]
1.460 + testdir = testdir.strip(os.sep)
1.461 + testRootAbs = os.path.abspath(os.path.join(testdir, testRoot))
1.462 + if isinstance(options.manifestFile, TestManifest):
1.463 + manifest = options.manifestFile
1.464 + elif options.manifestFile and os.path.isfile(options.manifestFile):
1.465 + manifestFileAbs = os.path.abspath(options.manifestFile)
1.466 + assert manifestFileAbs.startswith(testRootAbs)
1.467 + manifest = TestManifest([options.manifestFile], strict=False)
1.468 + else:
1.469 + masterName = self.getTestFlavor(options) + '.ini'
1.470 + masterPath = os.path.join(testdir, testRoot, masterName)
1.471 +
1.472 + if os.path.exists(masterPath):
1.473 + manifest = TestManifest([masterPath], strict=False)
1.474 +
1.475 + if manifest:
1.476 + # Python 2.6 doesn't allow unicode keys to be used for keyword
1.477 + # arguments. This gross hack works around the problem until we
1.478 + # rid ourselves of 2.6.
1.479 + info = {}
1.480 + for k, v in mozinfo.info.items():
1.481 + if isinstance(k, unicode):
1.482 + k = k.encode('ascii')
1.483 + info[k] = v
1.484 +
1.485 + # Bug 883858 - return all tests including disabled tests
1.486 + tests = manifest.active_tests(disabled=True, options=options, **info)
1.487 + paths = []
1.488 + testPath = self.getTestPath(options)
1.489 +
1.490 + for test in tests:
1.491 + pathAbs = os.path.abspath(test['path'])
1.492 + assert pathAbs.startswith(testRootAbs)
1.493 + tp = pathAbs[len(testRootAbs):].replace('\\', '/').strip('/')
1.494 +
1.495 + # Filter out tests if we are using --test-path
1.496 + if testPath and not tp.startswith(testPath):
1.497 + continue
1.498 +
1.499 + if not self.isTest(options, tp):
1.500 + print 'Warning: %s from manifest %s is not a valid test' % (test['name'], test['manifest'])
1.501 + continue
1.502 +
1.503 + testob = {'path': tp}
1.504 + if test.has_key('disabled'):
1.505 + testob['disabled'] = test['disabled']
1.506 + paths.append(testob)
1.507 +
1.508 + # Sort tests so they are run in a deterministic order.
1.509 + def path_sort(ob1, ob2):
1.510 + path1 = ob1['path'].split('/')
1.511 + path2 = ob2['path'].split('/')
1.512 + return cmp(path1, path2)
1.513 +
1.514 + paths.sort(path_sort)
1.515 +
1.516 + # Bug 883865 - add this functionality into manifestDestiny
1.517 + with open(os.path.join(testdir, 'tests.json'), 'w') as manifestFile:
1.518 + manifestFile.write(json.dumps({'tests': paths}))
1.519 + options.manifestFile = 'tests.json'
1.520 +
1.521 + return self.buildTestURL(options)
1.522 +
1.523 + def startWebSocketServer(self, options, debuggerInfo):
1.524 + """ Launch the websocket server """
1.525 + self.wsserver = WebSocketServer(options, SCRIPT_DIR, debuggerInfo)
1.526 + self.wsserver.start()
1.527 +
1.528 + def startWebServer(self, options):
1.529 + """Create the webserver and start it up"""
1.530 +
1.531 + self.server = MochitestServer(options)
1.532 + self.server.start()
1.533 +
1.534 + if options.pidFile != "":
1.535 + with open(options.pidFile + ".xpcshell.pid", 'w') as f:
1.536 + f.write("%s" % self.server._process.pid)
1.537 +
1.538 + def startServers(self, options, debuggerInfo):
1.539 + # start servers and set ports
1.540 + # TODO: pass these values, don't set on `self`
1.541 + self.webServer = options.webServer
1.542 + self.httpPort = options.httpPort
1.543 + self.sslPort = options.sslPort
1.544 + self.webSocketPort = options.webSocketPort
1.545 +
1.546 + # httpd-path is specified by standard makefile targets and may be specified
1.547 + # on the command line to select a particular version of httpd.js. If not
1.548 + # specified, try to select the one from hostutils.zip, as required in bug 882932.
1.549 + if not options.httpdPath:
1.550 + options.httpdPath = os.path.join(options.utilityPath, "components")
1.551 +
1.552 + self.startWebServer(options)
1.553 + self.startWebSocketServer(options, debuggerInfo)
1.554 +
1.555 + # start SSL pipe
1.556 + self.sslTunnel = SSLTunnel(options)
1.557 + self.sslTunnel.buildConfig(self.locations)
1.558 + self.sslTunnel.start()
1.559 +
1.560 + # If we're lucky, the server has fully started by now, and all paths are
1.561 + # ready, etc. However, xpcshell cold start times suck, at least for debug
1.562 + # builds. We'll try to connect to the server for awhile, and if we fail,
1.563 + # we'll try to kill the server and exit with an error.
1.564 + if self.server is not None:
1.565 + self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
1.566 +
1.567 + def stopServers(self):
1.568 + """Servers are no longer needed, and perhaps more importantly, anything they
1.569 + might spew to console might confuse things."""
1.570 + if self.server is not None:
1.571 + try:
1.572 + log.info('Stopping web server')
1.573 + self.server.stop()
1.574 + except Exception:
1.575 + log.exception('Exception when stopping web server')
1.576 +
1.577 + if self.wsserver is not None:
1.578 + try:
1.579 + log.info('Stopping web socket server')
1.580 + self.wsserver.stop()
1.581 + except Exception:
1.582 + log.exception('Exception when stopping web socket server');
1.583 +
1.584 + if self.sslTunnel is not None:
1.585 + try:
1.586 + log.info('Stopping ssltunnel')
1.587 + self.sslTunnel.stop()
1.588 + except Exception:
1.589 + log.exception('Exception stopping ssltunnel');
1.590 +
1.591 + def copyExtraFilesToProfile(self, options):
1.592 + "Copy extra files or dirs specified on the command line to the testing profile."
1.593 + for f in options.extraProfileFiles:
1.594 + abspath = self.getFullPath(f)
1.595 + if os.path.isfile(abspath):
1.596 + shutil.copy2(abspath, options.profilePath)
1.597 + elif os.path.isdir(abspath):
1.598 + dest = os.path.join(options.profilePath, os.path.basename(abspath))
1.599 + shutil.copytree(abspath, dest)
1.600 + else:
1.601 + log.warning("runtests.py | Failed to copy %s to profile", abspath)
1.602 +
1.603 + def installChromeJar(self, chrome, options):
1.604 + """
1.605 + copy mochijar directory to profile as an extension so we have chrome://mochikit for all harness code
1.606 + """
1.607 + # Write chrome.manifest.
1.608 + with open(os.path.join(options.profilePath, "extensions", "staged", "mochikit@mozilla.org", "chrome.manifest"), "a") as mfile:
1.609 + mfile.write(chrome)
1.610 +
1.611 + def addChromeToProfile(self, options):
1.612 + "Adds MochiKit chrome tests to the profile."
1.613 +
1.614 + # Create (empty) chrome directory.
1.615 + chromedir = os.path.join(options.profilePath, "chrome")
1.616 + os.mkdir(chromedir)
1.617 +
1.618 + # Write userChrome.css.
1.619 + chrome = """
1.620 +@namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */
1.621 +toolbar,
1.622 +toolbarpalette {
1.623 + background-color: rgb(235, 235, 235) !important;
1.624 +}
1.625 +toolbar#nav-bar {
1.626 + background-image: none !important;
1.627 +}
1.628 +"""
1.629 + with open(os.path.join(options.profilePath, "userChrome.css"), "a") as chromeFile:
1.630 + chromeFile.write(chrome)
1.631 +
1.632 + manifest = os.path.join(options.profilePath, "tests.manifest")
1.633 + with open(manifest, "w") as manifestFile:
1.634 + # Register chrome directory.
1.635 + chrometestDir = os.path.join(os.path.abspath("."), SCRIPT_DIR) + "/"
1.636 + if mozinfo.isWin:
1.637 + chrometestDir = "file:///" + chrometestDir.replace("\\", "/")
1.638 + manifestFile.write("content mochitests %s contentaccessible=yes\n" % chrometestDir)
1.639 +
1.640 + if options.testingModulesDir is not None:
1.641 + manifestFile.write("resource testing-common file:///%s\n" %
1.642 + options.testingModulesDir)
1.643 +
1.644 + # Call installChromeJar().
1.645 + if not os.path.isdir(os.path.join(SCRIPT_DIR, self.jarDir)):
1.646 + log.testFail("invalid setup: missing mochikit extension")
1.647 + return None
1.648 +
1.649 + # Support Firefox (browser), B2G (shell), SeaMonkey (navigator), and Webapp
1.650 + # Runtime (webapp).
1.651 + chrome = ""
1.652 + if options.browserChrome or options.chrome or options.a11y or options.webapprtChrome:
1.653 + chrome += """
1.654 +overlay chrome://browser/content/browser.xul chrome://mochikit/content/browser-test-overlay.xul
1.655 +overlay chrome://browser/content/shell.xhtml chrome://mochikit/content/browser-test-overlay.xul
1.656 +overlay chrome://navigator/content/navigator.xul chrome://mochikit/content/browser-test-overlay.xul
1.657 +overlay chrome://webapprt/content/webapp.xul chrome://mochikit/content/browser-test-overlay.xul
1.658 +"""
1.659 +
1.660 + self.installChromeJar(chrome, options)
1.661 + return manifest
1.662 +
1.663 + def getExtensionsToInstall(self, options):
1.664 + "Return a list of extensions to install in the profile"
1.665 + extensions = options.extensionsToInstall or []
1.666 + appDir = options.app[:options.app.rfind(os.sep)] if options.app else options.utilityPath
1.667 +
1.668 + extensionDirs = [
1.669 + # Extensions distributed with the test harness.
1.670 + os.path.normpath(os.path.join(SCRIPT_DIR, "extensions")),
1.671 + ]
1.672 + if appDir:
1.673 + # Extensions distributed with the application.
1.674 + extensionDirs.append(os.path.join(appDir, "distribution", "extensions"))
1.675 +
1.676 + for extensionDir in extensionDirs:
1.677 + if os.path.isdir(extensionDir):
1.678 + for dirEntry in os.listdir(extensionDir):
1.679 + if dirEntry not in options.extensionsToExclude:
1.680 + path = os.path.join(extensionDir, dirEntry)
1.681 + if os.path.isdir(path) or (os.path.isfile(path) and path.endswith(".xpi")):
1.682 + extensions.append(path)
1.683 +
1.684 + # append mochikit
1.685 + extensions.append(os.path.join(SCRIPT_DIR, self.jarDir))
1.686 + return extensions
1.687 +
1.688 +class SSLTunnel:
1.689 + def __init__(self, options):
1.690 + self.process = None
1.691 + self.utilityPath = options.utilityPath
1.692 + self.xrePath = options.xrePath
1.693 + self.certPath = options.certPath
1.694 + self.sslPort = options.sslPort
1.695 + self.httpPort = options.httpPort
1.696 + self.webServer = options.webServer
1.697 + self.webSocketPort = options.webSocketPort
1.698 +
1.699 + self.customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
1.700 + self.clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
1.701 + self.redirRE = re.compile("^redir=(?P<redirhost>[0-9a-zA-Z_ .]+)")
1.702 +
1.703 + def writeLocation(self, config, loc):
1.704 + for option in loc.options:
1.705 + match = self.customCertRE.match(option)
1.706 + if match:
1.707 + customcert = match.group("nickname");
1.708 + config.write("listen:%s:%s:%s:%s\n" %
1.709 + (loc.host, loc.port, self.sslPort, customcert))
1.710 +
1.711 + match = self.clientAuthRE.match(option)
1.712 + if match:
1.713 + clientauth = match.group("clientauth");
1.714 + config.write("clientauth:%s:%s:%s:%s\n" %
1.715 + (loc.host, loc.port, self.sslPort, clientauth))
1.716 +
1.717 + match = self.redirRE.match(option)
1.718 + if match:
1.719 + redirhost = match.group("redirhost")
1.720 + config.write("redirhost:%s:%s:%s:%s\n" %
1.721 + (loc.host, loc.port, self.sslPort, redirhost))
1.722 +
1.723 + def buildConfig(self, locations):
1.724 + """Create the ssltunnel configuration file"""
1.725 + configFd, self.configFile = tempfile.mkstemp(prefix="ssltunnel", suffix=".cfg")
1.726 + with os.fdopen(configFd, "w") as config:
1.727 + config.write("httpproxy:1\n")
1.728 + config.write("certdbdir:%s\n" % self.certPath)
1.729 + config.write("forward:127.0.0.1:%s\n" % self.httpPort)
1.730 + config.write("websocketserver:%s:%s\n" % (self.webServer, self.webSocketPort))
1.731 + config.write("listen:*:%s:pgo server certificate\n" % self.sslPort)
1.732 +
1.733 + for loc in locations:
1.734 + if loc.scheme == "https" and "nocert" not in loc.options:
1.735 + self.writeLocation(config, loc)
1.736 +
1.737 + def start(self):
1.738 + """ Starts the SSL Tunnel """
1.739 +
1.740 + # start ssltunnel to provide https:// URLs capability
1.741 + bin_suffix = mozinfo.info.get('bin_suffix', '')
1.742 + ssltunnel = os.path.join(self.utilityPath, "ssltunnel" + bin_suffix)
1.743 + if not os.path.exists(ssltunnel):
1.744 + log.error("INFO | runtests.py | expected to find ssltunnel at %s", ssltunnel)
1.745 + exit(1)
1.746 +
1.747 + env = environment(xrePath=self.xrePath)
1.748 + self.process = mozprocess.ProcessHandler([ssltunnel, self.configFile],
1.749 + env=env)
1.750 + self.process.run()
1.751 + log.info("INFO | runtests.py | SSL tunnel pid: %d", self.process.pid)
1.752 +
1.753 + def stop(self):
1.754 + """ Stops the SSL Tunnel and cleans up """
1.755 + if self.process is not None:
1.756 + self.process.kill()
1.757 + if os.path.exists(self.configFile):
1.758 + os.remove(self.configFile)
1.759 +
1.760 +class Mochitest(MochitestUtilsMixin):
1.761 + certdbNew = False
1.762 + sslTunnel = None
1.763 + vmwareHelper = None
1.764 + DEFAULT_TIMEOUT = 60.0
1.765 +
1.766 + # XXX use automation.py for test name to avoid breaking legacy
1.767 + # TODO: replace this with 'runtests.py' or 'mochitest' or the like
1.768 + test_name = 'automation.py'
1.769 +
1.770 + def __init__(self):
1.771 + super(Mochitest, self).__init__()
1.772 +
1.773 + # environment function for browserEnv
1.774 + self.environment = environment
1.775 +
1.776 + # Max time in seconds to wait for server startup before tests will fail -- if
1.777 + # this seems big, it's mostly for debug machines where cold startup
1.778 + # (particularly after a build) takes forever.
1.779 + self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90
1.780 +
1.781 + # metro browser sub process id
1.782 + self.browserProcessId = None
1.783 +
1.784 +
1.785 + self.haveDumpedScreen = False
1.786 +
1.787 + def extraPrefs(self, extraPrefs):
1.788 + """interpolate extra preferences from option strings"""
1.789 +
1.790 + try:
1.791 + return dict(parseKeyValue(extraPrefs, context='--setpref='))
1.792 + except KeyValueParseError, e:
1.793 + print str(e)
1.794 + sys.exit(1)
1.795 +
1.796 + def fillCertificateDB(self, options):
1.797 + # TODO: move -> mozprofile:
1.798 + # https://bugzilla.mozilla.org/show_bug.cgi?id=746243#c35
1.799 +
1.800 + pwfilePath = os.path.join(options.profilePath, ".crtdbpw")
1.801 + with open(pwfilePath, "w") as pwfile:
1.802 + pwfile.write("\n")
1.803 +
1.804 + # Pre-create the certification database for the profile
1.805 + env = self.environment(xrePath=options.xrePath)
1.806 + bin_suffix = mozinfo.info.get('bin_suffix', '')
1.807 + certutil = os.path.join(options.utilityPath, "certutil" + bin_suffix)
1.808 + pk12util = os.path.join(options.utilityPath, "pk12util" + bin_suffix)
1.809 +
1.810 + if self.certdbNew:
1.811 + # android and b2g use the new DB formats exclusively
1.812 + certdbPath = "sql:" + options.profilePath
1.813 + else:
1.814 + # desktop seems to use the old
1.815 + certdbPath = options.profilePath
1.816 +
1.817 + status = call([certutil, "-N", "-d", certdbPath, "-f", pwfilePath], env=env)
1.818 + if status:
1.819 + return status
1.820 +
1.821 + # Walk the cert directory and add custom CAs and client certs
1.822 + files = os.listdir(options.certPath)
1.823 + for item in files:
1.824 + root, ext = os.path.splitext(item)
1.825 + if ext == ".ca":
1.826 + trustBits = "CT,,"
1.827 + if root.endswith("-object"):
1.828 + trustBits = "CT,,CT"
1.829 + call([certutil, "-A", "-i", os.path.join(options.certPath, item),
1.830 + "-d", certdbPath, "-f", pwfilePath, "-n", root, "-t", trustBits],
1.831 + env=env)
1.832 + elif ext == ".client":
1.833 + call([pk12util, "-i", os.path.join(options.certPath, item),
1.834 + "-w", pwfilePath, "-d", certdbPath],
1.835 + env=env)
1.836 +
1.837 + os.unlink(pwfilePath)
1.838 + return 0
1.839 +
1.840 + def buildProfile(self, options):
1.841 + """ create the profile and add optional chrome bits and files if requested """
1.842 + if options.browserChrome and options.timeout:
1.843 + options.extraPrefs.append("testing.browserTestHarness.timeout=%d" % options.timeout)
1.844 + options.extraPrefs.append("browser.tabs.remote=%s" % ('true' if options.e10s else 'false'))
1.845 + options.extraPrefs.append("browser.tabs.remote.autostart=%s" % ('true' if options.e10s else 'false'))
1.846 +
1.847 + # get extensions to install
1.848 + extensions = self.getExtensionsToInstall(options)
1.849 +
1.850 + # web apps
1.851 + appsPath = os.path.join(SCRIPT_DIR, 'profile_data', 'webapps_mochitest.json')
1.852 + if os.path.exists(appsPath):
1.853 + with open(appsPath) as apps_file:
1.854 + apps = json.load(apps_file)
1.855 + else:
1.856 + apps = None
1.857 +
1.858 + # preferences
1.859 + prefsPath = os.path.join(SCRIPT_DIR, 'profile_data', 'prefs_general.js')
1.860 + prefs = dict(Preferences.read_prefs(prefsPath))
1.861 + prefs.update(self.extraPrefs(options.extraPrefs))
1.862 +
1.863 + # interpolate preferences
1.864 + interpolation = {"server": "%s:%s" % (options.webServer, options.httpPort)}
1.865 + prefs = json.loads(json.dumps(prefs) % interpolation)
1.866 + for pref in prefs:
1.867 + prefs[pref] = Preferences.cast(prefs[pref])
1.868 + # TODO: make this less hacky
1.869 + # https://bugzilla.mozilla.org/show_bug.cgi?id=913152
1.870 +
1.871 + # proxy
1.872 + proxy = {'remote': options.webServer,
1.873 + 'http': options.httpPort,
1.874 + 'https': options.sslPort,
1.875 + # use SSL port for legacy compatibility; see
1.876 + # - https://bugzilla.mozilla.org/show_bug.cgi?id=688667#c66
1.877 + # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221
1.878 + # - https://github.com/mozilla/mozbase/commit/43f9510e3d58bfed32790c82a57edac5f928474d
1.879 + # 'ws': str(self.webSocketPort)
1.880 + 'ws': options.sslPort
1.881 + }
1.882 +
1.883 +
1.884 + # create a profile
1.885 + self.profile = Profile(profile=options.profilePath,
1.886 + addons=extensions,
1.887 + locations=self.locations,
1.888 + preferences=prefs,
1.889 + apps=apps,
1.890 + proxy=proxy
1.891 + )
1.892 +
1.893 + # Fix options.profilePath for legacy consumers.
1.894 + options.profilePath = self.profile.profile
1.895 +
1.896 + manifest = self.addChromeToProfile(options)
1.897 + self.copyExtraFilesToProfile(options)
1.898 +
1.899 + # create certificate database for the profile
1.900 + # TODO: this should really be upstreamed somewhere, maybe mozprofile
1.901 + certificateStatus = self.fillCertificateDB(options)
1.902 + if certificateStatus:
1.903 + log.info("TEST-UNEXPECTED-FAIL | runtests.py | Certificate integration failed")
1.904 + return None
1.905 +
1.906 + return manifest
1.907 +
1.908 + def buildBrowserEnv(self, options, debugger=False):
1.909 + """build the environment variables for the specific test and operating system"""
1.910 + browserEnv = self.environment(xrePath=options.xrePath, debugger=debugger,
1.911 + dmdPath=options.dmdPath)
1.912 +
1.913 + # These variables are necessary for correct application startup; change
1.914 + # via the commandline at your own risk.
1.915 + browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
1.916 +
1.917 + # interpolate environment passed with options
1.918 + try:
1.919 + browserEnv.update(dict(parseKeyValue(options.environment, context='--setenv')))
1.920 + except KeyValueParseError, e:
1.921 + log.error(str(e))
1.922 + return
1.923 +
1.924 + browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
1.925 +
1.926 + if options.fatalAssertions:
1.927 + browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
1.928 +
1.929 + # Produce an NSPR log, is setup (see NSPR_LOG_MODULES global at the top of
1.930 + # this script).
1.931 + self.nsprLogs = NSPR_LOG_MODULES and "MOZ_UPLOAD_DIR" in os.environ
1.932 + if self.nsprLogs:
1.933 + browserEnv["NSPR_LOG_MODULES"] = NSPR_LOG_MODULES
1.934 +
1.935 + browserEnv["NSPR_LOG_FILE"] = "%s/nspr.log" % tempfile.gettempdir()
1.936 + browserEnv["GECKO_SEPARATE_NSPR_LOGS"] = "1"
1.937 +
1.938 + if debugger and not options.slowscript:
1.939 + browserEnv["JS_DISABLE_SLOW_SCRIPT_SIGNALS"] = "1"
1.940 +
1.941 + return browserEnv
1.942 +
1.943 + def cleanup(self, manifest, options):
1.944 + """ remove temporary files and profile """
1.945 + os.remove(manifest)
1.946 + del self.profile
1.947 + if options.pidFile != "":
1.948 + try:
1.949 + os.remove(options.pidFile)
1.950 + if os.path.exists(options.pidFile + ".xpcshell.pid"):
1.951 + os.remove(options.pidFile + ".xpcshell.pid")
1.952 + except:
1.953 + log.warn("cleaning up pidfile '%s' was unsuccessful from the test harness", options.pidFile)
1.954 +
1.955 + def dumpScreen(self, utilityPath):
1.956 + if self.haveDumpedScreen:
1.957 + log.info("Not taking screenshot here: see the one that was previously logged")
1.958 + return
1.959 + self.haveDumpedScreen = True
1.960 + dumpScreen(utilityPath)
1.961 +
1.962 + def killAndGetStack(self, processPID, utilityPath, debuggerInfo, dump_screen=False):
1.963 + """
1.964 + Kill the process, preferrably in a way that gets us a stack trace.
1.965 + Also attempts to obtain a screenshot before killing the process
1.966 + if specified.
1.967 + """
1.968 +
1.969 + if dump_screen:
1.970 + self.dumpScreen(utilityPath)
1.971 +
1.972 + if mozinfo.info.get('crashreporter', True) and not debuggerInfo:
1.973 + if mozinfo.isWin:
1.974 + # We should have a "crashinject" program in our utility path
1.975 + crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
1.976 + if os.path.exists(crashinject):
1.977 + status = subprocess.Popen([crashinject, str(processPID)]).wait()
1.978 + printstatus(status, "crashinject")
1.979 + if status == 0:
1.980 + return
1.981 + else:
1.982 + try:
1.983 + os.kill(processPID, signal.SIGABRT)
1.984 + except OSError:
1.985 + # https://bugzilla.mozilla.org/show_bug.cgi?id=921509
1.986 + log.info("Can't trigger Breakpad, process no longer exists")
1.987 + return
1.988 + log.info("Can't trigger Breakpad, just killing process")
1.989 + killPid(processPID)
1.990 +
1.991 + def checkForZombies(self, processLog, utilityPath, debuggerInfo):
1.992 + """Look for hung processes"""
1.993 +
1.994 + if not os.path.exists(processLog):
1.995 + log.info('Automation Error: PID log not found: %s', processLog)
1.996 + # Whilst no hung process was found, the run should still display as a failure
1.997 + return True
1.998 +
1.999 + # scan processLog for zombies
1.1000 + log.info('INFO | zombiecheck | Reading PID log: %s', processLog)
1.1001 + processList = []
1.1002 + pidRE = re.compile(r'launched child process (\d+)$')
1.1003 + with open(processLog) as processLogFD:
1.1004 + for line in processLogFD:
1.1005 + log.info(line.rstrip())
1.1006 + m = pidRE.search(line)
1.1007 + if m:
1.1008 + processList.append(int(m.group(1)))
1.1009 +
1.1010 + # kill zombies
1.1011 + foundZombie = False
1.1012 + for processPID in processList:
1.1013 + log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID)
1.1014 + if isPidAlive(processPID):
1.1015 + foundZombie = True
1.1016 + log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID)
1.1017 + self.killAndGetStack(processPID, utilityPath, debuggerInfo, dump_screen=not debuggerInfo)
1.1018 +
1.1019 + return foundZombie
1.1020 +
1.1021 + def startVMwareRecording(self, options):
1.1022 + """ starts recording inside VMware VM using the recording helper dll """
1.1023 + assert mozinfo.isWin
1.1024 + from ctypes import cdll
1.1025 + self.vmwareHelper = cdll.LoadLibrary(self.vmwareHelperPath)
1.1026 + if self.vmwareHelper is None:
1.1027 + log.warning("runtests.py | Failed to load "
1.1028 + "VMware recording helper")
1.1029 + return
1.1030 + log.info("runtests.py | Starting VMware recording.")
1.1031 + try:
1.1032 + self.vmwareHelper.StartRecording()
1.1033 + except Exception, e:
1.1034 + log.warning("runtests.py | Failed to start "
1.1035 + "VMware recording: (%s)" % str(e))
1.1036 + self.vmwareHelper = None
1.1037 +
1.1038 + def stopVMwareRecording(self):
1.1039 + """ stops recording inside VMware VM using the recording helper dll """
1.1040 + try:
1.1041 + assert mozinfo.isWin
1.1042 + if self.vmwareHelper is not None:
1.1043 + log.info("runtests.py | Stopping VMware recording.")
1.1044 + self.vmwareHelper.StopRecording()
1.1045 + except Exception, e:
1.1046 + log.warning("runtests.py | Failed to stop "
1.1047 + "VMware recording: (%s)" % str(e))
1.1048 + log.exception('Error stopping VMWare recording')
1.1049 +
1.1050 + self.vmwareHelper = None
1.1051 +
1.1052 + def runApp(self,
1.1053 + testUrl,
1.1054 + env,
1.1055 + app,
1.1056 + profile,
1.1057 + extraArgs,
1.1058 + utilityPath,
1.1059 + debuggerInfo=None,
1.1060 + symbolsPath=None,
1.1061 + timeout=-1,
1.1062 + onLaunch=None,
1.1063 + webapprtChrome=False,
1.1064 + hide_subtests=False,
1.1065 + screenshotOnFail=False):
1.1066 + """
1.1067 + Run the app, log the duration it took to execute, return the status code.
1.1068 + Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
1.1069 + """
1.1070 +
1.1071 + # debugger information
1.1072 + interactive = False
1.1073 + debug_args = None
1.1074 + if debuggerInfo:
1.1075 + interactive = debuggerInfo['interactive']
1.1076 + debug_args = [debuggerInfo['path']] + debuggerInfo['args']
1.1077 +
1.1078 + # fix default timeout
1.1079 + if timeout == -1:
1.1080 + timeout = self.DEFAULT_TIMEOUT
1.1081 +
1.1082 + # build parameters
1.1083 + is_test_build = mozinfo.info.get('tests_enabled', True)
1.1084 + bin_suffix = mozinfo.info.get('bin_suffix', '')
1.1085 +
1.1086 + # copy env so we don't munge the caller's environment
1.1087 + env = env.copy()
1.1088 +
1.1089 + # make sure we clean up after ourselves.
1.1090 + try:
1.1091 + # set process log environment variable
1.1092 + tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
1.1093 + os.close(tmpfd)
1.1094 + env["MOZ_PROCESS_LOG"] = processLog
1.1095 +
1.1096 + if interactive:
1.1097 + # If an interactive debugger is attached,
1.1098 + # don't use timeouts, and don't capture ctrl-c.
1.1099 + timeout = None
1.1100 + signal.signal(signal.SIGINT, lambda sigid, frame: None)
1.1101 +
1.1102 + # build command line
1.1103 + cmd = os.path.abspath(app)
1.1104 + args = list(extraArgs)
1.1105 + # TODO: mozrunner should use -foreground at least for mac
1.1106 + # https://bugzilla.mozilla.org/show_bug.cgi?id=916512
1.1107 + args.append('-foreground')
1.1108 + if testUrl:
1.1109 + if debuggerInfo and debuggerInfo['requiresEscapedArgs']:
1.1110 + testUrl = testUrl.replace("&", "\\&")
1.1111 + args.append(testUrl)
1.1112 +
1.1113 + if mozinfo.info["debug"] and not webapprtChrome:
1.1114 + shutdownLeaks = ShutdownLeaks(log.info)
1.1115 + else:
1.1116 + shutdownLeaks = None
1.1117 +
1.1118 + # create an instance to process the output
1.1119 + outputHandler = self.OutputHandler(harness=self,
1.1120 + utilityPath=utilityPath,
1.1121 + symbolsPath=symbolsPath,
1.1122 + dump_screen_on_timeout=not debuggerInfo,
1.1123 + dump_screen_on_fail=screenshotOnFail,
1.1124 + hide_subtests=hide_subtests,
1.1125 + shutdownLeaks=shutdownLeaks,
1.1126 + )
1.1127 +
1.1128 + def timeoutHandler():
1.1129 + outputHandler.log_output_buffer()
1.1130 + browserProcessId = outputHandler.browserProcessId
1.1131 + self.handleTimeout(timeout, proc, utilityPath, debuggerInfo, browserProcessId)
1.1132 + kp_kwargs = {'kill_on_timeout': False,
1.1133 + 'cwd': SCRIPT_DIR,
1.1134 + 'onTimeout': [timeoutHandler]}
1.1135 + kp_kwargs['processOutputLine'] = [outputHandler]
1.1136 +
1.1137 + # create mozrunner instance and start the system under test process
1.1138 + self.lastTestSeen = self.test_name
1.1139 + startTime = datetime.now()
1.1140 +
1.1141 + # b2g desktop requires FirefoxRunner even though appname is b2g
1.1142 + if mozinfo.info.get('appname') == 'b2g' and mozinfo.info.get('toolkit') != 'gonk':
1.1143 + runner_cls = mozrunner.FirefoxRunner
1.1144 + else:
1.1145 + runner_cls = mozrunner.runners.get(mozinfo.info.get('appname', 'firefox'),
1.1146 + mozrunner.Runner)
1.1147 + runner = runner_cls(profile=self.profile,
1.1148 + binary=cmd,
1.1149 + cmdargs=args,
1.1150 + env=env,
1.1151 + process_class=mozprocess.ProcessHandlerMixin,
1.1152 + kp_kwargs=kp_kwargs,
1.1153 + )
1.1154 +
1.1155 + # XXX work around bug 898379 until mozrunner is updated for m-c; see
1.1156 + # https://bugzilla.mozilla.org/show_bug.cgi?id=746243#c49
1.1157 + runner.kp_kwargs = kp_kwargs
1.1158 +
1.1159 + # start the runner
1.1160 + runner.start(debug_args=debug_args,
1.1161 + interactive=interactive,
1.1162 + outputTimeout=timeout)
1.1163 + proc = runner.process_handler
1.1164 + log.info("INFO | runtests.py | Application pid: %d", proc.pid)
1.1165 +
1.1166 + if onLaunch is not None:
1.1167 + # Allow callers to specify an onLaunch callback to be fired after the
1.1168 + # app is launched.
1.1169 + # We call onLaunch for b2g desktop mochitests so that we can
1.1170 + # run a Marionette script after gecko has completed startup.
1.1171 + onLaunch()
1.1172 +
1.1173 + # wait until app is finished
1.1174 + # XXX copy functionality from
1.1175 + # https://github.com/mozilla/mozbase/blob/master/mozrunner/mozrunner/runner.py#L61
1.1176 + # until bug 913970 is fixed regarding mozrunner `wait` not returning status
1.1177 + # see https://bugzilla.mozilla.org/show_bug.cgi?id=913970
1.1178 + status = proc.wait()
1.1179 + printstatus(status, "Main app process")
1.1180 + runner.process_handler = None
1.1181 +
1.1182 + if timeout is None:
1.1183 + didTimeout = False
1.1184 + else:
1.1185 + didTimeout = proc.didTimeout
1.1186 +
1.1187 + # finalize output handler
1.1188 + outputHandler.finish(didTimeout)
1.1189 +
1.1190 + # record post-test information
1.1191 + if status:
1.1192 + log.info("TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s", self.lastTestSeen, status)
1.1193 + else:
1.1194 + self.lastTestSeen = 'Main app process exited normally'
1.1195 +
1.1196 + log.info("INFO | runtests.py | Application ran for: %s", str(datetime.now() - startTime))
1.1197 +
1.1198 + # Do a final check for zombie child processes.
1.1199 + zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo)
1.1200 +
1.1201 + # check for crashes
1.1202 + minidump_path = os.path.join(self.profile.profile, "minidumps")
1.1203 + crashed = mozcrash.check_for_crashes(minidump_path,
1.1204 + symbolsPath,
1.1205 + test_name=self.lastTestSeen)
1.1206 +
1.1207 + if crashed or zombieProcesses:
1.1208 + status = 1
1.1209 +
1.1210 + finally:
1.1211 + # cleanup
1.1212 + if os.path.exists(processLog):
1.1213 + os.remove(processLog)
1.1214 +
1.1215 + return status
1.1216 +
1.1217 + def runTests(self, options, onLaunch=None):
1.1218 + """ Prepare, configure, run tests and cleanup """
1.1219 +
1.1220 + # get debugger info, a dict of:
1.1221 + # {'path': path to the debugger (string),
1.1222 + # 'interactive': whether the debugger is interactive or not (bool)
1.1223 + # 'args': arguments to the debugger (list)
1.1224 + # TODO: use mozrunner.local.debugger_arguments:
1.1225 + # https://github.com/mozilla/mozbase/blob/master/mozrunner/mozrunner/local.py#L42
1.1226 + debuggerInfo = getDebuggerInfo(self.oldcwd,
1.1227 + options.debugger,
1.1228 + options.debuggerArgs,
1.1229 + options.debuggerInteractive)
1.1230 +
1.1231 + self.leak_report_file = os.path.join(options.profilePath, "runtests_leaks.log")
1.1232 +
1.1233 + browserEnv = self.buildBrowserEnv(options, debuggerInfo is not None)
1.1234 + if browserEnv is None:
1.1235 + return 1
1.1236 +
1.1237 + # buildProfile sets self.profile .
1.1238 + # This relies on sideeffects and isn't very stateful:
1.1239 + # https://bugzilla.mozilla.org/show_bug.cgi?id=919300
1.1240 + manifest = self.buildProfile(options)
1.1241 + if manifest is None:
1.1242 + return 1
1.1243 +
1.1244 + try:
1.1245 + self.startServers(options, debuggerInfo)
1.1246 +
1.1247 + testURL = self.buildTestPath(options)
1.1248 + self.buildURLOptions(options, browserEnv)
1.1249 + if self.urlOpts:
1.1250 + testURL += "?" + "&".join(self.urlOpts)
1.1251 +
1.1252 + if options.webapprtContent:
1.1253 + options.browserArgs.extend(('-test-mode', testURL))
1.1254 + testURL = None
1.1255 +
1.1256 + if options.immersiveMode:
1.1257 + options.browserArgs.extend(('-firefoxpath', options.app))
1.1258 + options.app = self.immersiveHelperPath
1.1259 +
1.1260 + if options.jsdebugger:
1.1261 + options.browserArgs.extend(['-jsdebugger'])
1.1262 +
1.1263 + # Remove the leak detection file so it can't "leak" to the tests run.
1.1264 + # The file is not there if leak logging was not enabled in the application build.
1.1265 + if os.path.exists(self.leak_report_file):
1.1266 + os.remove(self.leak_report_file)
1.1267 +
1.1268 + # then again to actually run mochitest
1.1269 + if options.timeout:
1.1270 + timeout = options.timeout + 30
1.1271 + elif options.debugger or not options.autorun:
1.1272 + timeout = None
1.1273 + else:
1.1274 + timeout = 330.0 # default JS harness timeout is 300 seconds
1.1275 +
1.1276 + if options.vmwareRecording:
1.1277 + self.startVMwareRecording(options);
1.1278 +
1.1279 + log.info("runtests.py | Running tests: start.\n")
1.1280 + try:
1.1281 + status = self.runApp(testURL,
1.1282 + browserEnv,
1.1283 + options.app,
1.1284 + profile=self.profile,
1.1285 + extraArgs=options.browserArgs,
1.1286 + utilityPath=options.utilityPath,
1.1287 + debuggerInfo=debuggerInfo,
1.1288 + symbolsPath=options.symbolsPath,
1.1289 + timeout=timeout,
1.1290 + onLaunch=onLaunch,
1.1291 + webapprtChrome=options.webapprtChrome,
1.1292 + hide_subtests=options.hide_subtests,
1.1293 + screenshotOnFail=options.screenshotOnFail
1.1294 + )
1.1295 + except KeyboardInterrupt:
1.1296 + log.info("runtests.py | Received keyboard interrupt.\n");
1.1297 + status = -1
1.1298 + except:
1.1299 + traceback.print_exc()
1.1300 + log.error("Automation Error: Received unexpected exception while running application\n")
1.1301 + status = 1
1.1302 +
1.1303 + finally:
1.1304 + if options.vmwareRecording:
1.1305 + self.stopVMwareRecording();
1.1306 + self.stopServers()
1.1307 +
1.1308 + processLeakLog(self.leak_report_file, options.leakThreshold)
1.1309 +
1.1310 + if self.nsprLogs:
1.1311 + with zipfile.ZipFile("%s/nsprlog.zip" % browserEnv["MOZ_UPLOAD_DIR"], "w", zipfile.ZIP_DEFLATED) as logzip:
1.1312 + for logfile in glob.glob("%s/nspr*.log*" % tempfile.gettempdir()):
1.1313 + logzip.write(logfile)
1.1314 + os.remove(logfile)
1.1315 +
1.1316 + log.info("runtests.py | Running tests: end.")
1.1317 +
1.1318 + if manifest is not None:
1.1319 + self.cleanup(manifest, options)
1.1320 +
1.1321 + return status
1.1322 +
1.1323 + def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo, browserProcessId):
1.1324 + """handle process output timeout"""
1.1325 + # TODO: bug 913975 : _processOutput should call self.processOutputLine one more time one timeout (I think)
1.1326 + log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
1.1327 + browserProcessId = browserProcessId or proc.pid
1.1328 + self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo, dump_screen=not debuggerInfo)
1.1329 +
1.1330 + ### output processing
1.1331 +
1.1332 + class OutputHandler(object):
1.1333 + """line output handler for mozrunner"""
1.1334 + def __init__(self, harness, utilityPath, symbolsPath=None, dump_screen_on_timeout=True, dump_screen_on_fail=False,
1.1335 + hide_subtests=False, shutdownLeaks=None):
1.1336 + """
1.1337 + harness -- harness instance
1.1338 + dump_screen_on_timeout -- whether to dump the screen on timeout
1.1339 + """
1.1340 + self.harness = harness
1.1341 + self.output_buffer = []
1.1342 + self.running_test = False
1.1343 + self.utilityPath = utilityPath
1.1344 + self.symbolsPath = symbolsPath
1.1345 + self.dump_screen_on_timeout = dump_screen_on_timeout
1.1346 + self.dump_screen_on_fail = dump_screen_on_fail
1.1347 + self.hide_subtests = hide_subtests
1.1348 + self.shutdownLeaks = shutdownLeaks
1.1349 +
1.1350 + # perl binary to use
1.1351 + self.perl = which('perl')
1.1352 +
1.1353 + # With metro browser runs this script launches the metro test harness which launches the browser.
1.1354 + # The metro test harness hands back the real browser process id via log output which we need to
1.1355 + # pick up on and parse out. This variable tracks the real browser process id if we find it.
1.1356 + self.browserProcessId = None
1.1357 +
1.1358 + # stack fixer function and/or process
1.1359 + self.stackFixerFunction, self.stackFixerProcess = self.stackFixer()
1.1360 +
1.1361 + def processOutputLine(self, line):
1.1362 + """per line handler of output for mozprocess"""
1.1363 + for handler in self.outputHandlers():
1.1364 + line = handler(line)
1.1365 + __call__ = processOutputLine
1.1366 +
1.1367 + def outputHandlers(self):
1.1368 + """returns ordered list of output handlers"""
1.1369 + return [self.fix_stack,
1.1370 + self.format,
1.1371 + self.dumpScreenOnTimeout,
1.1372 + self.dumpScreenOnFail,
1.1373 + self.metro_subprocess_id,
1.1374 + self.trackShutdownLeaks,
1.1375 + self.check_test_failure,
1.1376 + self.log,
1.1377 + self.record_last_test,
1.1378 + ]
1.1379 +
1.1380 + def stackFixer(self):
1.1381 + """
1.1382 + return 2-tuple, (stackFixerFunction, StackFixerProcess),
1.1383 + if any, to use on the output lines
1.1384 + """
1.1385 +
1.1386 + if not mozinfo.info.get('debug'):
1.1387 + return None, None
1.1388 +
1.1389 + stackFixerFunction = stackFixerProcess = None
1.1390 +
1.1391 + def import_stackFixerModule(module_name):
1.1392 + sys.path.insert(0, self.utilityPath)
1.1393 + module = __import__(module_name, globals(), locals(), [])
1.1394 + sys.path.pop(0)
1.1395 + return module
1.1396 +
1.1397 + if self.symbolsPath and os.path.exists(self.symbolsPath):
1.1398 + # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files)
1.1399 + # This method is preferred for Tinderbox builds, since native symbols may have been stripped.
1.1400 + stackFixerModule = import_stackFixerModule('fix_stack_using_bpsyms')
1.1401 + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, self.symbolsPath)
1.1402 +
1.1403 + elif mozinfo.isLinux and self.perl:
1.1404 + # Run logsource through fix-linux-stack.pl (uses addr2line)
1.1405 + # This method is preferred for developer machines, so we don't have to run "make buildsymbols".
1.1406 + stackFixerCommand = [self.perl, os.path.join(self.utilityPath, "fix-linux-stack.pl")]
1.1407 + stackFixerProcess = subprocess.Popen(stackFixerCommand, stdin=subprocess.PIPE,
1.1408 + stdout=subprocess.PIPE)
1.1409 + def fixFunc(line):
1.1410 + stackFixerProcess.stdin.write(line + '\n')
1.1411 + return stackFixerProcess.stdout.readline().rstrip()
1.1412 +
1.1413 + stackFixerFunction = fixFunc
1.1414 +
1.1415 + return (stackFixerFunction, stackFixerProcess)
1.1416 +
1.1417 + def finish(self, didTimeout):
1.1418 + if self.stackFixerProcess:
1.1419 + self.stackFixerProcess.communicate()
1.1420 + status = self.stackFixerProcess.returncode
1.1421 + if status and not didTimeout:
1.1422 + log.info("TEST-UNEXPECTED-FAIL | runtests.py | Stack fixer process exited with code %d during test run", status)
1.1423 +
1.1424 + if self.shutdownLeaks:
1.1425 + self.shutdownLeaks.process()
1.1426 +
1.1427 + def log_output_buffer(self):
1.1428 + if self.output_buffer:
1.1429 + lines = [' %s' % line for line in self.output_buffer]
1.1430 + log.info("Buffered test output:\n%s" % '\n'.join(lines))
1.1431 +
1.1432 + # output line handlers:
1.1433 + # these take a line and return a line
1.1434 +
1.1435 + def fix_stack(self, line):
1.1436 + if self.stackFixerFunction:
1.1437 + return self.stackFixerFunction(line)
1.1438 + return line
1.1439 +
1.1440 + def format(self, line):
1.1441 + """format the line"""
1.1442 + return line.rstrip().decode("UTF-8", "ignore")
1.1443 +
1.1444 + def dumpScreenOnTimeout(self, line):
1.1445 + if not self.dump_screen_on_fail and self.dump_screen_on_timeout and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
1.1446 + self.log_output_buffer()
1.1447 + self.harness.dumpScreen(self.utilityPath)
1.1448 + return line
1.1449 +
1.1450 + def dumpScreenOnFail(self, line):
1.1451 + if self.dump_screen_on_fail and "TEST-UNEXPECTED-FAIL" in line:
1.1452 + self.log_output_buffer()
1.1453 + self.harness.dumpScreen(self.utilityPath)
1.1454 + return line
1.1455 +
1.1456 + def metro_subprocess_id(self, line):
1.1457 + """look for metro browser subprocess id"""
1.1458 + if "METRO_BROWSER_PROCESS" in line:
1.1459 + index = line.find("=")
1.1460 + if index != -1:
1.1461 + self.browserProcessId = line[index+1:].rstrip()
1.1462 + log.info("INFO | runtests.py | metro browser sub process id detected: %s", self.browserProcessId)
1.1463 + return line
1.1464 +
1.1465 + def trackShutdownLeaks(self, line):
1.1466 + if self.shutdownLeaks:
1.1467 + self.shutdownLeaks.log(line)
1.1468 + return line
1.1469 +
1.1470 + def check_test_failure(self, line):
1.1471 + if 'TEST-END' in line:
1.1472 + self.running_test = False
1.1473 + if any('TEST-UNEXPECTED' in l for l in self.output_buffer):
1.1474 + self.log_output_buffer()
1.1475 + return line
1.1476 +
1.1477 + def log(self, line):
1.1478 + if self.hide_subtests and self.running_test:
1.1479 + self.output_buffer.append(line)
1.1480 + else:
1.1481 + # hack to make separators align nicely, remove when we use mozlog
1.1482 + if self.hide_subtests and 'TEST-END' in line:
1.1483 + index = line.index('TEST-END') + len('TEST-END')
1.1484 + line = line[:index] + ' ' * (len('TEST-START')-len('TEST-END')) + line[index:]
1.1485 + log.info(line)
1.1486 + return line
1.1487 +
1.1488 + def record_last_test(self, line):
1.1489 + """record last test on harness"""
1.1490 + if "TEST-START" in line and "|" in line:
1.1491 + if not line.endswith('Shutdown'):
1.1492 + self.output_buffer = []
1.1493 + self.running_test = True
1.1494 + self.harness.lastTestSeen = line.split("|")[1].strip()
1.1495 + return line
1.1496 +
1.1497 +
1.1498 + def makeTestConfig(self, options):
1.1499 + "Creates a test configuration file for customizing test execution."
1.1500 + options.logFile = options.logFile.replace("\\", "\\\\")
1.1501 + options.testPath = options.testPath.replace("\\", "\\\\")
1.1502 + testRoot = self.getTestRoot(options)
1.1503 +
1.1504 + if "MOZ_HIDE_RESULTS_TABLE" in os.environ and os.environ["MOZ_HIDE_RESULTS_TABLE"] == "1":
1.1505 + options.hideResultsTable = True
1.1506 +
1.1507 + d = dict(options.__dict__)
1.1508 + d['testRoot'] = testRoot
1.1509 + content = json.dumps(d)
1.1510 +
1.1511 + with open(os.path.join(options.profilePath, "testConfig.js"), "w") as config:
1.1512 + config.write(content)
1.1513 +
1.1514 + def installExtensionFromPath(self, options, path, extensionID = None):
1.1515 + """install an extension to options.profilePath"""
1.1516 +
1.1517 + # TODO: currently extensionID is unused; see
1.1518 + # https://bugzilla.mozilla.org/show_bug.cgi?id=914267
1.1519 + # [mozprofile] make extensionID a parameter to install_from_path
1.1520 + # https://github.com/mozilla/mozbase/blob/master/mozprofile/mozprofile/addons.py#L169
1.1521 +
1.1522 + extensionPath = self.getFullPath(path)
1.1523 +
1.1524 + log.info("runtests.py | Installing extension at %s to %s." %
1.1525 + (extensionPath, options.profilePath))
1.1526 +
1.1527 + addons = AddonManager(options.profilePath)
1.1528 +
1.1529 + # XXX: del the __del__
1.1530 + # hack can be removed when mozprofile is mirrored to m-c ; see
1.1531 + # https://bugzilla.mozilla.org/show_bug.cgi?id=911218 :
1.1532 + # [mozprofile] AddonManager should only cleanup on __del__ optionally:
1.1533 + # https://github.com/mozilla/mozbase/blob/master/mozprofile/mozprofile/addons.py#L266
1.1534 + if hasattr(addons, '__del__'):
1.1535 + del addons.__del__
1.1536 +
1.1537 + addons.install_from_path(path)
1.1538 +
1.1539 + def installExtensionsToProfile(self, options):
1.1540 + "Install special testing extensions, application distributed extensions, and specified on the command line ones to testing profile."
1.1541 + for path in self.getExtensionsToInstall(options):
1.1542 + self.installExtensionFromPath(options, path)
1.1543 +
1.1544 +
1.1545 +def main():
1.1546 +
1.1547 + # parse command line options
1.1548 + mochitest = Mochitest()
1.1549 + parser = MochitestOptions()
1.1550 + options, args = parser.parse_args()
1.1551 + options = parser.verifyOptions(options, mochitest)
1.1552 + if options is None:
1.1553 + # parsing error
1.1554 + sys.exit(1)
1.1555 +
1.1556 + options.utilityPath = mochitest.getFullPath(options.utilityPath)
1.1557 + options.certPath = mochitest.getFullPath(options.certPath)
1.1558 + if options.symbolsPath and not isURL(options.symbolsPath):
1.1559 + options.symbolsPath = mochitest.getFullPath(options.symbolsPath)
1.1560 +
1.1561 + sys.exit(mochitest.runTests(options))
1.1562 +
1.1563 +if __name__ == "__main__":
1.1564 + main()
