diff -r 000000000000 -r 6474c204b198 build/automation.py.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/build/automation.py.in Wed Dec 31 06:09:35 2014 +0100 @@ -0,0 +1,988 @@ +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import with_statement +import codecs +import itertools +import json +import logging +import os +import re +import select +import shutil +import signal +import subprocess +import sys +import threading +import tempfile +import sqlite3 +from datetime import datetime, timedelta +from string import Template + +SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) +sys.path.insert(0, SCRIPT_DIR) +import automationutils + +# -------------------------------------------------------------- +# TODO: this is a hack for mozbase without virtualenv, remove with bug 849900 +# These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR +here = os.path.dirname(os.path.realpath(__file__)) +mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase')) + +if os.path.isdir(mozbase): + for package in os.listdir(mozbase): + package_path = os.path.join(mozbase, package) + if package_path not in sys.path: + sys.path.append(package_path) + +import mozcrash +from mozprofile import Profile, Preferences +from mozprofile.permissions import ServerLocations + +# --------------------------------------------------------------- + +_DEFAULT_PREFERENCE_FILE = os.path.join(SCRIPT_DIR, 'prefs_general.js') +_DEFAULT_APPS_FILE = os.path.join(SCRIPT_DIR, 'webapps_mochitest.json') + +_DEFAULT_WEB_SERVER = "127.0.0.1" +_DEFAULT_HTTP_PORT = 8888 +_DEFAULT_SSL_PORT = 4443 +_DEFAULT_WEBSOCKET_PORT = 9988 + +# from nsIPrincipal.idl +_APP_STATUS_NOT_INSTALLED = 0 +_APP_STATUS_INSTALLED = 1 +_APP_STATUS_PRIVILEGED = 2 +_APP_STATUS_CERTIFIED = 3 + +#expand _DIST_BIN = __XPC_BIN_PATH__ +#expand _IS_WIN32 = len("__WIN32__") != 0 +#expand _IS_MAC = __IS_MAC__ != 0 +#expand _IS_LINUX = __IS_LINUX__ != 0 +#ifdef IS_CYGWIN +#expand _IS_CYGWIN = __IS_CYGWIN__ == 1 +#else +_IS_CYGWIN = False +#endif +#expand _IS_CAMINO = __IS_CAMINO__ != 0 +#expand _BIN_SUFFIX = __BIN_SUFFIX__ +#expand _PERL = __PERL__ + +#expand _DEFAULT_APP = "./" + __BROWSER_PATH__ +#expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__ +#expand _IS_TEST_BUILD = __IS_TEST_BUILD__ +#expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__ +#expand _CRASHREPORTER = __CRASHREPORTER__ == 1 +#expand _IS_ASAN = __IS_ASAN__ == 1 + + +if _IS_WIN32: + import ctypes, ctypes.wintypes, time, msvcrt +else: + import errno + + +def getGlobalLog(): + return _log + +def resetGlobalLog(log): + while _log.handlers: + _log.removeHandler(_log.handlers[0]) + handler = logging.StreamHandler(log) + _log.setLevel(logging.INFO) + _log.addHandler(handler) + +# We use the logging system here primarily because it'll handle multiple +# threads, which is needed to process the output of the server and application +# processes simultaneously. +_log = logging.getLogger() +resetGlobalLog(sys.stdout) + + +################# +# PROFILE SETUP # +################# + +class SyntaxError(Exception): + "Signifies a syntax error on a particular line in server-locations.txt." + + def __init__(self, lineno, msg = None): + self.lineno = lineno + self.msg = msg + + def __str__(self): + s = "Syntax error on line " + str(self.lineno) + if self.msg: + s += ": %s." % self.msg + else: + s += "." + return s + + +class Location: + "Represents a location line in server-locations.txt." + + def __init__(self, scheme, host, port, options): + self.scheme = scheme + self.host = host + self.port = port + self.options = options + +class Automation(object): + """ + Runs the browser from a script, and provides useful utilities + for setting up the browser environment. + """ + + DIST_BIN = _DIST_BIN + IS_WIN32 = _IS_WIN32 + IS_MAC = _IS_MAC + IS_LINUX = _IS_LINUX + IS_CYGWIN = _IS_CYGWIN + IS_CAMINO = _IS_CAMINO + BIN_SUFFIX = _BIN_SUFFIX + PERL = _PERL + + UNIXISH = not IS_WIN32 and not IS_MAC + + DEFAULT_APP = _DEFAULT_APP + CERTS_SRC_DIR = _CERTS_SRC_DIR + IS_TEST_BUILD = _IS_TEST_BUILD + IS_DEBUG_BUILD = _IS_DEBUG_BUILD + CRASHREPORTER = _CRASHREPORTER + IS_ASAN = _IS_ASAN + + # timeout, in seconds + DEFAULT_TIMEOUT = 60.0 + DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER + DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT + DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT + DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT + + def __init__(self): + self.log = _log + self.lastTestSeen = "automation.py" + self.haveDumpedScreen = False + + def setServerInfo(self, + webServer = _DEFAULT_WEB_SERVER, + httpPort = _DEFAULT_HTTP_PORT, + sslPort = _DEFAULT_SSL_PORT, + webSocketPort = _DEFAULT_WEBSOCKET_PORT): + self.webServer = webServer + self.httpPort = httpPort + self.sslPort = sslPort + self.webSocketPort = webSocketPort + + @property + def __all__(self): + return [ + "UNIXISH", + "IS_WIN32", + "IS_MAC", + "log", + "runApp", + "Process", + "addCommonOptions", + "initializeProfile", + "DIST_BIN", + "DEFAULT_APP", + "CERTS_SRC_DIR", + "environment", + "IS_TEST_BUILD", + "IS_DEBUG_BUILD", + "DEFAULT_TIMEOUT", + ] + + class Process(subprocess.Popen): + """ + Represents our view of a subprocess. + It adds a kill() method which allows it to be stopped explicitly. + """ + + def __init__(self, + args, + bufsize=0, + executable=None, + stdin=None, + stdout=None, + stderr=None, + preexec_fn=None, + close_fds=False, + shell=False, + cwd=None, + env=None, + universal_newlines=False, + startupinfo=None, + creationflags=0): + _log.info("INFO | automation.py | Launching: %s", subprocess.list2cmdline(args)) + subprocess.Popen.__init__(self, args, bufsize, executable, + stdin, stdout, stderr, + preexec_fn, close_fds, + shell, cwd, env, + universal_newlines, startupinfo, creationflags) + self.log = _log + + def kill(self): + if Automation().IS_WIN32: + import platform + pid = "%i" % self.pid + if platform.release() == "2000": + # Windows 2000 needs 'kill.exe' from the + #'Windows 2000 Resource Kit tools'. (See bug 475455.) + try: + subprocess.Popen(["kill", "-f", pid]).wait() + except: + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid) + else: + # Windows XP and later. + subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait() + else: + os.kill(self.pid, signal.SIGKILL) + + def readLocations(self, locationsPath = "server-locations.txt"): + """ + Reads the locations at which the Mochitest HTTP server is available from + server-locations.txt. + """ + + locationFile = codecs.open(locationsPath, "r", "UTF-8") + + # Perhaps more detail than necessary, but it's the easiest way to make sure + # we get exactly the format we want. See server-locations.txt for the exact + # format guaranteed here. + lineRe = re.compile(r"^(?P[a-z][-a-z0-9+.]*)" + r"://" + r"(?P" + r"\d+\.\d+\.\d+\.\d+" + r"|" + r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*" + r"[a-z](?:[-a-z0-9]*[a-z0-9])?" + r")" + r":" + r"(?P\d+)" + r"(?:" + r"\s+" + r"(?P\S+(?:,\S+)*)" + r")?$") + locations = [] + lineno = 0 + seenPrimary = False + for line in locationFile: + lineno += 1 + if line.startswith("#") or line == "\n": + continue + + match = lineRe.match(line) + if not match: + raise SyntaxError(lineno) + + options = match.group("options") + if options: + options = options.split(",") + if "primary" in options: + if seenPrimary: + raise SyntaxError(lineno, "multiple primary locations") + seenPrimary = True + else: + options = [] + + locations.append(Location(match.group("scheme"), match.group("host"), + match.group("port"), options)) + + if not seenPrimary: + raise SyntaxError(lineno + 1, "missing primary location") + + return locations + + def setupPermissionsDatabase(self, profileDir, permissions): + # Included for reftest compatibility; + # see https://bugzilla.mozilla.org/show_bug.cgi?id=688667 + + # Open database and create table + permDB = sqlite3.connect(os.path.join(profileDir, "permissions.sqlite")) + cursor = permDB.cursor(); + + cursor.execute("PRAGMA user_version=3"); + + # SQL copied from nsPermissionManager.cpp + cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts ( + id INTEGER PRIMARY KEY, + host TEXT, + type TEXT, + permission INTEGER, + expireType INTEGER, + expireTime INTEGER, + appId INTEGER, + isInBrowserElement INTEGER)""") + + # Insert desired permissions + for perm in permissions.keys(): + for host,allow in permissions[perm]: + cursor.execute("INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)", + (host, perm, 1 if allow else 2)) + + # Commit and close + permDB.commit() + cursor.close() + + def initializeProfile(self, profileDir, + extraPrefs=None, + useServerLocations=False, + prefsPath=_DEFAULT_PREFERENCE_FILE, + appsPath=_DEFAULT_APPS_FILE, + addons=None): + " Sets up the standard testing profile." + + extraPrefs = extraPrefs or [] + + # create the profile + prefs = {} + locations = None + if useServerLocations: + locations = ServerLocations() + locations.read(os.path.abspath('server-locations.txt'), True) + else: + prefs['network.proxy.type'] = 0 + + prefs.update(Preferences.read_prefs(prefsPath)) + + for v in extraPrefs: + thispref = v.split("=", 1) + if len(thispref) < 2: + print "Error: syntax error in --setpref=" + v + sys.exit(1) + prefs[thispref[0]] = thispref[1] + + + interpolation = {"server": "%s:%s" % (self.webServer, self.httpPort)} + prefs = json.loads(json.dumps(prefs) % interpolation) + for pref in prefs: + prefs[pref] = Preferences.cast(prefs[pref]) + + # load apps + apps = None + if appsPath and os.path.exists(appsPath): + with open(appsPath, 'r') as apps_file: + apps = json.load(apps_file) + + proxy = {'remote': str(self.webServer), + 'http': str(self.httpPort), + 'https': str(self.sslPort), + # use SSL port for legacy compatibility; see + # - https://bugzilla.mozilla.org/show_bug.cgi?id=688667#c66 + # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221 + # 'ws': str(self.webSocketPort) + 'ws': str(self.sslPort) + } + + # return profile object + profile = Profile(profile=profileDir, + addons=addons, + locations=locations, + preferences=prefs, + restore=False, + apps=apps, + proxy=proxy) + return profile + + def addCommonOptions(self, parser): + "Adds command-line options which are common to mochitest and reftest." + + parser.add_option("--setpref", + action = "append", type = "string", + default = [], + dest = "extraPrefs", metavar = "PREF=VALUE", + help = "defines an extra user preference") + + def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath): + pwfilePath = os.path.join(profileDir, ".crtdbpw") + pwfile = open(pwfilePath, "w") + pwfile.write("\n") + pwfile.close() + + # Create head of the ssltunnel configuration file + sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg") + sslTunnelConfig = open(sslTunnelConfigPath, "w") + + sslTunnelConfig.write("httpproxy:1\n") + sslTunnelConfig.write("certdbdir:%s\n" % certPath) + sslTunnelConfig.write("forward:127.0.0.1:%s\n" % self.httpPort) + sslTunnelConfig.write("websocketserver:%s:%s\n" % (self.webServer, self.webSocketPort)) + sslTunnelConfig.write("listen:*:%s:pgo server certificate\n" % self.sslPort) + + # Configure automatic certificate and bind custom certificates, client authentication + locations = self.readLocations() + locations.pop(0) + for loc in locations: + if loc.scheme == "https" and "nocert" not in loc.options: + customCertRE = re.compile("^cert=(?P[0-9a-zA-Z_ ]+)") + clientAuthRE = re.compile("^clientauth=(?P[a-z]+)") + redirRE = re.compile("^redir=(?P[0-9a-zA-Z_ .]+)") + for option in loc.options: + match = customCertRE.match(option) + if match: + customcert = match.group("nickname"); + sslTunnelConfig.write("listen:%s:%s:%s:%s\n" % + (loc.host, loc.port, self.sslPort, customcert)) + + match = clientAuthRE.match(option) + if match: + clientauth = match.group("clientauth"); + sslTunnelConfig.write("clientauth:%s:%s:%s:%s\n" % + (loc.host, loc.port, self.sslPort, clientauth)) + + match = redirRE.match(option) + if match: + redirhost = match.group("redirhost") + sslTunnelConfig.write("redirhost:%s:%s:%s:%s\n" % + (loc.host, loc.port, self.sslPort, redirhost)) + + sslTunnelConfig.close() + + # Pre-create the certification database for the profile + env = self.environment(xrePath = xrePath) + certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX) + pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX) + + status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait() + automationutils.printstatus(status, "certutil") + if status != 0: + return status + + # Walk the cert directory and add custom CAs and client certs + files = os.listdir(certPath) + for item in files: + root, ext = os.path.splitext(item) + if ext == ".ca": + trustBits = "CT,," + if root.endswith("-object"): + trustBits = "CT,,CT" + status = self.Process([certutil, "-A", "-i", os.path.join(certPath, item), + "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits], + env = env).wait() + automationutils.printstatus(status, "certutil") + if ext == ".client": + status = self.Process([pk12util, "-i", os.path.join(certPath, item), "-w", + pwfilePath, "-d", profileDir], + env = env).wait() + automationutils.printstatus(status, "pk12util") + + os.unlink(pwfilePath) + return 0 + + def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None): + if xrePath == None: + xrePath = self.DIST_BIN + if env == None: + env = dict(os.environ) + + ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath)) + dmdLibrary = None + preloadEnvVar = None + if self.UNIXISH or self.IS_MAC: + envVar = "LD_LIBRARY_PATH" + preloadEnvVar = "LD_PRELOAD" + if self.IS_MAC: + envVar = "DYLD_LIBRARY_PATH" + dmdLibrary = "libdmd.dylib" + else: # unixish + env['MOZILLA_FIVE_HOME'] = xrePath + dmdLibrary = "libdmd.so" + if envVar in env: + ldLibraryPath = ldLibraryPath + ":" + env[envVar] + env[envVar] = ldLibraryPath + elif self.IS_WIN32: + env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath) + dmdLibrary = "dmd.dll" + preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" + + if dmdPath and dmdLibrary and preloadEnvVar: + env['DMD'] = '1' + env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) + + if crashreporter and not debugger: + env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' + env['MOZ_CRASHREPORTER'] = '1' + else: + env['MOZ_CRASHREPORTER_DISABLE'] = '1' + + # Crash on non-local network connections. + env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' + + env['GNOME_DISABLE_CRASH_DIALOG'] = '1' + env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' + env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1' + + # Set WebRTC logging in case it is not set yet + env.setdefault('NSPR_LOG_MODULES', 'signaling:5,mtransport:3') + env.setdefault('R_LOG_LEVEL', '5') + env.setdefault('R_LOG_DESTINATION', 'stderr') + env.setdefault('R_LOG_VERBOSE', '1') + + # ASan specific environment stuff + if self.IS_ASAN and (self.IS_LINUX or self.IS_MAC): + # Symbolizer support + llvmsym = os.path.join(xrePath, "llvm-symbolizer") + if os.path.isfile(llvmsym): + env["ASAN_SYMBOLIZER_PATH"] = llvmsym + self.log.info("INFO | automation.py | ASan using symbolizer at %s", llvmsym) + + try: + totalMemory = int(os.popen("free").readlines()[1].split()[1]) + + # Only 4 GB RAM or less available? Use custom ASan options to reduce + # the amount of resources required to do the tests. Standard options + # will otherwise lead to OOM conditions on the current test slaves. + if totalMemory <= 1024 * 1024 * 4: + self.log.info("INFO | automation.py | ASan running in low-memory configuration") + env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5" + else: + self.log.info("INFO | automation.py | ASan running in default memory configuration") + except OSError,err: + self.log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror) + except: + self.log.info("Failed determine available memory, disabling ASan low-memory configuration") + + return env + + def killPid(self, pid): + try: + os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM)) + except WindowsError: + self.log.info("Failed to kill process %d." % pid) + + if IS_WIN32: + PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe + GetLastError = ctypes.windll.kernel32.GetLastError + + def readWithTimeout(self, f, timeout): + """ + Try to read a line of output from the file object |f|. |f| must be a + pipe, like the |stdout| member of a subprocess.Popen object created + with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout| + is True if the read timed out, and False otherwise. If no output is + received within |timeout| seconds, returns a blank line. + """ + + if timeout is None: + timeout = 0 + + x = msvcrt.get_osfhandle(f.fileno()) + l = ctypes.c_long() + done = time.time() + timeout + + buffer = "" + while timeout == 0 or time.time() < done: + if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0: + err = self.GetLastError() + if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE + return ('', False) + else: + self.log.error("readWithTimeout got error: %d", err) + # read a character at a time, checking for eol. Return once we get there. + index = 0 + while index < l.value: + char = f.read(1) + buffer += char + if char == '\n': + return (buffer, False) + index = index + 1 + time.sleep(0.01) + return (buffer, True) + + def isPidAlive(self, pid): + STILL_ACTIVE = 259 + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) + if not pHandle: + return False + pExitCode = ctypes.wintypes.DWORD() + ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode)) + ctypes.windll.kernel32.CloseHandle(pHandle) + return pExitCode.value == STILL_ACTIVE + + else: + + def readWithTimeout(self, f, timeout): + """Try to read a line of output from the file object |f|. If no output + is received within |timeout| seconds, return a blank line. + Returns a tuple (line, did_timeout), where |did_timeout| is True + if the read timed out, and False otherwise.""" + (r, w, e) = select.select([f], [], [], timeout) + if len(r) == 0: + return ('', True) + return (f.readline(), False) + + def isPidAlive(self, pid): + try: + # kill(pid, 0) checks for a valid PID without actually sending a signal + # The method throws OSError if the PID is invalid, which we catch below. + os.kill(pid, 0) + + # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if + # the process terminates before we get to this point. + wpid, wstatus = os.waitpid(pid, os.WNOHANG) + return wpid == 0 + except OSError, err: + # Catch the errors we might expect from os.kill/os.waitpid, + # and re-raise any others + if err.errno == errno.ESRCH or err.errno == errno.ECHILD: + return False + raise + + def dumpScreen(self, utilityPath): + if self.haveDumpedScreen: + self.log.info("Not taking screenshot here: see the one that was previously logged") + return + + self.haveDumpedScreen = True; + automationutils.dumpScreen(utilityPath) + + + def killAndGetStack(self, processPID, utilityPath, debuggerInfo): + """Kill the process, preferrably in a way that gets us a stack trace. + Also attempts to obtain a screenshot before killing the process.""" + if not debuggerInfo: + self.dumpScreen(utilityPath) + self.killAndGetStackNoScreenshot(processPID, utilityPath, debuggerInfo) + + def killAndGetStackNoScreenshot(self, processPID, utilityPath, debuggerInfo): + """Kill the process, preferrably in a way that gets us a stack trace.""" + if self.CRASHREPORTER and not debuggerInfo: + if not self.IS_WIN32: + # ABRT will get picked up by Breakpad's signal handler + os.kill(processPID, signal.SIGABRT) + return + else: + # We should have a "crashinject" program in our utility path + crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe")) + if os.path.exists(crashinject): + status = subprocess.Popen([crashinject, str(processPID)]).wait() + automationutils.printstatus(status, "crashinject") + if status == 0: + return + self.log.info("Can't trigger Breakpad, just killing process") + self.killPid(processPID) + + def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath): + """ Look for timeout or crashes and return the status after the process terminates """ + stackFixerProcess = None + stackFixerFunction = None + didTimeout = False + hitMaxTime = False + if proc.stdout is None: + self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection") + else: + logsource = proc.stdout + + if self.IS_DEBUG_BUILD and symbolsPath and os.path.exists(symbolsPath): + # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files) + # This method is preferred for Tinderbox builds, since native symbols may have been stripped. + sys.path.insert(0, utilityPath) + import fix_stack_using_bpsyms as stackFixerModule + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath) + del sys.path[0] + elif self.IS_DEBUG_BUILD and self.IS_MAC and False: + # Run each line through a function in fix_macosx_stack.py (uses atos) + sys.path.insert(0, utilityPath) + import fix_macosx_stack as stackFixerModule + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line) + del sys.path[0] + elif self.IS_DEBUG_BUILD and self.IS_LINUX: + # Run logsource through fix-linux-stack.pl (uses addr2line) + # This method is preferred for developer machines, so we don't have to run "make buildsymbols". + stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")], + stdin=logsource, + stdout=subprocess.PIPE) + logsource = stackFixerProcess.stdout + + # With metro browser runs this script launches the metro test harness which launches the browser. + # The metro test harness hands back the real browser process id via log output which we need to + # pick up on and parse out. This variable tracks the real browser process id if we find it. + browserProcessId = -1 + + (line, didTimeout) = self.readWithTimeout(logsource, timeout) + while line != "" and not didTimeout: + if stackFixerFunction: + line = stackFixerFunction(line) + self.log.info(line.rstrip().decode("UTF-8", "ignore")) + if "TEST-START" in line and "|" in line: + self.lastTestSeen = line.split("|")[1].strip() + if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line: + self.dumpScreen(utilityPath) + + (line, didTimeout) = self.readWithTimeout(logsource, timeout) + + if "METRO_BROWSER_PROCESS" in line: + index = line.find("=") + if index: + browserProcessId = line[index+1:].rstrip() + self.log.info("INFO | automation.py | metro browser sub process id detected: %s", browserProcessId) + + if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime): + # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait(). + hitMaxTime = True + self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime)) + self.killAndGetStack(proc.pid, utilityPath, debuggerInfo) + if didTimeout: + if line: + self.log.info(line.rstrip().decode("UTF-8", "ignore")) + self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout)) + if browserProcessId == -1: + browserProcessId = proc.pid + self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo) + + status = proc.wait() + automationutils.printstatus(status, "Main app process") + if status == 0: + self.lastTestSeen = "Main app process exited normally" + if status != 0 and not didTimeout and not hitMaxTime: + self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status) + if stackFixerProcess is not None: + fixerStatus = stackFixerProcess.wait() + automationutils.printstatus(status, "stackFixerProcess") + if fixerStatus != 0 and not didTimeout and not hitMaxTime: + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus) + return status + + def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs): + """ build the application command line """ + + cmd = os.path.abspath(app) + if self.IS_MAC and not self.IS_CAMINO and os.path.exists(cmd + "-bin"): + # Prefer 'app-bin' in case 'app' is a shell script. + # We can remove this hack once bug 673899 etc are fixed. + cmd += "-bin" + + args = [] + + if debuggerInfo: + args.extend(debuggerInfo["args"]) + args.append(cmd) + cmd = os.path.abspath(debuggerInfo["path"]) + + if self.IS_MAC: + args.append("-foreground") + + if self.IS_CYGWIN: + profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"") + else: + profileDirectory = profileDir + "/" + + args.extend(("-no-remote", "-profile", profileDirectory)) + if testURL is not None: + if self.IS_CAMINO: + args.extend(("-url", testURL)) + else: + args.append((testURL)) + args.extend(extraArgs) + return cmd, args + + def checkForZombies(self, processLog, utilityPath, debuggerInfo): + """ Look for hung processes """ + if not os.path.exists(processLog): + self.log.info('Automation Error: PID log not found: %s', processLog) + # Whilst no hung process was found, the run should still display as a failure + return True + + foundZombie = False + self.log.info('INFO | zombiecheck | Reading PID log: %s', processLog) + processList = [] + pidRE = re.compile(r'launched child process (\d+)$') + processLogFD = open(processLog) + for line in processLogFD: + self.log.info(line.rstrip()) + m = pidRE.search(line) + if m: + processList.append(int(m.group(1))) + processLogFD.close() + + for processPID in processList: + self.log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID) + if self.isPidAlive(processPID): + foundZombie = True + self.log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID) + self.killAndGetStack(processPID, utilityPath, debuggerInfo) + return foundZombie + + def checkForCrashes(self, minidumpDir, symbolsPath): + return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen) + + def runApp(self, testURL, env, app, profileDir, extraArgs, + runSSLTunnel = False, utilityPath = None, + xrePath = None, certPath = None, + debuggerInfo = None, symbolsPath = None, + timeout = -1, maxTime = None, onLaunch = None, + webapprtChrome = False, hide_subtests=None, screenshotOnFail=False): + """ + Run the app, log the duration it took to execute, return the status code. + Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds. + """ + + if utilityPath == None: + utilityPath = self.DIST_BIN + if xrePath == None: + xrePath = self.DIST_BIN + if certPath == None: + certPath = self.CERTS_SRC_DIR + if timeout == -1: + timeout = self.DEFAULT_TIMEOUT + + # copy env so we don't munge the caller's environment + env = dict(env); + env["NO_EM_RESTART"] = "1" + tmpfd, processLog = tempfile.mkstemp(suffix='pidlog') + os.close(tmpfd) + env["MOZ_PROCESS_LOG"] = processLog + + if self.IS_TEST_BUILD and runSSLTunnel: + # create certificate database for the profile + certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath) + if certificateStatus != 0: + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Certificate integration failed") + return certificateStatus + + # start ssltunnel to provide https:// URLs capability + ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX) + ssltunnelProcess = self.Process([ssltunnel, + os.path.join(profileDir, "ssltunnel.cfg")], + env = self.environment(xrePath = xrePath)) + self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid) + + cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs) + startTime = datetime.now() + + if debuggerInfo and debuggerInfo["interactive"]: + # If an interactive debugger is attached, don't redirect output, + # don't use timeouts, and don't capture ctrl-c. + timeout = None + maxTime = None + outputPipe = None + signal.signal(signal.SIGINT, lambda sigid, frame: None) + else: + outputPipe = subprocess.PIPE + + self.lastTestSeen = "automation.py" + proc = self.Process([cmd] + args, + env = self.environment(env, xrePath = xrePath, + crashreporter = not debuggerInfo), + stdout = outputPipe, + stderr = subprocess.STDOUT) + self.log.info("INFO | automation.py | Application pid: %d", proc.pid) + + if onLaunch is not None: + # Allow callers to specify an onLaunch callback to be fired after the + # app is launched. + onLaunch() + + status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath) + self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime)) + + # Do a final check for zombie child processes. + zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo) + + crashed = self.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath) + + if crashed or zombieProcesses: + status = 1 + + if os.path.exists(processLog): + os.unlink(processLog) + + if self.IS_TEST_BUILD and runSSLTunnel: + ssltunnelProcess.kill() + + return status + + def getExtensionIDFromRDF(self, rdfSource): + """ + Retrieves the extension id from an install.rdf file (or string). + """ + from xml.dom.minidom import parse, parseString, Node + + if isinstance(rdfSource, file): + document = parse(rdfSource) + else: + document = parseString(rdfSource) + + # Find the element. There can be multiple tags + # within tags, so we have to check this way. + for rdfChild in document.documentElement.childNodes: + if rdfChild.nodeType == Node.ELEMENT_NODE and rdfChild.tagName == "Description": + for descChild in rdfChild.childNodes: + if descChild.nodeType == Node.ELEMENT_NODE and descChild.tagName == "em:id": + return descChild.childNodes[0].data + + return None + + def installExtension(self, extensionSource, profileDir, extensionID = None): + """ + Copies an extension into the extensions directory of the given profile. + extensionSource - the source location of the extension files. This can be either + a directory or a path to an xpi file. + profileDir - the profile directory we are copying into. We will create the + "extensions" directory there if it doesn't exist. + extensionID - the id of the extension to be used as the containing directory for the + extension, if extensionSource is a directory, i.e. + this is the name of the folder in the /extensions/ + """ + if not os.path.isdir(profileDir): + self.log.info("INFO | automation.py | Cannot install extension, invalid profileDir at: %s", profileDir) + return + + installRDFFilename = "install.rdf" + + extensionsRootDir = os.path.join(profileDir, "extensions", "staged") + if not os.path.isdir(extensionsRootDir): + os.makedirs(extensionsRootDir) + + if os.path.isfile(extensionSource): + reader = automationutils.ZipFileReader(extensionSource) + + for filename in reader.namelist(): + # Sanity check the zip file. + if os.path.isabs(filename): + self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi") + return + + # We may need to dig the extensionID out of the zip file... + if extensionID is None and filename == installRDFFilename: + extensionID = self.getExtensionIDFromRDF(reader.read(filename)) + + # We must know the extensionID now. + if extensionID is None: + self.log.info("INFO | automation.py | Cannot install extension, missing extensionID") + return + + # Make the extension directory. + extensionDir = os.path.join(extensionsRootDir, extensionID) + os.mkdir(extensionDir) + + # Extract all files. + reader.extractall(extensionDir) + + elif os.path.isdir(extensionSource): + if extensionID is None: + filename = os.path.join(extensionSource, installRDFFilename) + if os.path.isfile(filename): + with open(filename, "r") as installRDF: + extensionID = self.getExtensionIDFromRDF(installRDF) + + if extensionID is None: + self.log.info("INFO | automation.py | Cannot install extension, missing extensionID") + return + + # Copy extension tree into its own directory. + # "destination directory must not already exist". + shutil.copytree(extensionSource, os.path.join(extensionsRootDir, extensionID)) + + else: + self.log.info("INFO | automation.py | Cannot install extension, invalid extensionSource at: %s", extensionSource) + + def elf_arm(self, filename): + data = open(filename, 'rb').read(20) + return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM +