1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/build/automation.py.in Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,988 @@
1.4 +#
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +
1.9 +from __future__ import with_statement
1.10 +import codecs
1.11 +import itertools
1.12 +import json
1.13 +import logging
1.14 +import os
1.15 +import re
1.16 +import select
1.17 +import shutil
1.18 +import signal
1.19 +import subprocess
1.20 +import sys
1.21 +import threading
1.22 +import tempfile
1.23 +import sqlite3
1.24 +from datetime import datetime, timedelta
1.25 +from string import Template
1.26 +
1.27 +SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
1.28 +sys.path.insert(0, SCRIPT_DIR)
1.29 +import automationutils
1.30 +
1.31 +# --------------------------------------------------------------
1.32 +# TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
1.33 +# These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR
1.34 +here = os.path.dirname(os.path.realpath(__file__))
1.35 +mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
1.36 +
1.37 +if os.path.isdir(mozbase):
1.38 + for package in os.listdir(mozbase):
1.39 + package_path = os.path.join(mozbase, package)
1.40 + if package_path not in sys.path:
1.41 + sys.path.append(package_path)
1.42 +
1.43 +import mozcrash
1.44 +from mozprofile import Profile, Preferences
1.45 +from mozprofile.permissions import ServerLocations
1.46 +
1.47 +# ---------------------------------------------------------------
1.48 +
1.49 +_DEFAULT_PREFERENCE_FILE = os.path.join(SCRIPT_DIR, 'prefs_general.js')
1.50 +_DEFAULT_APPS_FILE = os.path.join(SCRIPT_DIR, 'webapps_mochitest.json')
1.51 +
1.52 +_DEFAULT_WEB_SERVER = "127.0.0.1"
1.53 +_DEFAULT_HTTP_PORT = 8888
1.54 +_DEFAULT_SSL_PORT = 4443
1.55 +_DEFAULT_WEBSOCKET_PORT = 9988
1.56 +
1.57 +# from nsIPrincipal.idl
1.58 +_APP_STATUS_NOT_INSTALLED = 0
1.59 +_APP_STATUS_INSTALLED = 1
1.60 +_APP_STATUS_PRIVILEGED = 2
1.61 +_APP_STATUS_CERTIFIED = 3
1.62 +
1.63 +#expand _DIST_BIN = __XPC_BIN_PATH__
1.64 +#expand _IS_WIN32 = len("__WIN32__") != 0
1.65 +#expand _IS_MAC = __IS_MAC__ != 0
1.66 +#expand _IS_LINUX = __IS_LINUX__ != 0
1.67 +#ifdef IS_CYGWIN
1.68 +#expand _IS_CYGWIN = __IS_CYGWIN__ == 1
1.69 +#else
1.70 +_IS_CYGWIN = False
1.71 +#endif
1.72 +#expand _IS_CAMINO = __IS_CAMINO__ != 0
1.73 +#expand _BIN_SUFFIX = __BIN_SUFFIX__
1.74 +#expand _PERL = __PERL__
1.75 +
1.76 +#expand _DEFAULT_APP = "./" + __BROWSER_PATH__
1.77 +#expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
1.78 +#expand _IS_TEST_BUILD = __IS_TEST_BUILD__
1.79 +#expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
1.80 +#expand _CRASHREPORTER = __CRASHREPORTER__ == 1
1.81 +#expand _IS_ASAN = __IS_ASAN__ == 1
1.82 +
1.83 +
1.84 +if _IS_WIN32:
1.85 + import ctypes, ctypes.wintypes, time, msvcrt
1.86 +else:
1.87 + import errno
1.88 +
1.89 +
1.90 +def getGlobalLog():
1.91 + return _log
1.92 +
1.93 +def resetGlobalLog(log):
1.94 + while _log.handlers:
1.95 + _log.removeHandler(_log.handlers[0])
1.96 + handler = logging.StreamHandler(log)
1.97 + _log.setLevel(logging.INFO)
1.98 + _log.addHandler(handler)
1.99 +
1.100 +# We use the logging system here primarily because it'll handle multiple
1.101 +# threads, which is needed to process the output of the server and application
1.102 +# processes simultaneously.
1.103 +_log = logging.getLogger()
1.104 +resetGlobalLog(sys.stdout)
1.105 +
1.106 +
1.107 +#################
1.108 +# PROFILE SETUP #
1.109 +#################
1.110 +
1.111 +class SyntaxError(Exception):
1.112 + "Signifies a syntax error on a particular line in server-locations.txt."
1.113 +
1.114 + def __init__(self, lineno, msg = None):
1.115 + self.lineno = lineno
1.116 + self.msg = msg
1.117 +
1.118 + def __str__(self):
1.119 + s = "Syntax error on line " + str(self.lineno)
1.120 + if self.msg:
1.121 + s += ": %s." % self.msg
1.122 + else:
1.123 + s += "."
1.124 + return s
1.125 +
1.126 +
1.127 +class Location:
1.128 + "Represents a location line in server-locations.txt."
1.129 +
1.130 + def __init__(self, scheme, host, port, options):
1.131 + self.scheme = scheme
1.132 + self.host = host
1.133 + self.port = port
1.134 + self.options = options
1.135 +
1.136 +class Automation(object):
1.137 + """
1.138 + Runs the browser from a script, and provides useful utilities
1.139 + for setting up the browser environment.
1.140 + """
1.141 +
1.142 + DIST_BIN = _DIST_BIN
1.143 + IS_WIN32 = _IS_WIN32
1.144 + IS_MAC = _IS_MAC
1.145 + IS_LINUX = _IS_LINUX
1.146 + IS_CYGWIN = _IS_CYGWIN
1.147 + IS_CAMINO = _IS_CAMINO
1.148 + BIN_SUFFIX = _BIN_SUFFIX
1.149 + PERL = _PERL
1.150 +
1.151 + UNIXISH = not IS_WIN32 and not IS_MAC
1.152 +
1.153 + DEFAULT_APP = _DEFAULT_APP
1.154 + CERTS_SRC_DIR = _CERTS_SRC_DIR
1.155 + IS_TEST_BUILD = _IS_TEST_BUILD
1.156 + IS_DEBUG_BUILD = _IS_DEBUG_BUILD
1.157 + CRASHREPORTER = _CRASHREPORTER
1.158 + IS_ASAN = _IS_ASAN
1.159 +
1.160 + # timeout, in seconds
1.161 + DEFAULT_TIMEOUT = 60.0
1.162 + DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER
1.163 + DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT
1.164 + DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT
1.165 + DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT
1.166 +
1.167 + def __init__(self):
1.168 + self.log = _log
1.169 + self.lastTestSeen = "automation.py"
1.170 + self.haveDumpedScreen = False
1.171 +
1.172 + def setServerInfo(self,
1.173 + webServer = _DEFAULT_WEB_SERVER,
1.174 + httpPort = _DEFAULT_HTTP_PORT,
1.175 + sslPort = _DEFAULT_SSL_PORT,
1.176 + webSocketPort = _DEFAULT_WEBSOCKET_PORT):
1.177 + self.webServer = webServer
1.178 + self.httpPort = httpPort
1.179 + self.sslPort = sslPort
1.180 + self.webSocketPort = webSocketPort
1.181 +
1.182 + @property
1.183 + def __all__(self):
1.184 + return [
1.185 + "UNIXISH",
1.186 + "IS_WIN32",
1.187 + "IS_MAC",
1.188 + "log",
1.189 + "runApp",
1.190 + "Process",
1.191 + "addCommonOptions",
1.192 + "initializeProfile",
1.193 + "DIST_BIN",
1.194 + "DEFAULT_APP",
1.195 + "CERTS_SRC_DIR",
1.196 + "environment",
1.197 + "IS_TEST_BUILD",
1.198 + "IS_DEBUG_BUILD",
1.199 + "DEFAULT_TIMEOUT",
1.200 + ]
1.201 +
1.202 + class Process(subprocess.Popen):
1.203 + """
1.204 + Represents our view of a subprocess.
1.205 + It adds a kill() method which allows it to be stopped explicitly.
1.206 + """
1.207 +
1.208 + def __init__(self,
1.209 + args,
1.210 + bufsize=0,
1.211 + executable=None,
1.212 + stdin=None,
1.213 + stdout=None,
1.214 + stderr=None,
1.215 + preexec_fn=None,
1.216 + close_fds=False,
1.217 + shell=False,
1.218 + cwd=None,
1.219 + env=None,
1.220 + universal_newlines=False,
1.221 + startupinfo=None,
1.222 + creationflags=0):
1.223 + _log.info("INFO | automation.py | Launching: %s", subprocess.list2cmdline(args))
1.224 + subprocess.Popen.__init__(self, args, bufsize, executable,
1.225 + stdin, stdout, stderr,
1.226 + preexec_fn, close_fds,
1.227 + shell, cwd, env,
1.228 + universal_newlines, startupinfo, creationflags)
1.229 + self.log = _log
1.230 +
1.231 + def kill(self):
1.232 + if Automation().IS_WIN32:
1.233 + import platform
1.234 + pid = "%i" % self.pid
1.235 + if platform.release() == "2000":
1.236 + # Windows 2000 needs 'kill.exe' from the
1.237 + #'Windows 2000 Resource Kit tools'. (See bug 475455.)
1.238 + try:
1.239 + subprocess.Popen(["kill", "-f", pid]).wait()
1.240 + except:
1.241 + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
1.242 + else:
1.243 + # Windows XP and later.
1.244 + subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
1.245 + else:
1.246 + os.kill(self.pid, signal.SIGKILL)
1.247 +
1.248 + def readLocations(self, locationsPath = "server-locations.txt"):
1.249 + """
1.250 + Reads the locations at which the Mochitest HTTP server is available from
1.251 + server-locations.txt.
1.252 + """
1.253 +
1.254 + locationFile = codecs.open(locationsPath, "r", "UTF-8")
1.255 +
1.256 + # Perhaps more detail than necessary, but it's the easiest way to make sure
1.257 + # we get exactly the format we want. See server-locations.txt for the exact
1.258 + # format guaranteed here.
1.259 + lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
1.260 + r"://"
1.261 + r"(?P<host>"
1.262 + r"\d+\.\d+\.\d+\.\d+"
1.263 + r"|"
1.264 + r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
1.265 + r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
1.266 + r")"
1.267 + r":"
1.268 + r"(?P<port>\d+)"
1.269 + r"(?:"
1.270 + r"\s+"
1.271 + r"(?P<options>\S+(?:,\S+)*)"
1.272 + r")?$")
1.273 + locations = []
1.274 + lineno = 0
1.275 + seenPrimary = False
1.276 + for line in locationFile:
1.277 + lineno += 1
1.278 + if line.startswith("#") or line == "\n":
1.279 + continue
1.280 +
1.281 + match = lineRe.match(line)
1.282 + if not match:
1.283 + raise SyntaxError(lineno)
1.284 +
1.285 + options = match.group("options")
1.286 + if options:
1.287 + options = options.split(",")
1.288 + if "primary" in options:
1.289 + if seenPrimary:
1.290 + raise SyntaxError(lineno, "multiple primary locations")
1.291 + seenPrimary = True
1.292 + else:
1.293 + options = []
1.294 +
1.295 + locations.append(Location(match.group("scheme"), match.group("host"),
1.296 + match.group("port"), options))
1.297 +
1.298 + if not seenPrimary:
1.299 + raise SyntaxError(lineno + 1, "missing primary location")
1.300 +
1.301 + return locations
1.302 +
1.303 + def setupPermissionsDatabase(self, profileDir, permissions):
1.304 + # Included for reftest compatibility;
1.305 + # see https://bugzilla.mozilla.org/show_bug.cgi?id=688667
1.306 +
1.307 + # Open database and create table
1.308 + permDB = sqlite3.connect(os.path.join(profileDir, "permissions.sqlite"))
1.309 + cursor = permDB.cursor();
1.310 +
1.311 + cursor.execute("PRAGMA user_version=3");
1.312 +
1.313 + # SQL copied from nsPermissionManager.cpp
1.314 + cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
1.315 + id INTEGER PRIMARY KEY,
1.316 + host TEXT,
1.317 + type TEXT,
1.318 + permission INTEGER,
1.319 + expireType INTEGER,
1.320 + expireTime INTEGER,
1.321 + appId INTEGER,
1.322 + isInBrowserElement INTEGER)""")
1.323 +
1.324 + # Insert desired permissions
1.325 + for perm in permissions.keys():
1.326 + for host,allow in permissions[perm]:
1.327 + cursor.execute("INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)",
1.328 + (host, perm, 1 if allow else 2))
1.329 +
1.330 + # Commit and close
1.331 + permDB.commit()
1.332 + cursor.close()
1.333 +
1.334 + def initializeProfile(self, profileDir,
1.335 + extraPrefs=None,
1.336 + useServerLocations=False,
1.337 + prefsPath=_DEFAULT_PREFERENCE_FILE,
1.338 + appsPath=_DEFAULT_APPS_FILE,
1.339 + addons=None):
1.340 + " Sets up the standard testing profile."
1.341 +
1.342 + extraPrefs = extraPrefs or []
1.343 +
1.344 + # create the profile
1.345 + prefs = {}
1.346 + locations = None
1.347 + if useServerLocations:
1.348 + locations = ServerLocations()
1.349 + locations.read(os.path.abspath('server-locations.txt'), True)
1.350 + else:
1.351 + prefs['network.proxy.type'] = 0
1.352 +
1.353 + prefs.update(Preferences.read_prefs(prefsPath))
1.354 +
1.355 + for v in extraPrefs:
1.356 + thispref = v.split("=", 1)
1.357 + if len(thispref) < 2:
1.358 + print "Error: syntax error in --setpref=" + v
1.359 + sys.exit(1)
1.360 + prefs[thispref[0]] = thispref[1]
1.361 +
1.362 +
1.363 + interpolation = {"server": "%s:%s" % (self.webServer, self.httpPort)}
1.364 + prefs = json.loads(json.dumps(prefs) % interpolation)
1.365 + for pref in prefs:
1.366 + prefs[pref] = Preferences.cast(prefs[pref])
1.367 +
1.368 + # load apps
1.369 + apps = None
1.370 + if appsPath and os.path.exists(appsPath):
1.371 + with open(appsPath, 'r') as apps_file:
1.372 + apps = json.load(apps_file)
1.373 +
1.374 + proxy = {'remote': str(self.webServer),
1.375 + 'http': str(self.httpPort),
1.376 + 'https': str(self.sslPort),
1.377 + # use SSL port for legacy compatibility; see
1.378 + # - https://bugzilla.mozilla.org/show_bug.cgi?id=688667#c66
1.379 + # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221
1.380 + # 'ws': str(self.webSocketPort)
1.381 + 'ws': str(self.sslPort)
1.382 + }
1.383 +
1.384 + # return profile object
1.385 + profile = Profile(profile=profileDir,
1.386 + addons=addons,
1.387 + locations=locations,
1.388 + preferences=prefs,
1.389 + restore=False,
1.390 + apps=apps,
1.391 + proxy=proxy)
1.392 + return profile
1.393 +
1.394 + def addCommonOptions(self, parser):
1.395 + "Adds command-line options which are common to mochitest and reftest."
1.396 +
1.397 + parser.add_option("--setpref",
1.398 + action = "append", type = "string",
1.399 + default = [],
1.400 + dest = "extraPrefs", metavar = "PREF=VALUE",
1.401 + help = "defines an extra user preference")
1.402 +
1.403 + def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
1.404 + pwfilePath = os.path.join(profileDir, ".crtdbpw")
1.405 + pwfile = open(pwfilePath, "w")
1.406 + pwfile.write("\n")
1.407 + pwfile.close()
1.408 +
1.409 + # Create head of the ssltunnel configuration file
1.410 + sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
1.411 + sslTunnelConfig = open(sslTunnelConfigPath, "w")
1.412 +
1.413 + sslTunnelConfig.write("httpproxy:1\n")
1.414 + sslTunnelConfig.write("certdbdir:%s\n" % certPath)
1.415 + sslTunnelConfig.write("forward:127.0.0.1:%s\n" % self.httpPort)
1.416 + sslTunnelConfig.write("websocketserver:%s:%s\n" % (self.webServer, self.webSocketPort))
1.417 + sslTunnelConfig.write("listen:*:%s:pgo server certificate\n" % self.sslPort)
1.418 +
1.419 + # Configure automatic certificate and bind custom certificates, client authentication
1.420 + locations = self.readLocations()
1.421 + locations.pop(0)
1.422 + for loc in locations:
1.423 + if loc.scheme == "https" and "nocert" not in loc.options:
1.424 + customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
1.425 + clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
1.426 + redirRE = re.compile("^redir=(?P<redirhost>[0-9a-zA-Z_ .]+)")
1.427 + for option in loc.options:
1.428 + match = customCertRE.match(option)
1.429 + if match:
1.430 + customcert = match.group("nickname");
1.431 + sslTunnelConfig.write("listen:%s:%s:%s:%s\n" %
1.432 + (loc.host, loc.port, self.sslPort, customcert))
1.433 +
1.434 + match = clientAuthRE.match(option)
1.435 + if match:
1.436 + clientauth = match.group("clientauth");
1.437 + sslTunnelConfig.write("clientauth:%s:%s:%s:%s\n" %
1.438 + (loc.host, loc.port, self.sslPort, clientauth))
1.439 +
1.440 + match = redirRE.match(option)
1.441 + if match:
1.442 + redirhost = match.group("redirhost")
1.443 + sslTunnelConfig.write("redirhost:%s:%s:%s:%s\n" %
1.444 + (loc.host, loc.port, self.sslPort, redirhost))
1.445 +
1.446 + sslTunnelConfig.close()
1.447 +
1.448 + # Pre-create the certification database for the profile
1.449 + env = self.environment(xrePath = xrePath)
1.450 + certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)
1.451 + pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)
1.452 +
1.453 + status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
1.454 + automationutils.printstatus(status, "certutil")
1.455 + if status != 0:
1.456 + return status
1.457 +
1.458 + # Walk the cert directory and add custom CAs and client certs
1.459 + files = os.listdir(certPath)
1.460 + for item in files:
1.461 + root, ext = os.path.splitext(item)
1.462 + if ext == ".ca":
1.463 + trustBits = "CT,,"
1.464 + if root.endswith("-object"):
1.465 + trustBits = "CT,,CT"
1.466 + status = self.Process([certutil, "-A", "-i", os.path.join(certPath, item),
1.467 + "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
1.468 + env = env).wait()
1.469 + automationutils.printstatus(status, "certutil")
1.470 + if ext == ".client":
1.471 + status = self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",
1.472 + pwfilePath, "-d", profileDir],
1.473 + env = env).wait()
1.474 + automationutils.printstatus(status, "pk12util")
1.475 +
1.476 + os.unlink(pwfilePath)
1.477 + return 0
1.478 +
1.479 + def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None):
1.480 + if xrePath == None:
1.481 + xrePath = self.DIST_BIN
1.482 + if env == None:
1.483 + env = dict(os.environ)
1.484 +
1.485 + ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
1.486 + dmdLibrary = None
1.487 + preloadEnvVar = None
1.488 + if self.UNIXISH or self.IS_MAC:
1.489 + envVar = "LD_LIBRARY_PATH"
1.490 + preloadEnvVar = "LD_PRELOAD"
1.491 + if self.IS_MAC:
1.492 + envVar = "DYLD_LIBRARY_PATH"
1.493 + dmdLibrary = "libdmd.dylib"
1.494 + else: # unixish
1.495 + env['MOZILLA_FIVE_HOME'] = xrePath
1.496 + dmdLibrary = "libdmd.so"
1.497 + if envVar in env:
1.498 + ldLibraryPath = ldLibraryPath + ":" + env[envVar]
1.499 + env[envVar] = ldLibraryPath
1.500 + elif self.IS_WIN32:
1.501 + env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath)
1.502 + dmdLibrary = "dmd.dll"
1.503 + preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
1.504 +
1.505 + if dmdPath and dmdLibrary and preloadEnvVar:
1.506 + env['DMD'] = '1'
1.507 + env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
1.508 +
1.509 + if crashreporter and not debugger:
1.510 + env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
1.511 + env['MOZ_CRASHREPORTER'] = '1'
1.512 + else:
1.513 + env['MOZ_CRASHREPORTER_DISABLE'] = '1'
1.514 +
1.515 + # Crash on non-local network connections.
1.516 + env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
1.517 +
1.518 + env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
1.519 + env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
1.520 + env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
1.521 +
1.522 + # Set WebRTC logging in case it is not set yet
1.523 + env.setdefault('NSPR_LOG_MODULES', 'signaling:5,mtransport:3')
1.524 + env.setdefault('R_LOG_LEVEL', '5')
1.525 + env.setdefault('R_LOG_DESTINATION', 'stderr')
1.526 + env.setdefault('R_LOG_VERBOSE', '1')
1.527 +
1.528 + # ASan specific environment stuff
1.529 + if self.IS_ASAN and (self.IS_LINUX or self.IS_MAC):
1.530 + # Symbolizer support
1.531 + llvmsym = os.path.join(xrePath, "llvm-symbolizer")
1.532 + if os.path.isfile(llvmsym):
1.533 + env["ASAN_SYMBOLIZER_PATH"] = llvmsym
1.534 + self.log.info("INFO | automation.py | ASan using symbolizer at %s", llvmsym)
1.535 +
1.536 + try:
1.537 + totalMemory = int(os.popen("free").readlines()[1].split()[1])
1.538 +
1.539 + # Only 4 GB RAM or less available? Use custom ASan options to reduce
1.540 + # the amount of resources required to do the tests. Standard options
1.541 + # will otherwise lead to OOM conditions on the current test slaves.
1.542 + if totalMemory <= 1024 * 1024 * 4:
1.543 + self.log.info("INFO | automation.py | ASan running in low-memory configuration")
1.544 + env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5"
1.545 + else:
1.546 + self.log.info("INFO | automation.py | ASan running in default memory configuration")
1.547 + except OSError,err:
1.548 + self.log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror)
1.549 + except:
1.550 + self.log.info("Failed determine available memory, disabling ASan low-memory configuration")
1.551 +
1.552 + return env
1.553 +
1.554 + def killPid(self, pid):
1.555 + try:
1.556 + os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
1.557 + except WindowsError:
1.558 + self.log.info("Failed to kill process %d." % pid)
1.559 +
1.560 + if IS_WIN32:
1.561 + PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
1.562 + GetLastError = ctypes.windll.kernel32.GetLastError
1.563 +
1.564 + def readWithTimeout(self, f, timeout):
1.565 + """
1.566 + Try to read a line of output from the file object |f|. |f| must be a
1.567 + pipe, like the |stdout| member of a subprocess.Popen object created
1.568 + with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout|
1.569 + is True if the read timed out, and False otherwise. If no output is
1.570 + received within |timeout| seconds, returns a blank line.
1.571 + """
1.572 +
1.573 + if timeout is None:
1.574 + timeout = 0
1.575 +
1.576 + x = msvcrt.get_osfhandle(f.fileno())
1.577 + l = ctypes.c_long()
1.578 + done = time.time() + timeout
1.579 +
1.580 + buffer = ""
1.581 + while timeout == 0 or time.time() < done:
1.582 + if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
1.583 + err = self.GetLastError()
1.584 + if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
1.585 + return ('', False)
1.586 + else:
1.587 + self.log.error("readWithTimeout got error: %d", err)
1.588 + # read a character at a time, checking for eol. Return once we get there.
1.589 + index = 0
1.590 + while index < l.value:
1.591 + char = f.read(1)
1.592 + buffer += char
1.593 + if char == '\n':
1.594 + return (buffer, False)
1.595 + index = index + 1
1.596 + time.sleep(0.01)
1.597 + return (buffer, True)
1.598 +
1.599 + def isPidAlive(self, pid):
1.600 + STILL_ACTIVE = 259
1.601 + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
1.602 + pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
1.603 + if not pHandle:
1.604 + return False
1.605 + pExitCode = ctypes.wintypes.DWORD()
1.606 + ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode))
1.607 + ctypes.windll.kernel32.CloseHandle(pHandle)
1.608 + return pExitCode.value == STILL_ACTIVE
1.609 +
1.610 + else:
1.611 +
1.612 + def readWithTimeout(self, f, timeout):
1.613 + """Try to read a line of output from the file object |f|. If no output
1.614 + is received within |timeout| seconds, return a blank line.
1.615 + Returns a tuple (line, did_timeout), where |did_timeout| is True
1.616 + if the read timed out, and False otherwise."""
1.617 + (r, w, e) = select.select([f], [], [], timeout)
1.618 + if len(r) == 0:
1.619 + return ('', True)
1.620 + return (f.readline(), False)
1.621 +
1.622 + def isPidAlive(self, pid):
1.623 + try:
1.624 + # kill(pid, 0) checks for a valid PID without actually sending a signal
1.625 + # The method throws OSError if the PID is invalid, which we catch below.
1.626 + os.kill(pid, 0)
1.627 +
1.628 + # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
1.629 + # the process terminates before we get to this point.
1.630 + wpid, wstatus = os.waitpid(pid, os.WNOHANG)
1.631 + return wpid == 0
1.632 + except OSError, err:
1.633 + # Catch the errors we might expect from os.kill/os.waitpid,
1.634 + # and re-raise any others
1.635 + if err.errno == errno.ESRCH or err.errno == errno.ECHILD:
1.636 + return False
1.637 + raise
1.638 +
1.639 + def dumpScreen(self, utilityPath):
1.640 + if self.haveDumpedScreen:
1.641 + self.log.info("Not taking screenshot here: see the one that was previously logged")
1.642 + return
1.643 +
1.644 + self.haveDumpedScreen = True;
1.645 + automationutils.dumpScreen(utilityPath)
1.646 +
1.647 +
1.648 + def killAndGetStack(self, processPID, utilityPath, debuggerInfo):
1.649 + """Kill the process, preferrably in a way that gets us a stack trace.
1.650 + Also attempts to obtain a screenshot before killing the process."""
1.651 + if not debuggerInfo:
1.652 + self.dumpScreen(utilityPath)
1.653 + self.killAndGetStackNoScreenshot(processPID, utilityPath, debuggerInfo)
1.654 +
1.655 + def killAndGetStackNoScreenshot(self, processPID, utilityPath, debuggerInfo):
1.656 + """Kill the process, preferrably in a way that gets us a stack trace."""
1.657 + if self.CRASHREPORTER and not debuggerInfo:
1.658 + if not self.IS_WIN32:
1.659 + # ABRT will get picked up by Breakpad's signal handler
1.660 + os.kill(processPID, signal.SIGABRT)
1.661 + return
1.662 + else:
1.663 + # We should have a "crashinject" program in our utility path
1.664 + crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
1.665 + if os.path.exists(crashinject):
1.666 + status = subprocess.Popen([crashinject, str(processPID)]).wait()
1.667 + automationutils.printstatus(status, "crashinject")
1.668 + if status == 0:
1.669 + return
1.670 + self.log.info("Can't trigger Breakpad, just killing process")
1.671 + self.killPid(processPID)
1.672 +
1.673 + def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath):
1.674 + """ Look for timeout or crashes and return the status after the process terminates """
1.675 + stackFixerProcess = None
1.676 + stackFixerFunction = None
1.677 + didTimeout = False
1.678 + hitMaxTime = False
1.679 + if proc.stdout is None:
1.680 + self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
1.681 + else:
1.682 + logsource = proc.stdout
1.683 +
1.684 + if self.IS_DEBUG_BUILD and symbolsPath and os.path.exists(symbolsPath):
1.685 + # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files)
1.686 + # This method is preferred for Tinderbox builds, since native symbols may have been stripped.
1.687 + sys.path.insert(0, utilityPath)
1.688 + import fix_stack_using_bpsyms as stackFixerModule
1.689 + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath)
1.690 + del sys.path[0]
1.691 + elif self.IS_DEBUG_BUILD and self.IS_MAC and False:
1.692 + # Run each line through a function in fix_macosx_stack.py (uses atos)
1.693 + sys.path.insert(0, utilityPath)
1.694 + import fix_macosx_stack as stackFixerModule
1.695 + stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line)
1.696 + del sys.path[0]
1.697 + elif self.IS_DEBUG_BUILD and self.IS_LINUX:
1.698 + # Run logsource through fix-linux-stack.pl (uses addr2line)
1.699 + # This method is preferred for developer machines, so we don't have to run "make buildsymbols".
1.700 + stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")],
1.701 + stdin=logsource,
1.702 + stdout=subprocess.PIPE)
1.703 + logsource = stackFixerProcess.stdout
1.704 +
1.705 + # With metro browser runs this script launches the metro test harness which launches the browser.
1.706 + # The metro test harness hands back the real browser process id via log output which we need to
1.707 + # pick up on and parse out. This variable tracks the real browser process id if we find it.
1.708 + browserProcessId = -1
1.709 +
1.710 + (line, didTimeout) = self.readWithTimeout(logsource, timeout)
1.711 + while line != "" and not didTimeout:
1.712 + if stackFixerFunction:
1.713 + line = stackFixerFunction(line)
1.714 + self.log.info(line.rstrip().decode("UTF-8", "ignore"))
1.715 + if "TEST-START" in line and "|" in line:
1.716 + self.lastTestSeen = line.split("|")[1].strip()
1.717 + if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
1.718 + self.dumpScreen(utilityPath)
1.719 +
1.720 + (line, didTimeout) = self.readWithTimeout(logsource, timeout)
1.721 +
1.722 + if "METRO_BROWSER_PROCESS" in line:
1.723 + index = line.find("=")
1.724 + if index:
1.725 + browserProcessId = line[index+1:].rstrip()
1.726 + self.log.info("INFO | automation.py | metro browser sub process id detected: %s", browserProcessId)
1.727 +
1.728 + if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):
1.729 + # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().
1.730 + hitMaxTime = True
1.731 + self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime))
1.732 + self.killAndGetStack(proc.pid, utilityPath, debuggerInfo)
1.733 + if didTimeout:
1.734 + if line:
1.735 + self.log.info(line.rstrip().decode("UTF-8", "ignore"))
1.736 + self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
1.737 + if browserProcessId == -1:
1.738 + browserProcessId = proc.pid
1.739 + self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo)
1.740 +
1.741 + status = proc.wait()
1.742 + automationutils.printstatus(status, "Main app process")
1.743 + if status == 0:
1.744 + self.lastTestSeen = "Main app process exited normally"
1.745 + if status != 0 and not didTimeout and not hitMaxTime:
1.746 + self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status)
1.747 + if stackFixerProcess is not None:
1.748 + fixerStatus = stackFixerProcess.wait()
1.749 + automationutils.printstatus(status, "stackFixerProcess")
1.750 + if fixerStatus != 0 and not didTimeout and not hitMaxTime:
1.751 + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus)
1.752 + return status
1.753 +
1.754 + def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
1.755 + """ build the application command line """
1.756 +
1.757 + cmd = os.path.abspath(app)
1.758 + if self.IS_MAC and not self.IS_CAMINO and os.path.exists(cmd + "-bin"):
1.759 + # Prefer 'app-bin' in case 'app' is a shell script.
1.760 + # We can remove this hack once bug 673899 etc are fixed.
1.761 + cmd += "-bin"
1.762 +
1.763 + args = []
1.764 +
1.765 + if debuggerInfo:
1.766 + args.extend(debuggerInfo["args"])
1.767 + args.append(cmd)
1.768 + cmd = os.path.abspath(debuggerInfo["path"])
1.769 +
1.770 + if self.IS_MAC:
1.771 + args.append("-foreground")
1.772 +
1.773 + if self.IS_CYGWIN:
1.774 + profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
1.775 + else:
1.776 + profileDirectory = profileDir + "/"
1.777 +
1.778 + args.extend(("-no-remote", "-profile", profileDirectory))
1.779 + if testURL is not None:
1.780 + if self.IS_CAMINO:
1.781 + args.extend(("-url", testURL))
1.782 + else:
1.783 + args.append((testURL))
1.784 + args.extend(extraArgs)
1.785 + return cmd, args
1.786 +
1.787 + def checkForZombies(self, processLog, utilityPath, debuggerInfo):
1.788 + """ Look for hung processes """
1.789 + if not os.path.exists(processLog):
1.790 + self.log.info('Automation Error: PID log not found: %s', processLog)
1.791 + # Whilst no hung process was found, the run should still display as a failure
1.792 + return True
1.793 +
1.794 + foundZombie = False
1.795 + self.log.info('INFO | zombiecheck | Reading PID log: %s', processLog)
1.796 + processList = []
1.797 + pidRE = re.compile(r'launched child process (\d+)$')
1.798 + processLogFD = open(processLog)
1.799 + for line in processLogFD:
1.800 + self.log.info(line.rstrip())
1.801 + m = pidRE.search(line)
1.802 + if m:
1.803 + processList.append(int(m.group(1)))
1.804 + processLogFD.close()
1.805 +
1.806 + for processPID in processList:
1.807 + self.log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID)
1.808 + if self.isPidAlive(processPID):
1.809 + foundZombie = True
1.810 + self.log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID)
1.811 + self.killAndGetStack(processPID, utilityPath, debuggerInfo)
1.812 + return foundZombie
1.813 +
1.814 + def checkForCrashes(self, minidumpDir, symbolsPath):
1.815 + return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen)
1.816 +
1.817 + def runApp(self, testURL, env, app, profileDir, extraArgs,
1.818 + runSSLTunnel = False, utilityPath = None,
1.819 + xrePath = None, certPath = None,
1.820 + debuggerInfo = None, symbolsPath = None,
1.821 + timeout = -1, maxTime = None, onLaunch = None,
1.822 + webapprtChrome = False, hide_subtests=None, screenshotOnFail=False):
1.823 + """
1.824 + Run the app, log the duration it took to execute, return the status code.
1.825 + Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
1.826 + """
1.827 +
1.828 + if utilityPath == None:
1.829 + utilityPath = self.DIST_BIN
1.830 + if xrePath == None:
1.831 + xrePath = self.DIST_BIN
1.832 + if certPath == None:
1.833 + certPath = self.CERTS_SRC_DIR
1.834 + if timeout == -1:
1.835 + timeout = self.DEFAULT_TIMEOUT
1.836 +
1.837 + # copy env so we don't munge the caller's environment
1.838 + env = dict(env);
1.839 + env["NO_EM_RESTART"] = "1"
1.840 + tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
1.841 + os.close(tmpfd)
1.842 + env["MOZ_PROCESS_LOG"] = processLog
1.843 +
1.844 + if self.IS_TEST_BUILD and runSSLTunnel:
1.845 + # create certificate database for the profile
1.846 + certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
1.847 + if certificateStatus != 0:
1.848 + self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Certificate integration failed")
1.849 + return certificateStatus
1.850 +
1.851 + # start ssltunnel to provide https:// URLs capability
1.852 + ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)
1.853 + ssltunnelProcess = self.Process([ssltunnel,
1.854 + os.path.join(profileDir, "ssltunnel.cfg")],
1.855 + env = self.environment(xrePath = xrePath))
1.856 + self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid)
1.857 +
1.858 + cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
1.859 + startTime = datetime.now()
1.860 +
1.861 + if debuggerInfo and debuggerInfo["interactive"]:
1.862 + # If an interactive debugger is attached, don't redirect output,
1.863 + # don't use timeouts, and don't capture ctrl-c.
1.864 + timeout = None
1.865 + maxTime = None
1.866 + outputPipe = None
1.867 + signal.signal(signal.SIGINT, lambda sigid, frame: None)
1.868 + else:
1.869 + outputPipe = subprocess.PIPE
1.870 +
1.871 + self.lastTestSeen = "automation.py"
1.872 + proc = self.Process([cmd] + args,
1.873 + env = self.environment(env, xrePath = xrePath,
1.874 + crashreporter = not debuggerInfo),
1.875 + stdout = outputPipe,
1.876 + stderr = subprocess.STDOUT)
1.877 + self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
1.878 +
1.879 + if onLaunch is not None:
1.880 + # Allow callers to specify an onLaunch callback to be fired after the
1.881 + # app is launched.
1.882 + onLaunch()
1.883 +
1.884 + status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath)
1.885 + self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
1.886 +
1.887 + # Do a final check for zombie child processes.
1.888 + zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo)
1.889 +
1.890 + crashed = self.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)
1.891 +
1.892 + if crashed or zombieProcesses:
1.893 + status = 1
1.894 +
1.895 + if os.path.exists(processLog):
1.896 + os.unlink(processLog)
1.897 +
1.898 + if self.IS_TEST_BUILD and runSSLTunnel:
1.899 + ssltunnelProcess.kill()
1.900 +
1.901 + return status
1.902 +
1.903 + def getExtensionIDFromRDF(self, rdfSource):
1.904 + """
1.905 + Retrieves the extension id from an install.rdf file (or string).
1.906 + """
1.907 + from xml.dom.minidom import parse, parseString, Node
1.908 +
1.909 + if isinstance(rdfSource, file):
1.910 + document = parse(rdfSource)
1.911 + else:
1.912 + document = parseString(rdfSource)
1.913 +
1.914 + # Find the <em:id> element. There can be multiple <em:id> tags
1.915 + # within <em:targetApplication> tags, so we have to check this way.
1.916 + for rdfChild in document.documentElement.childNodes:
1.917 + if rdfChild.nodeType == Node.ELEMENT_NODE and rdfChild.tagName == "Description":
1.918 + for descChild in rdfChild.childNodes:
1.919 + if descChild.nodeType == Node.ELEMENT_NODE and descChild.tagName == "em:id":
1.920 + return descChild.childNodes[0].data
1.921 +
1.922 + return None
1.923 +
1.924 + def installExtension(self, extensionSource, profileDir, extensionID = None):
1.925 + """
1.926 + Copies an extension into the extensions directory of the given profile.
1.927 + extensionSource - the source location of the extension files. This can be either
1.928 + a directory or a path to an xpi file.
1.929 + profileDir - the profile directory we are copying into. We will create the
1.930 + "extensions" directory there if it doesn't exist.
1.931 + extensionID - the id of the extension to be used as the containing directory for the
1.932 + extension, if extensionSource is a directory, i.e.
1.933 + this is the name of the folder in the <profileDir>/extensions/<extensionID>
1.934 + """
1.935 + if not os.path.isdir(profileDir):
1.936 + self.log.info("INFO | automation.py | Cannot install extension, invalid profileDir at: %s", profileDir)
1.937 + return
1.938 +
1.939 + installRDFFilename = "install.rdf"
1.940 +
1.941 + extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
1.942 + if not os.path.isdir(extensionsRootDir):
1.943 + os.makedirs(extensionsRootDir)
1.944 +
1.945 + if os.path.isfile(extensionSource):
1.946 + reader = automationutils.ZipFileReader(extensionSource)
1.947 +
1.948 + for filename in reader.namelist():
1.949 + # Sanity check the zip file.
1.950 + if os.path.isabs(filename):
1.951 + self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
1.952 + return
1.953 +
1.954 + # We may need to dig the extensionID out of the zip file...
1.955 + if extensionID is None and filename == installRDFFilename:
1.956 + extensionID = self.getExtensionIDFromRDF(reader.read(filename))
1.957 +
1.958 + # We must know the extensionID now.
1.959 + if extensionID is None:
1.960 + self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")
1.961 + return
1.962 +
1.963 + # Make the extension directory.
1.964 + extensionDir = os.path.join(extensionsRootDir, extensionID)
1.965 + os.mkdir(extensionDir)
1.966 +
1.967 + # Extract all files.
1.968 + reader.extractall(extensionDir)
1.969 +
1.970 + elif os.path.isdir(extensionSource):
1.971 + if extensionID is None:
1.972 + filename = os.path.join(extensionSource, installRDFFilename)
1.973 + if os.path.isfile(filename):
1.974 + with open(filename, "r") as installRDF:
1.975 + extensionID = self.getExtensionIDFromRDF(installRDF)
1.976 +
1.977 + if extensionID is None:
1.978 + self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")
1.979 + return
1.980 +
1.981 + # Copy extension tree into its own directory.
1.982 + # "destination directory must not already exist".
1.983 + shutil.copytree(extensionSource, os.path.join(extensionsRootDir, extensionID))
1.984 +
1.985 + else:
1.986 + self.log.info("INFO | automation.py | Cannot install extension, invalid extensionSource at: %s", extensionSource)
1.987 +
1.988 + def elf_arm(self, filename):
1.989 + data = open(filename, 'rb').read(20)
1.990 + return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM
1.991 +
