The Tor Browser: build/automation.py.in@b8a032363ba2

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

1 #

     2 # This Source Code Form is subject to the terms of the Mozilla Public

     3 # License, v. 2.0. If a copy of the MPL was not distributed with this

     4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     6 from __future__ import with_statement

     7 import codecs

     8 import itertools

     9 import json

    10 import logging

    11 import os

    12 import re

    13 import select

    14 import shutil

    15 import signal

    16 import subprocess

    17 import sys

    18 import threading

    19 import tempfile

    20 import sqlite3

    21 from datetime import datetime, timedelta

    22 from string import Template

    24 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))

    25 sys.path.insert(0, SCRIPT_DIR)

    26 import automationutils

    28 # --------------------------------------------------------------

    29 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900

    30 # These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR

    31 here = os.path.dirname(os.path.realpath(__file__))

    32 mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))

    34 if os.path.isdir(mozbase):

    35     for package in os.listdir(mozbase):

    36         package_path = os.path.join(mozbase, package)

    37         if package_path not in sys.path:

    38             sys.path.append(package_path)

    40 import mozcrash

    41 from mozprofile import Profile, Preferences

    42 from mozprofile.permissions import ServerLocations

    44 # ---------------------------------------------------------------

    46 _DEFAULT_PREFERENCE_FILE = os.path.join(SCRIPT_DIR, 'prefs_general.js')

    47 _DEFAULT_APPS_FILE = os.path.join(SCRIPT_DIR, 'webapps_mochitest.json')

    49 _DEFAULT_WEB_SERVER = "127.0.0.1"

    50 _DEFAULT_HTTP_PORT = 8888

    51 _DEFAULT_SSL_PORT = 4443

    52 _DEFAULT_WEBSOCKET_PORT = 9988

    54 # from nsIPrincipal.idl

    55 _APP_STATUS_NOT_INSTALLED = 0

    56 _APP_STATUS_INSTALLED     = 1

    57 _APP_STATUS_PRIVILEGED    = 2

    58 _APP_STATUS_CERTIFIED     = 3

    60 #expand _DIST_BIN = __XPC_BIN_PATH__

    61 #expand _IS_WIN32 = len("__WIN32__") != 0

    62 #expand _IS_MAC = __IS_MAC__ != 0

    63 #expand _IS_LINUX = __IS_LINUX__ != 0

    64 #ifdef IS_CYGWIN

    65 #expand _IS_CYGWIN = __IS_CYGWIN__ == 1

    66 #else

    67 _IS_CYGWIN = False

    68 #endif

    69 #expand _IS_CAMINO = __IS_CAMINO__ != 0

    70 #expand _BIN_SUFFIX = __BIN_SUFFIX__

    71 #expand _PERL = __PERL__

    73 #expand _DEFAULT_APP = "./" + __BROWSER_PATH__

    74 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__

    75 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__

    76 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__

    77 #expand _CRASHREPORTER = __CRASHREPORTER__ == 1

    78 #expand _IS_ASAN = __IS_ASAN__ == 1

    81 if _IS_WIN32:

    82   import ctypes, ctypes.wintypes, time, msvcrt

    83 else:

    84   import errno

    87 def getGlobalLog():

    88   return _log

    90 def resetGlobalLog(log):

    91   while _log.handlers:

    92     _log.removeHandler(_log.handlers[0])

    93   handler = logging.StreamHandler(log)

    94   _log.setLevel(logging.INFO)

    95   _log.addHandler(handler)

    97 # We use the logging system here primarily because it'll handle multiple

    98 # threads, which is needed to process the output of the server and application

    99 # processes simultaneously.

   100 _log = logging.getLogger()

   101 resetGlobalLog(sys.stdout)

   104 #################

   105 # PROFILE SETUP #

   106 #################

   108 class SyntaxError(Exception):

   109   "Signifies a syntax error on a particular line in server-locations.txt."

   111   def __init__(self, lineno, msg = None):

   112     self.lineno = lineno

   113     self.msg = msg

   115   def __str__(self):

   116     s = "Syntax error on line " + str(self.lineno)

   117     if self.msg:

   118       s += ": %s." % self.msg

   119     else:

   120       s += "."

   121     return s

   124 class Location:

   125   "Represents a location line in server-locations.txt."

   127   def __init__(self, scheme, host, port, options):

   128     self.scheme = scheme

   129     self.host = host

   130     self.port = port

   131     self.options = options

   133 class Automation(object):

   134   """

   135   Runs the browser from a script, and provides useful utilities

   136   for setting up the browser environment.

   137   """

   139   DIST_BIN = _DIST_BIN

   140   IS_WIN32 = _IS_WIN32

   141   IS_MAC = _IS_MAC

   142   IS_LINUX = _IS_LINUX

   143   IS_CYGWIN = _IS_CYGWIN

   144   IS_CAMINO = _IS_CAMINO

   145   BIN_SUFFIX = _BIN_SUFFIX

   146   PERL = _PERL

   148   UNIXISH = not IS_WIN32 and not IS_MAC

   150   DEFAULT_APP = _DEFAULT_APP

   151   CERTS_SRC_DIR = _CERTS_SRC_DIR

   152   IS_TEST_BUILD = _IS_TEST_BUILD

   153   IS_DEBUG_BUILD = _IS_DEBUG_BUILD

   154   CRASHREPORTER = _CRASHREPORTER

   155   IS_ASAN = _IS_ASAN

   157   # timeout, in seconds

   158   DEFAULT_TIMEOUT = 60.0

   159   DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER

   160   DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT

   161   DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT

   162   DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT

   164   def __init__(self):

   165     self.log = _log

   166     self.lastTestSeen = "automation.py"

   167     self.haveDumpedScreen = False

   169   def setServerInfo(self,

   170                     webServer = _DEFAULT_WEB_SERVER,

   171                     httpPort = _DEFAULT_HTTP_PORT,

   172                     sslPort = _DEFAULT_SSL_PORT,

   173                     webSocketPort = _DEFAULT_WEBSOCKET_PORT):

   174     self.webServer = webServer

   175     self.httpPort = httpPort

   176     self.sslPort = sslPort

   177     self.webSocketPort = webSocketPort

   179   @property

   180   def __all__(self):

   181     return [

   182            "UNIXISH",

   183            "IS_WIN32",

   184            "IS_MAC",

   185            "log",

   186            "runApp",

   187            "Process",

   188            "addCommonOptions",

   189            "initializeProfile",

   190            "DIST_BIN",

   191            "DEFAULT_APP",

   192            "CERTS_SRC_DIR",

   193            "environment",

   194            "IS_TEST_BUILD",

   195            "IS_DEBUG_BUILD",

   196            "DEFAULT_TIMEOUT",

   197           ]

   199   class Process(subprocess.Popen):

   200     """

   201     Represents our view of a subprocess.

   202     It adds a kill() method which allows it to be stopped explicitly.

   203     """

   205     def __init__(self,

   206                  args,

   207                  bufsize=0,

   208                  executable=None,

   209                  stdin=None,

   210                  stdout=None,

   211                  stderr=None,

   212                  preexec_fn=None,

   213                  close_fds=False,

   214                  shell=False,

   215                  cwd=None,

   216                  env=None,

   217                  universal_newlines=False,

   218                  startupinfo=None,

   219                  creationflags=0):

   220       _log.info("INFO | automation.py | Launching: %s", subprocess.list2cmdline(args))

   221       subprocess.Popen.__init__(self, args, bufsize, executable,

   222                                 stdin, stdout, stderr,

   223                                 preexec_fn, close_fds,

   224                                 shell, cwd, env,

   225                                 universal_newlines, startupinfo, creationflags)

   226       self.log = _log

   228     def kill(self):

   229       if Automation().IS_WIN32:

   230         import platform

   231         pid = "%i" % self.pid

   232         if platform.release() == "2000":

   233           # Windows 2000 needs 'kill.exe' from the

   234           #'Windows 2000 Resource Kit tools'. (See bug 475455.)

   235           try:

   236             subprocess.Popen(["kill", "-f", pid]).wait()

   237           except:

   238             self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)

   239         else:

   240           # Windows XP and later.

   241           subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()

   242       else:

   243         os.kill(self.pid, signal.SIGKILL)

   245   def readLocations(self, locationsPath = "server-locations.txt"):

   246     """

   247     Reads the locations at which the Mochitest HTTP server is available from

   248     server-locations.txt.

   249     """

   251     locationFile = codecs.open(locationsPath, "r", "UTF-8")

   253     # Perhaps more detail than necessary, but it's the easiest way to make sure

   254     # we get exactly the format we want.  See server-locations.txt for the exact

   255     # format guaranteed here.

   256     lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"

   257                       r"://"

   258                       r"(?P<host>"

   259                         r"\d+\.\d+\.\d+\.\d+"

   260                         r"|"

   261                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"

   262                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"

   263                       r")"

   264                       r":"

   265                       r"(?P<port>\d+)"

   266                       r"(?:"

   267                       r"\s+"

   268                       r"(?P<options>\S+(?:,\S+)*)"

   269                       r")?$")

   270     locations = []

   271     lineno = 0

   272     seenPrimary = False

   273     for line in locationFile:

   274       lineno += 1

   275       if line.startswith("#") or line == "\n":

   276         continue

   278       match = lineRe.match(line)

   279       if not match:

   280         raise SyntaxError(lineno)

   282       options = match.group("options")

   283       if options:

   284         options = options.split(",")

   285         if "primary" in options:

   286           if seenPrimary:

   287             raise SyntaxError(lineno, "multiple primary locations")

   288           seenPrimary = True

   289       else:

   290         options = []

   292       locations.append(Location(match.group("scheme"), match.group("host"),

   293                                 match.group("port"), options))

   295     if not seenPrimary:

   296       raise SyntaxError(lineno + 1, "missing primary location")

   298     return locations

   300   def setupPermissionsDatabase(self, profileDir, permissions):

   301     # Included for reftest compatibility;

   302     # see https://bugzilla.mozilla.org/show_bug.cgi?id=688667

   304     # Open database and create table

   305     permDB = sqlite3.connect(os.path.join(profileDir, "permissions.sqlite"))

   306     cursor = permDB.cursor();

   308     cursor.execute("PRAGMA user_version=3");

   310     # SQL copied from nsPermissionManager.cpp

   311     cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (

   312       id INTEGER PRIMARY KEY,

   313       host TEXT,

   314       type TEXT,

   315       permission INTEGER,

   316       expireType INTEGER,

   317       expireTime INTEGER,

   318       appId INTEGER,

   319       isInBrowserElement INTEGER)""")

   321     # Insert desired permissions

   322     for perm in permissions.keys():

   323       for host,allow in permissions[perm]:

   324         cursor.execute("INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)",

   325                        (host, perm, 1 if allow else 2))

   327     # Commit and close

   328     permDB.commit()

   329     cursor.close()

   331   def initializeProfile(self, profileDir,

   332                               extraPrefs=None,

   333                               useServerLocations=False,

   334                               prefsPath=_DEFAULT_PREFERENCE_FILE,

   335                               appsPath=_DEFAULT_APPS_FILE,

   336                               addons=None):

   337     " Sets up the standard testing profile."

   339     extraPrefs = extraPrefs or []

   341     # create the profile

   342     prefs = {}

   343     locations = None

   344     if useServerLocations:

   345         locations = ServerLocations()

   346         locations.read(os.path.abspath('server-locations.txt'), True)

   347     else:

   348       prefs['network.proxy.type'] = 0

   350     prefs.update(Preferences.read_prefs(prefsPath))

   352     for v in extraPrefs:

   353       thispref = v.split("=", 1)

   354       if len(thispref) < 2:

   355         print "Error: syntax error in --setpref=" + v

   356         sys.exit(1)

   357       prefs[thispref[0]] = thispref[1]

   360     interpolation = {"server": "%s:%s" % (self.webServer, self.httpPort)}

   361     prefs = json.loads(json.dumps(prefs) % interpolation)

   362     for pref in prefs:

   363         prefs[pref] = Preferences.cast(prefs[pref])

   365     # load apps

   366     apps = None

   367     if appsPath and os.path.exists(appsPath):

   368         with open(appsPath, 'r') as apps_file:

   369             apps = json.load(apps_file)

   371     proxy = {'remote': str(self.webServer),

   372              'http': str(self.httpPort),

   373              'https': str(self.sslPort),

   374     # use SSL port for legacy compatibility; see

   375     # - https://bugzilla.mozilla.org/show_bug.cgi?id=688667#c66

   376     # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221

   377     #             'ws': str(self.webSocketPort)

   378              'ws': str(self.sslPort)

   379              }

   381     # return profile object

   382     profile = Profile(profile=profileDir,

   383                       addons=addons,

   384                       locations=locations,

   385                       preferences=prefs,

   386                       restore=False,

   387                       apps=apps,

   388                       proxy=proxy)

   389     return profile

   391   def addCommonOptions(self, parser):

   392     "Adds command-line options which are common to mochitest and reftest."

   394     parser.add_option("--setpref",

   395                       action = "append", type = "string",

   396                       default = [],

   397                       dest = "extraPrefs", metavar = "PREF=VALUE",

   398                       help = "defines an extra user preference")

   400   def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):

   401     pwfilePath = os.path.join(profileDir, ".crtdbpw")

   402     pwfile = open(pwfilePath, "w")

   403     pwfile.write("\n")

   404     pwfile.close()

   406     # Create head of the ssltunnel configuration file

   407     sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")

   408     sslTunnelConfig = open(sslTunnelConfigPath, "w")

   410     sslTunnelConfig.write("httpproxy:1\n")

   411     sslTunnelConfig.write("certdbdir:%s\n" % certPath)

   412     sslTunnelConfig.write("forward:127.0.0.1:%s\n" % self.httpPort)

   413     sslTunnelConfig.write("websocketserver:%s:%s\n" % (self.webServer, self.webSocketPort))

   414     sslTunnelConfig.write("listen:*:%s:pgo server certificate\n" % self.sslPort)

   416     # Configure automatic certificate and bind custom certificates, client authentication

   417     locations = self.readLocations()

   418     locations.pop(0)

   419     for loc in locations:

   420       if loc.scheme == "https" and "nocert" not in loc.options:

   421         customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")

   422         clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")

   423         redirRE      = re.compile("^redir=(?P<redirhost>[0-9a-zA-Z_ .]+)")

   424         for option in loc.options:

   425           match = customCertRE.match(option)

   426           if match:

   427             customcert = match.group("nickname");

   428             sslTunnelConfig.write("listen:%s:%s:%s:%s\n" %

   429                       (loc.host, loc.port, self.sslPort, customcert))

   431           match = clientAuthRE.match(option)

   432           if match:

   433             clientauth = match.group("clientauth");

   434             sslTunnelConfig.write("clientauth:%s:%s:%s:%s\n" %

   435                       (loc.host, loc.port, self.sslPort, clientauth))

   437           match = redirRE.match(option)

   438           if match:

   439             redirhost = match.group("redirhost")

   440             sslTunnelConfig.write("redirhost:%s:%s:%s:%s\n" %

   441                       (loc.host, loc.port, self.sslPort, redirhost))

   443     sslTunnelConfig.close()

   445     # Pre-create the certification database for the profile

   446     env = self.environment(xrePath = xrePath)

   447     certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)

   448     pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)

   450     status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()

   451     automationutils.printstatus(status, "certutil")

   452     if status != 0:

   453       return status

   455     # Walk the cert directory and add custom CAs and client certs

   456     files = os.listdir(certPath)

   457     for item in files:

   458       root, ext = os.path.splitext(item)

   459       if ext == ".ca":

   460         trustBits = "CT,,"

   461         if root.endswith("-object"):

   462           trustBits = "CT,,CT"

   463         status = self.Process([certutil, "-A", "-i", os.path.join(certPath, item),

   464                     "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],

   465                     env = env).wait()

   466         automationutils.printstatus(status, "certutil")

   467       if ext == ".client":

   468         status = self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",

   469                     pwfilePath, "-d", profileDir],

   470                     env = env).wait()

   471         automationutils.printstatus(status, "pk12util")

   473     os.unlink(pwfilePath)

   474     return 0

   476   def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None):

   477     if xrePath == None:

   478       xrePath = self.DIST_BIN

   479     if env == None:

   480       env = dict(os.environ)

   482     ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))

   483     dmdLibrary = None

   484     preloadEnvVar = None

   485     if self.UNIXISH or self.IS_MAC:

   486       envVar = "LD_LIBRARY_PATH"

   487       preloadEnvVar = "LD_PRELOAD"

   488       if self.IS_MAC:

   489         envVar = "DYLD_LIBRARY_PATH"

   490         dmdLibrary = "libdmd.dylib"

   491       else: # unixish

   492         env['MOZILLA_FIVE_HOME'] = xrePath

   493         dmdLibrary = "libdmd.so"

   494       if envVar in env:

   495         ldLibraryPath = ldLibraryPath + ":" + env[envVar]

   496       env[envVar] = ldLibraryPath

   497     elif self.IS_WIN32:

   498       env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath)

   499       dmdLibrary = "dmd.dll"

   500       preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"

   502     if dmdPath and dmdLibrary and preloadEnvVar:

   503       env['DMD'] = '1'

   504       env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)

   506     if crashreporter and not debugger:

   507       env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'

   508       env['MOZ_CRASHREPORTER'] = '1'

   509     else:

   510       env['MOZ_CRASHREPORTER_DISABLE'] = '1'

   512     # Crash on non-local network connections.

   513     env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'

   515     env['GNOME_DISABLE_CRASH_DIALOG'] = '1'

   516     env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'

   517     env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'

   519     # Set WebRTC logging in case it is not set yet

   520     env.setdefault('NSPR_LOG_MODULES', 'signaling:5,mtransport:3')

   521     env.setdefault('R_LOG_LEVEL', '5')

   522     env.setdefault('R_LOG_DESTINATION', 'stderr')

   523     env.setdefault('R_LOG_VERBOSE', '1')

   525     # ASan specific environment stuff

   526     if self.IS_ASAN and (self.IS_LINUX or self.IS_MAC):

   527       # Symbolizer support

   528       llvmsym = os.path.join(xrePath, "llvm-symbolizer")

   529       if os.path.isfile(llvmsym):

   530         env["ASAN_SYMBOLIZER_PATH"] = llvmsym

   531         self.log.info("INFO | automation.py | ASan using symbolizer at %s", llvmsym)

   533       try:

   534         totalMemory = int(os.popen("free").readlines()[1].split()[1])

   536         # Only 4 GB RAM or less available? Use custom ASan options to reduce

   537         # the amount of resources required to do the tests. Standard options

   538         # will otherwise lead to OOM conditions on the current test slaves.

   539         if totalMemory <= 1024 * 1024 * 4:

   540           self.log.info("INFO | automation.py | ASan running in low-memory configuration")

   541           env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5"

   542         else:

   543           self.log.info("INFO | automation.py | ASan running in default memory configuration")

   544       except OSError,err:

   545         self.log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror)

   546       except:

   547         self.log.info("Failed determine available memory, disabling ASan low-memory configuration")

   549     return env

   551   def killPid(self, pid):

   552     try:

   553       os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))

   554     except WindowsError:

   555       self.log.info("Failed to kill process %d." % pid)

   557   if IS_WIN32:

   558     PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe

   559     GetLastError = ctypes.windll.kernel32.GetLastError

   561     def readWithTimeout(self, f, timeout):

   562       """

   563       Try to read a line of output from the file object |f|. |f| must be a

   564       pipe, like the |stdout| member of a subprocess.Popen object created

   565       with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout|

   566       is True if the read timed out, and False otherwise. If no output is

   567       received within |timeout| seconds, returns a blank line.

   568       """

   570       if timeout is None:

   571         timeout = 0

   573       x = msvcrt.get_osfhandle(f.fileno())

   574       l = ctypes.c_long()

   575       done = time.time() + timeout

   577       buffer = ""

   578       while timeout == 0 or time.time() < done:

   579         if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:

   580           err = self.GetLastError()

   581           if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE

   582             return ('', False)

   583           else:

   584             self.log.error("readWithTimeout got error: %d", err)

   585         # read a character at a time, checking for eol. Return once we get there.

   586         index = 0

   587         while index < l.value:

   588           char = f.read(1)

   589           buffer += char

   590           if char == '\n':

   591             return (buffer, False)

   592           index = index + 1

   593         time.sleep(0.01)

   594       return (buffer, True)

   596     def isPidAlive(self, pid):

   597       STILL_ACTIVE = 259

   598       PROCESS_QUERY_LIMITED_INFORMATION = 0x1000

   599       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)

   600       if not pHandle:

   601         return False

   602       pExitCode = ctypes.wintypes.DWORD()

   603       ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode))

   604       ctypes.windll.kernel32.CloseHandle(pHandle)

   605       return pExitCode.value == STILL_ACTIVE

   607   else:

   609     def readWithTimeout(self, f, timeout):

   610       """Try to read a line of output from the file object |f|. If no output

   611       is received within |timeout| seconds, return a blank line.

   612       Returns a tuple (line, did_timeout), where |did_timeout| is True

   613       if the read timed out, and False otherwise."""

   614       (r, w, e) = select.select([f], [], [], timeout)

   615       if len(r) == 0:

   616         return ('', True)

   617       return (f.readline(), False)

   619     def isPidAlive(self, pid):

   620       try:

   621         # kill(pid, 0) checks for a valid PID without actually sending a signal

   622         # The method throws OSError if the PID is invalid, which we catch below.

   623         os.kill(pid, 0)

   625         # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if

   626         # the process terminates before we get to this point.

   627         wpid, wstatus = os.waitpid(pid, os.WNOHANG)

   628         return wpid == 0

   629       except OSError, err:

   630         # Catch the errors we might expect from os.kill/os.waitpid,

   631         # and re-raise any others

   632         if err.errno == errno.ESRCH or err.errno == errno.ECHILD:

   633           return False

   634         raise

   636   def dumpScreen(self, utilityPath):

   637     if self.haveDumpedScreen:

   638       self.log.info("Not taking screenshot here: see the one that was previously logged")

   639       return

   641     self.haveDumpedScreen = True;

   642     automationutils.dumpScreen(utilityPath)

   645   def killAndGetStack(self, processPID, utilityPath, debuggerInfo):

   646     """Kill the process, preferrably in a way that gets us a stack trace.

   647        Also attempts to obtain a screenshot before killing the process."""

   648     if not debuggerInfo:

   649       self.dumpScreen(utilityPath)

   650     self.killAndGetStackNoScreenshot(processPID, utilityPath, debuggerInfo)

   652   def killAndGetStackNoScreenshot(self, processPID, utilityPath, debuggerInfo):

   653     """Kill the process, preferrably in a way that gets us a stack trace."""

   654     if self.CRASHREPORTER and not debuggerInfo:

   655       if not self.IS_WIN32:

   656         # ABRT will get picked up by Breakpad's signal handler

   657         os.kill(processPID, signal.SIGABRT)

   658         return

   659       else:

   660         # We should have a "crashinject" program in our utility path

   661         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))

   662         if os.path.exists(crashinject):

   663           status = subprocess.Popen([crashinject, str(processPID)]).wait()

   664           automationutils.printstatus(status, "crashinject")

   665           if status == 0:

   666             return

   667     self.log.info("Can't trigger Breakpad, just killing process")

   668     self.killPid(processPID)

   670   def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath):

   671     """ Look for timeout or crashes and return the status after the process terminates """

   672     stackFixerProcess = None

   673     stackFixerFunction = None

   674     didTimeout = False

   675     hitMaxTime = False

   676     if proc.stdout is None:

   677       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")

   678     else:

   679       logsource = proc.stdout

   681       if self.IS_DEBUG_BUILD and symbolsPath and os.path.exists(symbolsPath):

   682         # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files)

   683         # This method is preferred for Tinderbox builds, since native symbols may have been stripped.

   684         sys.path.insert(0, utilityPath)

   685         import fix_stack_using_bpsyms as stackFixerModule

   686         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath)

   687         del sys.path[0]

   688       elif self.IS_DEBUG_BUILD and self.IS_MAC and False:

   689         # Run each line through a function in fix_macosx_stack.py (uses atos)

   690         sys.path.insert(0, utilityPath)

   691         import fix_macosx_stack as stackFixerModule

   692         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line)

   693         del sys.path[0]

   694       elif self.IS_DEBUG_BUILD and self.IS_LINUX:

   695         # Run logsource through fix-linux-stack.pl (uses addr2line)

   696         # This method is preferred for developer machines, so we don't have to run "make buildsymbols".

   697         stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")],

   698                                          stdin=logsource,

   699                                          stdout=subprocess.PIPE)

   700         logsource = stackFixerProcess.stdout

   702       # With metro browser runs this script launches the metro test harness which launches the browser.

   703       # The metro test harness hands back the real browser process id via log output which we need to

   704       # pick up on and parse out. This variable tracks the real browser process id if we find it.

   705       browserProcessId = -1

   707       (line, didTimeout) = self.readWithTimeout(logsource, timeout)

   708       while line != "" and not didTimeout:

   709         if stackFixerFunction:

   710           line = stackFixerFunction(line)

   711         self.log.info(line.rstrip().decode("UTF-8", "ignore"))

   712         if "TEST-START" in line and "|" in line:

   713           self.lastTestSeen = line.split("|")[1].strip()

   714         if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:

   715           self.dumpScreen(utilityPath)

   717         (line, didTimeout) = self.readWithTimeout(logsource, timeout)

   719         if "METRO_BROWSER_PROCESS" in line:

   720           index = line.find("=")

   721           if index:

   722             browserProcessId = line[index+1:].rstrip()

   723             self.log.info("INFO | automation.py | metro browser sub process id detected: %s", browserProcessId)

   725         if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):

   726           # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().

   727           hitMaxTime = True

   728           self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime))

   729           self.killAndGetStack(proc.pid, utilityPath, debuggerInfo)

   730       if didTimeout:

   731         if line:

   732           self.log.info(line.rstrip().decode("UTF-8", "ignore"))

   733         self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))

   734         if browserProcessId == -1:

   735           browserProcessId = proc.pid

   736         self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo)

   738     status = proc.wait()

   739     automationutils.printstatus(status, "Main app process")

   740     if status == 0:

   741       self.lastTestSeen = "Main app process exited normally"

   742     if status != 0 and not didTimeout and not hitMaxTime:

   743       self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status)

   744     if stackFixerProcess is not None:

   745       fixerStatus = stackFixerProcess.wait()

   746       automationutils.printstatus(status, "stackFixerProcess")

   747       if fixerStatus != 0 and not didTimeout and not hitMaxTime:

   748         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus)

   749     return status

   751   def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):

   752     """ build the application command line """

   754     cmd = os.path.abspath(app)

   755     if self.IS_MAC and not self.IS_CAMINO and os.path.exists(cmd + "-bin"):

   756       # Prefer 'app-bin' in case 'app' is a shell script.

   757       # We can remove this hack once bug 673899 etc are fixed.

   758       cmd += "-bin"

   760     args = []

   762     if debuggerInfo:

   763       args.extend(debuggerInfo["args"])

   764       args.append(cmd)

   765       cmd = os.path.abspath(debuggerInfo["path"])

   767     if self.IS_MAC:

   768       args.append("-foreground")

   770     if self.IS_CYGWIN:

   771       profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")

   772     else:

   773       profileDirectory = profileDir + "/"

   775     args.extend(("-no-remote", "-profile", profileDirectory))

   776     if testURL is not None:

   777       if self.IS_CAMINO:

   778         args.extend(("-url", testURL))

   779       else:

   780         args.append((testURL))

   781     args.extend(extraArgs)

   782     return cmd, args

   784   def checkForZombies(self, processLog, utilityPath, debuggerInfo):

   785     """ Look for hung processes """

   786     if not os.path.exists(processLog):

   787       self.log.info('Automation Error: PID log not found: %s', processLog)

   788       # Whilst no hung process was found, the run should still display as a failure

   789       return True

   791     foundZombie = False

   792     self.log.info('INFO | zombiecheck | Reading PID log: %s', processLog)

   793     processList = []

   794     pidRE = re.compile(r'launched child process (\d+)$')

   795     processLogFD = open(processLog)

   796     for line in processLogFD:

   797       self.log.info(line.rstrip())

   798       m = pidRE.search(line)

   799       if m:

   800         processList.append(int(m.group(1)))

   801     processLogFD.close()

   803     for processPID in processList:

   804       self.log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID)

   805       if self.isPidAlive(processPID):

   806         foundZombie = True

   807         self.log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID)

   808         self.killAndGetStack(processPID, utilityPath, debuggerInfo)

   809     return foundZombie

   811   def checkForCrashes(self, minidumpDir, symbolsPath):

   812     return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen)

   814   def runApp(self, testURL, env, app, profileDir, extraArgs,

   815              runSSLTunnel = False, utilityPath = None,

   816              xrePath = None, certPath = None,

   817              debuggerInfo = None, symbolsPath = None,

   818              timeout = -1, maxTime = None, onLaunch = None,

   819              webapprtChrome = False, hide_subtests=None, screenshotOnFail=False):

   820     """

   821     Run the app, log the duration it took to execute, return the status code.

   822     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.

   823     """

   825     if utilityPath == None:

   826       utilityPath = self.DIST_BIN

   827     if xrePath == None:

   828       xrePath = self.DIST_BIN

   829     if certPath == None:

   830       certPath = self.CERTS_SRC_DIR

   831     if timeout == -1:

   832       timeout = self.DEFAULT_TIMEOUT

   834     # copy env so we don't munge the caller's environment

   835     env = dict(env);

   836     env["NO_EM_RESTART"] = "1"

   837     tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')

   838     os.close(tmpfd)

   839     env["MOZ_PROCESS_LOG"] = processLog

   841     if self.IS_TEST_BUILD and runSSLTunnel:

   842       # create certificate database for the profile

   843       certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)

   844       if certificateStatus != 0:

   845         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Certificate integration failed")

   846         return certificateStatus

   848       # start ssltunnel to provide https:// URLs capability

   849       ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)

   850       ssltunnelProcess = self.Process([ssltunnel,

   851                                os.path.join(profileDir, "ssltunnel.cfg")],

   852                                env = self.environment(xrePath = xrePath))

   853       self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid)

   855     cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)

   856     startTime = datetime.now()

   858     if debuggerInfo and debuggerInfo["interactive"]:

   859       # If an interactive debugger is attached, don't redirect output,

   860       # don't use timeouts, and don't capture ctrl-c.

   861       timeout = None

   862       maxTime = None

   863       outputPipe = None

   864       signal.signal(signal.SIGINT, lambda sigid, frame: None)

   865     else:

   866       outputPipe = subprocess.PIPE

   868     self.lastTestSeen = "automation.py"

   869     proc = self.Process([cmd] + args,

   870                  env = self.environment(env, xrePath = xrePath,

   871                                    crashreporter = not debuggerInfo),

   872                  stdout = outputPipe,

   873                  stderr = subprocess.STDOUT)

   874     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)

   876     if onLaunch is not None:

   877       # Allow callers to specify an onLaunch callback to be fired after the

   878       # app is launched.

   879       onLaunch()

   881     status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath)

   882     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))

   884     # Do a final check for zombie child processes.

   885     zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo)

   887     crashed = self.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)

   889     if crashed or zombieProcesses:

   890       status = 1

   892     if os.path.exists(processLog):

   893       os.unlink(processLog)

   895     if self.IS_TEST_BUILD and runSSLTunnel:

   896       ssltunnelProcess.kill()

   898     return status

   900   def getExtensionIDFromRDF(self, rdfSource):

   901     """

   902     Retrieves the extension id from an install.rdf file (or string).

   903     """

   904     from xml.dom.minidom import parse, parseString, Node

   906     if isinstance(rdfSource, file):

   907       document = parse(rdfSource)

   908     else:

   909       document = parseString(rdfSource)

   911     # Find the <em:id> element. There can be multiple <em:id> tags

   912     # within <em:targetApplication> tags, so we have to check this way.

   913     for rdfChild in document.documentElement.childNodes:

   914       if rdfChild.nodeType == Node.ELEMENT_NODE and rdfChild.tagName == "Description":

   915         for descChild in rdfChild.childNodes:

   916           if descChild.nodeType == Node.ELEMENT_NODE and descChild.tagName == "em:id":

   917             return descChild.childNodes[0].data

   919     return None

   921   def installExtension(self, extensionSource, profileDir, extensionID = None):

   922     """

   923     Copies an extension into the extensions directory of the given profile.

   924     extensionSource - the source location of the extension files.  This can be either

   925                       a directory or a path to an xpi file.

   926     profileDir      - the profile directory we are copying into.  We will create the

   927                       "extensions" directory there if it doesn't exist.

   928     extensionID     - the id of the extension to be used as the containing directory for the

   929                       extension, if extensionSource is a directory, i.e.

   930                   this is the name of the folder in the <profileDir>/extensions/<extensionID>

   931     """

   932     if not os.path.isdir(profileDir):

   933       self.log.info("INFO | automation.py | Cannot install extension, invalid profileDir at: %s", profileDir)

   934       return

   936     installRDFFilename = "install.rdf"

   938     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")

   939     if not os.path.isdir(extensionsRootDir):

   940       os.makedirs(extensionsRootDir)

   942     if os.path.isfile(extensionSource):

   943       reader = automationutils.ZipFileReader(extensionSource)

   945       for filename in reader.namelist():

   946         # Sanity check the zip file.

   947         if os.path.isabs(filename):

   948           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")

   949           return

   951         # We may need to dig the extensionID out of the zip file...

   952         if extensionID is None and filename == installRDFFilename:

   953           extensionID = self.getExtensionIDFromRDF(reader.read(filename))

   955       # We must know the extensionID now.

   956       if extensionID is None:

   957         self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")

   958         return

   960       # Make the extension directory.

   961       extensionDir = os.path.join(extensionsRootDir, extensionID)

   962       os.mkdir(extensionDir)

   964       # Extract all files.

   965       reader.extractall(extensionDir)

   967     elif os.path.isdir(extensionSource):

   968       if extensionID is None:

   969         filename = os.path.join(extensionSource, installRDFFilename)

   970         if os.path.isfile(filename):

   971           with open(filename, "r") as installRDF:

   972             extensionID = self.getExtensionIDFromRDF(installRDF)

   974         if extensionID is None:

   975           self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")

   976           return

   978       # Copy extension tree into its own directory.

   979       # "destination directory must not already exist".

   980       shutil.copytree(extensionSource, os.path.join(extensionsRootDir, extensionID))

   982     else:

   983       self.log.info("INFO | automation.py | Cannot install extension, invalid extensionSource at: %s", extensionSource)

   985   def elf_arm(self, filename):

   986     data = open(filename, 'rb').read(20)

   987     return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM

The Tor Browser / file revision

build/automation.py.in@b8a032363ba2

build/automation.py.in