The Tor Browser: layout/tools/reftest/runreftest.py@7e26c7da4463

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 """

     6 Runs the reftest test harness.

     7 """

     9 from optparse import OptionParser

    10 import collections

    11 import json

    12 import multiprocessing

    13 import os

    14 import re

    15 import shutil

    16 import subprocess

    17 import sys

    18 import threading

    20 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))

    21 sys.path.insert(0, SCRIPT_DIRECTORY)

    23 from automation import Automation

    24 from automationutils import (

    25         addCommonOptions,

    26         getDebuggerInfo,

    27         isURL,

    28         processLeakLog

    29 )

    30 import mozprofile

    32 def categoriesToRegex(categoryList):

    33   return "\\(" + ', '.join(["(?P<%s>\\d+) %s" % c for c in categoryList]) + "\\)"

    34 summaryLines = [('Successful', [('pass', 'pass'), ('loadOnly', 'load only')]),

    35                 ('Unexpected', [('fail', 'unexpected fail'),

    36                                 ('pass', 'unexpected pass'),

    37                                 ('asserts', 'unexpected asserts'),

    38                                 ('fixedAsserts', 'unexpected fixed asserts'),

    39                                 ('failedLoad', 'failed load'),

    40                                 ('exception', 'exception')]),

    41                 ('Known problems', [('knownFail', 'known fail'),

    42                                     ('knownAsserts', 'known asserts'),

    43                                     ('random', 'random'),

    44                                     ('skipped', 'skipped'),

    45                                     ('slow', 'slow')])]

    47 # Python's print is not threadsafe.

    48 printLock = threading.Lock()

    50 class ReftestThread(threading.Thread):

    51   def __init__(self, cmdlineArgs):

    52     threading.Thread.__init__(self)

    53     self.cmdlineArgs = cmdlineArgs

    54     self.summaryMatches = {}

    55     self.retcode = -1

    56     for text, _ in summaryLines:

    57       self.summaryMatches[text] = None

    59   def run(self):

    60     with printLock:

    61       print "Starting thread with", self.cmdlineArgs

    62       sys.stdout.flush()

    63     process = subprocess.Popen(self.cmdlineArgs, stdout=subprocess.PIPE)

    64     for chunk in self.chunkForMergedOutput(process.stdout):

    65       with printLock:

    66         print chunk,

    67         sys.stdout.flush()

    68     self.retcode = process.wait()

    70   def chunkForMergedOutput(self, logsource):

    71     """Gather lines together that should be printed as one atomic unit.

    72     Individual test results--anything between 'REFTEST TEST-START' and

    73     'REFTEST TEST-END' lines--are an atomic unit.  Lines with data from

    74     summaries are parsed and the data stored for later aggregation.

    75     Other lines are considered their own atomic units and are permitted

    76     to intermix freely."""

    77     testStartRegex = re.compile("^REFTEST TEST-START")

    78     testEndRegex = re.compile("^REFTEST TEST-END")

    79     summaryHeadRegex = re.compile("^REFTEST INFO \\| Result summary:")

    80     summaryRegexFormatString = "^REFTEST INFO \\| (?P<message>{text}): (?P<total>\\d+) {regex}"

    81     summaryRegexStrings = [summaryRegexFormatString.format(text=text,

    82                                                            regex=categoriesToRegex(categories))

    83                            for (text, categories) in summaryLines]

    84     summaryRegexes = [re.compile(regex) for regex in summaryRegexStrings]

    86     for line in logsource:

    87       if testStartRegex.search(line) is not None:

    88         chunkedLines = [line]

    89         for lineToBeChunked in logsource:

    90           chunkedLines.append(lineToBeChunked)

    91           if testEndRegex.search(lineToBeChunked) is not None:

    92             break

    93         yield ''.join(chunkedLines)

    94         continue

    96       haveSuppressedSummaryLine = False

    97       for regex in summaryRegexes:

    98         match = regex.search(line)

    99         if match is not None:

   100           self.summaryMatches[match.group('message')] = match

   101           haveSuppressedSummaryLine = True

   102           break

   103       if haveSuppressedSummaryLine:

   104         continue

   106       if summaryHeadRegex.search(line) is None:

   107         yield line

   109 class RefTest(object):

   111   oldcwd = os.getcwd()

   113   def __init__(self, automation=None):

   114     self.automation = automation or Automation()

   116   def getFullPath(self, path):

   117     "Get an absolute path relative to self.oldcwd."

   118     return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))

   120   def getManifestPath(self, path):

   121     "Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest"

   122     path = self.getFullPath(path)

   123     if os.path.isdir(path):

   124       defaultManifestPath = os.path.join(path, 'reftest.list')

   125       if os.path.exists(defaultManifestPath):

   126         path = defaultManifestPath

   127       else:

   128         defaultManifestPath = os.path.join(path, 'crashtests.list')

   129         if os.path.exists(defaultManifestPath):

   130           path = defaultManifestPath

   131     return path

   133   def makeJSString(self, s):

   134     return '"%s"' % re.sub(r'([\\"])', r'\\\1', s)

   136   def createReftestProfile(self, options, manifest, server='localhost',

   137                            special_powers=True, profile_to_clone=None):

   138     """

   139       Sets up a profile for reftest.

   140       'manifest' is the path to the reftest.list file we want to test with.  This is used in

   141       the remote subclass in remotereftest.py so we can write it to a preference for the

   142       bootstrap extension.

   143     """

   145     locations = mozprofile.permissions.ServerLocations()

   146     locations.add_host(server, port=0)

   147     locations.add_host('<file>', port=0)

   149     # Set preferences for communication between our command line arguments

   150     # and the reftest harness.  Preferences that are required for reftest

   151     # to work should instead be set in reftest-cmdline.js .

   152     prefs = {}

   153     prefs['reftest.timeout'] = options.timeout * 1000

   154     if options.totalChunks:

   155       prefs['reftest.totalChunks'] = options.totalChunks

   156     if options.thisChunk:

   157       prefs['reftest.thisChunk'] = options.thisChunk

   158     if options.logFile:

   159       prefs['reftest.logFile'] = options.logFile

   160     if options.ignoreWindowSize:

   161       prefs['reftest.ignoreWindowSize'] = True

   162     if options.filter:

   163       prefs['reftest.filter'] = options.filter

   164     if options.shuffle:

   165       prefs['reftest.shuffle'] = True

   166     prefs['reftest.focusFilterMode'] = options.focusFilterMode

   168     # Ensure that telemetry is disabled, so we don't connect to the telemetry

   169     # server in the middle of the tests.

   170     prefs['toolkit.telemetry.enabled'] = False

   171     # Likewise for safebrowsing.

   172     prefs['browser.safebrowsing.enabled'] = False

   173     prefs['browser.safebrowsing.malware.enabled'] = False

   174     # And for snippets.

   175     prefs['browser.snippets.enabled'] = False

   176     prefs['browser.snippets.syncPromo.enabled'] = False

   177     # And for useragent updates.

   178     prefs['general.useragent.updates.enabled'] = False

   179     # And for webapp updates.  Yes, it is supposed to be an integer.

   180     prefs['browser.webapps.checkForUpdates'] = 0

   182     if options.e10s:

   183       prefs['browser.tabs.remote.autostart'] = True

   185     for v in options.extraPrefs:

   186       thispref = v.split('=')

   187       if len(thispref) < 2:

   188         print "Error: syntax error in --setpref=" + v

   189         sys.exit(1)

   190       prefs[thispref[0]] = mozprofile.Preferences.cast(thispref[1].strip())

   192     # install the reftest extension bits into the profile

   193     addons = []

   194     addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest"))

   196     # I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with

   197     # release engineering and landing on multiple branches at once.

   198     if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')):

   199       addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers'))

   201     # Install distributed extensions, if application has any.

   202     distExtDir = os.path.join(options.app[ : options.app.rfind(os.sep)], "distribution", "extensions")

   203     if os.path.isdir(distExtDir):

   204       for f in os.listdir(distExtDir):

   205         addons.append(os.path.join(distExtDir, f))

   207     # Install custom extensions.

   208     for f in options.extensionsToInstall:

   209       addons.append(self.getFullPath(f))

   211     kwargs = { 'addons': addons,

   212                'preferences': prefs,

   213                'locations': locations }

   214     if profile_to_clone:

   215         profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)

   216     else:

   217         profile = mozprofile.Profile(**kwargs)

   219     self.copyExtraFilesToProfile(options, profile)

   220     return profile

   222   def buildBrowserEnv(self, options, profileDir):

   223     browserEnv = self.automation.environment(xrePath = options.xrePath, debugger=options.debugger)

   224     browserEnv["XPCOM_DEBUG_BREAK"] = "stack"

   226     for v in options.environment:

   227       ix = v.find("=")

   228       if ix <= 0:

   229         print "Error: syntax error in --setenv=" + v

   230         return None

   231       browserEnv[v[:ix]] = v[ix + 1:]

   233     # Enable leaks detection to its own log file.

   234     self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")

   235     browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile

   236     return browserEnv

   238   def cleanup(self, profileDir):

   239     if profileDir:

   240       shutil.rmtree(profileDir, True)

   242   def runTests(self, testPath, options, cmdlineArgs = None):

   243     if not options.runTestsInParallel:

   244       return self.runSerialTests(testPath, options, cmdlineArgs)

   246     cpuCount = multiprocessing.cpu_count()

   248     # We have the directive, technology, and machine to run multiple test instances.

   249     # Experimentation says that reftests are not overly CPU-intensive, so we can run

   250     # multiple jobs per CPU core.

   251     #

   252     # Our Windows machines in automation seem to get upset when we run a lot of

   253     # simultaneous tests on them, so tone things down there.

   254     if sys.platform == 'win32':

   255       jobsWithoutFocus = cpuCount

   256     else:

   257       jobsWithoutFocus = 2 * cpuCount

   259     totalJobs = jobsWithoutFocus + 1

   260     perProcessArgs = [sys.argv[:] for i in range(0, totalJobs)]

   262     # First job is only needs-focus tests.  Remaining jobs are non-needs-focus and chunked.

   263     perProcessArgs[0].insert(-1, "--focus-filter-mode=needs-focus")

   264     for (chunkNumber, jobArgs) in enumerate(perProcessArgs[1:], start=1):

   265       jobArgs[-1:-1] = ["--focus-filter-mode=non-needs-focus",

   266                         "--total-chunks=%d" % jobsWithoutFocus,

   267                         "--this-chunk=%d" % chunkNumber]

   269     for jobArgs in perProcessArgs:

   270       try:

   271         jobArgs.remove("--run-tests-in-parallel")

   272       except:

   273         pass

   274       jobArgs.insert(-1, "--no-run-tests-in-parallel")

   275       jobArgs[0:0] = [sys.executable, "-u"]

   277     threads = [ReftestThread(args) for args in perProcessArgs[1:]]

   278     for t in threads:

   279       t.start()

   281     while True:

   282       # The test harness in each individual thread will be doing timeout

   283       # handling on its own, so we shouldn't need to worry about any of

   284       # the threads hanging for arbitrarily long.

   285       for t in threads:

   286         t.join(10)

   287       if not any(t.is_alive() for t in threads):

   288         break

   290     # Run the needs-focus tests serially after the other ones, so we don't

   291     # have to worry about races between the needs-focus tests *actually*

   292     # needing focus and the dummy windows in the non-needs-focus tests

   293     # trying to focus themselves.

   294     focusThread = ReftestThread(perProcessArgs[0])

   295     focusThread.start()

   296     focusThread.join()

   298     # Output the summaries that the ReftestThread filters suppressed.

   299     summaryObjects = [collections.defaultdict(int) for s in summaryLines]

   300     for t in threads:

   301       for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):

   302         threadMatches = t.summaryMatches[text]

   303         for (attribute, description) in categories:

   304           amount = int(threadMatches.group(attribute) if threadMatches else 0)

   305           summaryObj[attribute] += amount

   306         amount = int(threadMatches.group('total') if threadMatches else 0)

   307         summaryObj['total'] += amount

   309     print 'REFTEST INFO | Result summary:'

   310     for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):

   311       details = ', '.join(["%d %s" % (summaryObj[attribute], description) for (attribute, description) in categories])

   312       print 'REFTEST INFO | ' + text + ': ' + str(summaryObj['total']) + ' (' +  details + ')'

   314     return int(any(t.retcode != 0 for t in threads))

   316   def runSerialTests(self, testPath, options, cmdlineArgs = None):

   317     debuggerInfo = getDebuggerInfo(self.oldcwd, options.debugger, options.debuggerArgs,

   318         options.debuggerInteractive);

   320     profileDir = None

   321     try:

   322       reftestlist = self.getManifestPath(testPath)

   323       if cmdlineArgs == None:

   324         cmdlineArgs = ['-reftest', reftestlist]

   325       profile = self.createReftestProfile(options, reftestlist)

   326       profileDir = profile.profile # name makes more sense

   328       # browser environment

   329       browserEnv = self.buildBrowserEnv(options, profileDir)

   331       self.automation.log.info("REFTEST INFO | runreftest.py | Running tests: start.\n")

   332       status = self.automation.runApp(None, browserEnv, options.app, profileDir,

   333                                  cmdlineArgs,

   334                                  utilityPath = options.utilityPath,

   335                                  xrePath=options.xrePath,

   336                                  debuggerInfo=debuggerInfo,

   337                                  symbolsPath=options.symbolsPath,

   338                                  # give the JS harness 30 seconds to deal

   339                                  # with its own timeouts

   340                                  timeout=options.timeout + 30.0)

   341       processLeakLog(self.leakLogFile, options.leakThreshold)

   342       self.automation.log.info("\nREFTEST INFO | runreftest.py | Running tests: end.")

   343     finally:

   344       self.cleanup(profileDir)

   345     return status

   347   def copyExtraFilesToProfile(self, options, profile):

   348     "Copy extra files or dirs specified on the command line to the testing profile."

   349     profileDir = profile.profile

   350     for f in options.extraProfileFiles:

   351       abspath = self.getFullPath(f)

   352       if os.path.isfile(abspath):

   353         if os.path.basename(abspath) == 'user.js':

   354           extra_prefs = mozprofile.Preferences.read_prefs(abspath)

   355           profile.set_preferences(extra_prefs)

   356         else:

   357           shutil.copy2(abspath, profileDir)

   358       elif os.path.isdir(abspath):

   359         dest = os.path.join(profileDir, os.path.basename(abspath))

   360         shutil.copytree(abspath, dest)

   361       else:

   362         self.automation.log.warning("WARNING | runreftest.py | Failed to copy %s to profile", abspath)

   363         continue

   366 class ReftestOptions(OptionParser):

   368   def __init__(self, automation=None):

   369     self.automation = automation or Automation()

   370     OptionParser.__init__(self)

   371     defaults = {}

   373     # we want to pass down everything from automation.__all__

   374     addCommonOptions(self,

   375                      defaults=dict(zip(self.automation.__all__,

   376                             [getattr(self.automation, x) for x in self.automation.__all__])))

   377     self.automation.addCommonOptions(self)

   378     self.add_option("--appname",

   379                     action = "store", type = "string", dest = "app",

   380                     default = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP),

   381                     help = "absolute path to application, overriding default")

   382     self.add_option("--extra-profile-file",

   383                     action = "append", dest = "extraProfileFiles",

   384                     default = [],

   385                     help = "copy specified files/dirs to testing profile")

   386     self.add_option("--timeout",

   387                     action = "store", dest = "timeout", type = "int",

   388                     default = 5 * 60, # 5 minutes per bug 479518

   389                     help = "reftest will timeout in specified number of seconds. [default %default s].")

   390     self.add_option("--leak-threshold",

   391                     action = "store", type = "int", dest = "leakThreshold",

   392                     default = 0,

   393                     help = "fail if the number of bytes leaked through "

   394                            "refcounted objects (or bytes in classes with "

   395                            "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "

   396                            "than the given number")

   397     self.add_option("--utility-path",

   398                     action = "store", type = "string", dest = "utilityPath",

   399                     default = self.automation.DIST_BIN,

   400                     help = "absolute path to directory containing utility "

   401                            "programs (xpcshell, ssltunnel, certutil)")

   402     defaults["utilityPath"] = self.automation.DIST_BIN

   404     self.add_option("--total-chunks",

   405                     type = "int", dest = "totalChunks",

   406                     help = "how many chunks to split the tests up into")

   407     defaults["totalChunks"] = None

   409     self.add_option("--this-chunk",

   410                     type = "int", dest = "thisChunk",

   411                     help = "which chunk to run between 1 and --total-chunks")

   412     defaults["thisChunk"] = None

   414     self.add_option("--log-file",

   415                     action = "store", type = "string", dest = "logFile",

   416                     default = None,

   417                     help = "file to log output to in addition to stdout")

   418     defaults["logFile"] = None

   420     self.add_option("--skip-slow-tests",

   421                     dest = "skipSlowTests", action = "store_true",

   422                     help = "skip tests marked as slow when running")

   423     defaults["skipSlowTests"] = False

   425     self.add_option("--ignore-window-size",

   426                     dest = "ignoreWindowSize", action = "store_true",

   427                     help = "ignore the window size, which may cause spurious failures and passes")

   428     defaults["ignoreWindowSize"] = False

   430     self.add_option("--install-extension",

   431                     action = "append", dest = "extensionsToInstall",

   432                     help = "install the specified extension in the testing profile. "

   433                            "The extension file's name should be <id>.xpi where <id> is "

   434                            "the extension's id as indicated in its install.rdf. "

   435                            "An optional path can be specified too.")

   436     defaults["extensionsToInstall"] = []

   438     self.add_option("--run-tests-in-parallel",

   439                     action = "store_true", dest = "runTestsInParallel",

   440                     help = "run tests in parallel if possible")

   441     self.add_option("--no-run-tests-in-parallel",

   442                     action = "store_false", dest = "runTestsInParallel",

   443                     help = "do not run tests in parallel")

   444     defaults["runTestsInParallel"] = False

   446     self.add_option("--setenv",

   447                     action = "append", type = "string",

   448                     dest = "environment", metavar = "NAME=VALUE",

   449                     help = "sets the given variable in the application's "

   450                            "environment")

   451     defaults["environment"] = []

   453     self.add_option("--filter",

   454                     action = "store", type="string", dest = "filter",

   455                     help = "specifies a regular expression (as could be passed to the JS "

   456                            "RegExp constructor) to test against URLs in the reftest manifest; "

   457                            "only test items that have a matching test URL will be run.")

   458     defaults["filter"] = None

   460     self.add_option("--shuffle",

   461                     action = "store_true", dest = "shuffle",

   462                     help = "run reftests in random order")

   463     defaults["shuffle"] = False

   465     self.add_option("--focus-filter-mode",

   466                     action = "store", type = "string", dest = "focusFilterMode",

   467                     help = "filters tests to run by whether they require focus. "

   468                            "Valid values are `all', `needs-focus', or `non-needs-focus'. "

   469                            "Defaults to `all'.")

   470     defaults["focusFilterMode"] = "all"

   472     self.add_option("--e10s",

   473                     action = "store_true",

   474                     dest = "e10s",

   475                     help = "enables content processes")

   476     defaults["e10s"] = False

   478     self.set_defaults(**defaults)

   480   def verifyCommonOptions(self, options, reftest):

   481     if options.totalChunks is not None and options.thisChunk is None:

   482       self.error("thisChunk must be specified when totalChunks is specified")

   484     if options.totalChunks:

   485       if not 1 <= options.thisChunk <= options.totalChunks:

   486         self.error("thisChunk must be between 1 and totalChunks")

   488     if options.logFile:

   489       options.logFile = reftest.getFullPath(options.logFile)

   491     if options.xrePath is not None:

   492       if not os.access(options.xrePath, os.F_OK):

   493         self.error("--xre-path '%s' not found" % options.xrePath)

   494       if not os.path.isdir(options.xrePath):

   495         self.error("--xre-path '%s' is not a directory" % options.xrePath)

   496       options.xrePath = reftest.getFullPath(options.xrePath)

   498     if options.runTestsInParallel:

   499       if options.logFile is not None:

   500         self.error("cannot specify logfile with parallel tests")

   501       if options.totalChunks is not None and options.thisChunk is None:

   502         self.error("cannot specify thisChunk or totalChunks with parallel tests")

   503       if options.focusFilterMode != "all":

   504         self.error("cannot specify focusFilterMode with parallel tests")

   505       if options.debugger is not None:

   506         self.error("cannot specify a debugger with parallel tests")

   508     return options

   510 def main():

   511   automation = Automation()

   512   parser = ReftestOptions(automation)

   513   reftest = RefTest(automation)

   515   options, args = parser.parse_args()

   516   if len(args) != 1:

   517     print >>sys.stderr, "No reftest.list specified."

   518     sys.exit(1)

   520   options = parser.verifyCommonOptions(options, reftest)

   522   options.app = reftest.getFullPath(options.app)

   523   if not os.path.exists(options.app):

   524     print """Error: Path %(app)s doesn't exist.

   525 Are you executing $objdir/_tests/reftest/runreftest.py?""" \

   526             % {"app": options.app}

   527     sys.exit(1)

   529   if options.xrePath is None:

   530     options.xrePath = os.path.dirname(options.app)

   532   if options.symbolsPath and not isURL(options.symbolsPath):

   533     options.symbolsPath = reftest.getFullPath(options.symbolsPath)

   534   options.utilityPath = reftest.getFullPath(options.utilityPath)

   536   sys.exit(reftest.runTests(args[0], options))

   538 if __name__ == "__main__":

   539   main()

The Tor Browser / file revision

layout/tools/reftest/runreftest.py@7e26c7da4463

layout/tools/reftest/runreftest.py