diff -r 000000000000 -r 6474c204b198 testing/mochitest/mochitest_options.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/testing/mochitest/mochitest_options.py Wed Dec 31 06:09:35 2014 +0100 @@ -0,0 +1,784 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozinfo +import moznetwork +import optparse +import os +import tempfile + +from automationutils import addCommonOptions, isURL +from mozprofile import DEFAULT_PORTS + +here = os.path.abspath(os.path.dirname(__file__)) + +try: + from mozbuild.base import MozbuildObject + build_obj = MozbuildObject.from_environment(cwd=here) +except ImportError: + build_obj = None + +__all__ = ["MochitestOptions", "B2GOptions"] + +VMWARE_RECORDING_HELPER_BASENAME = "vmwarerecordinghelper" + +class MochitestOptions(optparse.OptionParser): + """Usage instructions for runtests.py. + All arguments are optional. + If --chrome is specified, chrome tests will be run instead of web content tests. + If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. + See for details on the logging levels. + """ + + LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") + LEVEL_STRING = ", ".join(LOG_LEVELS) + mochitest_options = [ + [["--close-when-done"], + { "action": "store_true", + "dest": "closeWhenDone", + "default": False, + "help": "close the application when tests are done running", + }], + [["--appname"], + { "action": "store", + "type": "string", + "dest": "app", + "default": None, + "help": "absolute path to application, overriding default", + }], + [["--utility-path"], + { "action": "store", + "type": "string", + "dest": "utilityPath", + "default": build_obj.bindir if build_obj is not None else None, + "help": "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)", + }], + [["--certificate-path"], + { "action": "store", + "type": "string", + "dest": "certPath", + "help": "absolute path to directory containing certificate store to use testing profile", + "default": os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') if build_obj is not None else None, + }], + [["--log-file"], + { "action": "store", + "type": "string", + "dest": "logFile", + "metavar": "FILE", + "help": "file to which logging occurs", + "default": "", + }], + [["--hide-subtests"], + { "action": "store_true", + "dest": "hide_subtests", + "help": "only show subtest log output if there was a failure", + "default": False, + }], + [["--autorun"], + { "action": "store_true", + "dest": "autorun", + "help": "start running tests when the application starts", + "default": False, + }], + [["--timeout"], + { "type": "int", + "dest": "timeout", + "help": "per-test timeout in seconds", + "default": None, + }], + [["--total-chunks"], + { "type": "int", + "dest": "totalChunks", + "help": "how many chunks to split the tests up into", + "default": None, + }], + [["--this-chunk"], + { "type": "int", + "dest": "thisChunk", + "help": "which chunk to run", + "default": None, + }], + [["--chunk-by-dir"], + { "type": "int", + "dest": "chunkByDir", + "help": "group tests together in the same chunk that are in the same top chunkByDir directories", + "default": 0, + }], + [["--shuffle"], + { "dest": "shuffle", + "action": "store_true", + "help": "randomize test order", + "default": False, + }], + [["--console-level"], + { "action": "store", + "type": "choice", + "dest": "consoleLevel", + "choices": LOG_LEVELS, + "metavar": "LEVEL", + "help": "one of %s to determine the level of console " + "logging" % LEVEL_STRING, + "default": None, + }], + [["--file-level"], + { "action": "store", + "type": "choice", + "dest": "fileLevel", + "choices": LOG_LEVELS, + "metavar": "LEVEL", + "help": "one of %s to determine the level of file " + "logging if a file has been specified, defaulting " + "to INFO" % LEVEL_STRING, + "default": "INFO", + }], + [["--chrome"], + { "action": "store_true", + "dest": "chrome", + "help": "run chrome Mochitests", + "default": False, + }], + [["--ipcplugins"], + { "action": "store_true", + "dest": "ipcplugins", + "help": "run ipcplugins Mochitests", + "default": False, + }], + [["--test-path"], + { "action": "store", + "type": "string", + "dest": "testPath", + "help": "start in the given directory's tests", + "default": "", + }], + [["--start-at"], + { "action": "store", + "type": "string", + "dest": "startAt", + "help": "skip over tests until reaching the given test", + "default": "", + }], + [["--end-at"], + { "action": "store", + "type": "string", + "dest": "endAt", + "help": "don't run any tests after the given one", + "default": "", + }], + [["--browser-chrome"], + { "action": "store_true", + "dest": "browserChrome", + "help": "run browser chrome Mochitests", + "default": False, + }], + [["--subsuite"], + { "action": "store", + "dest": "subsuite", + "help": "subsuite of tests to run", + "default": "", + }], + [["--webapprt-content"], + { "action": "store_true", + "dest": "webapprtContent", + "help": "run WebappRT content tests", + "default": False, + }], + [["--webapprt-chrome"], + { "action": "store_true", + "dest": "webapprtChrome", + "help": "run WebappRT chrome tests", + "default": False, + }], + [["--a11y"], + { "action": "store_true", + "dest": "a11y", + "help": "run accessibility Mochitests", + "default": False, + }], + [["--setenv"], + { "action": "append", + "type": "string", + "dest": "environment", + "metavar": "NAME=VALUE", + "help": "sets the given variable in the application's " + "environment", + "default": [], + }], + [["--exclude-extension"], + { "action": "append", + "type": "string", + "dest": "extensionsToExclude", + "help": "excludes the given extension from being installed " + "in the test profile", + "default": [], + }], + [["--browser-arg"], + { "action": "append", + "type": "string", + "dest": "browserArgs", + "metavar": "ARG", + "help": "provides an argument to the test application", + "default": [], + }], + [["--leak-threshold"], + { "action": "store", + "type": "int", + "dest": "leakThreshold", + "metavar": "THRESHOLD", + "help": "fail if the number of bytes leaked through " + "refcounted objects (or bytes in classes with " + "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " + "than the given number", + "default": 0, + }], + [["--fatal-assertions"], + { "action": "store_true", + "dest": "fatalAssertions", + "help": "abort testing whenever an assertion is hit " + "(requires a debug build to be effective)", + "default": False, + }], + [["--extra-profile-file"], + { "action": "append", + "dest": "extraProfileFiles", + "help": "copy specified files/dirs to testing profile", + "default": [], + }], + [["--install-extension"], + { "action": "append", + "dest": "extensionsToInstall", + "help": "install the specified extension in the testing profile." + "The extension file's name should be .xpi where is" + "the extension's id as indicated in its install.rdf." + "An optional path can be specified too.", + "default": [], + }], + [["--profile-path"], + { "action": "store", + "type": "string", + "dest": "profilePath", + "help": "Directory where the profile will be stored." + "This directory will be deleted after the tests are finished", + "default": tempfile.mkdtemp(), + }], + [["--testing-modules-dir"], + { "action": "store", + "type": "string", + "dest": "testingModulesDir", + "help": "Directory where testing-only JS modules are located.", + "default": None, + }], + [["--use-vmware-recording"], + { "action": "store_true", + "dest": "vmwareRecording", + "help": "enables recording while the application is running " + "inside a VMware Workstation 7.0 or later VM", + "default": False, + }], + [["--repeat"], + { "action": "store", + "type": "int", + "dest": "repeat", + "metavar": "REPEAT", + "help": "repeats the test or set of tests the given number of times, ie: repeat: 1 will run the test twice.", + "default": 0, + }], + [["--run-until-failure"], + { "action": "store_true", + "dest": "runUntilFailure", + "help": "Run tests repeatedly and stops on the first time a test fails. " + "Default cap is 30 runs, which can be overwritten with the --repeat parameter.", + "default": False, + }], + [["--run-only-tests"], + { "action": "store", + "type": "string", + "dest": "runOnlyTests", + "help": "JSON list of tests that we only want to run. [DEPRECATED- please use --test-manifest]", + "default": None, + }], + [["--test-manifest"], + { "action": "store", + "type": "string", + "dest": "testManifest", + "help": "JSON list of tests to specify 'runtests'. Old format for mobile specific tests", + "default": None, + }], + [["--manifest"], + { "action": "store", + "type": "string", + "dest": "manifestFile", + "help": ".ini format of tests to run.", + "default": None, + }], + [["--failure-file"], + { "action": "store", + "type": "string", + "dest": "failureFile", + "help": "Filename of the output file where we can store a .json list of failures to be run in the future with --run-only-tests.", + "default": None, + }], + [["--run-slower"], + { "action": "store_true", + "dest": "runSlower", + "help": "Delay execution between test files.", + "default": False, + }], + [["--metro-immersive"], + { "action": "store_true", + "dest": "immersiveMode", + "help": "launches tests in immersive browser", + "default": False, + }], + [["--httpd-path"], + { "action": "store", + "type": "string", + "dest": "httpdPath", + "default": None, + "help": "path to the httpd.js file", + }], + [["--setpref"], + { "action": "append", + "type": "string", + "default": [], + "dest": "extraPrefs", + "metavar": "PREF=VALUE", + "help": "defines an extra user preference", + }], + [["--jsdebugger"], + { "action": "store_true", + "default": False, + "dest": "jsdebugger", + "help": "open the browser debugger", + }], + [["--debug-on-failure"], + { "action": "store_true", + "default": False, + "dest": "debugOnFailure", + "help": "breaks execution and enters the JS debugger on a test failure. Should be used together with --jsdebugger." + }], + [["--e10s"], + { "action": "store_true", + "default": False, + "dest": "e10s", + "help": "Run tests with electrolysis preferences and test filtering enabled.", + }], + [["--dmd-path"], + { "action": "store", + "default": None, + "dest": "dmdPath", + "help": "Specifies the path to the directory containing the shared library for DMD.", + }], + [["--dump-output-directory"], + { "action": "store", + "default": None, + "dest": "dumpOutputDirectory", + "help": "Specifies the directory in which to place dumped memory reports.", + }], + [["--dump-about-memory-after-test"], + { "action": "store_true", + "default": False, + "dest": "dumpAboutMemoryAfterTest", + "help": "Produce an about:memory dump after each test in the directory specified " + "by --dump-output-directory." + }], + [["--dump-dmd-after-test"], + { "action": "store_true", + "default": False, + "dest": "dumpDMDAfterTest", + "help": "Produce a DMD dump after each test in the directory specified " + "by --dump-output-directory." + }], + [["--slowscript"], + { "action": "store_true", + "default": False, + "dest": "slowscript", + "help": "Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; " + "when not set, recoverable but misleading SIGSEGV instances " + "may occur in Ion/Odin JIT code." + }], + [["--screenshot-on-fail"], + { "action": "store_true", + "default": False, + "dest": "screenshotOnFail", + "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots." + }], + [["--quiet"], + { "action": "store_true", + "default": False, + "dest": "quiet", + "help": "Do not print test log lines unless a failure occurs." + }], + [["--pidfile"], + { "action": "store", + "type": "string", + "dest": "pidFile", + "help": "name of the pidfile to generate", + "default": "", + }], + ] + + def __init__(self, **kwargs): + + optparse.OptionParser.__init__(self, **kwargs) + for option, value in self.mochitest_options: + self.add_option(*option, **value) + addCommonOptions(self) + self.set_usage(self.__doc__) + + def verifyOptions(self, options, mochitest): + """ verify correct options and cleanup paths """ + + mozinfo.update({"e10s": options.e10s}) # for test manifest parsing. + + if options.app is None: + if build_obj is not None: + options.app = build_obj.get_binary_path() + else: + self.error("could not find the application path, --appname must be specified") + + if options.totalChunks is not None and options.thisChunk is None: + self.error("thisChunk must be specified when totalChunks is specified") + + if options.totalChunks: + if not 1 <= options.thisChunk <= options.totalChunks: + self.error("thisChunk must be between 1 and totalChunks") + + if options.xrePath is None: + # default xrePath to the app path if not provided + # but only if an app path was explicitly provided + if options.app != self.defaults['app']: + options.xrePath = os.path.dirname(options.app) + elif build_obj is not None: + # otherwise default to dist/bin + options.xrePath = build_obj.bindir + else: + self.error("could not find xre directory, --xre-path must be specified") + + # allow relative paths + options.xrePath = mochitest.getFullPath(options.xrePath) + options.profilePath = mochitest.getFullPath(options.profilePath) + options.app = mochitest.getFullPath(options.app) + if options.dmdPath is not None: + options.dmdPath = mochitest.getFullPath(options.dmdPath) + + if not os.path.exists(options.app): + msg = """\ + Error: Path %(app)s doesn't exist. + Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" + self.error(msg % {"app": options.app}) + return None + + if options.utilityPath: + options.utilityPath = mochitest.getFullPath(options.utilityPath) + + if options.certPath: + options.certPath = mochitest.getFullPath(options.certPath) + + if options.symbolsPath and not isURL(options.symbolsPath): + options.symbolsPath = mochitest.getFullPath(options.symbolsPath) + + # Set server information on the options object + options.webServer = '127.0.0.1' + options.httpPort = DEFAULT_PORTS['http'] + options.sslPort = DEFAULT_PORTS['https'] + # options.webSocketPort = DEFAULT_PORTS['ws'] + options.webSocketPort = str(9988) # <- http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l30 + # The default websocket port is incorrect in mozprofile; it is + # set to the SSL proxy setting. See: + # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517 + + if options.vmwareRecording: + if not mozinfo.isWin: + self.error("use-vmware-recording is only supported on Windows.") + mochitest.vmwareHelperPath = os.path.join( + options.utilityPath, VMWARE_RECORDING_HELPER_BASENAME + ".dll") + if not os.path.exists(mochitest.vmwareHelperPath): + self.error("%s not found, cannot automate VMware recording." % + mochitest.vmwareHelperPath) + + if options.testManifest and options.runOnlyTests: + self.error("Please use --test-manifest only and not --run-only-tests") + + if options.runOnlyTests: + if not os.path.exists(os.path.abspath(os.path.join(here, options.runOnlyTests))): + self.error("unable to find --run-only-tests file '%s'" % options.runOnlyTests) + options.runOnly = True + options.testManifest = options.runOnlyTests + options.runOnlyTests = None + + if options.manifestFile and options.testManifest: + self.error("Unable to support both --manifest and --test-manifest/--run-only-tests at the same time") + + if options.webapprtContent and options.webapprtChrome: + self.error("Only one of --webapprt-content and --webapprt-chrome may be given.") + + if options.jsdebugger: + options.extraPrefs += [ + "devtools.debugger.remote-enabled=true", + "devtools.debugger.chrome-enabled=true", + "devtools.chrome.enabled=true", + "devtools.debugger.prompt-connection=false" + ] + options.autorun = False + + if options.debugOnFailure and not options.jsdebugger: + self.error("--debug-on-failure should be used together with --jsdebugger.") + + # Try to guess the testing modules directory. + # This somewhat grotesque hack allows the buildbot machines to find the + # modules directory without having to configure the buildbot hosts. This + # code should never be executed in local runs because the build system + # should always set the flag that populates this variable. If buildbot ever + # passes this argument, this code can be deleted. + if options.testingModulesDir is None: + possible = os.path.join(here, os.path.pardir, 'modules') + + if os.path.isdir(possible): + options.testingModulesDir = possible + + # Even if buildbot is updated, we still want this, as the path we pass in + # to the app must be absolute and have proper slashes. + if options.testingModulesDir is not None: + options.testingModulesDir = os.path.normpath(options.testingModulesDir) + + if not os.path.isabs(options.testingModulesDir): + options.testingModulesDir = os.path.abspath(options.testingModulesDir) + + if not os.path.isdir(options.testingModulesDir): + self.error('--testing-modules-dir not a directory: %s' % + options.testingModulesDir) + + options.testingModulesDir = options.testingModulesDir.replace('\\', '/') + if options.testingModulesDir[-1] != '/': + options.testingModulesDir += '/' + + if options.immersiveMode: + if not mozinfo.isWin: + self.error("immersive is only supported on Windows 8 and up.") + mochitest.immersiveHelperPath = os.path.join( + options.utilityPath, "metrotestharness.exe") + if not os.path.exists(mochitest.immersiveHelperPath): + self.error("%s not found, cannot launch immersive tests." % + mochitest.immersiveHelperPath) + + if options.runUntilFailure: + if not options.repeat: + options.repeat = 29 + + if options.dumpOutputDirectory is None: + options.dumpOutputDirectory = tempfile.gettempdir() + + if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest: + if not os.path.isdir(options.dumpOutputDirectory): + self.error('--dump-output-directory not a directory: %s' % + options.dumpOutputDirectory) + + return options + + +class B2GOptions(MochitestOptions): + b2g_options = [ + [["--b2gpath"], + { "action": "store", + "type": "string", + "dest": "b2gPath", + "help": "path to B2G repo or qemu dir", + "default": None, + }], + [["--desktop"], + { "action": "store_true", + "dest": "desktop", + "help": "Run the tests on a B2G desktop build", + "default": False, + }], + [["--marionette"], + { "action": "store", + "type": "string", + "dest": "marionette", + "help": "host:port to use when connecting to Marionette", + "default": None, + }], + [["--emulator"], + { "action": "store", + "type": "string", + "dest": "emulator", + "help": "Architecture of emulator to use: x86 or arm", + "default": None, + }], + [["--wifi"], + { "action": "store", + "type": "string", + "dest": "wifi", + "help": "Devine wifi configuration for on device mochitest", + "default": False, + }], + [["--sdcard"], + { "action": "store", + "type": "string", + "dest": "sdcard", + "help": "Define size of sdcard: 1MB, 50MB...etc", + "default": "10MB", + }], + [["--no-window"], + { "action": "store_true", + "dest": "noWindow", + "help": "Pass --no-window to the emulator", + "default": False, + }], + [["--adbpath"], + { "action": "store", + "type": "string", + "dest": "adbPath", + "help": "path to adb", + "default": "adb", + }], + [["--deviceIP"], + { "action": "store", + "type": "string", + "dest": "deviceIP", + "help": "ip address of remote device to test", + "default": None, + }], + [["--devicePort"], + { "action": "store", + "type": "string", + "dest": "devicePort", + "help": "port of remote device to test", + "default": 20701, + }], + [["--remote-logfile"], + { "action": "store", + "type": "string", + "dest": "remoteLogFile", + "help": "Name of log file on the device relative to the device root. \ + PLEASE ONLY USE A FILENAME.", + "default" : None, + }], + [["--remote-webserver"], + { "action": "store", + "type": "string", + "dest": "remoteWebServer", + "help": "ip address where the remote web server is hosted at", + "default": None, + }], + [["--http-port"], + { "action": "store", + "type": "string", + "dest": "httpPort", + "help": "ip address where the remote web server is hosted at", + "default": None, + }], + [["--ssl-port"], + { "action": "store", + "type": "string", + "dest": "sslPort", + "help": "ip address where the remote web server is hosted at", + "default": None, + }], + [["--gecko-path"], + { "action": "store", + "type": "string", + "dest": "geckoPath", + "help": "the path to a gecko distribution that should \ + be installed on the emulator prior to test", + "default": None, + }], + [["--profile"], + { "action": "store", + "type": "string", + "dest": "profile", + "help": "for desktop testing, the path to the \ + gaia profile to use", + "default": None, + }], + [["--logcat-dir"], + { "action": "store", + "type": "string", + "dest": "logcat_dir", + "help": "directory to store logcat dump files", + "default": None, + }], + [['--busybox'], + { "action": 'store', + "type": 'string', + "dest": 'busybox', + "help": "Path to busybox binary to install on device", + "default": None, + }], + [['--profile-data-dir'], + { "action": 'store', + "type": 'string', + "dest": 'profile_data_dir', + "help": "Path to a directory containing preference and other \ + data to be installed into the profile", + "default": os.path.join(here, 'profile_data'), + }], + ] + + def __init__(self): + MochitestOptions.__init__(self) + + for option in self.b2g_options: + self.add_option(*option[0], **option[1]) + + defaults = {} + defaults["httpPort"] = DEFAULT_PORTS['http'] + defaults["sslPort"] = DEFAULT_PORTS['https'] + defaults["remoteTestRoot"] = "/data/local/tests" + defaults["logFile"] = "mochitest.log" + defaults["autorun"] = True + defaults["closeWhenDone"] = True + defaults["testPath"] = "" + defaults["extensionsToExclude"] = ["specialpowers"] + self.set_defaults(**defaults) + + def verifyRemoteOptions(self, options): + if options.remoteWebServer == None: + if os.name != "nt": + options.remoteWebServer = moznetwork.get_ip() + else: + self.error("You must specify a --remote-webserver=") + options.webServer = options.remoteWebServer + + if options.geckoPath and not options.emulator: + self.error("You must specify --emulator if you specify --gecko-path") + + if options.logcat_dir and not options.emulator: + self.error("You must specify --emulator if you specify --logcat-dir") + + if not os.path.isdir(options.xrePath): + self.error("--xre-path '%s' is not a directory" % options.xrePath) + xpcshell = os.path.join(options.xrePath, 'xpcshell') + if not os.access(xpcshell, os.F_OK): + self.error('xpcshell not found at %s' % xpcshell) + if self.elf_arm(xpcshell): + self.error('--xre-path points to an ARM version of xpcshell; it ' + 'should instead point to a version that can run on ' + 'your desktop') + + if options.pidFile != "": + f = open(options.pidFile, 'w') + f.write("%s" % os.getpid()) + f.close() + + return options + + def verifyOptions(self, options, mochitest): + # since we are reusing verifyOptions, it will exit if App is not found + temp = options.app + options.app = __file__ + tempPort = options.httpPort + tempSSL = options.sslPort + tempIP = options.webServer + options = MochitestOptions.verifyOptions(self, options, mochitest) + options.webServer = tempIP + options.app = temp + options.sslPort = tempSSL + options.httpPort = tempPort + + return options + + def elf_arm(self, filename): + data = open(filename, 'rb').read(20) + return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM