1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/xpcshell/mach_commands.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,447 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +# Integrates the xpcshell test runner with mach.
1.9 +
1.10 +from __future__ import unicode_literals, print_function
1.11 +
1.12 +import mozpack.path
1.13 +import logging
1.14 +import os
1.15 +import shutil
1.16 +import sys
1.17 +import urllib2
1.18 +
1.19 +from StringIO import StringIO
1.20 +
1.21 +from mozbuild.base import (
1.22 + MachCommandBase,
1.23 + MozbuildObject,
1.24 + MachCommandConditions as conditions,
1.25 +)
1.26 +
1.27 +from mach.decorators import (
1.28 + CommandArgument,
1.29 + CommandProvider,
1.30 + Command,
1.31 +)
1.32 +
1.33 +ADB_NOT_FOUND = '''
1.34 +The %s command requires the adb binary to be on your path.
1.35 +
1.36 +If you have a B2G build, this can be found in
1.37 +'%s/out/host/<platform>/bin'.
1.38 +'''.lstrip()
1.39 +
1.40 +BUSYBOX_URL = 'http://www.busybox.net/downloads/binaries/latest/busybox-armv7l'
1.41 +
1.42 +
1.43 +if sys.version_info[0] < 3:
1.44 + unicode_type = unicode
1.45 +else:
1.46 + unicode_type = str
1.47 +
1.48 +# Simple filter to omit the message emitted as a test file begins.
1.49 +class TestStartFilter(logging.Filter):
1.50 + def filter(self, record):
1.51 + return not record.params['msg'].endswith("running test ...")
1.52 +
1.53 +# This should probably be consolidated with similar classes in other test
1.54 +# runners.
1.55 +class InvalidTestPathError(Exception):
1.56 + """Exception raised when the test path is not valid."""
1.57 +
1.58 +
1.59 +class XPCShellRunner(MozbuildObject):
1.60 + """Run xpcshell tests."""
1.61 + def run_suite(self, **kwargs):
1.62 + from manifestparser import TestManifest
1.63 + manifest = TestManifest(manifests=[os.path.join(self.topobjdir,
1.64 + '_tests', 'xpcshell', 'xpcshell.ini')])
1.65 +
1.66 + return self._run_xpcshell_harness(manifest=manifest, **kwargs)
1.67 +
1.68 + def run_test(self, test_paths, interactive=False,
1.69 + keep_going=False, sequential=False, shuffle=False,
1.70 + debugger=None, debuggerArgs=None, debuggerInteractive=None,
1.71 + rerun_failures=False,
1.72 + # ignore parameters from other platforms' options
1.73 + **kwargs):
1.74 + """Runs an individual xpcshell test."""
1.75 + from mozbuild.testing import TestResolver
1.76 + from manifestparser import TestManifest
1.77 +
1.78 + # TODO Bug 794506 remove once mach integrates with virtualenv.
1.79 + build_path = os.path.join(self.topobjdir, 'build')
1.80 + if build_path not in sys.path:
1.81 + sys.path.append(build_path)
1.82 +
1.83 + if test_paths == ['all']:
1.84 + self.run_suite(interactive=interactive,
1.85 + keep_going=keep_going, shuffle=shuffle, sequential=sequential,
1.86 + debugger=debugger, debuggerArgs=debuggerArgs,
1.87 + debuggerInteractive=debuggerInteractive,
1.88 + rerun_failures=rerun_failures)
1.89 + return
1.90 +
1.91 + resolver = self._spawn(TestResolver)
1.92 + tests = list(resolver.resolve_tests(paths=test_paths, flavor='xpcshell',
1.93 + cwd=self.cwd))
1.94 +
1.95 + if not tests:
1.96 + raise InvalidTestPathError('We could not find an xpcshell test '
1.97 + 'for the passed test path. Please select a path that is '
1.98 + 'a test file or is a directory containing xpcshell tests.')
1.99 +
1.100 + # Dynamically write out a manifest holding all the discovered tests.
1.101 + manifest = TestManifest()
1.102 + manifest.tests.extend(tests)
1.103 +
1.104 + args = {
1.105 + 'interactive': interactive,
1.106 + 'keep_going': keep_going,
1.107 + 'shuffle': shuffle,
1.108 + 'sequential': sequential,
1.109 + 'debugger': debugger,
1.110 + 'debuggerArgs': debuggerArgs,
1.111 + 'debuggerInteractive': debuggerInteractive,
1.112 + 'rerun_failures': rerun_failures,
1.113 + 'manifest': manifest,
1.114 + }
1.115 +
1.116 + return self._run_xpcshell_harness(**args)
1.117 +
1.118 + def _run_xpcshell_harness(self, manifest,
1.119 + test_path=None, shuffle=False, interactive=False,
1.120 + keep_going=False, sequential=False,
1.121 + debugger=None, debuggerArgs=None, debuggerInteractive=None,
1.122 + rerun_failures=False):
1.123 +
1.124 + # Obtain a reference to the xpcshell test runner.
1.125 + import runxpcshelltests
1.126 +
1.127 + dummy_log = StringIO()
1.128 + xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)
1.129 + self.log_manager.enable_unstructured()
1.130 +
1.131 + xpcshell_filter = TestStartFilter()
1.132 + self.log_manager.terminal_handler.addFilter(xpcshell_filter)
1.133 +
1.134 + tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')
1.135 + modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
1.136 + # We want output from the test to be written immediately if we are only
1.137 + # running a single test.
1.138 + verbose_output = test_path is not None or (manifest and len(manifest.test_paths())==1)
1.139 +
1.140 + args = {
1.141 + 'manifest': manifest,
1.142 + 'xpcshell': os.path.join(self.bindir, 'xpcshell'),
1.143 + 'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
1.144 + 'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
1.145 + 'interactive': interactive,
1.146 + 'keepGoing': keep_going,
1.147 + 'logfiles': False,
1.148 + 'sequential': sequential,
1.149 + 'shuffle': shuffle,
1.150 + 'testsRootDir': tests_dir,
1.151 + 'testingModulesDir': modules_dir,
1.152 + 'profileName': 'firefox',
1.153 + 'verbose': test_path is not None,
1.154 + 'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),
1.155 + 'xunitName': 'xpcshell',
1.156 + 'pluginsPath': os.path.join(self.distdir, 'plugins'),
1.157 + 'debugger': debugger,
1.158 + 'debuggerArgs': debuggerArgs,
1.159 + 'debuggerInteractive': debuggerInteractive,
1.160 + 'on_message': (lambda obj, msg: xpcshell.log.info(msg.decode('utf-8', 'replace'))) \
1.161 + if verbose_output else None,
1.162 + }
1.163 +
1.164 + if test_path is not None:
1.165 + args['testPath'] = test_path
1.166 +
1.167 + # A failure manifest is written by default. If --rerun-failures is
1.168 + # specified and a prior failure manifest is found, the prior manifest
1.169 + # will be run. A new failure manifest is always written over any
1.170 + # prior failure manifest.
1.171 + failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
1.172 + rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
1.173 + if os.path.exists(failure_manifest_path) and rerun_failures:
1.174 + shutil.move(failure_manifest_path, rerun_manifest_path)
1.175 + args['manifest'] = rerun_manifest_path
1.176 + elif os.path.exists(failure_manifest_path):
1.177 + os.remove(failure_manifest_path)
1.178 + elif rerun_failures:
1.179 + print("No failures were found to re-run.")
1.180 + return 0
1.181 + args['failureManifest'] = failure_manifest_path
1.182 +
1.183 + # Python through 2.7.2 has issues with unicode in some of the
1.184 + # arguments. Work around that.
1.185 + filtered_args = {}
1.186 + for k, v in args.items():
1.187 + if isinstance(v, unicode_type):
1.188 + v = v.encode('utf-8')
1.189 +
1.190 + if isinstance(k, unicode_type):
1.191 + k = k.encode('utf-8')
1.192 +
1.193 + filtered_args[k] = v
1.194 +
1.195 + result = xpcshell.runTests(**filtered_args)
1.196 +
1.197 + self.log_manager.terminal_handler.removeFilter(xpcshell_filter)
1.198 + self.log_manager.disable_unstructured()
1.199 +
1.200 + if not result and not xpcshell.sequential:
1.201 + print("Tests were run in parallel. Try running with --sequential "
1.202 + "to make sure the failures were not caused by this.")
1.203 + return int(not result)
1.204 +
1.205 +class AndroidXPCShellRunner(MozbuildObject):
1.206 + """Get specified DeviceManager"""
1.207 + def get_devicemanager(self, devicemanager, ip, port, remote_test_root):
1.208 + from mozdevice import devicemanagerADB, devicemanagerSUT
1.209 + dm = None
1.210 + if devicemanager == "adb":
1.211 + if ip:
1.212 + dm = devicemanagerADB.DeviceManagerADB(ip, port, packageName=None, deviceRoot=remote_test_root)
1.213 + else:
1.214 + dm = devicemanagerADB.DeviceManagerADB(packageName=None, deviceRoot=remote_test_root)
1.215 + else:
1.216 + if ip:
1.217 + dm = devicemanagerSUT.DeviceManagerSUT(ip, port, deviceRoot=remote_test_root)
1.218 + else:
1.219 + raise Exception("You must provide a device IP to connect to via the --ip option")
1.220 + return dm
1.221 +
1.222 + """Run Android xpcshell tests."""
1.223 + def run_test(self,
1.224 + test_paths, keep_going,
1.225 + devicemanager, ip, port, remote_test_root, no_setup, local_apk,
1.226 + # ignore parameters from other platforms' options
1.227 + **kwargs):
1.228 + # TODO Bug 794506 remove once mach integrates with virtualenv.
1.229 + build_path = os.path.join(self.topobjdir, 'build')
1.230 + if build_path not in sys.path:
1.231 + sys.path.append(build_path)
1.232 +
1.233 + import remotexpcshelltests
1.234 +
1.235 + dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)
1.236 +
1.237 + options = remotexpcshelltests.RemoteXPCShellOptions()
1.238 + options.shuffle = False
1.239 + options.sequential = True
1.240 + options.interactive = False
1.241 + options.debugger = None
1.242 + options.debuggerArgs = None
1.243 + options.setup = not no_setup
1.244 + options.keepGoing = keep_going
1.245 + options.objdir = self.topobjdir
1.246 + options.localLib = os.path.join(self.topobjdir, 'dist/fennec')
1.247 + options.localBin = os.path.join(self.topobjdir, 'dist/bin')
1.248 + options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')
1.249 + options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
1.250 + options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell_android.ini')
1.251 + options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
1.252 + if local_apk:
1.253 + options.localAPK = local_apk
1.254 + else:
1.255 + for file in os.listdir(os.path.join(options.objdir, "dist")):
1.256 + if file.endswith(".apk") and file.startswith("fennec"):
1.257 + options.localAPK = os.path.join(options.objdir, "dist")
1.258 + options.localAPK = os.path.join(options.localAPK, file)
1.259 + print ("using APK: " + options.localAPK)
1.260 + break
1.261 + else:
1.262 + raise Exception("You must specify an APK")
1.263 +
1.264 + if test_paths == ['all']:
1.265 + testdirs = []
1.266 + options.testPath = None
1.267 + options.verbose = False
1.268 + else:
1.269 + if len(test_paths) > 1:
1.270 + print('Warning: only the first test path argument will be used.')
1.271 + testdirs = test_paths[0]
1.272 + options.testPath = test_paths[0]
1.273 + options.verbose = True
1.274 + dummy_log = StringIO()
1.275 + xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, args=testdirs, log=dummy_log)
1.276 + self.log_manager.enable_unstructured()
1.277 +
1.278 + xpcshell_filter = TestStartFilter()
1.279 + self.log_manager.terminal_handler.addFilter(xpcshell_filter)
1.280 +
1.281 + result = xpcshell.runTests(xpcshell='xpcshell',
1.282 + testClass=remotexpcshelltests.RemoteXPCShellTestThread,
1.283 + testdirs=testdirs,
1.284 + mobileArgs=xpcshell.mobileArgs,
1.285 + **options.__dict__)
1.286 +
1.287 + self.log_manager.terminal_handler.removeFilter(xpcshell_filter)
1.288 + self.log_manager.disable_unstructured()
1.289 +
1.290 + return int(not result)
1.291 +
1.292 +class B2GXPCShellRunner(MozbuildObject):
1.293 + def __init__(self, *args, **kwargs):
1.294 + MozbuildObject.__init__(self, *args, **kwargs)
1.295 +
1.296 + # TODO Bug 794506 remove once mach integrates with virtualenv.
1.297 + build_path = os.path.join(self.topobjdir, 'build')
1.298 + if build_path not in sys.path:
1.299 + sys.path.append(build_path)
1.300 +
1.301 + build_path = os.path.join(self.topsrcdir, 'build')
1.302 + if build_path not in sys.path:
1.303 + sys.path.append(build_path)
1.304 +
1.305 + self.tests_dir = os.path.join(self.topobjdir, '_tests')
1.306 + self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell')
1.307 + self.bin_dir = os.path.join(self.distdir, 'bin')
1.308 +
1.309 + def _download_busybox(self, b2g_home):
1.310 + system_bin = os.path.join(b2g_home, 'out', 'target', 'product', 'generic', 'system', 'bin')
1.311 + busybox_path = os.path.join(system_bin, 'busybox')
1.312 +
1.313 + if os.path.isfile(busybox_path):
1.314 + return busybox_path
1.315 +
1.316 + if not os.path.isdir(system_bin):
1.317 + os.makedirs(system_bin)
1.318 +
1.319 + try:
1.320 + data = urllib2.urlopen(BUSYBOX_URL)
1.321 + except urllib2.URLError:
1.322 + print('There was a problem downloading busybox. Proceeding without it,' \
1.323 + 'initial setup will be slow.')
1.324 + return
1.325 +
1.326 + with open(busybox_path, 'wb') as f:
1.327 + f.write(data.read())
1.328 + return busybox_path
1.329 +
1.330 + def run_test(self, test_paths, b2g_home=None, busybox=None,
1.331 + # ignore parameters from other platforms' options
1.332 + **kwargs):
1.333 + try:
1.334 + import which
1.335 + which.which('adb')
1.336 + except which.WhichError:
1.337 + # TODO Find adb automatically if it isn't on the path
1.338 + print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
1.339 + sys.exit(1)
1.340 +
1.341 + test_path = None
1.342 + if test_paths:
1.343 + if len(test_paths) > 1:
1.344 + print('Warning: Only the first test path will be used.')
1.345 +
1.346 + test_path = self._wrap_path_argument(test_paths[0]).relpath()
1.347 +
1.348 + import runtestsb2g
1.349 + parser = runtestsb2g.B2GOptions()
1.350 + options, args = parser.parse_args([])
1.351 +
1.352 + options.b2g_path = b2g_home
1.353 + options.busybox = busybox or os.environ.get('BUSYBOX')
1.354 + options.emulator = 'arm'
1.355 + options.localLib = self.bin_dir
1.356 + options.localBin = self.bin_dir
1.357 + options.logcat_dir = self.xpcshell_dir
1.358 + options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell_b2g.ini')
1.359 + options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
1.360 + options.objdir = self.topobjdir
1.361 + options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),
1.362 + options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
1.363 + options.testsRootDir = self.xpcshell_dir
1.364 + options.testPath = test_path
1.365 + options.use_device_libs = True
1.366 +
1.367 + if not options.busybox:
1.368 + options.busybox = self._download_busybox(b2g_home)
1.369 +
1.370 + return runtestsb2g.run_remote_xpcshell(parser, options, args)
1.371 +
1.372 +def is_platform_supported(cls):
1.373 + """Must have a Firefox, Android or B2G build."""
1.374 + return conditions.is_android(cls) or \
1.375 + conditions.is_b2g(cls) or \
1.376 + conditions.is_firefox(cls)
1.377 +
1.378 +@CommandProvider
1.379 +class MachCommands(MachCommandBase):
1.380 + def __init__(self, context):
1.381 + MachCommandBase.__init__(self, context)
1.382 +
1.383 + for attr in ('b2g_home', 'device_name'):
1.384 + setattr(self, attr, getattr(context, attr, None))
1.385 +
1.386 + @Command('xpcshell-test', category='testing',
1.387 + conditions=[is_platform_supported],
1.388 + description='Run XPCOM Shell tests.')
1.389 + @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
1.390 + help='Test to run. Can be specified as a single JS file, a directory, '
1.391 + 'or omitted. If omitted, the entire test suite is executed.')
1.392 + @CommandArgument("--debugger", default=None, metavar='DEBUGGER',
1.393 + help = "Run xpcshell under the given debugger.")
1.394 + @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,
1.395 + dest = "debuggerArgs",
1.396 + help = "pass the given args to the debugger _before_ "
1.397 + "the application on the command line")
1.398 + @CommandArgument("--debugger-interactive", action = "store_true",
1.399 + dest = "debuggerInteractive",
1.400 + help = "prevents the test harness from redirecting "
1.401 + "stdout and stderr for interactive debuggers")
1.402 + @CommandArgument('--interactive', '-i', action='store_true',
1.403 + help='Open an xpcshell prompt before running tests.')
1.404 + @CommandArgument('--keep-going', '-k', action='store_true',
1.405 + help='Continue running tests after a SIGINT is received.')
1.406 + @CommandArgument('--sequential', action='store_true',
1.407 + help='Run the tests sequentially.')
1.408 + @CommandArgument('--shuffle', '-s', action='store_true',
1.409 + help='Randomize the execution order of tests.')
1.410 + @CommandArgument('--rerun-failures', action='store_true',
1.411 + help='Reruns failures from last time.')
1.412 + @CommandArgument('--devicemanager', default='adb', type=str,
1.413 + help='(Android) Type of devicemanager to use for communication: adb or sut')
1.414 + @CommandArgument('--ip', type=str, default=None,
1.415 + help='(Android) IP address of device')
1.416 + @CommandArgument('--port', type=int, default=20701,
1.417 + help='(Android) Port of device')
1.418 + @CommandArgument('--remote_test_root', type=str, default=None,
1.419 + help='(Android) Remote test root such as /mnt/sdcard or /data/local')
1.420 + @CommandArgument('--no-setup', action='store_true',
1.421 + help='(Android) Do not copy files to device')
1.422 + @CommandArgument('--local-apk', type=str, default=None,
1.423 + help='(Android) Use specified Fennec APK')
1.424 + @CommandArgument('--busybox', type=str, default=None,
1.425 + help='(B2G) Path to busybox binary (speeds up installation of tests).')
1.426 + def run_xpcshell_test(self, **params):
1.427 + from mozbuild.controller.building import BuildDriver
1.428 +
1.429 + # We should probably have a utility function to ensure the tree is
1.430 + # ready to run tests. Until then, we just create the state dir (in
1.431 + # case the tree wasn't built with mach).
1.432 + self._ensure_state_subdir_exists('.')
1.433 +
1.434 + driver = self._spawn(BuildDriver)
1.435 + driver.install_tests(remove=False)
1.436 +
1.437 + if conditions.is_android(self):
1.438 + xpcshell = self._spawn(AndroidXPCShellRunner)
1.439 + elif conditions.is_b2g(self):
1.440 + xpcshell = self._spawn(B2GXPCShellRunner)
1.441 + params['b2g_home'] = self.b2g_home
1.442 + else:
1.443 + xpcshell = self._spawn(XPCShellRunner)
1.444 + xpcshell.cwd = self._mach_context.cwd
1.445 +
1.446 + try:
1.447 + return xpcshell.run_test(**params)
1.448 + except InvalidTestPathError as e:
1.449 + print(e.message)
1.450 + return 1
