1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/layout/tools/reftest/mach_commands.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,434 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +from __future__ import unicode_literals
1.9 +
1.10 +import mozpack.path
1.11 +import os
1.12 +import re
1.13 +import sys
1.14 +import warnings
1.15 +import which
1.16 +
1.17 +from mozbuild.base import (
1.18 + MachCommandBase,
1.19 + MachCommandConditions as conditions,
1.20 + MozbuildObject,
1.21 +)
1.22 +
1.23 +from mach.decorators import (
1.24 + CommandArgument,
1.25 + CommandProvider,
1.26 + Command,
1.27 +)
1.28 +
1.29 +
1.30 +DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
1.31 +
1.32 +ADB_NOT_FOUND = '''
1.33 +The %s command requires the adb binary to be on your path.
1.34 +
1.35 +If you have a B2G build, this can be found in
1.36 +'%s/out/host/<platform>/bin'.
1.37 +'''.lstrip()
1.38 +
1.39 +GAIA_PROFILE_NOT_FOUND = '''
1.40 +The %s command requires a non-debug gaia profile. Either pass in --profile,
1.41 +or set the GAIA_PROFILE environment variable.
1.42 +
1.43 +If you do not have a non-debug gaia profile, you can build one:
1.44 + $ git clone https://github.com/mozilla-b2g/gaia
1.45 + $ cd gaia
1.46 + $ make
1.47 +
1.48 +The profile should be generated in a directory called 'profile'.
1.49 +'''.lstrip()
1.50 +
1.51 +GAIA_PROFILE_IS_DEBUG = '''
1.52 +The %s command requires a non-debug gaia profile. The specified profile,
1.53 +%s, is a debug profile.
1.54 +
1.55 +If you do not have a non-debug gaia profile, you can build one:
1.56 + $ git clone https://github.com/mozilla-b2g/gaia
1.57 + $ cd gaia
1.58 + $ make
1.59 +
1.60 +The profile should be generated in a directory called 'profile'.
1.61 +'''.lstrip()
1.62 +
1.63 +MARIONETTE_DISABLED = '''
1.64 +The %s command requires a marionette enabled build.
1.65 +
1.66 +Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application.
1.67 +Your currently active mozconfig is %s.
1.68 +'''.lstrip()
1.69 +
1.70 +class ReftestRunner(MozbuildObject):
1.71 + """Easily run reftests.
1.72 +
1.73 + This currently contains just the basics for running reftests. We may want
1.74 + to hook up result parsing, etc.
1.75 + """
1.76 + def __init__(self, *args, **kwargs):
1.77 + MozbuildObject.__init__(self, *args, **kwargs)
1.78 +
1.79 + # TODO Bug 794506 remove once mach integrates with virtualenv.
1.80 + build_path = os.path.join(self.topobjdir, 'build')
1.81 + if build_path not in sys.path:
1.82 + sys.path.append(build_path)
1.83 +
1.84 + self.tests_dir = os.path.join(self.topobjdir, '_tests')
1.85 + self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
1.86 +
1.87 + def _manifest_file(self, suite):
1.88 + """Returns the manifest file used for a given test suite."""
1.89 + files = {
1.90 + 'reftest': 'reftest.list',
1.91 + 'reftest-ipc': 'reftest.list',
1.92 + 'crashtest': 'crashtests.list',
1.93 + 'crashtest-ipc': 'crashtests.list',
1.94 + }
1.95 + assert suite in files
1.96 + return files[suite]
1.97 +
1.98 + def _find_manifest(self, suite, test_file):
1.99 + assert test_file
1.100 + path_arg = self._wrap_path_argument(test_file)
1.101 + relpath = path_arg.relpath()
1.102 +
1.103 + if os.path.isdir(path_arg.srcdir_path()):
1.104 + return mozpack.path.join(relpath, self._manifest_file(suite))
1.105 +
1.106 + if relpath.endswith('.list'):
1.107 + return relpath
1.108 +
1.109 + raise Exception('Running a single test is not currently supported')
1.110 +
1.111 + def _make_shell_string(self, s):
1.112 + return "'%s'" % re.sub("'", r"'\''", s)
1.113 +
1.114 + def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,
1.115 + suite=None, **kwargs):
1.116 + """Runs a b2g reftest.
1.117 +
1.118 + test_file is a path to a test file. It can be a relative path from the
1.119 + top source directory, an absolute filename, or a directory containing
1.120 + test files.
1.121 +
1.122 + suite is the type of reftest to run. It can be one of ('reftest',
1.123 + 'crashtest').
1.124 + """
1.125 + if suite not in ('reftest', 'crashtest'):
1.126 + raise Exception('None or unrecognized reftest suite type.')
1.127 +
1.128 + # Find the manifest file
1.129 + if not test_file:
1.130 + if suite == 'reftest':
1.131 + test_file = mozpack.path.join('layout', 'reftests')
1.132 + elif suite == 'crashtest':
1.133 + test_file = mozpack.path.join('testing', 'crashtest')
1.134 +
1.135 + if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
1.136 + test_file = mozpack.path.relpath(os.path.abspath(test_file),
1.137 + self.topsrcdir)
1.138 +
1.139 + manifest = self._find_manifest(suite, test_file)
1.140 + if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)):
1.141 + raise Exception('No manifest file was found at %s.' % manifest)
1.142 +
1.143 + # Need to chdir to reftest_dir otherwise imports fail below.
1.144 + os.chdir(self.reftest_dir)
1.145 +
1.146 + # The imp module can spew warnings if the modules below have
1.147 + # already been imported, ignore them.
1.148 + with warnings.catch_warnings():
1.149 + warnings.simplefilter('ignore')
1.150 +
1.151 + import imp
1.152 + path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
1.153 + with open(path, 'r') as fh:
1.154 + imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
1.155 + import reftest
1.156 +
1.157 + # Set up the reftest options.
1.158 + parser = reftest.B2GOptions()
1.159 + options, args = parser.parse_args([])
1.160 +
1.161 + # Tests need to be served from a subdirectory of the server. Symlink
1.162 + # topsrcdir here to get around this.
1.163 + tests = os.path.join(self.reftest_dir, 'tests')
1.164 + if not os.path.isdir(tests):
1.165 + os.symlink(self.topsrcdir, tests)
1.166 + args.insert(0, os.path.join('tests', manifest))
1.167 +
1.168 + for k, v in kwargs.iteritems():
1.169 + setattr(options, k, v)
1.170 +
1.171 + if conditions.is_b2g_desktop(self):
1.172 + if self.substs.get('ENABLE_MARIONETTE') != '1':
1.173 + print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
1.174 + self.mozconfig['path']))
1.175 + return 1
1.176 +
1.177 + options.profile = options.profile or os.environ.get('GAIA_PROFILE')
1.178 + if not options.profile:
1.179 + print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')
1.180 + return 1
1.181 +
1.182 + if os.path.isfile(os.path.join(options.profile, 'extensions', \
1.183 + 'httpd@gaiamobile.org')):
1.184 + print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
1.185 + options.profile))
1.186 + return 1
1.187 +
1.188 + options.desktop = True
1.189 + options.app = self.get_binary_path()
1.190 + if not options.app.endswith('-bin'):
1.191 + options.app = '%s-bin' % options.app
1.192 + if not os.path.isfile(options.app):
1.193 + options.app = options.app[:-len('-bin')]
1.194 +
1.195 + return reftest.run_desktop_reftests(parser, options, args)
1.196 +
1.197 +
1.198 + try:
1.199 + which.which('adb')
1.200 + except which.WhichError:
1.201 + # TODO Find adb automatically if it isn't on the path
1.202 + raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
1.203 +
1.204 + options.b2gPath = b2g_home
1.205 + options.logcat_dir = self.reftest_dir
1.206 + options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
1.207 + options.xrePath = xre_path
1.208 + options.ignoreWindowSize = True
1.209 + return reftest.run_remote_reftests(parser, options, args)
1.210 +
1.211 + def run_desktop_test(self, test_file=None, filter=None, suite=None,
1.212 + debugger=None, parallel=False, shuffle=False,
1.213 + e10s=False, this_chunk=None, total_chunks=None):
1.214 + """Runs a reftest.
1.215 +
1.216 + test_file is a path to a test file. It can be a relative path from the
1.217 + top source directory, an absolute filename, or a directory containing
1.218 + test files.
1.219 +
1.220 + filter is a regular expression (in JS syntax, as could be passed to the
1.221 + RegExp constructor) to select which reftests to run from the manifest.
1.222 +
1.223 + suite is the type of reftest to run. It can be one of ('reftest',
1.224 + 'crashtest').
1.225 +
1.226 + debugger is the program name (in $PATH) or the full path of the
1.227 + debugger to run.
1.228 +
1.229 + parallel indicates whether tests should be run in parallel or not.
1.230 +
1.231 + shuffle indicates whether to run tests in random order.
1.232 + """
1.233 +
1.234 + if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'):
1.235 + raise Exception('None or unrecognized reftest suite type.')
1.236 +
1.237 + env = {}
1.238 + extra_args = []
1.239 +
1.240 + if test_file:
1.241 + path = self._find_manifest(suite, test_file)
1.242 + if not os.path.exists(mozpack.path.join(self.topsrcdir, path)):
1.243 + raise Exception('No manifest file was found at %s.' % path)
1.244 + env[b'TEST_PATH'] = path
1.245 + if filter:
1.246 + extra_args.extend(['--filter', self._make_shell_string(filter)])
1.247 +
1.248 + pass_thru = False
1.249 +
1.250 + if debugger:
1.251 + extra_args.append('--debugger=%s' % debugger)
1.252 + pass_thru = True
1.253 +
1.254 + if parallel:
1.255 + extra_args.append('--run-tests-in-parallel')
1.256 +
1.257 + if shuffle:
1.258 + extra_args.append('--shuffle')
1.259 +
1.260 + if e10s:
1.261 + extra_args.append('--e10s')
1.262 +
1.263 + if this_chunk:
1.264 + extra_args.append('--this-chunk=%s' % this_chunk)
1.265 +
1.266 + if total_chunks:
1.267 + extra_args.append('--total-chunks=%s' % total_chunks)
1.268 +
1.269 + if extra_args:
1.270 + args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]
1.271 + args.extend(extra_args)
1.272 + env[b'EXTRA_TEST_ARGS'] = ' '.join(args)
1.273 +
1.274 + # TODO hook up harness via native Python
1.275 + return self._run_make(directory='.', target=suite, append_env=env,
1.276 + pass_thru=pass_thru, ensure_exit_code=False)
1.277 +
1.278 +
1.279 +def ReftestCommand(func):
1.280 + """Decorator that adds shared command arguments to reftest commands."""
1.281 +
1.282 + debugger = CommandArgument('--debugger', metavar='DEBUGGER',
1.283 + help=DEBUGGER_HELP)
1.284 + func = debugger(func)
1.285 +
1.286 + flter = CommandArgument('--filter', metavar='REGEX',
1.287 + help='A JS regular expression to match test URLs against, to select '
1.288 + 'a subset of tests to run.')
1.289 + func = flter(func)
1.290 +
1.291 + path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
1.292 + help='Reftest manifest file, or a directory in which to select '
1.293 + 'reftest.list. If omitted, the entire test suite is executed.')
1.294 + func = path(func)
1.295 +
1.296 + parallel = CommandArgument('--parallel', action='store_true',
1.297 + help='Run tests in parallel.')
1.298 + func = parallel(func)
1.299 +
1.300 + shuffle = CommandArgument('--shuffle', action='store_true',
1.301 + help='Run tests in random order.')
1.302 + func = shuffle(func)
1.303 +
1.304 + e10s = CommandArgument('--e10s', action='store_true',
1.305 + help='Use content processes.')
1.306 + func = e10s(func)
1.307 +
1.308 + totalChunks = CommandArgument('--total-chunks',
1.309 + help = 'How many chunks to split the tests up into.')
1.310 + func = totalChunks(func)
1.311 +
1.312 + thisChunk = CommandArgument('--this-chunk',
1.313 + help = 'Which chunk to run between 1 and --total-chunks.')
1.314 + func = thisChunk(func)
1.315 +
1.316 + return func
1.317 +
1.318 +def B2GCommand(func):
1.319 + """Decorator that adds shared command arguments to b2g mochitest commands."""
1.320 +
1.321 + busybox = CommandArgument('--busybox', default=None,
1.322 + help='Path to busybox binary to install on device')
1.323 + func = busybox(func)
1.324 +
1.325 + logcatdir = CommandArgument('--logcat-dir', default=None,
1.326 + help='directory to store logcat dump files')
1.327 + func = logcatdir(func)
1.328 +
1.329 + geckopath = CommandArgument('--gecko-path', default=None,
1.330 + help='the path to a gecko distribution that should \
1.331 + be installed on the emulator prior to test')
1.332 + func = geckopath(func)
1.333 +
1.334 + sdcard = CommandArgument('--sdcard', default="10MB",
1.335 + help='Define size of sdcard: 1MB, 50MB...etc')
1.336 + func = sdcard(func)
1.337 +
1.338 + emulator_res = CommandArgument('--emulator-res', default='800x1000',
1.339 + help='Emulator resolution of the format \'<width>x<height>\'')
1.340 + func = emulator_res(func)
1.341 +
1.342 + emulator = CommandArgument('--emulator', default='arm',
1.343 + help='Architecture of emulator to use: x86 or arm')
1.344 + func = emulator(func)
1.345 +
1.346 + marionette = CommandArgument('--marionette', default=None,
1.347 + help='host:port to use when connecting to Marionette')
1.348 + func = marionette(func)
1.349 +
1.350 + totalChunks = CommandArgument('--total-chunks', dest='totalChunks',
1.351 + help = 'How many chunks to split the tests up into.')
1.352 + func = totalChunks(func)
1.353 +
1.354 + thisChunk = CommandArgument('--this-chunk', dest='thisChunk',
1.355 + help = 'Which chunk to run between 1 and --total-chunks.')
1.356 + func = thisChunk(func)
1.357 +
1.358 + path = CommandArgument('test_file', default=None, nargs='?',
1.359 + metavar='TEST',
1.360 + help='Test to run. Can be specified as a single file, a ' \
1.361 + 'directory, or omitted. If omitted, the entire test suite is ' \
1.362 + 'executed.')
1.363 + func = path(func)
1.364 +
1.365 + return func
1.366 +
1.367 +
1.368 +@CommandProvider
1.369 +class MachCommands(MachCommandBase):
1.370 + @Command('reftest', category='testing', description='Run reftests.')
1.371 + @ReftestCommand
1.372 + def run_reftest(self, test_file, **kwargs):
1.373 + return self._run_reftest(test_file, suite='reftest', **kwargs)
1.374 +
1.375 + @Command('reftest-ipc', category='testing',
1.376 + description='Run IPC reftests.')
1.377 + @ReftestCommand
1.378 + def run_ipc(self, test_file, **kwargs):
1.379 + return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)
1.380 +
1.381 + @Command('crashtest', category='testing',
1.382 + description='Run crashtests.')
1.383 + @ReftestCommand
1.384 + def run_crashtest(self, test_file, **kwargs):
1.385 + return self._run_reftest(test_file, suite='crashtest', **kwargs)
1.386 +
1.387 + @Command('crashtest-ipc', category='testing',
1.388 + description='Run IPC crashtests.')
1.389 + @ReftestCommand
1.390 + def run_crashtest_ipc(self, test_file, **kwargs):
1.391 + return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
1.392 +
1.393 + def _run_reftest(self, test_file=None, suite=None, **kwargs):
1.394 + reftest = self._spawn(ReftestRunner)
1.395 + return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
1.396 +
1.397 +
1.398 +# TODO For now b2g commands will only work with the emulator,
1.399 +# they should be modified to work with all devices.
1.400 +def is_emulator(cls):
1.401 + """Emulator needs to be configured."""
1.402 + return cls.device_name.find('emulator') == 0
1.403 +
1.404 +
1.405 +@CommandProvider
1.406 +class B2GCommands(MachCommandBase):
1.407 + def __init__(self, context):
1.408 + MachCommandBase.__init__(self, context)
1.409 +
1.410 + for attr in ('b2g_home', 'xre_path', 'device_name'):
1.411 + setattr(self, attr, getattr(context, attr, None))
1.412 +
1.413 + @Command('reftest-remote', category='testing',
1.414 + description='Run a remote reftest.',
1.415 + conditions=[conditions.is_b2g, is_emulator])
1.416 + @B2GCommand
1.417 + def run_reftest_remote(self, test_file, **kwargs):
1.418 + return self._run_reftest(test_file, suite='reftest', **kwargs)
1.419 +
1.420 + @Command('reftest-b2g-desktop', category='testing',
1.421 + description='Run a b2g desktop reftest.',
1.422 + conditions=[conditions.is_b2g_desktop])
1.423 + @B2GCommand
1.424 + def run_reftest_b2g_desktop(self, test_file, **kwargs):
1.425 + return self._run_reftest(test_file, suite='reftest', **kwargs)
1.426 +
1.427 + @Command('crashtest-remote', category='testing',
1.428 + description='Run a remote crashtest.',
1.429 + conditions=[conditions.is_b2g, is_emulator])
1.430 + @B2GCommand
1.431 + def run_crashtest_remote(self, test_file, **kwargs):
1.432 + return self._run_reftest(test_file, suite='crashtest', **kwargs)
1.433 +
1.434 + def _run_reftest(self, test_file=None, suite=None, **kwargs):
1.435 + reftest = self._spawn(ReftestRunner)
1.436 + return reftest.run_b2g_test(self.b2g_home, self.xre_path,
1.437 + test_file, suite=suite, **kwargs)
