The Tor Browser / file comparison
comparison: layout/tools/reftest/mach_commands.py
layout/tools/reftest/mach_commands.py
- branch
- TOR_BUG_9701
- changeset 14
- 925c144e1f1f
equal
deleted
inserted
replaced
| -1:000000000000 | 0:98153a7dca3e |
|---|---|
| 1 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 2 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 4 | |
| 5 from __future__ import unicode_literals | |
| 6 | |
| 7 import mozpack.path | |
| 8 import os | |
| 9 import re | |
| 10 import sys | |
| 11 import warnings | |
| 12 import which | |
| 13 | |
| 14 from mozbuild.base import ( | |
| 15 MachCommandBase, | |
| 16 MachCommandConditions as conditions, | |
| 17 MozbuildObject, | |
| 18 ) | |
| 19 | |
| 20 from mach.decorators import ( | |
| 21 CommandArgument, | |
| 22 CommandProvider, | |
| 23 Command, | |
| 24 ) | |
| 25 | |
| 26 | |
| 27 DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.' | |
| 28 | |
| 29 ADB_NOT_FOUND = ''' | |
| 30 The %s command requires the adb binary to be on your path. | |
| 31 | |
| 32 If you have a B2G build, this can be found in | |
| 33 '%s/out/host/<platform>/bin'. | |
| 34 '''.lstrip() | |
| 35 | |
| 36 GAIA_PROFILE_NOT_FOUND = ''' | |
| 37 The %s command requires a non-debug gaia profile. Either pass in --profile, | |
| 38 or set the GAIA_PROFILE environment variable. | |
| 39 | |
| 40 If you do not have a non-debug gaia profile, you can build one: | |
| 41 $ git clone https://github.com/mozilla-b2g/gaia | |
| 42 $ cd gaia | |
| 43 $ make | |
| 44 | |
| 45 The profile should be generated in a directory called 'profile'. | |
| 46 '''.lstrip() | |
| 47 | |
| 48 GAIA_PROFILE_IS_DEBUG = ''' | |
| 49 The %s command requires a non-debug gaia profile. The specified profile, | |
| 50 %s, is a debug profile. | |
| 51 | |
| 52 If you do not have a non-debug gaia profile, you can build one: | |
| 53 $ git clone https://github.com/mozilla-b2g/gaia | |
| 54 $ cd gaia | |
| 55 $ make | |
| 56 | |
| 57 The profile should be generated in a directory called 'profile'. | |
| 58 '''.lstrip() | |
| 59 | |
| 60 MARIONETTE_DISABLED = ''' | |
| 61 The %s command requires a marionette enabled build. | |
| 62 | |
| 63 Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application. | |
| 64 Your currently active mozconfig is %s. | |
| 65 '''.lstrip() | |
| 66 | |
| 67 class ReftestRunner(MozbuildObject): | |
| 68 """Easily run reftests. | |
| 69 | |
| 70 This currently contains just the basics for running reftests. We may want | |
| 71 to hook up result parsing, etc. | |
| 72 """ | |
| 73 def __init__(self, *args, **kwargs): | |
| 74 MozbuildObject.__init__(self, *args, **kwargs) | |
| 75 | |
| 76 # TODO Bug 794506 remove once mach integrates with virtualenv. | |
| 77 build_path = os.path.join(self.topobjdir, 'build') | |
| 78 if build_path not in sys.path: | |
| 79 sys.path.append(build_path) | |
| 80 | |
| 81 self.tests_dir = os.path.join(self.topobjdir, '_tests') | |
| 82 self.reftest_dir = os.path.join(self.tests_dir, 'reftest') | |
| 83 | |
| 84 def _manifest_file(self, suite): | |
| 85 """Returns the manifest file used for a given test suite.""" | |
| 86 files = { | |
| 87 'reftest': 'reftest.list', | |
| 88 'reftest-ipc': 'reftest.list', | |
| 89 'crashtest': 'crashtests.list', | |
| 90 'crashtest-ipc': 'crashtests.list', | |
| 91 } | |
| 92 assert suite in files | |
| 93 return files[suite] | |
| 94 | |
| 95 def _find_manifest(self, suite, test_file): | |
| 96 assert test_file | |
| 97 path_arg = self._wrap_path_argument(test_file) | |
| 98 relpath = path_arg.relpath() | |
| 99 | |
| 100 if os.path.isdir(path_arg.srcdir_path()): | |
| 101 return mozpack.path.join(relpath, self._manifest_file(suite)) | |
| 102 | |
| 103 if relpath.endswith('.list'): | |
| 104 return relpath | |
| 105 | |
| 106 raise Exception('Running a single test is not currently supported') | |
| 107 | |
| 108 def _make_shell_string(self, s): | |
| 109 return "'%s'" % re.sub("'", r"'\''", s) | |
| 110 | |
| 111 def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None, | |
| 112 suite=None, **kwargs): | |
| 113 """Runs a b2g reftest. | |
| 114 | |
| 115 test_file is a path to a test file. It can be a relative path from the | |
| 116 top source directory, an absolute filename, or a directory containing | |
| 117 test files. | |
| 118 | |
| 119 suite is the type of reftest to run. It can be one of ('reftest', | |
| 120 'crashtest'). | |
| 121 """ | |
| 122 if suite not in ('reftest', 'crashtest'): | |
| 123 raise Exception('None or unrecognized reftest suite type.') | |
| 124 | |
| 125 # Find the manifest file | |
| 126 if not test_file: | |
| 127 if suite == 'reftest': | |
| 128 test_file = mozpack.path.join('layout', 'reftests') | |
| 129 elif suite == 'crashtest': | |
| 130 test_file = mozpack.path.join('testing', 'crashtest') | |
| 131 | |
| 132 if not os.path.exists(os.path.join(self.topsrcdir, test_file)): | |
| 133 test_file = mozpack.path.relpath(os.path.abspath(test_file), | |
| 134 self.topsrcdir) | |
| 135 | |
| 136 manifest = self._find_manifest(suite, test_file) | |
| 137 if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)): | |
| 138 raise Exception('No manifest file was found at %s.' % manifest) | |
| 139 | |
| 140 # Need to chdir to reftest_dir otherwise imports fail below. | |
| 141 os.chdir(self.reftest_dir) | |
| 142 | |
| 143 # The imp module can spew warnings if the modules below have | |
| 144 # already been imported, ignore them. | |
| 145 with warnings.catch_warnings(): | |
| 146 warnings.simplefilter('ignore') | |
| 147 | |
| 148 import imp | |
| 149 path = os.path.join(self.reftest_dir, 'runreftestb2g.py') | |
| 150 with open(path, 'r') as fh: | |
| 151 imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE)) | |
| 152 import reftest | |
| 153 | |
| 154 # Set up the reftest options. | |
| 155 parser = reftest.B2GOptions() | |
| 156 options, args = parser.parse_args([]) | |
| 157 | |
| 158 # Tests need to be served from a subdirectory of the server. Symlink | |
| 159 # topsrcdir here to get around this. | |
| 160 tests = os.path.join(self.reftest_dir, 'tests') | |
| 161 if not os.path.isdir(tests): | |
| 162 os.symlink(self.topsrcdir, tests) | |
| 163 args.insert(0, os.path.join('tests', manifest)) | |
| 164 | |
| 165 for k, v in kwargs.iteritems(): | |
| 166 setattr(options, k, v) | |
| 167 | |
| 168 if conditions.is_b2g_desktop(self): | |
| 169 if self.substs.get('ENABLE_MARIONETTE') != '1': | |
| 170 print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop', | |
| 171 self.mozconfig['path'])) | |
| 172 return 1 | |
| 173 | |
| 174 options.profile = options.profile or os.environ.get('GAIA_PROFILE') | |
| 175 if not options.profile: | |
| 176 print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop') | |
| 177 return 1 | |
| 178 | |
| 179 if os.path.isfile(os.path.join(options.profile, 'extensions', \ | |
| 180 'httpd@gaiamobile.org')): | |
| 181 print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop', | |
| 182 options.profile)) | |
| 183 return 1 | |
| 184 | |
| 185 options.desktop = True | |
| 186 options.app = self.get_binary_path() | |
| 187 if not options.app.endswith('-bin'): | |
| 188 options.app = '%s-bin' % options.app | |
| 189 if not os.path.isfile(options.app): | |
| 190 options.app = options.app[:-len('-bin')] | |
| 191 | |
| 192 return reftest.run_desktop_reftests(parser, options, args) | |
| 193 | |
| 194 | |
| 195 try: | |
| 196 which.which('adb') | |
| 197 except which.WhichError: | |
| 198 # TODO Find adb automatically if it isn't on the path | |
| 199 raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home)) | |
| 200 | |
| 201 options.b2gPath = b2g_home | |
| 202 options.logcat_dir = self.reftest_dir | |
| 203 options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver') | |
| 204 options.xrePath = xre_path | |
| 205 options.ignoreWindowSize = True | |
| 206 return reftest.run_remote_reftests(parser, options, args) | |
| 207 | |
| 208 def run_desktop_test(self, test_file=None, filter=None, suite=None, | |
| 209 debugger=None, parallel=False, shuffle=False, | |
| 210 e10s=False, this_chunk=None, total_chunks=None): | |
| 211 """Runs a reftest. | |
| 212 | |
| 213 test_file is a path to a test file. It can be a relative path from the | |
| 214 top source directory, an absolute filename, or a directory containing | |
| 215 test files. | |
| 216 | |
| 217 filter is a regular expression (in JS syntax, as could be passed to the | |
| 218 RegExp constructor) to select which reftests to run from the manifest. | |
| 219 | |
| 220 suite is the type of reftest to run. It can be one of ('reftest', | |
| 221 'crashtest'). | |
| 222 | |
| 223 debugger is the program name (in $PATH) or the full path of the | |
| 224 debugger to run. | |
| 225 | |
| 226 parallel indicates whether tests should be run in parallel or not. | |
| 227 | |
| 228 shuffle indicates whether to run tests in random order. | |
| 229 """ | |
| 230 | |
| 231 if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'): | |
| 232 raise Exception('None or unrecognized reftest suite type.') | |
| 233 | |
| 234 env = {} | |
| 235 extra_args = [] | |
| 236 | |
| 237 if test_file: | |
| 238 path = self._find_manifest(suite, test_file) | |
| 239 if not os.path.exists(mozpack.path.join(self.topsrcdir, path)): | |
| 240 raise Exception('No manifest file was found at %s.' % path) | |
| 241 env[b'TEST_PATH'] = path | |
| 242 if filter: | |
| 243 extra_args.extend(['--filter', self._make_shell_string(filter)]) | |
| 244 | |
| 245 pass_thru = False | |
| 246 | |
| 247 if debugger: | |
| 248 extra_args.append('--debugger=%s' % debugger) | |
| 249 pass_thru = True | |
| 250 | |
| 251 if parallel: | |
| 252 extra_args.append('--run-tests-in-parallel') | |
| 253 | |
| 254 if shuffle: | |
| 255 extra_args.append('--shuffle') | |
| 256 | |
| 257 if e10s: | |
| 258 extra_args.append('--e10s') | |
| 259 | |
| 260 if this_chunk: | |
| 261 extra_args.append('--this-chunk=%s' % this_chunk) | |
| 262 | |
| 263 if total_chunks: | |
| 264 extra_args.append('--total-chunks=%s' % total_chunks) | |
| 265 | |
| 266 if extra_args: | |
| 267 args = [os.environ.get(b'EXTRA_TEST_ARGS', '')] | |
| 268 args.extend(extra_args) | |
| 269 env[b'EXTRA_TEST_ARGS'] = ' '.join(args) | |
| 270 | |
| 271 # TODO hook up harness via native Python | |
| 272 return self._run_make(directory='.', target=suite, append_env=env, | |
| 273 pass_thru=pass_thru, ensure_exit_code=False) | |
| 274 | |
| 275 | |
| 276 def ReftestCommand(func): | |
| 277 """Decorator that adds shared command arguments to reftest commands.""" | |
| 278 | |
| 279 debugger = CommandArgument('--debugger', metavar='DEBUGGER', | |
| 280 help=DEBUGGER_HELP) | |
| 281 func = debugger(func) | |
| 282 | |
| 283 flter = CommandArgument('--filter', metavar='REGEX', | |
| 284 help='A JS regular expression to match test URLs against, to select ' | |
| 285 'a subset of tests to run.') | |
| 286 func = flter(func) | |
| 287 | |
| 288 path = CommandArgument('test_file', nargs='?', metavar='MANIFEST', | |
| 289 help='Reftest manifest file, or a directory in which to select ' | |
| 290 'reftest.list. If omitted, the entire test suite is executed.') | |
| 291 func = path(func) | |
| 292 | |
| 293 parallel = CommandArgument('--parallel', action='store_true', | |
| 294 help='Run tests in parallel.') | |
| 295 func = parallel(func) | |
| 296 | |
| 297 shuffle = CommandArgument('--shuffle', action='store_true', | |
| 298 help='Run tests in random order.') | |
| 299 func = shuffle(func) | |
| 300 | |
| 301 e10s = CommandArgument('--e10s', action='store_true', | |
| 302 help='Use content processes.') | |
| 303 func = e10s(func) | |
| 304 | |
| 305 totalChunks = CommandArgument('--total-chunks', | |
| 306 help = 'How many chunks to split the tests up into.') | |
| 307 func = totalChunks(func) | |
| 308 | |
| 309 thisChunk = CommandArgument('--this-chunk', | |
| 310 help = 'Which chunk to run between 1 and --total-chunks.') | |
| 311 func = thisChunk(func) | |
| 312 | |
| 313 return func | |
| 314 | |
| 315 def B2GCommand(func): | |
| 316 """Decorator that adds shared command arguments to b2g mochitest commands.""" | |
| 317 | |
| 318 busybox = CommandArgument('--busybox', default=None, | |
| 319 help='Path to busybox binary to install on device') | |
| 320 func = busybox(func) | |
| 321 | |
| 322 logcatdir = CommandArgument('--logcat-dir', default=None, | |
| 323 help='directory to store logcat dump files') | |
| 324 func = logcatdir(func) | |
| 325 | |
| 326 geckopath = CommandArgument('--gecko-path', default=None, | |
| 327 help='the path to a gecko distribution that should \ | |
| 328 be installed on the emulator prior to test') | |
| 329 func = geckopath(func) | |
| 330 | |
| 331 sdcard = CommandArgument('--sdcard', default="10MB", | |
| 332 help='Define size of sdcard: 1MB, 50MB...etc') | |
| 333 func = sdcard(func) | |
| 334 | |
| 335 emulator_res = CommandArgument('--emulator-res', default='800x1000', | |
| 336 help='Emulator resolution of the format \'<width>x<height>\'') | |
| 337 func = emulator_res(func) | |
| 338 | |
| 339 emulator = CommandArgument('--emulator', default='arm', | |
| 340 help='Architecture of emulator to use: x86 or arm') | |
| 341 func = emulator(func) | |
| 342 | |
| 343 marionette = CommandArgument('--marionette', default=None, | |
| 344 help='host:port to use when connecting to Marionette') | |
| 345 func = marionette(func) | |
| 346 | |
| 347 totalChunks = CommandArgument('--total-chunks', dest='totalChunks', | |
| 348 help = 'How many chunks to split the tests up into.') | |
| 349 func = totalChunks(func) | |
| 350 | |
| 351 thisChunk = CommandArgument('--this-chunk', dest='thisChunk', | |
| 352 help = 'Which chunk to run between 1 and --total-chunks.') | |
| 353 func = thisChunk(func) | |
| 354 | |
| 355 path = CommandArgument('test_file', default=None, nargs='?', | |
| 356 metavar='TEST', | |
| 357 help='Test to run. Can be specified as a single file, a ' \ | |
| 358 'directory, or omitted. If omitted, the entire test suite is ' \ | |
| 359 'executed.') | |
| 360 func = path(func) | |
| 361 | |
| 362 return func | |
| 363 | |
| 364 | |
| 365 @CommandProvider | |
| 366 class MachCommands(MachCommandBase): | |
| 367 @Command('reftest', category='testing', description='Run reftests.') | |
| 368 @ReftestCommand | |
| 369 def run_reftest(self, test_file, **kwargs): | |
| 370 return self._run_reftest(test_file, suite='reftest', **kwargs) | |
| 371 | |
| 372 @Command('reftest-ipc', category='testing', | |
| 373 description='Run IPC reftests.') | |
| 374 @ReftestCommand | |
| 375 def run_ipc(self, test_file, **kwargs): | |
| 376 return self._run_reftest(test_file, suite='reftest-ipc', **kwargs) | |
| 377 | |
| 378 @Command('crashtest', category='testing', | |
| 379 description='Run crashtests.') | |
| 380 @ReftestCommand | |
| 381 def run_crashtest(self, test_file, **kwargs): | |
| 382 return self._run_reftest(test_file, suite='crashtest', **kwargs) | |
| 383 | |
| 384 @Command('crashtest-ipc', category='testing', | |
| 385 description='Run IPC crashtests.') | |
| 386 @ReftestCommand | |
| 387 def run_crashtest_ipc(self, test_file, **kwargs): | |
| 388 return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs) | |
| 389 | |
| 390 def _run_reftest(self, test_file=None, suite=None, **kwargs): | |
| 391 reftest = self._spawn(ReftestRunner) | |
| 392 return reftest.run_desktop_test(test_file, suite=suite, **kwargs) | |
| 393 | |
| 394 | |
| 395 # TODO For now b2g commands will only work with the emulator, | |
| 396 # they should be modified to work with all devices. | |
| 397 def is_emulator(cls): | |
| 398 """Emulator needs to be configured.""" | |
| 399 return cls.device_name.find('emulator') == 0 | |
| 400 | |
| 401 | |
| 402 @CommandProvider | |
| 403 class B2GCommands(MachCommandBase): | |
| 404 def __init__(self, context): | |
| 405 MachCommandBase.__init__(self, context) | |
| 406 | |
| 407 for attr in ('b2g_home', 'xre_path', 'device_name'): | |
| 408 setattr(self, attr, getattr(context, attr, None)) | |
| 409 | |
| 410 @Command('reftest-remote', category='testing', | |
| 411 description='Run a remote reftest.', | |
| 412 conditions=[conditions.is_b2g, is_emulator]) | |
| 413 @B2GCommand | |
| 414 def run_reftest_remote(self, test_file, **kwargs): | |
| 415 return self._run_reftest(test_file, suite='reftest', **kwargs) | |
| 416 | |
| 417 @Command('reftest-b2g-desktop', category='testing', | |
| 418 description='Run a b2g desktop reftest.', | |
| 419 conditions=[conditions.is_b2g_desktop]) | |
| 420 @B2GCommand | |
| 421 def run_reftest_b2g_desktop(self, test_file, **kwargs): | |
| 422 return self._run_reftest(test_file, suite='reftest', **kwargs) | |
| 423 | |
| 424 @Command('crashtest-remote', category='testing', | |
| 425 description='Run a remote crashtest.', | |
| 426 conditions=[conditions.is_b2g, is_emulator]) | |
| 427 @B2GCommand | |
| 428 def run_crashtest_remote(self, test_file, **kwargs): | |
| 429 return self._run_reftest(test_file, suite='crashtest', **kwargs) | |
| 430 | |
| 431 def _run_reftest(self, test_file=None, suite=None, **kwargs): | |
| 432 reftest = self._spawn(ReftestRunner) | |
| 433 return reftest.run_b2g_test(self.b2g_home, self.xre_path, | |
| 434 test_file, suite=suite, **kwargs) |
