The Tor Browser / file comparison
comparison: testing/xpcshell/mach_commands.py
testing/xpcshell/mach_commands.py
- changeset 2
- 7e26c7da4463
equal
deleted
inserted
replaced
| -1:000000000000 | 0:92d24384cc05 |
|---|---|
| 1 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 2 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 4 | |
| 5 # Integrates the xpcshell test runner with mach. | |
| 6 | |
| 7 from __future__ import unicode_literals, print_function | |
| 8 | |
| 9 import mozpack.path | |
| 10 import logging | |
| 11 import os | |
| 12 import shutil | |
| 13 import sys | |
| 14 import urllib2 | |
| 15 | |
| 16 from StringIO import StringIO | |
| 17 | |
| 18 from mozbuild.base import ( | |
| 19 MachCommandBase, | |
| 20 MozbuildObject, | |
| 21 MachCommandConditions as conditions, | |
| 22 ) | |
| 23 | |
| 24 from mach.decorators import ( | |
| 25 CommandArgument, | |
| 26 CommandProvider, | |
| 27 Command, | |
| 28 ) | |
| 29 | |
| 30 ADB_NOT_FOUND = ''' | |
| 31 The %s command requires the adb binary to be on your path. | |
| 32 | |
| 33 If you have a B2G build, this can be found in | |
| 34 '%s/out/host/<platform>/bin'. | |
| 35 '''.lstrip() | |
| 36 | |
| 37 BUSYBOX_URL = 'http://www.busybox.net/downloads/binaries/latest/busybox-armv7l' | |
| 38 | |
| 39 | |
| 40 if sys.version_info[0] < 3: | |
| 41 unicode_type = unicode | |
| 42 else: | |
| 43 unicode_type = str | |
| 44 | |
| 45 # Simple filter to omit the message emitted as a test file begins. | |
| 46 class TestStartFilter(logging.Filter): | |
| 47 def filter(self, record): | |
| 48 return not record.params['msg'].endswith("running test ...") | |
| 49 | |
| 50 # This should probably be consolidated with similar classes in other test | |
| 51 # runners. | |
| 52 class InvalidTestPathError(Exception): | |
| 53 """Exception raised when the test path is not valid.""" | |
| 54 | |
| 55 | |
| 56 class XPCShellRunner(MozbuildObject): | |
| 57 """Run xpcshell tests.""" | |
| 58 def run_suite(self, **kwargs): | |
| 59 from manifestparser import TestManifest | |
| 60 manifest = TestManifest(manifests=[os.path.join(self.topobjdir, | |
| 61 '_tests', 'xpcshell', 'xpcshell.ini')]) | |
| 62 | |
| 63 return self._run_xpcshell_harness(manifest=manifest, **kwargs) | |
| 64 | |
| 65 def run_test(self, test_paths, interactive=False, | |
| 66 keep_going=False, sequential=False, shuffle=False, | |
| 67 debugger=None, debuggerArgs=None, debuggerInteractive=None, | |
| 68 rerun_failures=False, | |
| 69 # ignore parameters from other platforms' options | |
| 70 **kwargs): | |
| 71 """Runs an individual xpcshell test.""" | |
| 72 from mozbuild.testing import TestResolver | |
| 73 from manifestparser import TestManifest | |
| 74 | |
| 75 # TODO Bug 794506 remove once mach integrates with virtualenv. | |
| 76 build_path = os.path.join(self.topobjdir, 'build') | |
| 77 if build_path not in sys.path: | |
| 78 sys.path.append(build_path) | |
| 79 | |
| 80 if test_paths == ['all']: | |
| 81 self.run_suite(interactive=interactive, | |
| 82 keep_going=keep_going, shuffle=shuffle, sequential=sequential, | |
| 83 debugger=debugger, debuggerArgs=debuggerArgs, | |
| 84 debuggerInteractive=debuggerInteractive, | |
| 85 rerun_failures=rerun_failures) | |
| 86 return | |
| 87 | |
| 88 resolver = self._spawn(TestResolver) | |
| 89 tests = list(resolver.resolve_tests(paths=test_paths, flavor='xpcshell', | |
| 90 cwd=self.cwd)) | |
| 91 | |
| 92 if not tests: | |
| 93 raise InvalidTestPathError('We could not find an xpcshell test ' | |
| 94 'for the passed test path. Please select a path that is ' | |
| 95 'a test file or is a directory containing xpcshell tests.') | |
| 96 | |
| 97 # Dynamically write out a manifest holding all the discovered tests. | |
| 98 manifest = TestManifest() | |
| 99 manifest.tests.extend(tests) | |
| 100 | |
| 101 args = { | |
| 102 'interactive': interactive, | |
| 103 'keep_going': keep_going, | |
| 104 'shuffle': shuffle, | |
| 105 'sequential': sequential, | |
| 106 'debugger': debugger, | |
| 107 'debuggerArgs': debuggerArgs, | |
| 108 'debuggerInteractive': debuggerInteractive, | |
| 109 'rerun_failures': rerun_failures, | |
| 110 'manifest': manifest, | |
| 111 } | |
| 112 | |
| 113 return self._run_xpcshell_harness(**args) | |
| 114 | |
| 115 def _run_xpcshell_harness(self, manifest, | |
| 116 test_path=None, shuffle=False, interactive=False, | |
| 117 keep_going=False, sequential=False, | |
| 118 debugger=None, debuggerArgs=None, debuggerInteractive=None, | |
| 119 rerun_failures=False): | |
| 120 | |
| 121 # Obtain a reference to the xpcshell test runner. | |
| 122 import runxpcshelltests | |
| 123 | |
| 124 dummy_log = StringIO() | |
| 125 xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log) | |
| 126 self.log_manager.enable_unstructured() | |
| 127 | |
| 128 xpcshell_filter = TestStartFilter() | |
| 129 self.log_manager.terminal_handler.addFilter(xpcshell_filter) | |
| 130 | |
| 131 tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell') | |
| 132 modules_dir = os.path.join(self.topobjdir, '_tests', 'modules') | |
| 133 # We want output from the test to be written immediately if we are only | |
| 134 # running a single test. | |
| 135 verbose_output = test_path is not None or (manifest and len(manifest.test_paths())==1) | |
| 136 | |
| 137 args = { | |
| 138 'manifest': manifest, | |
| 139 'xpcshell': os.path.join(self.bindir, 'xpcshell'), | |
| 140 'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'), | |
| 141 'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'), | |
| 142 'interactive': interactive, | |
| 143 'keepGoing': keep_going, | |
| 144 'logfiles': False, | |
| 145 'sequential': sequential, | |
| 146 'shuffle': shuffle, | |
| 147 'testsRootDir': tests_dir, | |
| 148 'testingModulesDir': modules_dir, | |
| 149 'profileName': 'firefox', | |
| 150 'verbose': test_path is not None, | |
| 151 'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'), | |
| 152 'xunitName': 'xpcshell', | |
| 153 'pluginsPath': os.path.join(self.distdir, 'plugins'), | |
| 154 'debugger': debugger, | |
| 155 'debuggerArgs': debuggerArgs, | |
| 156 'debuggerInteractive': debuggerInteractive, | |
| 157 'on_message': (lambda obj, msg: xpcshell.log.info(msg.decode('utf-8', 'replace'))) \ | |
| 158 if verbose_output else None, | |
| 159 } | |
| 160 | |
| 161 if test_path is not None: | |
| 162 args['testPath'] = test_path | |
| 163 | |
| 164 # A failure manifest is written by default. If --rerun-failures is | |
| 165 # specified and a prior failure manifest is found, the prior manifest | |
| 166 # will be run. A new failure manifest is always written over any | |
| 167 # prior failure manifest. | |
| 168 failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini') | |
| 169 rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini') | |
| 170 if os.path.exists(failure_manifest_path) and rerun_failures: | |
| 171 shutil.move(failure_manifest_path, rerun_manifest_path) | |
| 172 args['manifest'] = rerun_manifest_path | |
| 173 elif os.path.exists(failure_manifest_path): | |
| 174 os.remove(failure_manifest_path) | |
| 175 elif rerun_failures: | |
| 176 print("No failures were found to re-run.") | |
| 177 return 0 | |
| 178 args['failureManifest'] = failure_manifest_path | |
| 179 | |
| 180 # Python through 2.7.2 has issues with unicode in some of the | |
| 181 # arguments. Work around that. | |
| 182 filtered_args = {} | |
| 183 for k, v in args.items(): | |
| 184 if isinstance(v, unicode_type): | |
| 185 v = v.encode('utf-8') | |
| 186 | |
| 187 if isinstance(k, unicode_type): | |
| 188 k = k.encode('utf-8') | |
| 189 | |
| 190 filtered_args[k] = v | |
| 191 | |
| 192 result = xpcshell.runTests(**filtered_args) | |
| 193 | |
| 194 self.log_manager.terminal_handler.removeFilter(xpcshell_filter) | |
| 195 self.log_manager.disable_unstructured() | |
| 196 | |
| 197 if not result and not xpcshell.sequential: | |
| 198 print("Tests were run in parallel. Try running with --sequential " | |
| 199 "to make sure the failures were not caused by this.") | |
| 200 return int(not result) | |
| 201 | |
| 202 class AndroidXPCShellRunner(MozbuildObject): | |
| 203 """Get specified DeviceManager""" | |
| 204 def get_devicemanager(self, devicemanager, ip, port, remote_test_root): | |
| 205 from mozdevice import devicemanagerADB, devicemanagerSUT | |
| 206 dm = None | |
| 207 if devicemanager == "adb": | |
| 208 if ip: | |
| 209 dm = devicemanagerADB.DeviceManagerADB(ip, port, packageName=None, deviceRoot=remote_test_root) | |
| 210 else: | |
| 211 dm = devicemanagerADB.DeviceManagerADB(packageName=None, deviceRoot=remote_test_root) | |
| 212 else: | |
| 213 if ip: | |
| 214 dm = devicemanagerSUT.DeviceManagerSUT(ip, port, deviceRoot=remote_test_root) | |
| 215 else: | |
| 216 raise Exception("You must provide a device IP to connect to via the --ip option") | |
| 217 return dm | |
| 218 | |
| 219 """Run Android xpcshell tests.""" | |
| 220 def run_test(self, | |
| 221 test_paths, keep_going, | |
| 222 devicemanager, ip, port, remote_test_root, no_setup, local_apk, | |
| 223 # ignore parameters from other platforms' options | |
| 224 **kwargs): | |
| 225 # TODO Bug 794506 remove once mach integrates with virtualenv. | |
| 226 build_path = os.path.join(self.topobjdir, 'build') | |
| 227 if build_path not in sys.path: | |
| 228 sys.path.append(build_path) | |
| 229 | |
| 230 import remotexpcshelltests | |
| 231 | |
| 232 dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root) | |
| 233 | |
| 234 options = remotexpcshelltests.RemoteXPCShellOptions() | |
| 235 options.shuffle = False | |
| 236 options.sequential = True | |
| 237 options.interactive = False | |
| 238 options.debugger = None | |
| 239 options.debuggerArgs = None | |
| 240 options.setup = not no_setup | |
| 241 options.keepGoing = keep_going | |
| 242 options.objdir = self.topobjdir | |
| 243 options.localLib = os.path.join(self.topobjdir, 'dist/fennec') | |
| 244 options.localBin = os.path.join(self.topobjdir, 'dist/bin') | |
| 245 options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules') | |
| 246 options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json') | |
| 247 options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell_android.ini') | |
| 248 options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols') | |
| 249 if local_apk: | |
| 250 options.localAPK = local_apk | |
| 251 else: | |
| 252 for file in os.listdir(os.path.join(options.objdir, "dist")): | |
| 253 if file.endswith(".apk") and file.startswith("fennec"): | |
| 254 options.localAPK = os.path.join(options.objdir, "dist") | |
| 255 options.localAPK = os.path.join(options.localAPK, file) | |
| 256 print ("using APK: " + options.localAPK) | |
| 257 break | |
| 258 else: | |
| 259 raise Exception("You must specify an APK") | |
| 260 | |
| 261 if test_paths == ['all']: | |
| 262 testdirs = [] | |
| 263 options.testPath = None | |
| 264 options.verbose = False | |
| 265 else: | |
| 266 if len(test_paths) > 1: | |
| 267 print('Warning: only the first test path argument will be used.') | |
| 268 testdirs = test_paths[0] | |
| 269 options.testPath = test_paths[0] | |
| 270 options.verbose = True | |
| 271 dummy_log = StringIO() | |
| 272 xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, args=testdirs, log=dummy_log) | |
| 273 self.log_manager.enable_unstructured() | |
| 274 | |
| 275 xpcshell_filter = TestStartFilter() | |
| 276 self.log_manager.terminal_handler.addFilter(xpcshell_filter) | |
| 277 | |
| 278 result = xpcshell.runTests(xpcshell='xpcshell', | |
| 279 testClass=remotexpcshelltests.RemoteXPCShellTestThread, | |
| 280 testdirs=testdirs, | |
| 281 mobileArgs=xpcshell.mobileArgs, | |
| 282 **options.__dict__) | |
| 283 | |
| 284 self.log_manager.terminal_handler.removeFilter(xpcshell_filter) | |
| 285 self.log_manager.disable_unstructured() | |
| 286 | |
| 287 return int(not result) | |
| 288 | |
| 289 class B2GXPCShellRunner(MozbuildObject): | |
| 290 def __init__(self, *args, **kwargs): | |
| 291 MozbuildObject.__init__(self, *args, **kwargs) | |
| 292 | |
| 293 # TODO Bug 794506 remove once mach integrates with virtualenv. | |
| 294 build_path = os.path.join(self.topobjdir, 'build') | |
| 295 if build_path not in sys.path: | |
| 296 sys.path.append(build_path) | |
| 297 | |
| 298 build_path = os.path.join(self.topsrcdir, 'build') | |
| 299 if build_path not in sys.path: | |
| 300 sys.path.append(build_path) | |
| 301 | |
| 302 self.tests_dir = os.path.join(self.topobjdir, '_tests') | |
| 303 self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell') | |
| 304 self.bin_dir = os.path.join(self.distdir, 'bin') | |
| 305 | |
| 306 def _download_busybox(self, b2g_home): | |
| 307 system_bin = os.path.join(b2g_home, 'out', 'target', 'product', 'generic', 'system', 'bin') | |
| 308 busybox_path = os.path.join(system_bin, 'busybox') | |
| 309 | |
| 310 if os.path.isfile(busybox_path): | |
| 311 return busybox_path | |
| 312 | |
| 313 if not os.path.isdir(system_bin): | |
| 314 os.makedirs(system_bin) | |
| 315 | |
| 316 try: | |
| 317 data = urllib2.urlopen(BUSYBOX_URL) | |
| 318 except urllib2.URLError: | |
| 319 print('There was a problem downloading busybox. Proceeding without it,' \ | |
| 320 'initial setup will be slow.') | |
| 321 return | |
| 322 | |
| 323 with open(busybox_path, 'wb') as f: | |
| 324 f.write(data.read()) | |
| 325 return busybox_path | |
| 326 | |
| 327 def run_test(self, test_paths, b2g_home=None, busybox=None, | |
| 328 # ignore parameters from other platforms' options | |
| 329 **kwargs): | |
| 330 try: | |
| 331 import which | |
| 332 which.which('adb') | |
| 333 except which.WhichError: | |
| 334 # TODO Find adb automatically if it isn't on the path | |
| 335 print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home)) | |
| 336 sys.exit(1) | |
| 337 | |
| 338 test_path = None | |
| 339 if test_paths: | |
| 340 if len(test_paths) > 1: | |
| 341 print('Warning: Only the first test path will be used.') | |
| 342 | |
| 343 test_path = self._wrap_path_argument(test_paths[0]).relpath() | |
| 344 | |
| 345 import runtestsb2g | |
| 346 parser = runtestsb2g.B2GOptions() | |
| 347 options, args = parser.parse_args([]) | |
| 348 | |
| 349 options.b2g_path = b2g_home | |
| 350 options.busybox = busybox or os.environ.get('BUSYBOX') | |
| 351 options.emulator = 'arm' | |
| 352 options.localLib = self.bin_dir | |
| 353 options.localBin = self.bin_dir | |
| 354 options.logcat_dir = self.xpcshell_dir | |
| 355 options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell_b2g.ini') | |
| 356 options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json') | |
| 357 options.objdir = self.topobjdir | |
| 358 options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'), | |
| 359 options.testingModulesDir = os.path.join(self.tests_dir, 'modules') | |
| 360 options.testsRootDir = self.xpcshell_dir | |
| 361 options.testPath = test_path | |
| 362 options.use_device_libs = True | |
| 363 | |
| 364 if not options.busybox: | |
| 365 options.busybox = self._download_busybox(b2g_home) | |
| 366 | |
| 367 return runtestsb2g.run_remote_xpcshell(parser, options, args) | |
| 368 | |
| 369 def is_platform_supported(cls): | |
| 370 """Must have a Firefox, Android or B2G build.""" | |
| 371 return conditions.is_android(cls) or \ | |
| 372 conditions.is_b2g(cls) or \ | |
| 373 conditions.is_firefox(cls) | |
| 374 | |
| 375 @CommandProvider | |
| 376 class MachCommands(MachCommandBase): | |
| 377 def __init__(self, context): | |
| 378 MachCommandBase.__init__(self, context) | |
| 379 | |
| 380 for attr in ('b2g_home', 'device_name'): | |
| 381 setattr(self, attr, getattr(context, attr, None)) | |
| 382 | |
| 383 @Command('xpcshell-test', category='testing', | |
| 384 conditions=[is_platform_supported], | |
| 385 description='Run XPCOM Shell tests.') | |
| 386 @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST', | |
| 387 help='Test to run. Can be specified as a single JS file, a directory, ' | |
| 388 'or omitted. If omitted, the entire test suite is executed.') | |
| 389 @CommandArgument("--debugger", default=None, metavar='DEBUGGER', | |
| 390 help = "Run xpcshell under the given debugger.") | |
| 391 @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str, | |
| 392 dest = "debuggerArgs", | |
| 393 help = "pass the given args to the debugger _before_ " | |
| 394 "the application on the command line") | |
| 395 @CommandArgument("--debugger-interactive", action = "store_true", | |
| 396 dest = "debuggerInteractive", | |
| 397 help = "prevents the test harness from redirecting " | |
| 398 "stdout and stderr for interactive debuggers") | |
| 399 @CommandArgument('--interactive', '-i', action='store_true', | |
| 400 help='Open an xpcshell prompt before running tests.') | |
| 401 @CommandArgument('--keep-going', '-k', action='store_true', | |
| 402 help='Continue running tests after a SIGINT is received.') | |
| 403 @CommandArgument('--sequential', action='store_true', | |
| 404 help='Run the tests sequentially.') | |
| 405 @CommandArgument('--shuffle', '-s', action='store_true', | |
| 406 help='Randomize the execution order of tests.') | |
| 407 @CommandArgument('--rerun-failures', action='store_true', | |
| 408 help='Reruns failures from last time.') | |
| 409 @CommandArgument('--devicemanager', default='adb', type=str, | |
| 410 help='(Android) Type of devicemanager to use for communication: adb or sut') | |
| 411 @CommandArgument('--ip', type=str, default=None, | |
| 412 help='(Android) IP address of device') | |
| 413 @CommandArgument('--port', type=int, default=20701, | |
| 414 help='(Android) Port of device') | |
| 415 @CommandArgument('--remote_test_root', type=str, default=None, | |
| 416 help='(Android) Remote test root such as /mnt/sdcard or /data/local') | |
| 417 @CommandArgument('--no-setup', action='store_true', | |
| 418 help='(Android) Do not copy files to device') | |
| 419 @CommandArgument('--local-apk', type=str, default=None, | |
| 420 help='(Android) Use specified Fennec APK') | |
| 421 @CommandArgument('--busybox', type=str, default=None, | |
| 422 help='(B2G) Path to busybox binary (speeds up installation of tests).') | |
| 423 def run_xpcshell_test(self, **params): | |
| 424 from mozbuild.controller.building import BuildDriver | |
| 425 | |
| 426 # We should probably have a utility function to ensure the tree is | |
| 427 # ready to run tests. Until then, we just create the state dir (in | |
| 428 # case the tree wasn't built with mach). | |
| 429 self._ensure_state_subdir_exists('.') | |
| 430 | |
| 431 driver = self._spawn(BuildDriver) | |
| 432 driver.install_tests(remove=False) | |
| 433 | |
| 434 if conditions.is_android(self): | |
| 435 xpcshell = self._spawn(AndroidXPCShellRunner) | |
| 436 elif conditions.is_b2g(self): | |
| 437 xpcshell = self._spawn(B2GXPCShellRunner) | |
| 438 params['b2g_home'] = self.b2g_home | |
| 439 else: | |
| 440 xpcshell = self._spawn(XPCShellRunner) | |
| 441 xpcshell.cwd = self._mach_context.cwd | |
| 442 | |
| 443 try: | |
| 444 return xpcshell.run_test(**params) | |
| 445 except InvalidTestPathError as e: | |
| 446 print(e.message) | |
| 447 return 1 |
