michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: # Integrates the xpcshell test runner with mach.
michael@0: 
michael@0: from __future__ import unicode_literals, print_function
michael@0: 
michael@0: import mozpack.path
michael@0: import logging
michael@0: import os
michael@0: import shutil
michael@0: import sys
michael@0: import urllib2
michael@0: 
michael@0: from StringIO import StringIO
michael@0: 
michael@0: from mozbuild.base import (
michael@0:     MachCommandBase,
michael@0:     MozbuildObject,
michael@0:     MachCommandConditions as conditions,
michael@0: )
michael@0: 
michael@0: from mach.decorators import (
michael@0:     CommandArgument,
michael@0:     CommandProvider,
michael@0:     Command,
michael@0: )
michael@0: 
michael@0: ADB_NOT_FOUND = '''
michael@0: The %s command requires the adb binary to be on your path.
michael@0: 
michael@0: If you have a B2G build, this can be found in
michael@0: '%s/out/host/<platform>/bin'.
michael@0: '''.lstrip()
michael@0: 
michael@0: BUSYBOX_URL = 'http://www.busybox.net/downloads/binaries/latest/busybox-armv7l'
michael@0: 
michael@0: 
michael@0: if sys.version_info[0] < 3:
michael@0:     unicode_type = unicode
michael@0: else:
michael@0:     unicode_type = str
michael@0: 
michael@0: # Simple filter to omit the message emitted as a test file begins.
michael@0: class TestStartFilter(logging.Filter):
michael@0:     def filter(self, record):
michael@0:         return not record.params['msg'].endswith("running test ...")
michael@0: 
michael@0: # This should probably be consolidated with similar classes in other test
michael@0: # runners.
michael@0: class InvalidTestPathError(Exception):
michael@0:     """Exception raised when the test path is not valid."""
michael@0: 
michael@0: 
michael@0: class XPCShellRunner(MozbuildObject):
michael@0:     """Run xpcshell tests."""
michael@0:     def run_suite(self, **kwargs):
michael@0:         from manifestparser import TestManifest
michael@0:         manifest = TestManifest(manifests=[os.path.join(self.topobjdir,
michael@0:             '_tests', 'xpcshell', 'xpcshell.ini')])
michael@0: 
michael@0:         return self._run_xpcshell_harness(manifest=manifest, **kwargs)
michael@0: 
michael@0:     def run_test(self, test_paths, interactive=False,
michael@0:                  keep_going=False, sequential=False, shuffle=False,
michael@0:                  debugger=None, debuggerArgs=None, debuggerInteractive=None,
michael@0:                  rerun_failures=False,
michael@0:                  # ignore parameters from other platforms' options
michael@0:                  **kwargs):
michael@0:         """Runs an individual xpcshell test."""
michael@0:         from mozbuild.testing import TestResolver
michael@0:         from manifestparser import TestManifest
michael@0: 
michael@0:         # TODO Bug 794506 remove once mach integrates with virtualenv.
michael@0:         build_path = os.path.join(self.topobjdir, 'build')
michael@0:         if build_path not in sys.path:
michael@0:             sys.path.append(build_path)
michael@0: 
michael@0:         if test_paths == ['all']:
michael@0:             self.run_suite(interactive=interactive,
michael@0:                            keep_going=keep_going, shuffle=shuffle, sequential=sequential,
michael@0:                            debugger=debugger, debuggerArgs=debuggerArgs,
michael@0:                            debuggerInteractive=debuggerInteractive,
michael@0:                            rerun_failures=rerun_failures)
michael@0:             return
michael@0: 
michael@0:         resolver = self._spawn(TestResolver)
michael@0:         tests = list(resolver.resolve_tests(paths=test_paths, flavor='xpcshell',
michael@0:             cwd=self.cwd))
michael@0: 
michael@0:         if not tests:
michael@0:             raise InvalidTestPathError('We could not find an xpcshell test '
michael@0:                 'for the passed test path. Please select a path that is '
michael@0:                 'a test file or is a directory containing xpcshell tests.')
michael@0: 
michael@0:         # Dynamically write out a manifest holding all the discovered tests.
michael@0:         manifest = TestManifest()
michael@0:         manifest.tests.extend(tests)
michael@0: 
michael@0:         args = {
michael@0:             'interactive': interactive,
michael@0:             'keep_going': keep_going,
michael@0:             'shuffle': shuffle,
michael@0:             'sequential': sequential,
michael@0:             'debugger': debugger,
michael@0:             'debuggerArgs': debuggerArgs,
michael@0:             'debuggerInteractive': debuggerInteractive,
michael@0:             'rerun_failures': rerun_failures,
michael@0:             'manifest': manifest,
michael@0:         }
michael@0: 
michael@0:         return self._run_xpcshell_harness(**args)
michael@0: 
michael@0:     def _run_xpcshell_harness(self, manifest,
michael@0:                               test_path=None, shuffle=False, interactive=False,
michael@0:                               keep_going=False, sequential=False,
michael@0:                               debugger=None, debuggerArgs=None, debuggerInteractive=None,
michael@0:                               rerun_failures=False):
michael@0: 
michael@0:         # Obtain a reference to the xpcshell test runner.
michael@0:         import runxpcshelltests
michael@0: 
michael@0:         dummy_log = StringIO()
michael@0:         xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)
michael@0:         self.log_manager.enable_unstructured()
michael@0: 
michael@0:         xpcshell_filter = TestStartFilter()
michael@0:         self.log_manager.terminal_handler.addFilter(xpcshell_filter)
michael@0: 
michael@0:         tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')
michael@0:         modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
michael@0:         # We want output from the test to be written immediately if we are only
michael@0:         # running a single test.
michael@0:         verbose_output = test_path is not None or (manifest and len(manifest.test_paths())==1)
michael@0: 
michael@0:         args = {
michael@0:             'manifest': manifest,
michael@0:             'xpcshell': os.path.join(self.bindir, 'xpcshell'),
michael@0:             'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
michael@0:             'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
michael@0:             'interactive': interactive,
michael@0:             'keepGoing': keep_going,
michael@0:             'logfiles': False,
michael@0:             'sequential': sequential,
michael@0:             'shuffle': shuffle,
michael@0:             'testsRootDir': tests_dir,
michael@0:             'testingModulesDir': modules_dir,
michael@0:             'profileName': 'firefox',
michael@0:             'verbose': test_path is not None,
michael@0:             'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),
michael@0:             'xunitName': 'xpcshell',
michael@0:             'pluginsPath': os.path.join(self.distdir, 'plugins'),
michael@0:             'debugger': debugger,
michael@0:             'debuggerArgs': debuggerArgs,
michael@0:             'debuggerInteractive': debuggerInteractive,
michael@0:             'on_message': (lambda obj, msg: xpcshell.log.info(msg.decode('utf-8', 'replace'))) \
michael@0:                             if verbose_output else None,
michael@0:         }
michael@0: 
michael@0:         if test_path is not None:
michael@0:             args['testPath'] = test_path
michael@0: 
michael@0:         # A failure manifest is written by default. If --rerun-failures is
michael@0:         # specified and a prior failure manifest is found, the prior manifest
michael@0:         # will be run. A new failure manifest is always written over any
michael@0:         # prior failure manifest.
michael@0:         failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
michael@0:         rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
michael@0:         if os.path.exists(failure_manifest_path) and rerun_failures:
michael@0:             shutil.move(failure_manifest_path, rerun_manifest_path)
michael@0:             args['manifest'] = rerun_manifest_path
michael@0:         elif os.path.exists(failure_manifest_path):
michael@0:             os.remove(failure_manifest_path)
michael@0:         elif rerun_failures:
michael@0:             print("No failures were found to re-run.")
michael@0:             return 0
michael@0:         args['failureManifest'] = failure_manifest_path
michael@0: 
michael@0:         # Python through 2.7.2 has issues with unicode in some of the
michael@0:         # arguments. Work around that.
michael@0:         filtered_args = {}
michael@0:         for k, v in args.items():
michael@0:             if isinstance(v, unicode_type):
michael@0:                 v = v.encode('utf-8')
michael@0: 
michael@0:             if isinstance(k, unicode_type):
michael@0:                 k = k.encode('utf-8')
michael@0: 
michael@0:             filtered_args[k] = v
michael@0: 
michael@0:         result = xpcshell.runTests(**filtered_args)
michael@0: 
michael@0:         self.log_manager.terminal_handler.removeFilter(xpcshell_filter)
michael@0:         self.log_manager.disable_unstructured()
michael@0: 
michael@0:         if not result and not xpcshell.sequential:
michael@0:             print("Tests were run in parallel. Try running with --sequential "
michael@0:                   "to make sure the failures were not caused by this.")
michael@0:         return int(not result)
michael@0: 
michael@0: class AndroidXPCShellRunner(MozbuildObject):
michael@0:     """Get specified DeviceManager"""
michael@0:     def get_devicemanager(self, devicemanager, ip, port, remote_test_root):
michael@0:         from mozdevice import devicemanagerADB, devicemanagerSUT
michael@0:         dm = None
michael@0:         if devicemanager == "adb":
michael@0:             if ip:
michael@0:                 dm = devicemanagerADB.DeviceManagerADB(ip, port, packageName=None, deviceRoot=remote_test_root)
michael@0:             else:
michael@0:                 dm = devicemanagerADB.DeviceManagerADB(packageName=None, deviceRoot=remote_test_root)
michael@0:         else:
michael@0:             if ip:
michael@0:                 dm = devicemanagerSUT.DeviceManagerSUT(ip, port, deviceRoot=remote_test_root)
michael@0:             else:
michael@0:                 raise Exception("You must provide a device IP to connect to via the --ip option")
michael@0:         return dm
michael@0: 
michael@0:     """Run Android xpcshell tests."""
michael@0:     def run_test(self,
michael@0:                  test_paths, keep_going,
michael@0:                  devicemanager, ip, port, remote_test_root, no_setup, local_apk,
michael@0:                  # ignore parameters from other platforms' options
michael@0:                  **kwargs):
michael@0:         # TODO Bug 794506 remove once mach integrates with virtualenv.
michael@0:         build_path = os.path.join(self.topobjdir, 'build')
michael@0:         if build_path not in sys.path:
michael@0:             sys.path.append(build_path)
michael@0: 
michael@0:         import remotexpcshelltests
michael@0: 
michael@0:         dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)
michael@0: 
michael@0:         options = remotexpcshelltests.RemoteXPCShellOptions()
michael@0:         options.shuffle = False
michael@0:         options.sequential = True
michael@0:         options.interactive = False
michael@0:         options.debugger = None
michael@0:         options.debuggerArgs = None
michael@0:         options.setup = not no_setup
michael@0:         options.keepGoing = keep_going
michael@0:         options.objdir = self.topobjdir
michael@0:         options.localLib = os.path.join(self.topobjdir, 'dist/fennec')
michael@0:         options.localBin = os.path.join(self.topobjdir, 'dist/bin')
michael@0:         options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')
michael@0:         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
michael@0:         options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell_android.ini')
michael@0:         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
michael@0:         if local_apk:
michael@0:             options.localAPK = local_apk
michael@0:         else:
michael@0:             for file in os.listdir(os.path.join(options.objdir, "dist")):
michael@0:                 if file.endswith(".apk") and file.startswith("fennec"):
michael@0:                     options.localAPK = os.path.join(options.objdir, "dist")
michael@0:                     options.localAPK = os.path.join(options.localAPK, file)
michael@0:                     print ("using APK: " + options.localAPK)
michael@0:                     break
michael@0:             else:
michael@0:                 raise Exception("You must specify an APK")
michael@0: 
michael@0:         if test_paths == ['all']:
michael@0:             testdirs = []
michael@0:             options.testPath = None
michael@0:             options.verbose = False
michael@0:         else:
michael@0:             if len(test_paths) > 1:
michael@0:                 print('Warning: only the first test path argument will be used.')
michael@0:             testdirs = test_paths[0]
michael@0:             options.testPath = test_paths[0]
michael@0:             options.verbose = True
michael@0:         dummy_log = StringIO()
michael@0:         xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, args=testdirs, log=dummy_log)
michael@0:         self.log_manager.enable_unstructured()
michael@0: 
michael@0:         xpcshell_filter = TestStartFilter()
michael@0:         self.log_manager.terminal_handler.addFilter(xpcshell_filter)
michael@0: 
michael@0:         result = xpcshell.runTests(xpcshell='xpcshell',
michael@0:                       testClass=remotexpcshelltests.RemoteXPCShellTestThread,
michael@0:                       testdirs=testdirs,
michael@0:                       mobileArgs=xpcshell.mobileArgs,
michael@0:                       **options.__dict__)
michael@0: 
michael@0:         self.log_manager.terminal_handler.removeFilter(xpcshell_filter)
michael@0:         self.log_manager.disable_unstructured()
michael@0: 
michael@0:         return int(not result)
michael@0: 
michael@0: class B2GXPCShellRunner(MozbuildObject):
michael@0:     def __init__(self, *args, **kwargs):
michael@0:         MozbuildObject.__init__(self, *args, **kwargs)
michael@0: 
michael@0:         # TODO Bug 794506 remove once mach integrates with virtualenv.
michael@0:         build_path = os.path.join(self.topobjdir, 'build')
michael@0:         if build_path not in sys.path:
michael@0:             sys.path.append(build_path)
michael@0: 
michael@0:         build_path = os.path.join(self.topsrcdir, 'build')
michael@0:         if build_path not in sys.path:
michael@0:             sys.path.append(build_path)
michael@0: 
michael@0:         self.tests_dir = os.path.join(self.topobjdir, '_tests')
michael@0:         self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell')
michael@0:         self.bin_dir = os.path.join(self.distdir, 'bin')
michael@0: 
michael@0:     def _download_busybox(self, b2g_home):
michael@0:         system_bin = os.path.join(b2g_home, 'out', 'target', 'product', 'generic', 'system', 'bin')
michael@0:         busybox_path = os.path.join(system_bin, 'busybox')
michael@0: 
michael@0:         if os.path.isfile(busybox_path):
michael@0:             return busybox_path
michael@0: 
michael@0:         if not os.path.isdir(system_bin):
michael@0:             os.makedirs(system_bin)
michael@0: 
michael@0:         try:
michael@0:             data = urllib2.urlopen(BUSYBOX_URL)
michael@0:         except urllib2.URLError:
michael@0:             print('There was a problem downloading busybox. Proceeding without it,' \
michael@0:                   'initial setup will be slow.')
michael@0:             return
michael@0: 
michael@0:         with open(busybox_path, 'wb') as f:
michael@0:             f.write(data.read())
michael@0:         return busybox_path
michael@0: 
michael@0:     def run_test(self, test_paths, b2g_home=None, busybox=None,
michael@0:                  # ignore parameters from other platforms' options
michael@0:                  **kwargs):
michael@0:         try:
michael@0:             import which
michael@0:             which.which('adb')
michael@0:         except which.WhichError:
michael@0:             # TODO Find adb automatically if it isn't on the path
michael@0:             print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
michael@0:             sys.exit(1)
michael@0: 
michael@0:         test_path = None
michael@0:         if test_paths:
michael@0:             if len(test_paths) > 1:
michael@0:                 print('Warning: Only the first test path will be used.')
michael@0: 
michael@0:             test_path = self._wrap_path_argument(test_paths[0]).relpath()
michael@0: 
michael@0:         import runtestsb2g
michael@0:         parser = runtestsb2g.B2GOptions()
michael@0:         options, args = parser.parse_args([])
michael@0: 
michael@0:         options.b2g_path = b2g_home
michael@0:         options.busybox = busybox or os.environ.get('BUSYBOX')
michael@0:         options.emulator = 'arm'
michael@0:         options.localLib = self.bin_dir
michael@0:         options.localBin = self.bin_dir
michael@0:         options.logcat_dir = self.xpcshell_dir
michael@0:         options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell_b2g.ini')
michael@0:         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
michael@0:         options.objdir = self.topobjdir
michael@0:         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),
michael@0:         options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
michael@0:         options.testsRootDir = self.xpcshell_dir
michael@0:         options.testPath = test_path
michael@0:         options.use_device_libs = True
michael@0: 
michael@0:         if not options.busybox:
michael@0:             options.busybox = self._download_busybox(b2g_home)
michael@0: 
michael@0:         return runtestsb2g.run_remote_xpcshell(parser, options, args)
michael@0: 
michael@0: def is_platform_supported(cls):
michael@0:     """Must have a Firefox, Android or B2G build."""
michael@0:     return conditions.is_android(cls) or \
michael@0:            conditions.is_b2g(cls) or \
michael@0:            conditions.is_firefox(cls)
michael@0: 
michael@0: @CommandProvider
michael@0: class MachCommands(MachCommandBase):
michael@0:     def __init__(self, context):
michael@0:         MachCommandBase.__init__(self, context)
michael@0: 
michael@0:         for attr in ('b2g_home', 'device_name'):
michael@0:             setattr(self, attr, getattr(context, attr, None))
michael@0: 
michael@0:     @Command('xpcshell-test', category='testing',
michael@0:         conditions=[is_platform_supported],
michael@0:         description='Run XPCOM Shell tests.')
michael@0:     @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
michael@0:         help='Test to run. Can be specified as a single JS file, a directory, '
michael@0:              'or omitted. If omitted, the entire test suite is executed.')
michael@0:     @CommandArgument("--debugger", default=None, metavar='DEBUGGER',
michael@0:                      help = "Run xpcshell under the given debugger.")
michael@0:     @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,
michael@0:                      dest = "debuggerArgs",
michael@0:                      help = "pass the given args to the debugger _before_ "
michael@0:                             "the application on the command line")
michael@0:     @CommandArgument("--debugger-interactive", action = "store_true",
michael@0:                      dest = "debuggerInteractive",
michael@0:                      help = "prevents the test harness from redirecting "
michael@0:                             "stdout and stderr for interactive debuggers")
michael@0:     @CommandArgument('--interactive', '-i', action='store_true',
michael@0:         help='Open an xpcshell prompt before running tests.')
michael@0:     @CommandArgument('--keep-going', '-k', action='store_true',
michael@0:         help='Continue running tests after a SIGINT is received.')
michael@0:     @CommandArgument('--sequential', action='store_true',
michael@0:         help='Run the tests sequentially.')
michael@0:     @CommandArgument('--shuffle', '-s', action='store_true',
michael@0:         help='Randomize the execution order of tests.')
michael@0:     @CommandArgument('--rerun-failures', action='store_true',
michael@0:         help='Reruns failures from last time.')
michael@0:     @CommandArgument('--devicemanager', default='adb', type=str,
michael@0:         help='(Android) Type of devicemanager to use for communication: adb or sut')
michael@0:     @CommandArgument('--ip', type=str, default=None,
michael@0:         help='(Android) IP address of device')
michael@0:     @CommandArgument('--port', type=int, default=20701,
michael@0:         help='(Android) Port of device')
michael@0:     @CommandArgument('--remote_test_root', type=str, default=None,
michael@0:         help='(Android) Remote test root such as /mnt/sdcard or /data/local')
michael@0:     @CommandArgument('--no-setup', action='store_true',
michael@0:         help='(Android) Do not copy files to device')
michael@0:     @CommandArgument('--local-apk', type=str, default=None,
michael@0:         help='(Android) Use specified Fennec APK')
michael@0:     @CommandArgument('--busybox', type=str, default=None,
michael@0:         help='(B2G) Path to busybox binary (speeds up installation of tests).')
michael@0:     def run_xpcshell_test(self, **params):
michael@0:         from mozbuild.controller.building import BuildDriver
michael@0: 
michael@0:         # We should probably have a utility function to ensure the tree is
michael@0:         # ready to run tests. Until then, we just create the state dir (in
michael@0:         # case the tree wasn't built with mach).
michael@0:         self._ensure_state_subdir_exists('.')
michael@0: 
michael@0:         driver = self._spawn(BuildDriver)
michael@0:         driver.install_tests(remove=False)
michael@0: 
michael@0:         if conditions.is_android(self):
michael@0:             xpcshell = self._spawn(AndroidXPCShellRunner)
michael@0:         elif conditions.is_b2g(self):
michael@0:             xpcshell = self._spawn(B2GXPCShellRunner)
michael@0:             params['b2g_home'] = self.b2g_home
michael@0:         else:
michael@0:             xpcshell = self._spawn(XPCShellRunner)
michael@0:         xpcshell.cwd = self._mach_context.cwd
michael@0: 
michael@0:         try:
michael@0:             return xpcshell.run_test(**params)
michael@0:         except InvalidTestPathError as e:
michael@0:             print(e.message)
michael@0:             return 1