michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: from __future__ import unicode_literals
michael@0: 
michael@0: import mozpack.path
michael@0: import os
michael@0: import re
michael@0: import sys
michael@0: import warnings
michael@0: import which
michael@0: 
michael@0: from mozbuild.base import (
michael@0:     MachCommandBase,
michael@0:     MachCommandConditions as conditions,
michael@0:     MozbuildObject,
michael@0: )
michael@0: 
michael@0: from mach.decorators import (
michael@0:     CommandArgument,
michael@0:     CommandProvider,
michael@0:     Command,
michael@0: )
michael@0: 
michael@0: 
michael@0: DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
michael@0: 
michael@0: ADB_NOT_FOUND = '''
michael@0: The %s command requires the adb binary to be on your path.
michael@0: 
michael@0: If you have a B2G build, this can be found in
michael@0: '%s/out/host/<platform>/bin'.
michael@0: '''.lstrip()
michael@0: 
michael@0: GAIA_PROFILE_NOT_FOUND = '''
michael@0: The %s command requires a non-debug gaia profile. Either pass in --profile,
michael@0: or set the GAIA_PROFILE environment variable.
michael@0: 
michael@0: If you do not have a non-debug gaia profile, you can build one:
michael@0:     $ git clone https://github.com/mozilla-b2g/gaia
michael@0:     $ cd gaia
michael@0:     $ make
michael@0: 
michael@0: The profile should be generated in a directory called 'profile'.
michael@0: '''.lstrip()
michael@0: 
michael@0: GAIA_PROFILE_IS_DEBUG = '''
michael@0: The %s command requires a non-debug gaia profile. The specified profile,
michael@0: %s, is a debug profile.
michael@0: 
michael@0: If you do not have a non-debug gaia profile, you can build one:
michael@0:     $ git clone https://github.com/mozilla-b2g/gaia
michael@0:     $ cd gaia
michael@0:     $ make
michael@0: 
michael@0: The profile should be generated in a directory called 'profile'.
michael@0: '''.lstrip()
michael@0: 
michael@0: MARIONETTE_DISABLED = '''
michael@0: The %s command requires a marionette enabled build.
michael@0: 
michael@0: Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application.
michael@0: Your currently active mozconfig is %s.
michael@0: '''.lstrip()
michael@0: 
michael@0: class ReftestRunner(MozbuildObject):
michael@0:     """Easily run reftests.
michael@0: 
michael@0:     This currently contains just the basics for running reftests. We may want
michael@0:     to hook up result parsing, etc.
michael@0:     """
michael@0:     def __init__(self, *args, **kwargs):
michael@0:         MozbuildObject.__init__(self, *args, **kwargs)
michael@0: 
michael@0:         # TODO Bug 794506 remove once mach integrates with virtualenv.
michael@0:         build_path = os.path.join(self.topobjdir, 'build')
michael@0:         if build_path not in sys.path:
michael@0:             sys.path.append(build_path)
michael@0: 
michael@0:         self.tests_dir = os.path.join(self.topobjdir, '_tests')
michael@0:         self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
michael@0: 
michael@0:     def _manifest_file(self, suite):
michael@0:         """Returns the manifest file used for a given test suite."""
michael@0:         files = {
michael@0:           'reftest': 'reftest.list',
michael@0:           'reftest-ipc': 'reftest.list',
michael@0:           'crashtest': 'crashtests.list',
michael@0:           'crashtest-ipc': 'crashtests.list',
michael@0:         }
michael@0:         assert suite in files
michael@0:         return files[suite]
michael@0: 
michael@0:     def _find_manifest(self, suite, test_file):
michael@0:         assert test_file
michael@0:         path_arg = self._wrap_path_argument(test_file)
michael@0:         relpath = path_arg.relpath()
michael@0: 
michael@0:         if os.path.isdir(path_arg.srcdir_path()):
michael@0:             return mozpack.path.join(relpath, self._manifest_file(suite))
michael@0: 
michael@0:         if relpath.endswith('.list'):
michael@0:             return relpath
michael@0: 
michael@0:         raise Exception('Running a single test is not currently supported')
michael@0: 
michael@0:     def _make_shell_string(self, s):
michael@0:         return "'%s'" % re.sub("'", r"'\''", s)
michael@0: 
michael@0:     def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,
michael@0:                      suite=None, **kwargs):
michael@0:         """Runs a b2g reftest.
michael@0: 
michael@0:         test_file is a path to a test file. It can be a relative path from the
michael@0:         top source directory, an absolute filename, or a directory containing
michael@0:         test files.
michael@0: 
michael@0:         suite is the type of reftest to run. It can be one of ('reftest',
michael@0:         'crashtest').
michael@0:         """
michael@0:         if suite not in ('reftest', 'crashtest'):
michael@0:             raise Exception('None or unrecognized reftest suite type.')
michael@0: 
michael@0:         # Find the manifest file
michael@0:         if not test_file:
michael@0:             if suite == 'reftest':
michael@0:                 test_file = mozpack.path.join('layout', 'reftests')
michael@0:             elif suite == 'crashtest':
michael@0:                 test_file = mozpack.path.join('testing', 'crashtest')
michael@0: 
michael@0:         if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
michael@0:             test_file = mozpack.path.relpath(os.path.abspath(test_file),
michael@0:                                              self.topsrcdir)
michael@0: 
michael@0:         manifest = self._find_manifest(suite, test_file)
michael@0:         if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)):
michael@0:             raise Exception('No manifest file was found at %s.' % manifest)
michael@0: 
michael@0:         # Need to chdir to reftest_dir otherwise imports fail below.
michael@0:         os.chdir(self.reftest_dir)
michael@0: 
michael@0:         # The imp module can spew warnings if the modules below have
michael@0:         # already been imported, ignore them.
michael@0:         with warnings.catch_warnings():
michael@0:             warnings.simplefilter('ignore')
michael@0: 
michael@0:             import imp
michael@0:             path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
michael@0:             with open(path, 'r') as fh:
michael@0:                 imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
michael@0:             import reftest
michael@0: 
michael@0:         # Set up the reftest options.
michael@0:         parser = reftest.B2GOptions()
michael@0:         options, args = parser.parse_args([])
michael@0: 
michael@0:         # Tests need to be served from a subdirectory of the server. Symlink
michael@0:         # topsrcdir here to get around this.
michael@0:         tests = os.path.join(self.reftest_dir, 'tests')
michael@0:         if not os.path.isdir(tests):
michael@0:             os.symlink(self.topsrcdir, tests)
michael@0:         args.insert(0, os.path.join('tests', manifest))
michael@0: 
michael@0:         for k, v in kwargs.iteritems():
michael@0:             setattr(options, k, v)
michael@0: 
michael@0:         if conditions.is_b2g_desktop(self):
michael@0:             if self.substs.get('ENABLE_MARIONETTE') != '1':
michael@0:                 print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
michael@0:                                              self.mozconfig['path']))
michael@0:                 return 1
michael@0: 
michael@0:             options.profile = options.profile or os.environ.get('GAIA_PROFILE')
michael@0:             if not options.profile:
michael@0:                 print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')
michael@0:                 return 1
michael@0: 
michael@0:             if os.path.isfile(os.path.join(options.profile, 'extensions', \
michael@0:                     'httpd@gaiamobile.org')):
michael@0:                 print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
michael@0:                                                options.profile))
michael@0:                 return 1
michael@0: 
michael@0:             options.desktop = True
michael@0:             options.app = self.get_binary_path()
michael@0:             if not options.app.endswith('-bin'):
michael@0:                 options.app = '%s-bin' % options.app
michael@0:             if not os.path.isfile(options.app):
michael@0:                 options.app = options.app[:-len('-bin')]
michael@0: 
michael@0:             return reftest.run_desktop_reftests(parser, options, args)
michael@0: 
michael@0: 
michael@0:         try:
michael@0:             which.which('adb')
michael@0:         except which.WhichError:
michael@0:             # TODO Find adb automatically if it isn't on the path
michael@0:             raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
michael@0: 
michael@0:         options.b2gPath = b2g_home
michael@0:         options.logcat_dir = self.reftest_dir
michael@0:         options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
michael@0:         options.xrePath = xre_path
michael@0:         options.ignoreWindowSize = True
michael@0:         return reftest.run_remote_reftests(parser, options, args)
michael@0: 
michael@0:     def run_desktop_test(self, test_file=None, filter=None, suite=None,
michael@0:             debugger=None, parallel=False, shuffle=False,
michael@0:             e10s=False, this_chunk=None, total_chunks=None):
michael@0:         """Runs a reftest.
michael@0: 
michael@0:         test_file is a path to a test file. It can be a relative path from the
michael@0:         top source directory, an absolute filename, or a directory containing
michael@0:         test files.
michael@0: 
michael@0:         filter is a regular expression (in JS syntax, as could be passed to the
michael@0:         RegExp constructor) to select which reftests to run from the manifest.
michael@0: 
michael@0:         suite is the type of reftest to run. It can be one of ('reftest',
michael@0:         'crashtest').
michael@0: 
michael@0:         debugger is the program name (in $PATH) or the full path of the
michael@0:         debugger to run.
michael@0: 
michael@0:         parallel indicates whether tests should be run in parallel or not.
michael@0: 
michael@0:         shuffle indicates whether to run tests in random order.
michael@0:         """
michael@0: 
michael@0:         if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'):
michael@0:             raise Exception('None or unrecognized reftest suite type.')
michael@0: 
michael@0:         env = {}
michael@0:         extra_args = []
michael@0: 
michael@0:         if test_file:
michael@0:             path = self._find_manifest(suite, test_file)
michael@0:             if not os.path.exists(mozpack.path.join(self.topsrcdir, path)):
michael@0:                 raise Exception('No manifest file was found at %s.' % path)
michael@0:             env[b'TEST_PATH'] = path
michael@0:         if filter:
michael@0:             extra_args.extend(['--filter', self._make_shell_string(filter)])
michael@0: 
michael@0:         pass_thru = False
michael@0: 
michael@0:         if debugger:
michael@0:             extra_args.append('--debugger=%s' % debugger)
michael@0:             pass_thru = True
michael@0: 
michael@0:         if parallel:
michael@0:             extra_args.append('--run-tests-in-parallel')
michael@0: 
michael@0:         if shuffle:
michael@0:             extra_args.append('--shuffle')
michael@0: 
michael@0:         if e10s:
michael@0:             extra_args.append('--e10s')
michael@0: 
michael@0:         if this_chunk:
michael@0:             extra_args.append('--this-chunk=%s' % this_chunk)
michael@0: 
michael@0:         if total_chunks:
michael@0:             extra_args.append('--total-chunks=%s' % total_chunks)
michael@0: 
michael@0:         if extra_args:
michael@0:             args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]
michael@0:             args.extend(extra_args)
michael@0:             env[b'EXTRA_TEST_ARGS'] = ' '.join(args)
michael@0: 
michael@0:         # TODO hook up harness via native Python
michael@0:         return self._run_make(directory='.', target=suite, append_env=env,
michael@0:             pass_thru=pass_thru, ensure_exit_code=False)
michael@0: 
michael@0: 
michael@0: def ReftestCommand(func):
michael@0:     """Decorator that adds shared command arguments to reftest commands."""
michael@0: 
michael@0:     debugger = CommandArgument('--debugger', metavar='DEBUGGER',
michael@0:         help=DEBUGGER_HELP)
michael@0:     func = debugger(func)
michael@0: 
michael@0:     flter = CommandArgument('--filter', metavar='REGEX',
michael@0:         help='A JS regular expression to match test URLs against, to select '
michael@0:              'a subset of tests to run.')
michael@0:     func = flter(func)
michael@0: 
michael@0:     path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
michael@0:         help='Reftest manifest file, or a directory in which to select '
michael@0:              'reftest.list. If omitted, the entire test suite is executed.')
michael@0:     func = path(func)
michael@0: 
michael@0:     parallel = CommandArgument('--parallel', action='store_true',
michael@0:         help='Run tests in parallel.')
michael@0:     func = parallel(func)
michael@0: 
michael@0:     shuffle = CommandArgument('--shuffle', action='store_true',
michael@0:         help='Run tests in random order.')
michael@0:     func = shuffle(func)
michael@0: 
michael@0:     e10s = CommandArgument('--e10s', action='store_true',
michael@0:         help='Use content processes.')
michael@0:     func = e10s(func)
michael@0: 
michael@0:     totalChunks = CommandArgument('--total-chunks',
michael@0:         help = 'How many chunks to split the tests up into.')
michael@0:     func = totalChunks(func)
michael@0: 
michael@0:     thisChunk = CommandArgument('--this-chunk',
michael@0:         help = 'Which chunk to run between 1 and --total-chunks.')
michael@0:     func = thisChunk(func)
michael@0: 
michael@0:     return func
michael@0: 
michael@0: def B2GCommand(func):
michael@0:     """Decorator that adds shared command arguments to b2g mochitest commands."""
michael@0: 
michael@0:     busybox = CommandArgument('--busybox', default=None,
michael@0:         help='Path to busybox binary to install on device')
michael@0:     func = busybox(func)
michael@0: 
michael@0:     logcatdir = CommandArgument('--logcat-dir', default=None,
michael@0:         help='directory to store logcat dump files')
michael@0:     func = logcatdir(func)
michael@0: 
michael@0:     geckopath = CommandArgument('--gecko-path', default=None,
michael@0:         help='the path to a gecko distribution that should \
michael@0:               be installed on the emulator prior to test')
michael@0:     func = geckopath(func)
michael@0: 
michael@0:     sdcard = CommandArgument('--sdcard', default="10MB",
michael@0:         help='Define size of sdcard: 1MB, 50MB...etc')
michael@0:     func = sdcard(func)
michael@0: 
michael@0:     emulator_res = CommandArgument('--emulator-res', default='800x1000',
michael@0:         help='Emulator resolution of the format \'<width>x<height>\'')
michael@0:     func = emulator_res(func)
michael@0: 
michael@0:     emulator = CommandArgument('--emulator', default='arm',
michael@0:         help='Architecture of emulator to use: x86 or arm')
michael@0:     func = emulator(func)
michael@0: 
michael@0:     marionette = CommandArgument('--marionette', default=None,
michael@0:         help='host:port to use when connecting to Marionette')
michael@0:     func = marionette(func)
michael@0: 
michael@0:     totalChunks = CommandArgument('--total-chunks', dest='totalChunks',
michael@0:         help = 'How many chunks to split the tests up into.')
michael@0:     func = totalChunks(func)
michael@0: 
michael@0:     thisChunk = CommandArgument('--this-chunk', dest='thisChunk',
michael@0:         help = 'Which chunk to run between 1 and --total-chunks.')
michael@0:     func = thisChunk(func)
michael@0: 
michael@0:     path = CommandArgument('test_file', default=None, nargs='?',
michael@0:         metavar='TEST',
michael@0:         help='Test to run. Can be specified as a single file, a ' \
michael@0:             'directory, or omitted. If omitted, the entire test suite is ' \
michael@0:             'executed.')
michael@0:     func = path(func)
michael@0: 
michael@0:     return func
michael@0: 
michael@0: 
michael@0: @CommandProvider
michael@0: class MachCommands(MachCommandBase):
michael@0:     @Command('reftest', category='testing', description='Run reftests.')
michael@0:     @ReftestCommand
michael@0:     def run_reftest(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='reftest', **kwargs)
michael@0: 
michael@0:     @Command('reftest-ipc', category='testing',
michael@0:         description='Run IPC reftests.')
michael@0:     @ReftestCommand
michael@0:     def run_ipc(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)
michael@0: 
michael@0:     @Command('crashtest', category='testing',
michael@0:         description='Run crashtests.')
michael@0:     @ReftestCommand
michael@0:     def run_crashtest(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='crashtest', **kwargs)
michael@0: 
michael@0:     @Command('crashtest-ipc', category='testing',
michael@0:         description='Run IPC crashtests.')
michael@0:     @ReftestCommand
michael@0:     def run_crashtest_ipc(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
michael@0: 
michael@0:     def _run_reftest(self, test_file=None, suite=None, **kwargs):
michael@0:         reftest = self._spawn(ReftestRunner)
michael@0:         return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
michael@0: 
michael@0: 
michael@0: # TODO For now b2g commands will only work with the emulator,
michael@0: # they should be modified to work with all devices.
michael@0: def is_emulator(cls):
michael@0:     """Emulator needs to be configured."""
michael@0:     return cls.device_name.find('emulator') == 0
michael@0: 
michael@0: 
michael@0: @CommandProvider
michael@0: class B2GCommands(MachCommandBase):
michael@0:     def __init__(self, context):
michael@0:         MachCommandBase.__init__(self, context)
michael@0: 
michael@0:         for attr in ('b2g_home', 'xre_path', 'device_name'):
michael@0:             setattr(self, attr, getattr(context, attr, None))
michael@0: 
michael@0:     @Command('reftest-remote', category='testing',
michael@0:         description='Run a remote reftest.',
michael@0:         conditions=[conditions.is_b2g, is_emulator])
michael@0:     @B2GCommand
michael@0:     def run_reftest_remote(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='reftest', **kwargs)
michael@0: 
michael@0:     @Command('reftest-b2g-desktop', category='testing',
michael@0:         description='Run a b2g desktop reftest.',
michael@0:         conditions=[conditions.is_b2g_desktop])
michael@0:     @B2GCommand
michael@0:     def run_reftest_b2g_desktop(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='reftest', **kwargs)
michael@0: 
michael@0:     @Command('crashtest-remote', category='testing',
michael@0:         description='Run a remote crashtest.',
michael@0:         conditions=[conditions.is_b2g, is_emulator])
michael@0:     @B2GCommand
michael@0:     def run_crashtest_remote(self, test_file, **kwargs):
michael@0:         return self._run_reftest(test_file, suite='crashtest', **kwargs)
michael@0: 
michael@0:     def _run_reftest(self, test_file=None, suite=None, **kwargs):
michael@0:         reftest = self._spawn(ReftestRunner)
michael@0:         return reftest.run_b2g_test(self.b2g_home, self.xre_path,
michael@0:             test_file, suite=suite, **kwargs)