The Tor Browser: layout/tools/reftest/mach

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 from __future__ import unicode_literals

     7 import mozpack.path

     8 import os

     9 import re

    10 import sys

    11 import warnings

    12 import which

    14 from mozbuild.base import (

    15     MachCommandBase,

    16     MachCommandConditions as conditions,

    17     MozbuildObject,

    18 )

    20 from mach.decorators import (

    21     CommandArgument,

    22     CommandProvider,

    23     Command,

    24 )

    27 DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'

    29 ADB_NOT_FOUND = '''

    30 The %s command requires the adb binary to be on your path.

    32 If you have a B2G build, this can be found in

    33 '%s/out/host/<platform>/bin'.

    34 '''.lstrip()

    36 GAIA_PROFILE_NOT_FOUND = '''

    37 The %s command requires a non-debug gaia profile. Either pass in --profile,

    38 or set the GAIA_PROFILE environment variable.

    40 If you do not have a non-debug gaia profile, you can build one:

    41     $ git clone https://github.com/mozilla-b2g/gaia

    42     $ cd gaia

    43     $ make

    45 The profile should be generated in a directory called 'profile'.

    46 '''.lstrip()

    48 GAIA_PROFILE_IS_DEBUG = '''

    49 The %s command requires a non-debug gaia profile. The specified profile,

    50 %s, is a debug profile.

    52 If you do not have a non-debug gaia profile, you can build one:

    53     $ git clone https://github.com/mozilla-b2g/gaia

    54     $ cd gaia

    55     $ make

    57 The profile should be generated in a directory called 'profile'.

    58 '''.lstrip()

    60 MARIONETTE_DISABLED = '''

    61 The %s command requires a marionette enabled build.

    63 Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application.

    64 Your currently active mozconfig is %s.

    65 '''.lstrip()

    67 class ReftestRunner(MozbuildObject):

    68     """Easily run reftests.

    70     This currently contains just the basics for running reftests. We may want

    71     to hook up result parsing, etc.

    72     """

    73     def __init__(self, *args, **kwargs):

    74         MozbuildObject.__init__(self, *args, **kwargs)

    76         # TODO Bug 794506 remove once mach integrates with virtualenv.

    77         build_path = os.path.join(self.topobjdir, 'build')

    78         if build_path not in sys.path:

    79             sys.path.append(build_path)

    81         self.tests_dir = os.path.join(self.topobjdir, '_tests')

    82         self.reftest_dir = os.path.join(self.tests_dir, 'reftest')

    84     def _manifest_file(self, suite):

    85         """Returns the manifest file used for a given test suite."""

    86         files = {

    87           'reftest': 'reftest.list',

    88           'reftest-ipc': 'reftest.list',

    89           'crashtest': 'crashtests.list',

    90           'crashtest-ipc': 'crashtests.list',

    91         }

    92         assert suite in files

    93         return files[suite]

    95     def _find_manifest(self, suite, test_file):

    96         assert test_file

    97         path_arg = self._wrap_path_argument(test_file)

    98         relpath = path_arg.relpath()

   100         if os.path.isdir(path_arg.srcdir_path()):

   101             return mozpack.path.join(relpath, self._manifest_file(suite))

   103         if relpath.endswith('.list'):

   104             return relpath

   106         raise Exception('Running a single test is not currently supported')

   108     def _make_shell_string(self, s):

   109         return "'%s'" % re.sub("'", r"'\''", s)

   111     def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,

   112                      suite=None, **kwargs):

   113         """Runs a b2g reftest.

   115         test_file is a path to a test file. It can be a relative path from the

   116         top source directory, an absolute filename, or a directory containing

   117         test files.

   119         suite is the type of reftest to run. It can be one of ('reftest',

   120         'crashtest').

   121         """

   122         if suite not in ('reftest', 'crashtest'):

   123             raise Exception('None or unrecognized reftest suite type.')

   125         # Find the manifest file

   126         if not test_file:

   127             if suite == 'reftest':

   128                 test_file = mozpack.path.join('layout', 'reftests')

   129             elif suite == 'crashtest':

   130                 test_file = mozpack.path.join('testing', 'crashtest')

   132         if not os.path.exists(os.path.join(self.topsrcdir, test_file)):

   133             test_file = mozpack.path.relpath(os.path.abspath(test_file),

   134                                              self.topsrcdir)

   136         manifest = self._find_manifest(suite, test_file)

   137         if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)):

   138             raise Exception('No manifest file was found at %s.' % manifest)

   140         # Need to chdir to reftest_dir otherwise imports fail below.

   141         os.chdir(self.reftest_dir)

   143         # The imp module can spew warnings if the modules below have

   144         # already been imported, ignore them.

   145         with warnings.catch_warnings():

   146             warnings.simplefilter('ignore')

   148             import imp

   149             path = os.path.join(self.reftest_dir, 'runreftestb2g.py')

   150             with open(path, 'r') as fh:

   151                 imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))

   152             import reftest

   154         # Set up the reftest options.

   155         parser = reftest.B2GOptions()

   156         options, args = parser.parse_args([])

   158         # Tests need to be served from a subdirectory of the server. Symlink

   159         # topsrcdir here to get around this.

   160         tests = os.path.join(self.reftest_dir, 'tests')

   161         if not os.path.isdir(tests):

   162             os.symlink(self.topsrcdir, tests)

   163         args.insert(0, os.path.join('tests', manifest))

   165         for k, v in kwargs.iteritems():

   166             setattr(options, k, v)

   168         if conditions.is_b2g_desktop(self):

   169             if self.substs.get('ENABLE_MARIONETTE') != '1':

   170                 print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',

   171                                              self.mozconfig['path']))

   172                 return 1

   174             options.profile = options.profile or os.environ.get('GAIA_PROFILE')

   175             if not options.profile:

   176                 print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')

   177                 return 1

   179             if os.path.isfile(os.path.join(options.profile, 'extensions', \

   180                     'httpd@gaiamobile.org')):

   181                 print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',

   182                                                options.profile))

   183                 return 1

   185             options.desktop = True

   186             options.app = self.get_binary_path()

   187             if not options.app.endswith('-bin'):

   188                 options.app = '%s-bin' % options.app

   189             if not os.path.isfile(options.app):

   190                 options.app = options.app[:-len('-bin')]

   192             return reftest.run_desktop_reftests(parser, options, args)

   195         try:

   196             which.which('adb')

   197         except which.WhichError:

   198             # TODO Find adb automatically if it isn't on the path

   199             raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))

   201         options.b2gPath = b2g_home

   202         options.logcat_dir = self.reftest_dir

   203         options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')

   204         options.xrePath = xre_path

   205         options.ignoreWindowSize = True

   206         return reftest.run_remote_reftests(parser, options, args)

   208     def run_desktop_test(self, test_file=None, filter=None, suite=None,

   209             debugger=None, parallel=False, shuffle=False,

   210             e10s=False, this_chunk=None, total_chunks=None):

   211         """Runs a reftest.

   213         test_file is a path to a test file. It can be a relative path from the

   214         top source directory, an absolute filename, or a directory containing

   215         test files.

   217         filter is a regular expression (in JS syntax, as could be passed to the

   218         RegExp constructor) to select which reftests to run from the manifest.

   220         suite is the type of reftest to run. It can be one of ('reftest',

   221         'crashtest').

   223         debugger is the program name (in $PATH) or the full path of the

   224         debugger to run.

   226         parallel indicates whether tests should be run in parallel or not.

   228         shuffle indicates whether to run tests in random order.

   229         """

   231         if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'):

   232             raise Exception('None or unrecognized reftest suite type.')

   234         env = {}

   235         extra_args = []

   237         if test_file:

   238             path = self._find_manifest(suite, test_file)

   239             if not os.path.exists(mozpack.path.join(self.topsrcdir, path)):

   240                 raise Exception('No manifest file was found at %s.' % path)

   241             env[b'TEST_PATH'] = path

   242         if filter:

   243             extra_args.extend(['--filter', self._make_shell_string(filter)])

   245         pass_thru = False

   247         if debugger:

   248             extra_args.append('--debugger=%s' % debugger)

   249             pass_thru = True

   251         if parallel:

   252             extra_args.append('--run-tests-in-parallel')

   254         if shuffle:

   255             extra_args.append('--shuffle')

   257         if e10s:

   258             extra_args.append('--e10s')

   260         if this_chunk:

   261             extra_args.append('--this-chunk=%s' % this_chunk)

   263         if total_chunks:

   264             extra_args.append('--total-chunks=%s' % total_chunks)

   266         if extra_args:

   267             args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]

   268             args.extend(extra_args)

   269             env[b'EXTRA_TEST_ARGS'] = ' '.join(args)

   271         # TODO hook up harness via native Python

   272         return self._run_make(directory='.', target=suite, append_env=env,

   273             pass_thru=pass_thru, ensure_exit_code=False)

   276 def ReftestCommand(func):

   277     """Decorator that adds shared command arguments to reftest commands."""

   279     debugger = CommandArgument('--debugger', metavar='DEBUGGER',

   280         help=DEBUGGER_HELP)

   281     func = debugger(func)

   283     flter = CommandArgument('--filter', metavar='REGEX',

   284         help='A JS regular expression to match test URLs against, to select '

   285              'a subset of tests to run.')

   286     func = flter(func)

   288     path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',

   289         help='Reftest manifest file, or a directory in which to select '

   290              'reftest.list. If omitted, the entire test suite is executed.')

   291     func = path(func)

   293     parallel = CommandArgument('--parallel', action='store_true',

   294         help='Run tests in parallel.')

   295     func = parallel(func)

   297     shuffle = CommandArgument('--shuffle', action='store_true',

   298         help='Run tests in random order.')

   299     func = shuffle(func)

   301     e10s = CommandArgument('--e10s', action='store_true',

   302         help='Use content processes.')

   303     func = e10s(func)

   305     totalChunks = CommandArgument('--total-chunks',

   306         help = 'How many chunks to split the tests up into.')

   307     func = totalChunks(func)

   309     thisChunk = CommandArgument('--this-chunk',

   310         help = 'Which chunk to run between 1 and --total-chunks.')

   311     func = thisChunk(func)

   313     return func

   315 def B2GCommand(func):

   316     """Decorator that adds shared command arguments to b2g mochitest commands."""

   318     busybox = CommandArgument('--busybox', default=None,

   319         help='Path to busybox binary to install on device')

   320     func = busybox(func)

   322     logcatdir = CommandArgument('--logcat-dir', default=None,

   323         help='directory to store logcat dump files')

   324     func = logcatdir(func)

   326     geckopath = CommandArgument('--gecko-path', default=None,

   327         help='the path to a gecko distribution that should \

   328               be installed on the emulator prior to test')

   329     func = geckopath(func)

   331     sdcard = CommandArgument('--sdcard', default="10MB",

   332         help='Define size of sdcard: 1MB, 50MB...etc')

   333     func = sdcard(func)

   335     emulator_res = CommandArgument('--emulator-res', default='800x1000',

   336         help='Emulator resolution of the format \'<width>x<height>\'')

   337     func = emulator_res(func)

   339     emulator = CommandArgument('--emulator', default='arm',

   340         help='Architecture of emulator to use: x86 or arm')

   341     func = emulator(func)

   343     marionette = CommandArgument('--marionette', default=None,

   344         help='host:port to use when connecting to Marionette')

   345     func = marionette(func)

   347     totalChunks = CommandArgument('--total-chunks', dest='totalChunks',

   348         help = 'How many chunks to split the tests up into.')

   349     func = totalChunks(func)

   351     thisChunk = CommandArgument('--this-chunk', dest='thisChunk',

   352         help = 'Which chunk to run between 1 and --total-chunks.')

   353     func = thisChunk(func)

   355     path = CommandArgument('test_file', default=None, nargs='?',

   356         metavar='TEST',

   357         help='Test to run. Can be specified as a single file, a ' \

   358             'directory, or omitted. If omitted, the entire test suite is ' \

   359             'executed.')

   360     func = path(func)

   362     return func

   365 @CommandProvider

   366 class MachCommands(MachCommandBase):

   367     @Command('reftest', category='testing', description='Run reftests.')

   368     @ReftestCommand

   369     def run_reftest(self, test_file, **kwargs):

   370         return self._run_reftest(test_file, suite='reftest', **kwargs)

   372     @Command('reftest-ipc', category='testing',

   373         description='Run IPC reftests.')

   374     @ReftestCommand

   375     def run_ipc(self, test_file, **kwargs):

   376         return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)

   378     @Command('crashtest', category='testing',

   379         description='Run crashtests.')

   380     @ReftestCommand

   381     def run_crashtest(self, test_file, **kwargs):

   382         return self._run_reftest(test_file, suite='crashtest', **kwargs)

   384     @Command('crashtest-ipc', category='testing',

   385         description='Run IPC crashtests.')

   386     @ReftestCommand

   387     def run_crashtest_ipc(self, test_file, **kwargs):

   388         return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)

   390     def _run_reftest(self, test_file=None, suite=None, **kwargs):

   391         reftest = self._spawn(ReftestRunner)

   392         return reftest.run_desktop_test(test_file, suite=suite, **kwargs)

   395 # TODO For now b2g commands will only work with the emulator,

   396 # they should be modified to work with all devices.

   397 def is_emulator(cls):

   398     """Emulator needs to be configured."""

   399     return cls.device_name.find('emulator') == 0

   402 @CommandProvider

   403 class B2GCommands(MachCommandBase):

   404     def __init__(self, context):

   405         MachCommandBase.__init__(self, context)

   407         for attr in ('b2g_home', 'xre_path', 'device_name'):

   408             setattr(self, attr, getattr(context, attr, None))

   410     @Command('reftest-remote', category='testing',

   411         description='Run a remote reftest.',

   412         conditions=[conditions.is_b2g, is_emulator])

   413     @B2GCommand

   414     def run_reftest_remote(self, test_file, **kwargs):

   415         return self._run_reftest(test_file, suite='reftest', **kwargs)

   417     @Command('reftest-b2g-desktop', category='testing',

   418         description='Run a b2g desktop reftest.',

   419         conditions=[conditions.is_b2g_desktop])

   420     @B2GCommand

   421     def run_reftest_b2g_desktop(self, test_file, **kwargs):

   422         return self._run_reftest(test_file, suite='reftest', **kwargs)

   424     @Command('crashtest-remote', category='testing',

   425         description='Run a remote crashtest.',

   426         conditions=[conditions.is_b2g, is_emulator])

   427     @B2GCommand

   428     def run_crashtest_remote(self, test_file, **kwargs):

   429         return self._run_reftest(test_file, suite='crashtest', **kwargs)

   431     def _run_reftest(self, test_file=None, suite=None, **kwargs):

   432         reftest = self._spawn(ReftestRunner)

   433         return reftest.run_b2g_test(self.b2g_home, self.xre_path,

   434             test_file, suite=suite, **kwargs)

The Tor Browser / file revision

layout/tools/reftest/mach_commands.py@925c144e1f1f

layout/tools/reftest/mach_commands.py