The Tor Browser: testing/xpcshell/mach

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 # Integrates the xpcshell test runner with mach.

     7 from __future__ import unicode_literals, print_function

     9 import mozpack.path

    10 import logging

    11 import os

    12 import shutil

    13 import sys

    14 import urllib2

    16 from StringIO import StringIO

    18 from mozbuild.base import (

    19     MachCommandBase,

    20     MozbuildObject,

    21     MachCommandConditions as conditions,

    22 )

    24 from mach.decorators import (

    25     CommandArgument,

    26     CommandProvider,

    27     Command,

    28 )

    30 ADB_NOT_FOUND = '''

    31 The %s command requires the adb binary to be on your path.

    33 If you have a B2G build, this can be found in

    34 '%s/out/host/<platform>/bin'.

    35 '''.lstrip()

    37 BUSYBOX_URL = 'http://www.busybox.net/downloads/binaries/latest/busybox-armv7l'

    40 if sys.version_info[0] < 3:

    41     unicode_type = unicode

    42 else:

    43     unicode_type = str

    45 # Simple filter to omit the message emitted as a test file begins.

    46 class TestStartFilter(logging.Filter):

    47     def filter(self, record):

    48         return not record.params['msg'].endswith("running test ...")

    50 # This should probably be consolidated with similar classes in other test

    51 # runners.

    52 class InvalidTestPathError(Exception):

    53     """Exception raised when the test path is not valid."""

    56 class XPCShellRunner(MozbuildObject):

    57     """Run xpcshell tests."""

    58     def run_suite(self, **kwargs):

    59         from manifestparser import TestManifest

    60         manifest = TestManifest(manifests=[os.path.join(self.topobjdir,

    61             '_tests', 'xpcshell', 'xpcshell.ini')])

    63         return self._run_xpcshell_harness(manifest=manifest, **kwargs)

    65     def run_test(self, test_paths, interactive=False,

    66                  keep_going=False, sequential=False, shuffle=False,

    67                  debugger=None, debuggerArgs=None, debuggerInteractive=None,

    68                  rerun_failures=False,

    69                  # ignore parameters from other platforms' options

    70                  **kwargs):

    71         """Runs an individual xpcshell test."""

    72         from mozbuild.testing import TestResolver

    73         from manifestparser import TestManifest

    75         # TODO Bug 794506 remove once mach integrates with virtualenv.

    76         build_path = os.path.join(self.topobjdir, 'build')

    77         if build_path not in sys.path:

    78             sys.path.append(build_path)

    80         if test_paths == ['all']:

    81             self.run_suite(interactive=interactive,

    82                            keep_going=keep_going, shuffle=shuffle, sequential=sequential,

    83                            debugger=debugger, debuggerArgs=debuggerArgs,

    84                            debuggerInteractive=debuggerInteractive,

    85                            rerun_failures=rerun_failures)

    86             return

    88         resolver = self._spawn(TestResolver)

    89         tests = list(resolver.resolve_tests(paths=test_paths, flavor='xpcshell',

    90             cwd=self.cwd))

    92         if not tests:

    93             raise InvalidTestPathError('We could not find an xpcshell test '

    94                 'for the passed test path. Please select a path that is '

    95                 'a test file or is a directory containing xpcshell tests.')

    97         # Dynamically write out a manifest holding all the discovered tests.

    98         manifest = TestManifest()

    99         manifest.tests.extend(tests)

   101         args = {

   102             'interactive': interactive,

   103             'keep_going': keep_going,

   104             'shuffle': shuffle,

   105             'sequential': sequential,

   106             'debugger': debugger,

   107             'debuggerArgs': debuggerArgs,

   108             'debuggerInteractive': debuggerInteractive,

   109             'rerun_failures': rerun_failures,

   110             'manifest': manifest,

   111         }

   113         return self._run_xpcshell_harness(**args)

   115     def _run_xpcshell_harness(self, manifest,

   116                               test_path=None, shuffle=False, interactive=False,

   117                               keep_going=False, sequential=False,

   118                               debugger=None, debuggerArgs=None, debuggerInteractive=None,

   119                               rerun_failures=False):

   121         # Obtain a reference to the xpcshell test runner.

   122         import runxpcshelltests

   124         dummy_log = StringIO()

   125         xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)

   126         self.log_manager.enable_unstructured()

   128         xpcshell_filter = TestStartFilter()

   129         self.log_manager.terminal_handler.addFilter(xpcshell_filter)

   131         tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')

   132         modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')

   133         # We want output from the test to be written immediately if we are only

   134         # running a single test.

   135         verbose_output = test_path is not None or (manifest and len(manifest.test_paths())==1)

   137         args = {

   138             'manifest': manifest,

   139             'xpcshell': os.path.join(self.bindir, 'xpcshell'),

   140             'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),

   141             'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),

   142             'interactive': interactive,

   143             'keepGoing': keep_going,

   144             'logfiles': False,

   145             'sequential': sequential,

   146             'shuffle': shuffle,

   147             'testsRootDir': tests_dir,

   148             'testingModulesDir': modules_dir,

   149             'profileName': 'firefox',

   150             'verbose': test_path is not None,

   151             'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),

   152             'xunitName': 'xpcshell',

   153             'pluginsPath': os.path.join(self.distdir, 'plugins'),

   154             'debugger': debugger,

   155             'debuggerArgs': debuggerArgs,

   156             'debuggerInteractive': debuggerInteractive,

   157             'on_message': (lambda obj, msg: xpcshell.log.info(msg.decode('utf-8', 'replace'))) \

   158                             if verbose_output else None,

   159         }

   161         if test_path is not None:

   162             args['testPath'] = test_path

   164         # A failure manifest is written by default. If --rerun-failures is

   165         # specified and a prior failure manifest is found, the prior manifest

   166         # will be run. A new failure manifest is always written over any

   167         # prior failure manifest.

   168         failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')

   169         rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')

   170         if os.path.exists(failure_manifest_path) and rerun_failures:

   171             shutil.move(failure_manifest_path, rerun_manifest_path)

   172             args['manifest'] = rerun_manifest_path

   173         elif os.path.exists(failure_manifest_path):

   174             os.remove(failure_manifest_path)

   175         elif rerun_failures:

   176             print("No failures were found to re-run.")

   177             return 0

   178         args['failureManifest'] = failure_manifest_path

   180         # Python through 2.7.2 has issues with unicode in some of the

   181         # arguments. Work around that.

   182         filtered_args = {}

   183         for k, v in args.items():

   184             if isinstance(v, unicode_type):

   185                 v = v.encode('utf-8')

   187             if isinstance(k, unicode_type):

   188                 k = k.encode('utf-8')

   190             filtered_args[k] = v

   192         result = xpcshell.runTests(**filtered_args)

   194         self.log_manager.terminal_handler.removeFilter(xpcshell_filter)

   195         self.log_manager.disable_unstructured()

   197         if not result and not xpcshell.sequential:

   198             print("Tests were run in parallel. Try running with --sequential "

   199                   "to make sure the failures were not caused by this.")

   200         return int(not result)

   202 class AndroidXPCShellRunner(MozbuildObject):

   203     """Get specified DeviceManager"""

   204     def get_devicemanager(self, devicemanager, ip, port, remote_test_root):

   205         from mozdevice import devicemanagerADB, devicemanagerSUT

   206         dm = None

   207         if devicemanager == "adb":

   208             if ip:

   209                 dm = devicemanagerADB.DeviceManagerADB(ip, port, packageName=None, deviceRoot=remote_test_root)

   210             else:

   211                 dm = devicemanagerADB.DeviceManagerADB(packageName=None, deviceRoot=remote_test_root)

   212         else:

   213             if ip:

   214                 dm = devicemanagerSUT.DeviceManagerSUT(ip, port, deviceRoot=remote_test_root)

   215             else:

   216                 raise Exception("You must provide a device IP to connect to via the --ip option")

   217         return dm

   219     """Run Android xpcshell tests."""

   220     def run_test(self,

   221                  test_paths, keep_going,

   222                  devicemanager, ip, port, remote_test_root, no_setup, local_apk,

   223                  # ignore parameters from other platforms' options

   224                  **kwargs):

   225         # TODO Bug 794506 remove once mach integrates with virtualenv.

   226         build_path = os.path.join(self.topobjdir, 'build')

   227         if build_path not in sys.path:

   228             sys.path.append(build_path)

   230         import remotexpcshelltests

   232         dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)

   234         options = remotexpcshelltests.RemoteXPCShellOptions()

   235         options.shuffle = False

   236         options.sequential = True

   237         options.interactive = False

   238         options.debugger = None

   239         options.debuggerArgs = None

   240         options.setup = not no_setup

   241         options.keepGoing = keep_going

   242         options.objdir = self.topobjdir

   243         options.localLib = os.path.join(self.topobjdir, 'dist/fennec')

   244         options.localBin = os.path.join(self.topobjdir, 'dist/bin')

   245         options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')

   246         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')

   247         options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell_android.ini')

   248         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')

   249         if local_apk:

   250             options.localAPK = local_apk

   251         else:

   252             for file in os.listdir(os.path.join(options.objdir, "dist")):

   253                 if file.endswith(".apk") and file.startswith("fennec"):

   254                     options.localAPK = os.path.join(options.objdir, "dist")

   255                     options.localAPK = os.path.join(options.localAPK, file)

   256                     print ("using APK: " + options.localAPK)

   257                     break

   258             else:

   259                 raise Exception("You must specify an APK")

   261         if test_paths == ['all']:

   262             testdirs = []

   263             options.testPath = None

   264             options.verbose = False

   265         else:

   266             if len(test_paths) > 1:

   267                 print('Warning: only the first test path argument will be used.')

   268             testdirs = test_paths[0]

   269             options.testPath = test_paths[0]

   270             options.verbose = True

   271         dummy_log = StringIO()

   272         xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, args=testdirs, log=dummy_log)

   273         self.log_manager.enable_unstructured()

   275         xpcshell_filter = TestStartFilter()

   276         self.log_manager.terminal_handler.addFilter(xpcshell_filter)

   278         result = xpcshell.runTests(xpcshell='xpcshell',

   279                       testClass=remotexpcshelltests.RemoteXPCShellTestThread,

   280                       testdirs=testdirs,

   281                       mobileArgs=xpcshell.mobileArgs,

   282                       **options.__dict__)

   284         self.log_manager.terminal_handler.removeFilter(xpcshell_filter)

   285         self.log_manager.disable_unstructured()

   287         return int(not result)

   289 class B2GXPCShellRunner(MozbuildObject):

   290     def __init__(self, *args, **kwargs):

   291         MozbuildObject.__init__(self, *args, **kwargs)

   293         # TODO Bug 794506 remove once mach integrates with virtualenv.

   294         build_path = os.path.join(self.topobjdir, 'build')

   295         if build_path not in sys.path:

   296             sys.path.append(build_path)

   298         build_path = os.path.join(self.topsrcdir, 'build')

   299         if build_path not in sys.path:

   300             sys.path.append(build_path)

   302         self.tests_dir = os.path.join(self.topobjdir, '_tests')

   303         self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell')

   304         self.bin_dir = os.path.join(self.distdir, 'bin')

   306     def _download_busybox(self, b2g_home):

   307         system_bin = os.path.join(b2g_home, 'out', 'target', 'product', 'generic', 'system', 'bin')

   308         busybox_path = os.path.join(system_bin, 'busybox')

   310         if os.path.isfile(busybox_path):

   311             return busybox_path

   313         if not os.path.isdir(system_bin):

   314             os.makedirs(system_bin)

   316         try:

   317             data = urllib2.urlopen(BUSYBOX_URL)

   318         except urllib2.URLError:

   319             print('There was a problem downloading busybox. Proceeding without it,' \

   320                   'initial setup will be slow.')

   321             return

   323         with open(busybox_path, 'wb') as f:

   324             f.write(data.read())

   325         return busybox_path

   327     def run_test(self, test_paths, b2g_home=None, busybox=None,

   328                  # ignore parameters from other platforms' options

   329                  **kwargs):

   330         try:

   331             import which

   332             which.which('adb')

   333         except which.WhichError:

   334             # TODO Find adb automatically if it isn't on the path

   335             print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))

   336             sys.exit(1)

   338         test_path = None

   339         if test_paths:

   340             if len(test_paths) > 1:

   341                 print('Warning: Only the first test path will be used.')

   343             test_path = self._wrap_path_argument(test_paths[0]).relpath()

   345         import runtestsb2g

   346         parser = runtestsb2g.B2GOptions()

   347         options, args = parser.parse_args([])

   349         options.b2g_path = b2g_home

   350         options.busybox = busybox or os.environ.get('BUSYBOX')

   351         options.emulator = 'arm'

   352         options.localLib = self.bin_dir

   353         options.localBin = self.bin_dir

   354         options.logcat_dir = self.xpcshell_dir

   355         options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell_b2g.ini')

   356         options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')

   357         options.objdir = self.topobjdir

   358         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),

   359         options.testingModulesDir = os.path.join(self.tests_dir, 'modules')

   360         options.testsRootDir = self.xpcshell_dir

   361         options.testPath = test_path

   362         options.use_device_libs = True

   364         if not options.busybox:

   365             options.busybox = self._download_busybox(b2g_home)

   367         return runtestsb2g.run_remote_xpcshell(parser, options, args)

   369 def is_platform_supported(cls):

   370     """Must have a Firefox, Android or B2G build."""

   371     return conditions.is_android(cls) or \

   372            conditions.is_b2g(cls) or \

   373            conditions.is_firefox(cls)

   375 @CommandProvider

   376 class MachCommands(MachCommandBase):

   377     def __init__(self, context):

   378         MachCommandBase.__init__(self, context)

   380         for attr in ('b2g_home', 'device_name'):

   381             setattr(self, attr, getattr(context, attr, None))

   383     @Command('xpcshell-test', category='testing',

   384         conditions=[is_platform_supported],

   385         description='Run XPCOM Shell tests.')

   386     @CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',

   387         help='Test to run. Can be specified as a single JS file, a directory, '

   388              'or omitted. If omitted, the entire test suite is executed.')

   389     @CommandArgument("--debugger", default=None, metavar='DEBUGGER',

   390                      help = "Run xpcshell under the given debugger.")

   391     @CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,

   392                      dest = "debuggerArgs",

   393                      help = "pass the given args to the debugger _before_ "

   394                             "the application on the command line")

   395     @CommandArgument("--debugger-interactive", action = "store_true",

   396                      dest = "debuggerInteractive",

   397                      help = "prevents the test harness from redirecting "

   398                             "stdout and stderr for interactive debuggers")

   399     @CommandArgument('--interactive', '-i', action='store_true',

   400         help='Open an xpcshell prompt before running tests.')

   401     @CommandArgument('--keep-going', '-k', action='store_true',

   402         help='Continue running tests after a SIGINT is received.')

   403     @CommandArgument('--sequential', action='store_true',

   404         help='Run the tests sequentially.')

   405     @CommandArgument('--shuffle', '-s', action='store_true',

   406         help='Randomize the execution order of tests.')

   407     @CommandArgument('--rerun-failures', action='store_true',

   408         help='Reruns failures from last time.')

   409     @CommandArgument('--devicemanager', default='adb', type=str,

   410         help='(Android) Type of devicemanager to use for communication: adb or sut')

   411     @CommandArgument('--ip', type=str, default=None,

   412         help='(Android) IP address of device')

   413     @CommandArgument('--port', type=int, default=20701,

   414         help='(Android) Port of device')

   415     @CommandArgument('--remote_test_root', type=str, default=None,

   416         help='(Android) Remote test root such as /mnt/sdcard or /data/local')

   417     @CommandArgument('--no-setup', action='store_true',

   418         help='(Android) Do not copy files to device')

   419     @CommandArgument('--local-apk', type=str, default=None,

   420         help='(Android) Use specified Fennec APK')

   421     @CommandArgument('--busybox', type=str, default=None,

   422         help='(B2G) Path to busybox binary (speeds up installation of tests).')

   423     def run_xpcshell_test(self, **params):

   424         from mozbuild.controller.building import BuildDriver

   426         # We should probably have a utility function to ensure the tree is

   427         # ready to run tests. Until then, we just create the state dir (in

   428         # case the tree wasn't built with mach).

   429         self._ensure_state_subdir_exists('.')

   431         driver = self._spawn(BuildDriver)

   432         driver.install_tests(remove=False)

   434         if conditions.is_android(self):

   435             xpcshell = self._spawn(AndroidXPCShellRunner)

   436         elif conditions.is_b2g(self):

   437             xpcshell = self._spawn(B2GXPCShellRunner)

   438             params['b2g_home'] = self.b2g_home

   439         else:

   440             xpcshell = self._spawn(XPCShellRunner)

   441         xpcshell.cwd = self._mach_context.cwd

   443         try:

   444             return xpcshell.run_test(**params)

   445         except InvalidTestPathError as e:

   446             print(e.message)

   447             return 1

The Tor Browser / file revision

testing/xpcshell/mach_commands.py@7e26c7da4463

testing/xpcshell/mach_commands.py