1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/media/webrtc/trunk/build/android/pylib/single_test_runner.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,343 @@
1.4 +# Copyright (c) 2012 The Chromium Authors. All rights reserved.
1.5 +# Use of this source code is governed by a BSD-style license that can be
1.6 +# found in the LICENSE file.
1.7 +
1.8 +import glob
1.9 +import logging
1.10 +import os
1.11 +import sys
1.12 +
1.13 +from base_test_runner import BaseTestRunner
1.14 +import debug_info
1.15 +import constants
1.16 +import perf_tests_helper
1.17 +import run_tests_helper
1.18 +from test_package_apk import TestPackageApk
1.19 +from test_package_executable import TestPackageExecutable
1.20 +from test_result import TestResults
1.21 +
1.22 +
1.23 +class SingleTestRunner(BaseTestRunner):
1.24 + """Single test suite attached to a single device.
1.25 +
1.26 + Args:
1.27 + device: Device to run the tests.
1.28 + test_suite: A specific test suite to run, empty to run all.
1.29 + gtest_filter: A gtest_filter flag.
1.30 + test_arguments: Additional arguments to pass to the test binary.
1.31 + timeout: Timeout for each test.
1.32 + rebaseline: Whether or not to run tests in isolation and update the filter.
1.33 + performance_test: Whether or not performance test(s).
1.34 + cleanup_test_files: Whether or not to cleanup test files on device.
1.35 + tool: Name of the Valgrind tool.
1.36 + shard_index: index number of the shard on which the test suite will run.
1.37 + dump_debug_info: Whether or not to dump debug information.
1.38 + build_type: 'Release' or 'Debug'.
1.39 + """
1.40 +
1.41 + def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout,
1.42 + rebaseline, performance_test, cleanup_test_files, tool_name,
1.43 + shard_index, dump_debug_info, fast_and_loose, build_type):
1.44 + BaseTestRunner.__init__(self, device, tool_name, shard_index, build_type)
1.45 + self._running_on_emulator = self.device.startswith('emulator')
1.46 + self._gtest_filter = gtest_filter
1.47 + self._test_arguments = test_arguments
1.48 + self.test_results = TestResults()
1.49 + if dump_debug_info:
1.50 + self.dump_debug_info = debug_info.GTestDebugInfo(self.adb, device,
1.51 + os.path.basename(test_suite), gtest_filter)
1.52 + else:
1.53 + self.dump_debug_info = None
1.54 + self.fast_and_loose = fast_and_loose
1.55 +
1.56 + logging.warning('Test suite: ' + test_suite)
1.57 + if os.path.splitext(test_suite)[1] == '.apk':
1.58 + self.test_package = TestPackageApk(self.adb, device,
1.59 + test_suite, timeout, rebaseline, performance_test, cleanup_test_files,
1.60 + self.tool, self.dump_debug_info)
1.61 + else:
1.62 + self.test_package = TestPackageExecutable(
1.63 + self.adb, device,
1.64 + test_suite, timeout, rebaseline, performance_test, cleanup_test_files,
1.65 + self.tool, self.dump_debug_info)
1.66 + self._performance_test_setup = None
1.67 + if performance_test:
1.68 + self._performance_test_setup = perf_tests_helper.PerfTestSetup(self.adb)
1.69 +
1.70 + def _TestSuiteRequiresMockTestServer(self):
1.71 + """Returns True if the test suite requires mock test server."""
1.72 + return False
1.73 + # TODO(yfriedman): Disabled because of flakiness.
1.74 + # (self.test_package.test_suite_basename == 'unit_tests' or
1.75 + # self.test_package.test_suite_basename == 'net_unittests' or
1.76 + # False)
1.77 +
1.78 + def _GetFilterFileName(self):
1.79 + """Returns the filename of gtest filter."""
1.80 + return os.path.join(sys.path[0], 'gtest_filter',
1.81 + self.test_package.test_suite_basename + '_disabled')
1.82 +
1.83 + def _GetAdditionalEmulatorFilterName(self):
1.84 + """Returns the filename of additional gtest filter for emulator."""
1.85 + return os.path.join(sys.path[0], 'gtest_filter',
1.86 + self.test_package.test_suite_basename +
1.87 + '_emulator_additional_disabled')
1.88 +
1.89 + def GetDisabledTests(self):
1.90 + """Returns a list of disabled tests.
1.91 +
1.92 + Returns:
1.93 + A list of disabled tests obtained from gtest_filter/test_suite_disabled.
1.94 + """
1.95 + disabled_tests = run_tests_helper.GetExpectations(self._GetFilterFileName())
1.96 + if self._running_on_emulator:
1.97 + # Append emulator's filter file.
1.98 + disabled_tests.extend(run_tests_helper.GetExpectations(
1.99 + self._GetAdditionalEmulatorFilterName()))
1.100 + return disabled_tests
1.101 +
1.102 + def UpdateFilter(self, failed_tests):
1.103 + """Updates test_suite_disabled file with the new filter (deletes if empty).
1.104 +
1.105 + If running in Emulator, only the failed tests which are not in the normal
1.106 + filter returned by _GetFilterFileName() are written to emulator's
1.107 + additional filter file.
1.108 +
1.109 + Args:
1.110 + failed_tests: A sorted list of failed tests.
1.111 + """
1.112 + disabled_tests = []
1.113 + if not self._running_on_emulator:
1.114 + filter_file_name = self._GetFilterFileName()
1.115 + else:
1.116 + filter_file_name = self._GetAdditionalEmulatorFilterName()
1.117 + disabled_tests.extend(
1.118 + run_tests_helper.GetExpectations(self._GetFilterFileName()))
1.119 + logging.info('About to update emulator\'s additional filter (%s).'
1.120 + % filter_file_name)
1.121 +
1.122 + new_failed_tests = []
1.123 + if failed_tests:
1.124 + for test in failed_tests:
1.125 + if test.name not in disabled_tests:
1.126 + new_failed_tests.append(test.name)
1.127 +
1.128 + if not new_failed_tests:
1.129 + if os.path.exists(filter_file_name):
1.130 + os.unlink(filter_file_name)
1.131 + return
1.132 +
1.133 + filter_file = file(filter_file_name, 'w')
1.134 + if self._running_on_emulator:
1.135 + filter_file.write('# Addtional list of suppressions from emulator\n')
1.136 + else:
1.137 + filter_file.write('# List of suppressions\n')
1.138 + filter_file.write('# This file was automatically generated by %s\n'
1.139 + % sys.argv[0])
1.140 + filter_file.write('\n'.join(sorted(new_failed_tests)))
1.141 + filter_file.write('\n')
1.142 + filter_file.close()
1.143 +
1.144 + def GetDataFilesForTestSuite(self):
1.145 + """Returns a list of data files/dirs needed by the test suite."""
1.146 + # Ideally, we'd just push all test data. However, it has >100MB, and a lot
1.147 + # of the files are not relevant (some are used for browser_tests, others for
1.148 + # features not supported, etc..).
1.149 + if self.test_package.test_suite_basename in ['base_unittests',
1.150 + 'sql_unittests',
1.151 + 'unit_tests']:
1.152 + test_files = [
1.153 + 'base/data/file_util_unittest',
1.154 + 'base/data/json/bom_feff.json',
1.155 + 'chrome/test/data/download-test1.lib',
1.156 + 'chrome/test/data/extensions/bad_magic.crx',
1.157 + 'chrome/test/data/extensions/good.crx',
1.158 + 'chrome/test/data/extensions/icon1.png',
1.159 + 'chrome/test/data/extensions/icon2.png',
1.160 + 'chrome/test/data/extensions/icon3.png',
1.161 + 'chrome/test/data/extensions/allow_silent_upgrade/',
1.162 + 'chrome/test/data/extensions/app/',
1.163 + 'chrome/test/data/extensions/bad/',
1.164 + 'chrome/test/data/extensions/effective_host_permissions/',
1.165 + 'chrome/test/data/extensions/empty_manifest/',
1.166 + 'chrome/test/data/extensions/good/Extensions/',
1.167 + 'chrome/test/data/extensions/manifest_tests/',
1.168 + 'chrome/test/data/extensions/page_action/',
1.169 + 'chrome/test/data/extensions/permissions/',
1.170 + 'chrome/test/data/extensions/script_and_capture/',
1.171 + 'chrome/test/data/extensions/unpacker/',
1.172 + 'chrome/test/data/bookmarks/',
1.173 + 'chrome/test/data/components/',
1.174 + 'chrome/test/data/extensions/json_schema_test.js',
1.175 + 'chrome/test/data/History/',
1.176 + 'chrome/test/data/json_schema_validator/',
1.177 + 'chrome/test/data/pref_service/',
1.178 + 'chrome/test/data/serializer_nested_test.js',
1.179 + 'chrome/test/data/serializer_test.js',
1.180 + 'chrome/test/data/serializer_test_nowhitespace.js',
1.181 + 'chrome/test/data/top_sites/',
1.182 + 'chrome/test/data/web_app_info/',
1.183 + 'chrome/test/data/web_database',
1.184 + 'chrome/test/data/webui/',
1.185 + 'chrome/test/data/zip',
1.186 + 'chrome/third_party/mock4js/',
1.187 + 'content/browser/gpu/software_rendering_list.json',
1.188 + 'net/data/cache_tests/insert_load1',
1.189 + 'net/data/cache_tests/dirty_entry5',
1.190 + 'net/data/ssl/certificates/',
1.191 + 'ui/base/test/data/data_pack_unittest',
1.192 + ]
1.193 + if self.test_package.test_suite_basename == 'unit_tests':
1.194 + test_files += ['chrome/test/data/simple_open_search.xml']
1.195 + # The following are spell check data. Now only list the data under
1.196 + # third_party/hunspell_dictionaries which are used by unit tests.
1.197 + old_cwd = os.getcwd()
1.198 + os.chdir(constants.CHROME_DIR)
1.199 + test_files += glob.glob('third_party/hunspell_dictionaries/*.bdic')
1.200 + os.chdir(old_cwd)
1.201 + return test_files
1.202 + elif self.test_package.test_suite_basename == 'net_unittests':
1.203 + return [
1.204 + 'net/data/cache_tests',
1.205 + 'net/data/filter_unittests',
1.206 + 'net/data/ftp',
1.207 + 'net/data/proxy_resolver_v8_unittest',
1.208 + 'net/data/ssl/certificates',
1.209 + 'net/data/url_request_unittest/',
1.210 + 'net/data/proxy_script_fetcher_unittest'
1.211 + ]
1.212 + elif self.test_package.test_suite_basename == 'ui_tests':
1.213 + return [
1.214 + 'chrome/test/data/dromaeo',
1.215 + 'chrome/test/data/json2.js',
1.216 + 'chrome/test/data/sunspider',
1.217 + 'chrome/test/data/v8_benchmark',
1.218 + 'chrome/test/perf/sunspider_uitest.js',
1.219 + 'chrome/test/perf/v8_benchmark_uitest.js',
1.220 + ]
1.221 + elif self.test_package.test_suite_basename == 'page_cycler_tests':
1.222 + data = [
1.223 + 'tools/page_cycler',
1.224 + 'data/page_cycler',
1.225 + ]
1.226 + for d in data:
1.227 + if not os.path.exists(d):
1.228 + raise Exception('Page cycler data not found.')
1.229 + return data
1.230 + elif self.test_package.test_suite_basename == 'webkit_unit_tests':
1.231 + return [
1.232 + 'third_party/WebKit/Source/WebKit/chromium/tests/data',
1.233 + ]
1.234 + elif self.test_package.test_suite_basename == 'content_unittests':
1.235 + return [
1.236 + 'content/test/data/gpu/webgl_conformance_test_expectations.txt',
1.237 + 'net/data/ssl/certificates/',
1.238 + 'webkit/data/dom_storage/webcore_test_database.localstorage',
1.239 + 'third_party/hyphen/hyph_en_US.dic',
1.240 + ]
1.241 + elif self.test_package.test_suite_basename == 'media_unittests':
1.242 + return [
1.243 + 'media/test/data',
1.244 + ]
1.245 + return []
1.246 +
1.247 + def LaunchHelperToolsForTestSuite(self):
1.248 + """Launches helper tools for the test suite.
1.249 +
1.250 + Sometimes one test may need to run some helper tools first in order to
1.251 + successfully complete the test.
1.252 + """
1.253 + if self._TestSuiteRequiresMockTestServer():
1.254 + self.LaunchChromeTestServerSpawner()
1.255 +
1.256 + def StripAndCopyFiles(self):
1.257 + """Strips and copies the required data files for the test suite."""
1.258 + self.test_package.StripAndCopyExecutable()
1.259 + self.test_package.PushDataAndPakFiles()
1.260 + self.tool.CopyFiles()
1.261 + test_data = self.GetDataFilesForTestSuite()
1.262 + if test_data and not self.fast_and_loose:
1.263 + # Make sure SD card is ready.
1.264 + self.adb.WaitForSdCardReady(20)
1.265 + for data in test_data:
1.266 + self.CopyTestData([data], self.adb.GetExternalStorage())
1.267 +
1.268 + def RunTestsWithFilter(self):
1.269 + """Runs a tests via a small, temporary shell script."""
1.270 + self.test_package.CreateTestRunnerScript(self._gtest_filter,
1.271 + self._test_arguments)
1.272 + self.test_results = self.test_package.RunTestsAndListResults()
1.273 +
1.274 + def RebaselineTests(self):
1.275 + """Runs all available tests, restarting in case of failures."""
1.276 + if self._gtest_filter:
1.277 + all_tests = set(self._gtest_filter.split(':'))
1.278 + else:
1.279 + all_tests = set(self.test_package.GetAllTests())
1.280 + failed_results = set()
1.281 + executed_results = set()
1.282 + while True:
1.283 + executed_names = set([f.name for f in executed_results])
1.284 + self._gtest_filter = ':'.join(all_tests - executed_names)
1.285 + self.RunTestsWithFilter()
1.286 + failed_results.update(self.test_results.crashed,
1.287 + self.test_results.failed)
1.288 + executed_results.update(self.test_results.crashed,
1.289 + self.test_results.failed,
1.290 + self.test_results.ok)
1.291 + executed_names = set([f.name for f in executed_results])
1.292 + logging.info('*' * 80)
1.293 + logging.info(self.device)
1.294 + logging.info('Executed: ' + str(len(executed_names)) + ' of ' +
1.295 + str(len(all_tests)))
1.296 + logging.info('Failed so far: ' + str(len(failed_results)) + ' ' +
1.297 + str([f.name for f in failed_results]))
1.298 + logging.info('Remaining: ' + str(len(all_tests - executed_names)) + ' ' +
1.299 + str(all_tests - executed_names))
1.300 + logging.info('*' * 80)
1.301 + if executed_names == all_tests:
1.302 + break
1.303 + self.test_results = TestResults.FromRun(
1.304 + ok=list(executed_results - failed_results),
1.305 + failed=list(failed_results))
1.306 +
1.307 + def RunTests(self):
1.308 + """Runs all tests (in rebaseline mode, runs each test in isolation).
1.309 +
1.310 + Returns:
1.311 + A TestResults object.
1.312 + """
1.313 + if self.test_package.rebaseline:
1.314 + self.RebaselineTests()
1.315 + else:
1.316 + if not self._gtest_filter:
1.317 + self._gtest_filter = ('-' + ':'.join(self.GetDisabledTests()) + ':' +
1.318 + ':'.join(['*.' + x + '*' for x in
1.319 + self.test_package.GetDisabledPrefixes()]))
1.320 + self.RunTestsWithFilter()
1.321 + return self.test_results
1.322 +
1.323 + def SetUp(self):
1.324 + """Sets up necessary test enviroment for the test suite."""
1.325 + super(SingleTestRunner, self).SetUp()
1.326 + self.adb.ClearApplicationState(constants.CHROME_PACKAGE)
1.327 + if self._performance_test_setup:
1.328 + self._performance_test_setup.SetUp()
1.329 + if self.dump_debug_info:
1.330 + self.dump_debug_info.StartRecordingLog(True)
1.331 + self.StripAndCopyFiles()
1.332 + self.LaunchHelperToolsForTestSuite()
1.333 + self.tool.SetupEnvironment()
1.334 +
1.335 + def TearDown(self):
1.336 + """Cleans up the test enviroment for the test suite."""
1.337 + self.tool.CleanUpEnvironment()
1.338 + if self.test_package.cleanup_test_files:
1.339 + self.adb.RemovePushedFiles()
1.340 + if self.dump_debug_info:
1.341 + self.dump_debug_info.StopRecordingLog()
1.342 + if self._performance_test_setup:
1.343 + self._performance_test_setup.TearDown()
1.344 + if self.dump_debug_info:
1.345 + self.dump_debug_info.ArchiveNewCrashFiles()
1.346 + super(SingleTestRunner, self).TearDown()
