michael@0: # Copyright (c) 2012 The Chromium Authors. All rights reserved. michael@0: # Use of this source code is governed by a BSD-style license that can be michael@0: # found in the LICENSE file. michael@0: michael@0: import glob michael@0: import logging michael@0: import os michael@0: import sys michael@0: michael@0: from base_test_runner import BaseTestRunner michael@0: import debug_info michael@0: import constants michael@0: import perf_tests_helper michael@0: import run_tests_helper michael@0: from test_package_apk import TestPackageApk michael@0: from test_package_executable import TestPackageExecutable michael@0: from test_result import TestResults michael@0: michael@0: michael@0: class SingleTestRunner(BaseTestRunner): michael@0: """Single test suite attached to a single device. michael@0: michael@0: Args: michael@0: device: Device to run the tests. michael@0: test_suite: A specific test suite to run, empty to run all. michael@0: gtest_filter: A gtest_filter flag. michael@0: test_arguments: Additional arguments to pass to the test binary. michael@0: timeout: Timeout for each test. michael@0: rebaseline: Whether or not to run tests in isolation and update the filter. michael@0: performance_test: Whether or not performance test(s). michael@0: cleanup_test_files: Whether or not to cleanup test files on device. michael@0: tool: Name of the Valgrind tool. michael@0: shard_index: index number of the shard on which the test suite will run. michael@0: dump_debug_info: Whether or not to dump debug information. michael@0: build_type: 'Release' or 'Debug'. michael@0: """ michael@0: michael@0: def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout, michael@0: rebaseline, performance_test, cleanup_test_files, tool_name, michael@0: shard_index, dump_debug_info, fast_and_loose, build_type): michael@0: BaseTestRunner.__init__(self, device, tool_name, shard_index, build_type) michael@0: self._running_on_emulator = self.device.startswith('emulator') michael@0: self._gtest_filter = gtest_filter michael@0: self._test_arguments = test_arguments michael@0: self.test_results = TestResults() michael@0: if dump_debug_info: michael@0: self.dump_debug_info = debug_info.GTestDebugInfo(self.adb, device, michael@0: os.path.basename(test_suite), gtest_filter) michael@0: else: michael@0: self.dump_debug_info = None michael@0: self.fast_and_loose = fast_and_loose michael@0: michael@0: logging.warning('Test suite: ' + test_suite) michael@0: if os.path.splitext(test_suite)[1] == '.apk': michael@0: self.test_package = TestPackageApk(self.adb, device, michael@0: test_suite, timeout, rebaseline, performance_test, cleanup_test_files, michael@0: self.tool, self.dump_debug_info) michael@0: else: michael@0: self.test_package = TestPackageExecutable( michael@0: self.adb, device, michael@0: test_suite, timeout, rebaseline, performance_test, cleanup_test_files, michael@0: self.tool, self.dump_debug_info) michael@0: self._performance_test_setup = None michael@0: if performance_test: michael@0: self._performance_test_setup = perf_tests_helper.PerfTestSetup(self.adb) michael@0: michael@0: def _TestSuiteRequiresMockTestServer(self): michael@0: """Returns True if the test suite requires mock test server.""" michael@0: return False michael@0: # TODO(yfriedman): Disabled because of flakiness. michael@0: # (self.test_package.test_suite_basename == 'unit_tests' or michael@0: # self.test_package.test_suite_basename == 'net_unittests' or michael@0: # False) michael@0: michael@0: def _GetFilterFileName(self): michael@0: """Returns the filename of gtest filter.""" michael@0: return os.path.join(sys.path[0], 'gtest_filter', michael@0: self.test_package.test_suite_basename + '_disabled') michael@0: michael@0: def _GetAdditionalEmulatorFilterName(self): michael@0: """Returns the filename of additional gtest filter for emulator.""" michael@0: return os.path.join(sys.path[0], 'gtest_filter', michael@0: self.test_package.test_suite_basename + michael@0: '_emulator_additional_disabled') michael@0: michael@0: def GetDisabledTests(self): michael@0: """Returns a list of disabled tests. michael@0: michael@0: Returns: michael@0: A list of disabled tests obtained from gtest_filter/test_suite_disabled. michael@0: """ michael@0: disabled_tests = run_tests_helper.GetExpectations(self._GetFilterFileName()) michael@0: if self._running_on_emulator: michael@0: # Append emulator's filter file. michael@0: disabled_tests.extend(run_tests_helper.GetExpectations( michael@0: self._GetAdditionalEmulatorFilterName())) michael@0: return disabled_tests michael@0: michael@0: def UpdateFilter(self, failed_tests): michael@0: """Updates test_suite_disabled file with the new filter (deletes if empty). michael@0: michael@0: If running in Emulator, only the failed tests which are not in the normal michael@0: filter returned by _GetFilterFileName() are written to emulator's michael@0: additional filter file. michael@0: michael@0: Args: michael@0: failed_tests: A sorted list of failed tests. michael@0: """ michael@0: disabled_tests = [] michael@0: if not self._running_on_emulator: michael@0: filter_file_name = self._GetFilterFileName() michael@0: else: michael@0: filter_file_name = self._GetAdditionalEmulatorFilterName() michael@0: disabled_tests.extend( michael@0: run_tests_helper.GetExpectations(self._GetFilterFileName())) michael@0: logging.info('About to update emulator\'s additional filter (%s).' michael@0: % filter_file_name) michael@0: michael@0: new_failed_tests = [] michael@0: if failed_tests: michael@0: for test in failed_tests: michael@0: if test.name not in disabled_tests: michael@0: new_failed_tests.append(test.name) michael@0: michael@0: if not new_failed_tests: michael@0: if os.path.exists(filter_file_name): michael@0: os.unlink(filter_file_name) michael@0: return michael@0: michael@0: filter_file = file(filter_file_name, 'w') michael@0: if self._running_on_emulator: michael@0: filter_file.write('# Addtional list of suppressions from emulator\n') michael@0: else: michael@0: filter_file.write('# List of suppressions\n') michael@0: filter_file.write('# This file was automatically generated by %s\n' michael@0: % sys.argv[0]) michael@0: filter_file.write('\n'.join(sorted(new_failed_tests))) michael@0: filter_file.write('\n') michael@0: filter_file.close() michael@0: michael@0: def GetDataFilesForTestSuite(self): michael@0: """Returns a list of data files/dirs needed by the test suite.""" michael@0: # Ideally, we'd just push all test data. However, it has >100MB, and a lot michael@0: # of the files are not relevant (some are used for browser_tests, others for michael@0: # features not supported, etc..). michael@0: if self.test_package.test_suite_basename in ['base_unittests', michael@0: 'sql_unittests', michael@0: 'unit_tests']: michael@0: test_files = [ michael@0: 'base/data/file_util_unittest', michael@0: 'base/data/json/bom_feff.json', michael@0: 'chrome/test/data/download-test1.lib', michael@0: 'chrome/test/data/extensions/bad_magic.crx', michael@0: 'chrome/test/data/extensions/good.crx', michael@0: 'chrome/test/data/extensions/icon1.png', michael@0: 'chrome/test/data/extensions/icon2.png', michael@0: 'chrome/test/data/extensions/icon3.png', michael@0: 'chrome/test/data/extensions/allow_silent_upgrade/', michael@0: 'chrome/test/data/extensions/app/', michael@0: 'chrome/test/data/extensions/bad/', michael@0: 'chrome/test/data/extensions/effective_host_permissions/', michael@0: 'chrome/test/data/extensions/empty_manifest/', michael@0: 'chrome/test/data/extensions/good/Extensions/', michael@0: 'chrome/test/data/extensions/manifest_tests/', michael@0: 'chrome/test/data/extensions/page_action/', michael@0: 'chrome/test/data/extensions/permissions/', michael@0: 'chrome/test/data/extensions/script_and_capture/', michael@0: 'chrome/test/data/extensions/unpacker/', michael@0: 'chrome/test/data/bookmarks/', michael@0: 'chrome/test/data/components/', michael@0: 'chrome/test/data/extensions/json_schema_test.js', michael@0: 'chrome/test/data/History/', michael@0: 'chrome/test/data/json_schema_validator/', michael@0: 'chrome/test/data/pref_service/', michael@0: 'chrome/test/data/serializer_nested_test.js', michael@0: 'chrome/test/data/serializer_test.js', michael@0: 'chrome/test/data/serializer_test_nowhitespace.js', michael@0: 'chrome/test/data/top_sites/', michael@0: 'chrome/test/data/web_app_info/', michael@0: 'chrome/test/data/web_database', michael@0: 'chrome/test/data/webui/', michael@0: 'chrome/test/data/zip', michael@0: 'chrome/third_party/mock4js/', michael@0: 'content/browser/gpu/software_rendering_list.json', michael@0: 'net/data/cache_tests/insert_load1', michael@0: 'net/data/cache_tests/dirty_entry5', michael@0: 'net/data/ssl/certificates/', michael@0: 'ui/base/test/data/data_pack_unittest', michael@0: ] michael@0: if self.test_package.test_suite_basename == 'unit_tests': michael@0: test_files += ['chrome/test/data/simple_open_search.xml'] michael@0: # The following are spell check data. Now only list the data under michael@0: # third_party/hunspell_dictionaries which are used by unit tests. michael@0: old_cwd = os.getcwd() michael@0: os.chdir(constants.CHROME_DIR) michael@0: test_files += glob.glob('third_party/hunspell_dictionaries/*.bdic') michael@0: os.chdir(old_cwd) michael@0: return test_files michael@0: elif self.test_package.test_suite_basename == 'net_unittests': michael@0: return [ michael@0: 'net/data/cache_tests', michael@0: 'net/data/filter_unittests', michael@0: 'net/data/ftp', michael@0: 'net/data/proxy_resolver_v8_unittest', michael@0: 'net/data/ssl/certificates', michael@0: 'net/data/url_request_unittest/', michael@0: 'net/data/proxy_script_fetcher_unittest' michael@0: ] michael@0: elif self.test_package.test_suite_basename == 'ui_tests': michael@0: return [ michael@0: 'chrome/test/data/dromaeo', michael@0: 'chrome/test/data/json2.js', michael@0: 'chrome/test/data/sunspider', michael@0: 'chrome/test/data/v8_benchmark', michael@0: 'chrome/test/perf/sunspider_uitest.js', michael@0: 'chrome/test/perf/v8_benchmark_uitest.js', michael@0: ] michael@0: elif self.test_package.test_suite_basename == 'page_cycler_tests': michael@0: data = [ michael@0: 'tools/page_cycler', michael@0: 'data/page_cycler', michael@0: ] michael@0: for d in data: michael@0: if not os.path.exists(d): michael@0: raise Exception('Page cycler data not found.') michael@0: return data michael@0: elif self.test_package.test_suite_basename == 'webkit_unit_tests': michael@0: return [ michael@0: 'third_party/WebKit/Source/WebKit/chromium/tests/data', michael@0: ] michael@0: elif self.test_package.test_suite_basename == 'content_unittests': michael@0: return [ michael@0: 'content/test/data/gpu/webgl_conformance_test_expectations.txt', michael@0: 'net/data/ssl/certificates/', michael@0: 'webkit/data/dom_storage/webcore_test_database.localstorage', michael@0: 'third_party/hyphen/hyph_en_US.dic', michael@0: ] michael@0: elif self.test_package.test_suite_basename == 'media_unittests': michael@0: return [ michael@0: 'media/test/data', michael@0: ] michael@0: return [] michael@0: michael@0: def LaunchHelperToolsForTestSuite(self): michael@0: """Launches helper tools for the test suite. michael@0: michael@0: Sometimes one test may need to run some helper tools first in order to michael@0: successfully complete the test. michael@0: """ michael@0: if self._TestSuiteRequiresMockTestServer(): michael@0: self.LaunchChromeTestServerSpawner() michael@0: michael@0: def StripAndCopyFiles(self): michael@0: """Strips and copies the required data files for the test suite.""" michael@0: self.test_package.StripAndCopyExecutable() michael@0: self.test_package.PushDataAndPakFiles() michael@0: self.tool.CopyFiles() michael@0: test_data = self.GetDataFilesForTestSuite() michael@0: if test_data and not self.fast_and_loose: michael@0: # Make sure SD card is ready. michael@0: self.adb.WaitForSdCardReady(20) michael@0: for data in test_data: michael@0: self.CopyTestData([data], self.adb.GetExternalStorage()) michael@0: michael@0: def RunTestsWithFilter(self): michael@0: """Runs a tests via a small, temporary shell script.""" michael@0: self.test_package.CreateTestRunnerScript(self._gtest_filter, michael@0: self._test_arguments) michael@0: self.test_results = self.test_package.RunTestsAndListResults() michael@0: michael@0: def RebaselineTests(self): michael@0: """Runs all available tests, restarting in case of failures.""" michael@0: if self._gtest_filter: michael@0: all_tests = set(self._gtest_filter.split(':')) michael@0: else: michael@0: all_tests = set(self.test_package.GetAllTests()) michael@0: failed_results = set() michael@0: executed_results = set() michael@0: while True: michael@0: executed_names = set([f.name for f in executed_results]) michael@0: self._gtest_filter = ':'.join(all_tests - executed_names) michael@0: self.RunTestsWithFilter() michael@0: failed_results.update(self.test_results.crashed, michael@0: self.test_results.failed) michael@0: executed_results.update(self.test_results.crashed, michael@0: self.test_results.failed, michael@0: self.test_results.ok) michael@0: executed_names = set([f.name for f in executed_results]) michael@0: logging.info('*' * 80) michael@0: logging.info(self.device) michael@0: logging.info('Executed: ' + str(len(executed_names)) + ' of ' + michael@0: str(len(all_tests))) michael@0: logging.info('Failed so far: ' + str(len(failed_results)) + ' ' + michael@0: str([f.name for f in failed_results])) michael@0: logging.info('Remaining: ' + str(len(all_tests - executed_names)) + ' ' + michael@0: str(all_tests - executed_names)) michael@0: logging.info('*' * 80) michael@0: if executed_names == all_tests: michael@0: break michael@0: self.test_results = TestResults.FromRun( michael@0: ok=list(executed_results - failed_results), michael@0: failed=list(failed_results)) michael@0: michael@0: def RunTests(self): michael@0: """Runs all tests (in rebaseline mode, runs each test in isolation). michael@0: michael@0: Returns: michael@0: A TestResults object. michael@0: """ michael@0: if self.test_package.rebaseline: michael@0: self.RebaselineTests() michael@0: else: michael@0: if not self._gtest_filter: michael@0: self._gtest_filter = ('-' + ':'.join(self.GetDisabledTests()) + ':' + michael@0: ':'.join(['*.' + x + '*' for x in michael@0: self.test_package.GetDisabledPrefixes()])) michael@0: self.RunTestsWithFilter() michael@0: return self.test_results michael@0: michael@0: def SetUp(self): michael@0: """Sets up necessary test enviroment for the test suite.""" michael@0: super(SingleTestRunner, self).SetUp() michael@0: self.adb.ClearApplicationState(constants.CHROME_PACKAGE) michael@0: if self._performance_test_setup: michael@0: self._performance_test_setup.SetUp() michael@0: if self.dump_debug_info: michael@0: self.dump_debug_info.StartRecordingLog(True) michael@0: self.StripAndCopyFiles() michael@0: self.LaunchHelperToolsForTestSuite() michael@0: self.tool.SetupEnvironment() michael@0: michael@0: def TearDown(self): michael@0: """Cleans up the test enviroment for the test suite.""" michael@0: self.tool.CleanUpEnvironment() michael@0: if self.test_package.cleanup_test_files: michael@0: self.adb.RemovePushedFiles() michael@0: if self.dump_debug_info: michael@0: self.dump_debug_info.StopRecordingLog() michael@0: if self._performance_test_setup: michael@0: self._performance_test_setup.TearDown() michael@0: if self.dump_debug_info: michael@0: self.dump_debug_info.ArchiveNewCrashFiles() michael@0: super(SingleTestRunner, self).TearDown()