1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/media/webrtc/trunk/build/android/pylib/test_package.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,200 @@
1.4 +# Copyright (c) 2012 The Chromium Authors. All rights reserved.
1.5 +# Use of this source code is governed by a BSD-style license that can be
1.6 +# found in the LICENSE file.
1.7 +
1.8 +
1.9 +import logging
1.10 +import re
1.11 +import os
1.12 +
1.13 +import constants
1.14 +from perf_tests_helper import PrintPerfResult
1.15 +from pylib import pexpect
1.16 +from test_result import BaseTestResult, TestResults
1.17 +
1.18 +
1.19 +# TODO(bulach): TestPackage, TestPackageExecutable and
1.20 +# TestPackageApk are a work in progress related to making the native tests
1.21 +# run as a NDK-app from an APK rather than a stand-alone executable.
1.22 +class TestPackage(object):
1.23 + """A helper base class for both APK and stand-alone executables.
1.24 +
1.25 + Args:
1.26 + adb: ADB interface the tests are using.
1.27 + device: Device to run the tests.
1.28 + test_suite: A specific test suite to run, empty to run all.
1.29 + timeout: Timeout for each test.
1.30 + rebaseline: Whether or not to run tests in isolation and update the filter.
1.31 + performance_test: Whether or not performance test(s).
1.32 + cleanup_test_files: Whether or not to cleanup test files on device.
1.33 + tool: Name of the Valgrind tool.
1.34 + dump_debug_info: A debug_info object.
1.35 + """
1.36 +
1.37 + def __init__(self, adb, device, test_suite, timeout, rebaseline,
1.38 + performance_test, cleanup_test_files, tool, dump_debug_info):
1.39 + self.adb = adb
1.40 + self.device = device
1.41 + self.test_suite_full = test_suite
1.42 + self.test_suite = os.path.splitext(test_suite)[0]
1.43 + self.test_suite_basename = self._GetTestSuiteBaseName()
1.44 + self.test_suite_dirname = os.path.dirname(
1.45 + self.test_suite.split(self.test_suite_basename)[0])
1.46 + self.rebaseline = rebaseline
1.47 + self.performance_test = performance_test
1.48 + self.cleanup_test_files = cleanup_test_files
1.49 + self.tool = tool
1.50 + if timeout == 0:
1.51 + timeout = 60
1.52 + # On a VM (e.g. chromium buildbots), this timeout is way too small.
1.53 + if os.environ.get('BUILDBOT_SLAVENAME'):
1.54 + timeout = timeout * 2
1.55 + self.timeout = timeout * self.tool.GetTimeoutScale()
1.56 + self.dump_debug_info = dump_debug_info
1.57 +
1.58 + def _BeginGetIOStats(self):
1.59 + """Gets I/O statistics before running test.
1.60 +
1.61 + Return:
1.62 + I/O stats object.The I/O stats object may be None if the test is not
1.63 + performance test.
1.64 + """
1.65 + initial_io_stats = None
1.66 + # Try to get the disk I/O statistics for all performance tests.
1.67 + if self.performance_test and not self.rebaseline:
1.68 + initial_io_stats = self.adb.GetIoStats()
1.69 + return initial_io_stats
1.70 +
1.71 + def _EndGetIOStats(self, initial_io_stats):
1.72 + """Gets I/O statistics after running test and calcuate the I/O delta.
1.73 +
1.74 + Args:
1.75 + initial_io_stats: I/O stats object got from _BeginGetIOStats.
1.76 +
1.77 + Return:
1.78 + String for formated diso I/O statistics.
1.79 + """
1.80 + disk_io = ''
1.81 + if self.performance_test and initial_io_stats:
1.82 + final_io_stats = self.adb.GetIoStats()
1.83 + for stat in final_io_stats:
1.84 + disk_io += '\n' + PrintPerfResult(stat, stat,
1.85 + [final_io_stats[stat] -
1.86 + initial_io_stats[stat]],
1.87 + stat.split('_')[1],
1.88 + print_to_stdout=False)
1.89 + logging.info(disk_io)
1.90 + return disk_io
1.91 +
1.92 + def GetDisabledPrefixes(self):
1.93 + return ['DISABLED_', 'FLAKY_', 'FAILS_']
1.94 +
1.95 + def _ParseGTestListTests(self, all_tests):
1.96 + ret = []
1.97 + current = ''
1.98 + disabled_prefixes = self.GetDisabledPrefixes()
1.99 + for test in all_tests:
1.100 + if not test:
1.101 + continue
1.102 + if test[0] != ' ' and not test.endswith('.'):
1.103 + # Ignore any lines with unexpected format.
1.104 + continue
1.105 + if test[0] != ' ' and test.endswith('.'):
1.106 + current = test
1.107 + continue
1.108 + if 'YOU HAVE' in test:
1.109 + break
1.110 + test_name = test[2:]
1.111 + if not any([test_name.startswith(x) for x in disabled_prefixes]):
1.112 + ret += [current + test_name]
1.113 + return ret
1.114 +
1.115 + def PushDataAndPakFiles(self):
1.116 + external_storage = self.adb.GetExternalStorage()
1.117 + if (self.test_suite_basename == 'ui_unittests' or
1.118 + self.test_suite_basename == 'unit_tests'):
1.119 + self.adb.PushIfNeeded(
1.120 + self.test_suite_dirname + '/chrome.pak',
1.121 + external_storage + '/paks/chrome.pak')
1.122 + self.adb.PushIfNeeded(
1.123 + self.test_suite_dirname + '/locales/en-US.pak',
1.124 + external_storage + '/paks/en-US.pak')
1.125 + if self.test_suite_basename == 'unit_tests':
1.126 + self.adb.PushIfNeeded(
1.127 + self.test_suite_dirname + '/resources.pak',
1.128 + external_storage + '/paks/resources.pak')
1.129 + self.adb.PushIfNeeded(
1.130 + self.test_suite_dirname + '/chrome_100_percent.pak',
1.131 + external_storage + '/paks/chrome_100_percent.pak')
1.132 + self.adb.PushIfNeeded(self.test_suite_dirname + '/test_data',
1.133 + external_storage + '/test_data')
1.134 + if self.test_suite_basename == 'content_unittests':
1.135 + self.adb.PushIfNeeded(
1.136 + self.test_suite_dirname + '/content_resources.pak',
1.137 + external_storage + '/paks/content_resources.pak')
1.138 +
1.139 + def _WatchTestOutput(self, p):
1.140 + """Watches the test output.
1.141 + Args:
1.142 + p: the process generating output as created by pexpect.spawn.
1.143 + """
1.144 + ok_tests = []
1.145 + failed_tests = []
1.146 + crashed_tests = []
1.147 + timed_out = False
1.148 + overall_fail = False
1.149 + re_run = re.compile('\[ RUN \] ?(.*)\r\n')
1.150 + # APK tests rely on the PASSED tag.
1.151 + re_passed = re.compile('\[ PASSED \] ?(.*)\r\n')
1.152 + # Signal handlers are installed before starting tests
1.153 + # to output the CRASHED marker when a crash happens.
1.154 + re_crash = re.compile('\[ CRASHED \](.*)\r\n')
1.155 + re_fail = re.compile('\[ FAILED \] ?(.*)\r\n')
1.156 + re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
1.157 + re_ok = re.compile('\[ OK \] ?(.*?) .*\r\n')
1.158 + io_stats_before = self._BeginGetIOStats()
1.159 + try:
1.160 + while True:
1.161 + found = p.expect([re_run, re_passed, re_runner_fail],
1.162 + timeout=self.timeout)
1.163 + if found == 1: # matched PASSED.
1.164 + break
1.165 + if found == 2: # RUNNER_FAILED
1.166 + logging.error('RUNNER_FAILED')
1.167 + overall_fail = True
1.168 + break
1.169 + if self.dump_debug_info:
1.170 + self.dump_debug_info.TakeScreenshot('_Test_Start_Run_')
1.171 + full_test_name = p.match.group(1).replace('\r', '')
1.172 + found = p.expect([re_ok, re_fail, re_crash], timeout=self.timeout)
1.173 + if found == 0: # re_ok
1.174 + if full_test_name == p.match.group(1).replace('\r', ''):
1.175 + ok_tests += [BaseTestResult(full_test_name, p.before)]
1.176 + continue
1.177 + if found == 2: # re_crash
1.178 + crashed_tests += [BaseTestResult(full_test_name, p.before)]
1.179 + overall_fail = True
1.180 + break
1.181 + # The test failed.
1.182 + failed_tests += [BaseTestResult(full_test_name, p.before)]
1.183 + except pexpect.EOF:
1.184 + logging.error('Test terminated - EOF')
1.185 + except pexpect.TIMEOUT:
1.186 + logging.error('Test terminated after %d second timeout.',
1.187 + self.timeout)
1.188 + timed_out = True
1.189 + finally:
1.190 + p.close()
1.191 + if not self.rebaseline:
1.192 + ok_tests += self._EndGetIOStats(io_stats_before)
1.193 + ret_code = self._GetGTestReturnCode()
1.194 + if ret_code:
1.195 + failed_tests += [BaseTestResult('gtest exit code: %d' % ret_code,
1.196 + 'pexpect.before: %s'
1.197 + '\npexpect.after: %s'
1.198 + % (p.before,
1.199 + p.after))]
1.200 + # Create TestResults and return
1.201 + return TestResults.FromRun(ok=ok_tests, failed=failed_tests,
1.202 + crashed=crashed_tests, timed_out=timed_out,
1.203 + overall_fail=overall_fail)
