media/webrtc/trunk/build/android/pylib/test_package.py@8bccb770b82d
media/webrtc/trunk/build/android/pylib/test_package.py
Wed, 31 Dec 2014 13:27:57 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 13:27:57 +0100
- branch
- TOR_BUG_3246
- changeset 6
- 8bccb770b82d
- permissions
- -rw-r--r--
Ignore runtime configuration files generated during quality assurance.
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
6 import logging
7 import re
8 import os
10 import constants
11 from perf_tests_helper import PrintPerfResult
12 from pylib import pexpect
13 from test_result import BaseTestResult, TestResults
16 # TODO(bulach): TestPackage, TestPackageExecutable and
17 # TestPackageApk are a work in progress related to making the native tests
18 # run as a NDK-app from an APK rather than a stand-alone executable.
19 class TestPackage(object):
20 """A helper base class for both APK and stand-alone executables.
22 Args:
23 adb: ADB interface the tests are using.
24 device: Device to run the tests.
25 test_suite: A specific test suite to run, empty to run all.
26 timeout: Timeout for each test.
27 rebaseline: Whether or not to run tests in isolation and update the filter.
28 performance_test: Whether or not performance test(s).
29 cleanup_test_files: Whether or not to cleanup test files on device.
30 tool: Name of the Valgrind tool.
31 dump_debug_info: A debug_info object.
32 """
34 def __init__(self, adb, device, test_suite, timeout, rebaseline,
35 performance_test, cleanup_test_files, tool, dump_debug_info):
36 self.adb = adb
37 self.device = device
38 self.test_suite_full = test_suite
39 self.test_suite = os.path.splitext(test_suite)[0]
40 self.test_suite_basename = self._GetTestSuiteBaseName()
41 self.test_suite_dirname = os.path.dirname(
42 self.test_suite.split(self.test_suite_basename)[0])
43 self.rebaseline = rebaseline
44 self.performance_test = performance_test
45 self.cleanup_test_files = cleanup_test_files
46 self.tool = tool
47 if timeout == 0:
48 timeout = 60
49 # On a VM (e.g. chromium buildbots), this timeout is way too small.
50 if os.environ.get('BUILDBOT_SLAVENAME'):
51 timeout = timeout * 2
52 self.timeout = timeout * self.tool.GetTimeoutScale()
53 self.dump_debug_info = dump_debug_info
55 def _BeginGetIOStats(self):
56 """Gets I/O statistics before running test.
58 Return:
59 I/O stats object.The I/O stats object may be None if the test is not
60 performance test.
61 """
62 initial_io_stats = None
63 # Try to get the disk I/O statistics for all performance tests.
64 if self.performance_test and not self.rebaseline:
65 initial_io_stats = self.adb.GetIoStats()
66 return initial_io_stats
68 def _EndGetIOStats(self, initial_io_stats):
69 """Gets I/O statistics after running test and calcuate the I/O delta.
71 Args:
72 initial_io_stats: I/O stats object got from _BeginGetIOStats.
74 Return:
75 String for formated diso I/O statistics.
76 """
77 disk_io = ''
78 if self.performance_test and initial_io_stats:
79 final_io_stats = self.adb.GetIoStats()
80 for stat in final_io_stats:
81 disk_io += '\n' + PrintPerfResult(stat, stat,
82 [final_io_stats[stat] -
83 initial_io_stats[stat]],
84 stat.split('_')[1],
85 print_to_stdout=False)
86 logging.info(disk_io)
87 return disk_io
89 def GetDisabledPrefixes(self):
90 return ['DISABLED_', 'FLAKY_', 'FAILS_']
92 def _ParseGTestListTests(self, all_tests):
93 ret = []
94 current = ''
95 disabled_prefixes = self.GetDisabledPrefixes()
96 for test in all_tests:
97 if not test:
98 continue
99 if test[0] != ' ' and not test.endswith('.'):
100 # Ignore any lines with unexpected format.
101 continue
102 if test[0] != ' ' and test.endswith('.'):
103 current = test
104 continue
105 if 'YOU HAVE' in test:
106 break
107 test_name = test[2:]
108 if not any([test_name.startswith(x) for x in disabled_prefixes]):
109 ret += [current + test_name]
110 return ret
112 def PushDataAndPakFiles(self):
113 external_storage = self.adb.GetExternalStorage()
114 if (self.test_suite_basename == 'ui_unittests' or
115 self.test_suite_basename == 'unit_tests'):
116 self.adb.PushIfNeeded(
117 self.test_suite_dirname + '/chrome.pak',
118 external_storage + '/paks/chrome.pak')
119 self.adb.PushIfNeeded(
120 self.test_suite_dirname + '/locales/en-US.pak',
121 external_storage + '/paks/en-US.pak')
122 if self.test_suite_basename == 'unit_tests':
123 self.adb.PushIfNeeded(
124 self.test_suite_dirname + '/resources.pak',
125 external_storage + '/paks/resources.pak')
126 self.adb.PushIfNeeded(
127 self.test_suite_dirname + '/chrome_100_percent.pak',
128 external_storage + '/paks/chrome_100_percent.pak')
129 self.adb.PushIfNeeded(self.test_suite_dirname + '/test_data',
130 external_storage + '/test_data')
131 if self.test_suite_basename == 'content_unittests':
132 self.adb.PushIfNeeded(
133 self.test_suite_dirname + '/content_resources.pak',
134 external_storage + '/paks/content_resources.pak')
136 def _WatchTestOutput(self, p):
137 """Watches the test output.
138 Args:
139 p: the process generating output as created by pexpect.spawn.
140 """
141 ok_tests = []
142 failed_tests = []
143 crashed_tests = []
144 timed_out = False
145 overall_fail = False
146 re_run = re.compile('\[ RUN \] ?(.*)\r\n')
147 # APK tests rely on the PASSED tag.
148 re_passed = re.compile('\[ PASSED \] ?(.*)\r\n')
149 # Signal handlers are installed before starting tests
150 # to output the CRASHED marker when a crash happens.
151 re_crash = re.compile('\[ CRASHED \](.*)\r\n')
152 re_fail = re.compile('\[ FAILED \] ?(.*)\r\n')
153 re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
154 re_ok = re.compile('\[ OK \] ?(.*?) .*\r\n')
155 io_stats_before = self._BeginGetIOStats()
156 try:
157 while True:
158 found = p.expect([re_run, re_passed, re_runner_fail],
159 timeout=self.timeout)
160 if found == 1: # matched PASSED.
161 break
162 if found == 2: # RUNNER_FAILED
163 logging.error('RUNNER_FAILED')
164 overall_fail = True
165 break
166 if self.dump_debug_info:
167 self.dump_debug_info.TakeScreenshot('_Test_Start_Run_')
168 full_test_name = p.match.group(1).replace('\r', '')
169 found = p.expect([re_ok, re_fail, re_crash], timeout=self.timeout)
170 if found == 0: # re_ok
171 if full_test_name == p.match.group(1).replace('\r', ''):
172 ok_tests += [BaseTestResult(full_test_name, p.before)]
173 continue
174 if found == 2: # re_crash
175 crashed_tests += [BaseTestResult(full_test_name, p.before)]
176 overall_fail = True
177 break
178 # The test failed.
179 failed_tests += [BaseTestResult(full_test_name, p.before)]
180 except pexpect.EOF:
181 logging.error('Test terminated - EOF')
182 except pexpect.TIMEOUT:
183 logging.error('Test terminated after %d second timeout.',
184 self.timeout)
185 timed_out = True
186 finally:
187 p.close()
188 if not self.rebaseline:
189 ok_tests += self._EndGetIOStats(io_stats_before)
190 ret_code = self._GetGTestReturnCode()
191 if ret_code:
192 failed_tests += [BaseTestResult('gtest exit code: %d' % ret_code,
193 'pexpect.before: %s'
194 '\npexpect.after: %s'
195 % (p.before,
196 p.after))]
197 # Create TestResults and return
198 return TestResults.FromRun(ok=ok_tests, failed=failed_tests,
199 crashed=crashed_tests, timed_out=timed_out,
200 overall_fail=overall_fail)
