media/webrtc/trunk/build/android/pylib/run_java_tests.py@8bccb770b82d
media/webrtc/trunk/build/android/pylib/run_java_tests.py
Wed, 31 Dec 2014 13:27:57 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 13:27:57 +0100
- branch
- TOR_BUG_3246
- changeset 6
- 8bccb770b82d
- permissions
- -rw-r--r--
Ignore runtime configuration files generated during quality assurance.
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Runs the Java tests. See more information on run_instrumentation_tests.py."""
7 import fnmatch
8 import logging
9 import os
10 import re
11 import shutil
12 import sys
13 import time
15 import android_commands
16 import apk_info
17 from base_test_runner import BaseTestRunner
18 from base_test_sharder import BaseTestSharder, SetTestsContainer
19 import cmd_helper
20 import constants
21 import errors
22 from forwarder import Forwarder
23 from json_perf_parser import GetAverageRunInfoFromJSONString
24 from perf_tests_helper import PrintPerfResult
25 import sharded_tests_queue
26 from test_result import SingleTestResult, TestResults
27 import valgrind_tools
29 _PERF_TEST_ANNOTATION = 'PerfTest'
32 class FatalTestException(Exception):
33 """A fatal test exception."""
34 pass
37 def _TestNameToExpectation(test_name):
38 # A test name is a Package.Path.Class#testName; convert to what we use in
39 # the expectation file.
40 return '.'.join(test_name.replace('#', '.').split('.')[-2:])
43 def FilterTests(test_names, pattern_list, inclusive):
44 """Filters |test_names| using a list of patterns.
46 Args:
47 test_names: A list of test names.
48 pattern_list: A list of patterns.
49 inclusive: If True, returns the tests that match any pattern. if False,
50 returns the tests that do not match any pattern.
51 Returns:
52 A list of test names.
53 """
54 ret = []
55 for t in test_names:
56 has_match = False
57 for pattern in pattern_list:
58 has_match = has_match or fnmatch.fnmatch(_TestNameToExpectation(t),
59 pattern)
60 if has_match == inclusive:
61 ret += [t]
62 return ret
65 class TestRunner(BaseTestRunner):
66 """Responsible for running a series of tests connected to a single device."""
68 _DEVICE_DATA_DIR = 'chrome/test/data'
69 _EMMA_JAR = os.path.join(os.environ.get('ANDROID_BUILD_TOP', ''),
70 'external/emma/lib/emma.jar')
71 _COVERAGE_MERGED_FILENAME = 'unittest_coverage.es'
72 _COVERAGE_WEB_ROOT_DIR = os.environ.get('EMMA_WEB_ROOTDIR')
73 _COVERAGE_FILENAME = 'coverage.ec'
74 _COVERAGE_RESULT_PATH = ('/data/data/com.google.android.apps.chrome/files/' +
75 _COVERAGE_FILENAME)
76 _COVERAGE_META_INFO_PATH = os.path.join(os.environ.get('ANDROID_BUILD_TOP',
77 ''),
78 'out/target/common/obj/APPS',
79 'Chrome_intermediates/coverage.em')
80 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
81 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
82 '/chrome-profile*')
83 _DEVICE_HAS_TEST_FILES = {}
85 def __init__(self, options, device, tests_iter, coverage, shard_index, apks,
86 ports_to_forward):
87 """Create a new TestRunner.
89 Args:
90 options: An options object with the following required attributes:
91 - build_type: 'Release' or 'Debug'.
92 - install_apk: Re-installs the apk if opted.
93 - save_perf_json: Whether or not to save the JSON file from UI perf
94 tests.
95 - screenshot_failures: Take a screenshot for a test failure
96 - tool: Name of the Valgrind tool.
97 - wait_for_debugger: blocks until the debugger is connected.
98 device: Attached android device.
99 tests_iter: A list of tests to be run.
100 coverage: Collects coverage information if opted.
101 shard_index: shard # for this TestRunner, used to create unique port
102 numbers.
103 apks: A list of ApkInfo objects need to be installed. The first element
104 should be the tests apk, the rests could be the apks used in test.
105 The default is ChromeTest.apk.
106 ports_to_forward: A list of port numbers for which to set up forwarders.
107 Can be optionally requested by a test case.
108 Raises:
109 FatalTestException: if coverage metadata is not available.
110 """
111 BaseTestRunner.__init__(
112 self, device, options.tool, shard_index, options.build_type)
114 if not apks:
115 apks = [apk_info.ApkInfo(options.test_apk_path,
116 options.test_apk_jar_path)]
118 self.build_type = options.build_type
119 self.install_apk = options.install_apk
120 self.save_perf_json = options.save_perf_json
121 self.screenshot_failures = options.screenshot_failures
122 self.wait_for_debugger = options.wait_for_debugger
124 self.tests_iter = tests_iter
125 self.coverage = coverage
126 self.apks = apks
127 self.test_apk = apks[0]
128 self.instrumentation_class_path = self.test_apk.GetPackageName()
129 self.ports_to_forward = ports_to_forward
131 self.test_results = TestResults()
132 self.forwarder = None
134 if self.coverage:
135 if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
136 os.remove(TestRunner._COVERAGE_MERGED_FILENAME)
137 if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH):
138 raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
139 ' : Coverage meta info [' +
140 TestRunner._COVERAGE_META_INFO_PATH +
141 '] does not exist.')
142 if (not TestRunner._COVERAGE_WEB_ROOT_DIR or
143 not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)):
144 raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
145 ' : Path specified in $EMMA_WEB_ROOTDIR [' +
146 TestRunner._COVERAGE_WEB_ROOT_DIR +
147 '] does not exist.')
149 def _GetTestsIter(self):
150 if not self.tests_iter:
151 # multiprocessing.Queue can't be pickled across processes if we have it as
152 # a member set during constructor. Grab one here instead.
153 self.tests_iter = (BaseTestSharder.tests_container)
154 assert self.tests_iter
155 return self.tests_iter
157 def CopyTestFilesOnce(self):
158 """Pushes the test data files to the device. Installs the apk if opted."""
159 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
160 logging.warning('Already copied test files to device %s, skipping.',
161 self.device)
162 return
163 host_test_files = [
164 ('android_webview/test/data/device_files', 'webview'),
165 ('content/test/data/android/device_files', 'content'),
166 ('chrome/test/data/android/device_files', 'chrome')
167 ]
168 for (host_src, dst_layer) in host_test_files:
169 host_test_files_path = constants.CHROME_DIR + '/' + host_src
170 if os.path.exists(host_test_files_path):
171 self.adb.PushIfNeeded(host_test_files_path,
172 self.adb.GetExternalStorage() + '/' +
173 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
174 if self.install_apk:
175 for apk in self.apks:
176 self.adb.ManagedInstall(apk.GetApkPath(),
177 package_name=apk.GetPackageName())
178 self.tool.CopyFiles()
179 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
181 def SaveCoverageData(self, test):
182 """Saves the Emma coverage data before it's overwritten by the next test.
184 Args:
185 test: the test whose coverage data is collected.
186 """
187 if not self.coverage:
188 return
189 if not self.adb.Adb().Pull(TestRunner._COVERAGE_RESULT_PATH,
190 constants.CHROME_DIR):
191 logging.error('ERROR: Unable to find file ' +
192 TestRunner._COVERAGE_RESULT_PATH +
193 ' on the device for test ' + test)
194 pulled_coverage_file = os.path.join(constants.CHROME_DIR,
195 TestRunner._COVERAGE_FILENAME)
196 if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
197 cmd = ['java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'merge',
198 '-in', pulled_coverage_file,
199 '-in', TestRunner._COVERAGE_MERGED_FILENAME,
200 '-out', TestRunner._COVERAGE_MERGED_FILENAME]
201 cmd_helper.RunCmd(cmd)
202 else:
203 shutil.copy(pulled_coverage_file,
204 TestRunner._COVERAGE_MERGED_FILENAME)
205 os.remove(pulled_coverage_file)
207 def GenerateCoverageReportIfNeeded(self):
208 """Uses the Emma to generate a coverage report and a html page."""
209 if not self.coverage:
210 return
211 cmd = ['java', '-classpath', TestRunner._EMMA_JAR,
212 'emma', 'report', '-r', 'html',
213 '-in', TestRunner._COVERAGE_MERGED_FILENAME,
214 '-in', TestRunner._COVERAGE_META_INFO_PATH]
215 cmd_helper.RunCmd(cmd)
216 new_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
217 time.strftime('Coverage_for_%Y_%m_%d_%a_%H:%M'))
218 shutil.copytree('coverage', new_dir)
220 latest_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
221 'Latest_Coverage_Run')
222 if os.path.exists(latest_dir):
223 shutil.rmtree(latest_dir)
224 os.mkdir(latest_dir)
225 webserver_new_index = os.path.join(new_dir, 'index.html')
226 webserver_new_files = os.path.join(new_dir, '_files')
227 webserver_latest_index = os.path.join(latest_dir, 'index.html')
228 webserver_latest_files = os.path.join(latest_dir, '_files')
229 # Setup new softlinks to last result.
230 os.symlink(webserver_new_index, webserver_latest_index)
231 os.symlink(webserver_new_files, webserver_latest_files)
232 cmd_helper.RunCmd(['chmod', '755', '-R', latest_dir, new_dir])
234 def _GetInstrumentationArgs(self):
235 ret = {}
236 if self.coverage:
237 ret['coverage'] = 'true'
238 if self.wait_for_debugger:
239 ret['debug'] = 'true'
240 return ret
242 def _TakeScreenshot(self, test):
243 """Takes a screenshot from the device."""
244 screenshot_tool = os.path.join(constants.CHROME_DIR,
245 'third_party/android_tools/sdk/tools/monkeyrunner')
246 screenshot_script = os.path.join(constants.CHROME_DIR,
247 'build/android/monkeyrunner_screenshot.py')
248 screenshot_path = os.path.join(constants.CHROME_DIR,
249 'out_screenshots')
250 if not os.path.exists(screenshot_path):
251 os.mkdir(screenshot_path)
252 screenshot_name = os.path.join(screenshot_path, test + '.png')
253 logging.info('Taking screenshot named %s', screenshot_name)
254 cmd_helper.RunCmd([screenshot_tool, screenshot_script,
255 '--serial', self.device,
256 '--file', screenshot_name])
258 def SetUp(self):
259 """Sets up the test harness and device before all tests are run."""
260 super(TestRunner, self).SetUp()
261 if not self.adb.IsRootEnabled():
262 logging.warning('Unable to enable java asserts for %s, non rooted device',
263 self.device)
264 else:
265 if self.adb.SetJavaAssertsEnabled(enable=True):
266 self.adb.Reboot(full_reboot=False)
268 # We give different default value to launch HTTP server based on shard index
269 # because it may have race condition when multiple processes are trying to
270 # launch lighttpd with same port at same time.
271 http_server_ports = self.LaunchTestHttpServer(
272 os.path.join(constants.CHROME_DIR),
273 (constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index))
274 if self.ports_to_forward:
275 port_pairs = [(port, port) for port in self.ports_to_forward]
276 # We need to remember which ports the HTTP server is using, since the
277 # forwarder will stomp on them otherwise.
278 port_pairs.append(http_server_ports)
279 self.forwarder = Forwarder(
280 self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type)
281 self.CopyTestFilesOnce()
282 self.flags.AddFlags(['--enable-test-intents'])
284 def TearDown(self):
285 """Cleans up the test harness and saves outstanding data from test run."""
286 if self.forwarder:
287 self.forwarder.Close()
288 self.GenerateCoverageReportIfNeeded()
289 super(TestRunner, self).TearDown()
291 def TestSetup(self, test):
292 """Sets up the test harness for running a particular test.
294 Args:
295 test: The name of the test that will be run.
296 """
297 self.SetupPerfMonitoringIfNeeded(test)
298 self._SetupIndividualTestTimeoutScale(test)
299 self.tool.SetupEnvironment()
301 # Make sure the forwarder is still running.
302 self.RestartHttpServerForwarderIfNecessary()
304 def _IsPerfTest(self, test):
305 """Determines whether a test is a performance test.
307 Args:
308 test: The name of the test to be checked.
310 Returns:
311 Whether the test is annotated as a performance test.
312 """
313 return _PERF_TEST_ANNOTATION in self.test_apk.GetTestAnnotations(test)
315 def SetupPerfMonitoringIfNeeded(self, test):
316 """Sets up performance monitoring if the specified test requires it.
318 Args:
319 test: The name of the test to be run.
320 """
321 if not self._IsPerfTest(test):
322 return
323 self.adb.Adb().SendCommand('shell rm ' +
324 TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX)
325 self.adb.StartMonitoringLogcat()
327 def TestTeardown(self, test, test_result):
328 """Cleans up the test harness after running a particular test.
330 Depending on the options of this TestRunner this might handle coverage
331 tracking or performance tracking. This method will only be called if the
332 test passed.
334 Args:
335 test: The name of the test that was just run.
336 test_result: result for this test.
337 """
339 self.tool.CleanUpEnvironment()
341 # The logic below relies on the test passing.
342 if not test_result or test_result.GetStatusCode():
343 return
345 self.TearDownPerfMonitoring(test)
346 self.SaveCoverageData(test)
348 def TearDownPerfMonitoring(self, test):
349 """Cleans up performance monitoring if the specified test required it.
351 Args:
352 test: The name of the test that was just run.
353 Raises:
354 FatalTestException: if there's anything wrong with the perf data.
355 """
356 if not self._IsPerfTest(test):
357 return
358 raw_test_name = test.split('#')[1]
360 # Wait and grab annotation data so we can figure out which traces to parse
361 regex = self.adb.WaitForLogMatch(re.compile('\*\*PERFANNOTATION\(' +
362 raw_test_name +
363 '\)\:(.*)'), None)
365 # If the test is set to run on a specific device type only (IE: only
366 # tablet or phone) and it is being run on the wrong device, the test
367 # just quits and does not do anything. The java test harness will still
368 # print the appropriate annotation for us, but will add --NORUN-- for
369 # us so we know to ignore the results.
370 # The --NORUN-- tag is managed by MainActivityTestBase.java
371 if regex.group(1) != '--NORUN--':
373 # Obtain the relevant perf data. The data is dumped to a
374 # JSON formatted file.
375 json_string = self.adb.GetFileContents(
376 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt')
378 if json_string:
379 json_string = '\n'.join(json_string)
380 else:
381 raise FatalTestException('Perf file does not exist or is empty')
383 if self.save_perf_json:
384 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
385 with open(json_local_file, 'w') as f:
386 f.write(json_string)
387 logging.info('Saving Perf UI JSON from test ' +
388 test + ' to ' + json_local_file)
390 raw_perf_data = regex.group(1).split(';')
392 for raw_perf_set in raw_perf_data:
393 if raw_perf_set:
394 perf_set = raw_perf_set.split(',')
395 if len(perf_set) != 3:
396 raise FatalTestException('Unexpected number of tokens in '
397 'perf annotation string: ' + raw_perf_set)
399 # Process the performance data
400 result = GetAverageRunInfoFromJSONString(json_string, perf_set[0])
402 PrintPerfResult(perf_set[1], perf_set[2],
403 [result['average']], result['units'])
405 def _SetupIndividualTestTimeoutScale(self, test):
406 timeout_scale = self._GetIndividualTestTimeoutScale(test)
407 valgrind_tools.SetChromeTimeoutScale(self.adb, timeout_scale)
409 def _GetIndividualTestTimeoutScale(self, test):
410 """Returns the timeout scale for the given |test|."""
411 annotations = self.apks[0].GetTestAnnotations(test)
412 timeout_scale = 1
413 if 'TimeoutScale' in annotations:
414 for annotation in annotations:
415 scale_match = re.match('TimeoutScale:([0-9]+)', annotation)
416 if scale_match:
417 timeout_scale = int(scale_match.group(1))
418 if self.wait_for_debugger:
419 timeout_scale *= 100
420 return timeout_scale
422 def _GetIndividualTestTimeoutSecs(self, test):
423 """Returns the timeout in seconds for the given |test|."""
424 annotations = self.apks[0].GetTestAnnotations(test)
425 if 'Manual' in annotations:
426 return 600 * 60
427 if 'External' in annotations:
428 return 10 * 60
429 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
430 return 5 * 60
431 if 'MediumTest' in annotations:
432 return 3 * 60
433 return 1 * 60
435 def RunTests(self):
436 """Runs the tests, generating the coverage if needed.
438 Returns:
439 A TestResults object.
440 """
441 instrumentation_path = (self.instrumentation_class_path +
442 '/android.test.InstrumentationTestRunner')
443 instrumentation_args = self._GetInstrumentationArgs()
444 for test in self._GetTestsIter():
445 test_result = None
446 start_date_ms = None
447 try:
448 self.TestSetup(test)
449 start_date_ms = int(time.time()) * 1000
450 args_with_filter = dict(instrumentation_args)
451 args_with_filter['class'] = test
452 # |test_results| is a list that should contain
453 # a single TestResult object.
454 logging.warn(args_with_filter)
455 (test_results, _) = self.adb.Adb().StartInstrumentation(
456 instrumentation_path=instrumentation_path,
457 instrumentation_args=args_with_filter,
458 timeout_time=(self._GetIndividualTestTimeoutSecs(test) *
459 self._GetIndividualTestTimeoutScale(test) *
460 self.tool.GetTimeoutScale()))
461 duration_ms = int(time.time()) * 1000 - start_date_ms
462 assert len(test_results) == 1
463 test_result = test_results[0]
464 status_code = test_result.GetStatusCode()
465 if status_code:
466 log = test_result.GetFailureReason()
467 if not log:
468 log = 'No information.'
469 if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0:
470 self._TakeScreenshot(test)
471 self.test_results.failed += [SingleTestResult(test, start_date_ms,
472 duration_ms, log)]
473 else:
474 result = [SingleTestResult(test, start_date_ms, duration_ms)]
475 self.test_results.ok += result
476 # Catch exceptions thrown by StartInstrumentation().
477 # See ../../third_party/android/testrunner/adb_interface.py
478 except (errors.WaitForResponseTimedOutError,
479 errors.DeviceUnresponsiveError,
480 errors.InstrumentationError), e:
481 if start_date_ms:
482 duration_ms = int(time.time()) * 1000 - start_date_ms
483 else:
484 start_date_ms = int(time.time()) * 1000
485 duration_ms = 0
486 message = str(e)
487 if not message:
488 message = 'No information.'
489 self.test_results.crashed += [SingleTestResult(test, start_date_ms,
490 duration_ms,
491 message)]
492 test_result = None
493 self.TestTeardown(test, test_result)
494 return self.test_results
497 class TestSharder(BaseTestSharder):
498 """Responsible for sharding the tests on the connected devices."""
500 def __init__(self, attached_devices, options, tests, apks):
501 BaseTestSharder.__init__(self, attached_devices)
502 self.options = options
503 self.tests = tests
504 self.apks = apks
506 def SetupSharding(self, tests):
507 """Called before starting the shards."""
508 SetTestsContainer(sharded_tests_queue.ShardedTestsQueue(
509 len(self.attached_devices), tests))
511 def CreateShardedTestRunner(self, device, index):
512 """Creates a sharded test runner.
514 Args:
515 device: Device serial where this shard will run.
516 index: Index of this device in the pool.
518 Returns:
519 A TestRunner object.
520 """
521 return TestRunner(self.options, device, None, False, index, self.apks, [])
524 def DispatchJavaTests(options, apks):
525 """Dispatches Java tests onto connected device(s).
527 If possible, this method will attempt to shard the tests to
528 all connected devices. Otherwise, dispatch and run tests on one device.
530 Args:
531 options: Command line options.
532 apks: list of APKs to use.
534 Returns:
535 A TestResults object holding the results of the Java tests.
537 Raises:
538 FatalTestException: when there's no attached the devices.
539 """
540 test_apk = apks[0]
541 if options.annotation:
542 available_tests = test_apk.GetAnnotatedTests(options.annotation)
543 if len(options.annotation) == 1 and options.annotation[0] == 'SmallTest':
544 tests_without_annotation = [
545 m for m in
546 test_apk.GetTestMethods()
547 if not test_apk.GetTestAnnotations(m) and
548 not apk_info.ApkInfo.IsPythonDrivenTest(m)]
549 if tests_without_annotation:
550 tests_without_annotation.sort()
551 logging.warning('The following tests do not contain any annotation. '
552 'Assuming "SmallTest":\n%s',
553 '\n'.join(tests_without_annotation))
554 available_tests += tests_without_annotation
555 else:
556 available_tests = [m for m in test_apk.GetTestMethods()
557 if not apk_info.ApkInfo.IsPythonDrivenTest(m)]
558 coverage = os.environ.get('EMMA_INSTRUMENT') == 'true'
560 tests = []
561 if options.test_filter:
562 # |available_tests| are in adb instrument format: package.path.class#test.
563 filter_without_hash = options.test_filter.replace('#', '.')
564 tests = [t for t in available_tests
565 if filter_without_hash in t.replace('#', '.')]
566 else:
567 tests = available_tests
569 if not tests:
570 logging.warning('No Java tests to run with current args.')
571 return TestResults()
573 tests *= options.number_of_runs
575 attached_devices = android_commands.GetAttachedDevices()
576 test_results = TestResults()
578 if not attached_devices:
579 raise FatalTestException('You have no devices attached or visible!')
580 if options.device:
581 attached_devices = [options.device]
583 logging.info('Will run: %s', str(tests))
585 if len(attached_devices) > 1 and (coverage or options.wait_for_debugger):
586 logging.warning('Coverage / debugger can not be sharded, '
587 'using first available device')
588 attached_devices = attached_devices[:1]
589 sharder = TestSharder(attached_devices, options, tests, apks)
590 test_results = sharder.RunShardedTests()
591 return test_results
