1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/media/webrtc/trunk/build/android/pylib/test_result.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,193 @@
1.4 +# Copyright (c) 2012 The Chromium Authors. All rights reserved.
1.5 +# Use of this source code is governed by a BSD-style license that can be
1.6 +# found in the LICENSE file.
1.7 +
1.8 +
1.9 +import json
1.10 +import logging
1.11 +import os
1.12 +import time
1.13 +import traceback
1.14 +
1.15 +import buildbot_report
1.16 +import constants
1.17 +
1.18 +
1.19 +class BaseTestResult(object):
1.20 + """A single result from a unit test."""
1.21 +
1.22 + def __init__(self, name, log):
1.23 + self.name = name
1.24 + self.log = log.replace('\r', '')
1.25 +
1.26 +
1.27 +class SingleTestResult(BaseTestResult):
1.28 + """Result information for a single test.
1.29 +
1.30 + Args:
1.31 + full_name: Full name of the test.
1.32 + start_date: Date in milliseconds when the test began running.
1.33 + dur: Duration of the test run in milliseconds.
1.34 + log: An optional string listing any errors.
1.35 + """
1.36 +
1.37 + def __init__(self, full_name, start_date, dur, log=''):
1.38 + BaseTestResult.__init__(self, full_name, log)
1.39 + name_pieces = full_name.rsplit('#')
1.40 + if len(name_pieces) > 1:
1.41 + self.test_name = name_pieces[1]
1.42 + self.class_name = name_pieces[0]
1.43 + else:
1.44 + self.class_name = full_name
1.45 + self.test_name = full_name
1.46 + self.start_date = start_date
1.47 + self.dur = dur
1.48 +
1.49 +
1.50 +class TestResults(object):
1.51 + """Results of a test run."""
1.52 +
1.53 + def __init__(self):
1.54 + self.ok = []
1.55 + self.failed = []
1.56 + self.crashed = []
1.57 + self.unknown = []
1.58 + self.timed_out = False
1.59 + self.overall_fail = False
1.60 +
1.61 + @staticmethod
1.62 + def FromRun(ok=None, failed=None, crashed=None, timed_out=False,
1.63 + overall_fail=False):
1.64 + ret = TestResults()
1.65 + ret.ok = ok or []
1.66 + ret.failed = failed or []
1.67 + ret.crashed = crashed or []
1.68 + ret.timed_out = timed_out
1.69 + ret.overall_fail = overall_fail
1.70 + return ret
1.71 +
1.72 + @staticmethod
1.73 + def FromTestResults(results):
1.74 + """Combines a list of results in a single TestResults object."""
1.75 + ret = TestResults()
1.76 + for t in results:
1.77 + ret.ok += t.ok
1.78 + ret.failed += t.failed
1.79 + ret.crashed += t.crashed
1.80 + ret.unknown += t.unknown
1.81 + if t.timed_out:
1.82 + ret.timed_out = True
1.83 + if t.overall_fail:
1.84 + ret.overall_fail = True
1.85 + return ret
1.86 +
1.87 + @staticmethod
1.88 + def FromPythonException(test_name, start_date_ms, exc_info):
1.89 + """Constructs a TestResults with exception information for the given test.
1.90 +
1.91 + Args:
1.92 + test_name: name of the test which raised an exception.
1.93 + start_date_ms: the starting time for the test.
1.94 + exc_info: exception info, ostensibly from sys.exc_info().
1.95 +
1.96 + Returns:
1.97 + A TestResults object with a SingleTestResult in the failed list.
1.98 + """
1.99 + exc_type, exc_value, exc_traceback = exc_info
1.100 + trace_info = ''.join(traceback.format_exception(exc_type, exc_value,
1.101 + exc_traceback))
1.102 + log_msg = 'Exception:\n' + trace_info
1.103 + duration_ms = (int(time.time()) * 1000) - start_date_ms
1.104 +
1.105 + exc_result = SingleTestResult(
1.106 + full_name='PythonWrapper#' + test_name,
1.107 + start_date=start_date_ms,
1.108 + dur=duration_ms,
1.109 + log=(str(exc_type) + ' ' + log_msg))
1.110 +
1.111 + results = TestResults()
1.112 + results.failed.append(exc_result)
1.113 + return results
1.114 +
1.115 + def _Log(self, sorted_list):
1.116 + for t in sorted_list:
1.117 + logging.critical(t.name)
1.118 + if t.log:
1.119 + logging.critical(t.log)
1.120 +
1.121 + def GetAllBroken(self):
1.122 + """Returns the all broken tests including failed, crashed, unknown."""
1.123 + return self.failed + self.crashed + self.unknown
1.124 +
1.125 + def LogFull(self, test_group, test_suite, build_type):
1.126 + """Output broken test logs, summarize in a log file and the test output."""
1.127 + # Output all broken tests or 'passed' if none broken.
1.128 + logging.critical('*' * 80)
1.129 + logging.critical('Final result')
1.130 + if self.failed:
1.131 + logging.critical('Failed:')
1.132 + self._Log(sorted(self.failed))
1.133 + if self.crashed:
1.134 + logging.critical('Crashed:')
1.135 + self._Log(sorted(self.crashed))
1.136 + if self.unknown:
1.137 + logging.critical('Unknown:')
1.138 + self._Log(sorted(self.unknown))
1.139 + if not self.GetAllBroken():
1.140 + logging.critical('Passed')
1.141 + logging.critical('*' * 80)
1.142 +
1.143 + # Summarize in a log file, if tests are running on bots.
1.144 + if test_group and test_suite and os.environ.get('BUILDBOT_BUILDERNAME'):
1.145 + log_file_path = os.path.join(constants.CHROME_DIR, 'out',
1.146 + build_type, 'test_logs')
1.147 + if not os.path.exists(log_file_path):
1.148 + os.mkdir(log_file_path)
1.149 + full_file_name = os.path.join(log_file_path, test_group)
1.150 + if not os.path.exists(full_file_name):
1.151 + with open(full_file_name, 'w') as log_file:
1.152 + print >> log_file, '\n%s results for %s build %s:' % (
1.153 + test_group, os.environ.get('BUILDBOT_BUILDERNAME'),
1.154 + os.environ.get('BUILDBOT_BUILDNUMBER'))
1.155 + log_contents = [' %s result : %d tests ran' % (test_suite,
1.156 + len(self.ok) +
1.157 + len(self.failed) +
1.158 + len(self.crashed) +
1.159 + len(self.unknown))]
1.160 + content_pairs = [('passed', len(self.ok)), ('failed', len(self.failed)),
1.161 + ('crashed', len(self.crashed))]
1.162 + for (result, count) in content_pairs:
1.163 + if count:
1.164 + log_contents.append(', %d tests %s' % (count, result))
1.165 + with open(full_file_name, 'a') as log_file:
1.166 + print >> log_file, ''.join(log_contents)
1.167 + content = {'test_group': test_group,
1.168 + 'ok': [t.name for t in self.ok],
1.169 + 'failed': [t.name for t in self.failed],
1.170 + 'crashed': [t.name for t in self.failed],
1.171 + 'unknown': [t.name for t in self.unknown],}
1.172 + with open(os.path.join(log_file_path, 'results.json'), 'a') as json_file:
1.173 + print >> json_file, json.dumps(content)
1.174 +
1.175 + # Summarize in the test output.
1.176 + summary_string = 'Summary:\n'
1.177 + summary_string += 'RAN=%d\n' % (len(self.ok) + len(self.failed) +
1.178 + len(self.crashed) + len(self.unknown))
1.179 + summary_string += 'PASSED=%d\n' % (len(self.ok))
1.180 + summary_string += 'FAILED=%d %s\n' % (len(self.failed),
1.181 + [t.name for t in self.failed])
1.182 + summary_string += 'CRASHED=%d %s\n' % (len(self.crashed),
1.183 + [t.name for t in self.crashed])
1.184 + summary_string += 'UNKNOWN=%d %s\n' % (len(self.unknown),
1.185 + [t.name for t in self.unknown])
1.186 + logging.critical(summary_string)
1.187 + return summary_string
1.188 +
1.189 + def PrintAnnotation(self):
1.190 + """Print buildbot annotations for test results."""
1.191 + if self.timed_out:
1.192 + buildbot_report.PrintWarning()
1.193 + elif self.failed or self.crashed or self.overall_fail:
1.194 + buildbot_report.PrintError()
1.195 + else:
1.196 + print 'Step success!' # No annotation needed
