michael@0: # Copyright (c) 2012 The Chromium Authors. All rights reserved. michael@0: # Use of this source code is governed by a BSD-style license that can be michael@0: # found in the LICENSE file. michael@0: michael@0: michael@0: import json michael@0: import logging michael@0: import os michael@0: import time michael@0: import traceback michael@0: michael@0: import buildbot_report michael@0: import constants michael@0: michael@0: michael@0: class BaseTestResult(object): michael@0: """A single result from a unit test.""" michael@0: michael@0: def __init__(self, name, log): michael@0: self.name = name michael@0: self.log = log.replace('\r', '') michael@0: michael@0: michael@0: class SingleTestResult(BaseTestResult): michael@0: """Result information for a single test. michael@0: michael@0: Args: michael@0: full_name: Full name of the test. michael@0: start_date: Date in milliseconds when the test began running. michael@0: dur: Duration of the test run in milliseconds. michael@0: log: An optional string listing any errors. michael@0: """ michael@0: michael@0: def __init__(self, full_name, start_date, dur, log=''): michael@0: BaseTestResult.__init__(self, full_name, log) michael@0: name_pieces = full_name.rsplit('#') michael@0: if len(name_pieces) > 1: michael@0: self.test_name = name_pieces[1] michael@0: self.class_name = name_pieces[0] michael@0: else: michael@0: self.class_name = full_name michael@0: self.test_name = full_name michael@0: self.start_date = start_date michael@0: self.dur = dur michael@0: michael@0: michael@0: class TestResults(object): michael@0: """Results of a test run.""" michael@0: michael@0: def __init__(self): michael@0: self.ok = [] michael@0: self.failed = [] michael@0: self.crashed = [] michael@0: self.unknown = [] michael@0: self.timed_out = False michael@0: self.overall_fail = False michael@0: michael@0: @staticmethod michael@0: def FromRun(ok=None, failed=None, crashed=None, timed_out=False, michael@0: overall_fail=False): michael@0: ret = TestResults() michael@0: ret.ok = ok or [] michael@0: ret.failed = failed or [] michael@0: ret.crashed = crashed or [] michael@0: ret.timed_out = timed_out michael@0: ret.overall_fail = overall_fail michael@0: return ret michael@0: michael@0: @staticmethod michael@0: def FromTestResults(results): michael@0: """Combines a list of results in a single TestResults object.""" michael@0: ret = TestResults() michael@0: for t in results: michael@0: ret.ok += t.ok michael@0: ret.failed += t.failed michael@0: ret.crashed += t.crashed michael@0: ret.unknown += t.unknown michael@0: if t.timed_out: michael@0: ret.timed_out = True michael@0: if t.overall_fail: michael@0: ret.overall_fail = True michael@0: return ret michael@0: michael@0: @staticmethod michael@0: def FromPythonException(test_name, start_date_ms, exc_info): michael@0: """Constructs a TestResults with exception information for the given test. michael@0: michael@0: Args: michael@0: test_name: name of the test which raised an exception. michael@0: start_date_ms: the starting time for the test. michael@0: exc_info: exception info, ostensibly from sys.exc_info(). michael@0: michael@0: Returns: michael@0: A TestResults object with a SingleTestResult in the failed list. michael@0: """ michael@0: exc_type, exc_value, exc_traceback = exc_info michael@0: trace_info = ''.join(traceback.format_exception(exc_type, exc_value, michael@0: exc_traceback)) michael@0: log_msg = 'Exception:\n' + trace_info michael@0: duration_ms = (int(time.time()) * 1000) - start_date_ms michael@0: michael@0: exc_result = SingleTestResult( michael@0: full_name='PythonWrapper#' + test_name, michael@0: start_date=start_date_ms, michael@0: dur=duration_ms, michael@0: log=(str(exc_type) + ' ' + log_msg)) michael@0: michael@0: results = TestResults() michael@0: results.failed.append(exc_result) michael@0: return results michael@0: michael@0: def _Log(self, sorted_list): michael@0: for t in sorted_list: michael@0: logging.critical(t.name) michael@0: if t.log: michael@0: logging.critical(t.log) michael@0: michael@0: def GetAllBroken(self): michael@0: """Returns the all broken tests including failed, crashed, unknown.""" michael@0: return self.failed + self.crashed + self.unknown michael@0: michael@0: def LogFull(self, test_group, test_suite, build_type): michael@0: """Output broken test logs, summarize in a log file and the test output.""" michael@0: # Output all broken tests or 'passed' if none broken. michael@0: logging.critical('*' * 80) michael@0: logging.critical('Final result') michael@0: if self.failed: michael@0: logging.critical('Failed:') michael@0: self._Log(sorted(self.failed)) michael@0: if self.crashed: michael@0: logging.critical('Crashed:') michael@0: self._Log(sorted(self.crashed)) michael@0: if self.unknown: michael@0: logging.critical('Unknown:') michael@0: self._Log(sorted(self.unknown)) michael@0: if not self.GetAllBroken(): michael@0: logging.critical('Passed') michael@0: logging.critical('*' * 80) michael@0: michael@0: # Summarize in a log file, if tests are running on bots. michael@0: if test_group and test_suite and os.environ.get('BUILDBOT_BUILDERNAME'): michael@0: log_file_path = os.path.join(constants.CHROME_DIR, 'out', michael@0: build_type, 'test_logs') michael@0: if not os.path.exists(log_file_path): michael@0: os.mkdir(log_file_path) michael@0: full_file_name = os.path.join(log_file_path, test_group) michael@0: if not os.path.exists(full_file_name): michael@0: with open(full_file_name, 'w') as log_file: michael@0: print >> log_file, '\n%s results for %s build %s:' % ( michael@0: test_group, os.environ.get('BUILDBOT_BUILDERNAME'), michael@0: os.environ.get('BUILDBOT_BUILDNUMBER')) michael@0: log_contents = [' %s result : %d tests ran' % (test_suite, michael@0: len(self.ok) + michael@0: len(self.failed) + michael@0: len(self.crashed) + michael@0: len(self.unknown))] michael@0: content_pairs = [('passed', len(self.ok)), ('failed', len(self.failed)), michael@0: ('crashed', len(self.crashed))] michael@0: for (result, count) in content_pairs: michael@0: if count: michael@0: log_contents.append(', %d tests %s' % (count, result)) michael@0: with open(full_file_name, 'a') as log_file: michael@0: print >> log_file, ''.join(log_contents) michael@0: content = {'test_group': test_group, michael@0: 'ok': [t.name for t in self.ok], michael@0: 'failed': [t.name for t in self.failed], michael@0: 'crashed': [t.name for t in self.failed], michael@0: 'unknown': [t.name for t in self.unknown],} michael@0: with open(os.path.join(log_file_path, 'results.json'), 'a') as json_file: michael@0: print >> json_file, json.dumps(content) michael@0: michael@0: # Summarize in the test output. michael@0: summary_string = 'Summary:\n' michael@0: summary_string += 'RAN=%d\n' % (len(self.ok) + len(self.failed) + michael@0: len(self.crashed) + len(self.unknown)) michael@0: summary_string += 'PASSED=%d\n' % (len(self.ok)) michael@0: summary_string += 'FAILED=%d %s\n' % (len(self.failed), michael@0: [t.name for t in self.failed]) michael@0: summary_string += 'CRASHED=%d %s\n' % (len(self.crashed), michael@0: [t.name for t in self.crashed]) michael@0: summary_string += 'UNKNOWN=%d %s\n' % (len(self.unknown), michael@0: [t.name for t in self.unknown]) michael@0: logging.critical(summary_string) michael@0: return summary_string michael@0: michael@0: def PrintAnnotation(self): michael@0: """Print buildbot annotations for test results.""" michael@0: if self.timed_out: michael@0: buildbot_report.PrintWarning() michael@0: elif self.failed or self.crashed or self.overall_fail: michael@0: buildbot_report.PrintError() michael@0: else: michael@0: print 'Step success!' # No annotation needed