The Tor Browser / file comparison
comparison: js/src/tests/lib/results.py
js/src/tests/lib/results.py
- branch
- TOR_BUG_3246
- changeset 7
- 129ffea94266
equal
deleted
inserted
replaced
| -1:000000000000 | 0:e3944e815f15 |
|---|---|
| 1 import re | |
| 2 from progressbar import NullProgressBar, ProgressBar | |
| 3 import pipes | |
| 4 | |
| 5 # subprocess.list2cmdline does not properly escape for sh-like shells | |
| 6 def escape_cmdline(args): | |
| 7 return ' '.join([ pipes.quote(a) for a in args ]) | |
| 8 | |
| 9 class TestOutput: | |
| 10 """Output from a test run.""" | |
| 11 def __init__(self, test, cmd, out, err, rc, dt, timed_out): | |
| 12 self.test = test # Test | |
| 13 self.cmd = cmd # str: command line of test | |
| 14 self.out = out # str: stdout | |
| 15 self.err = err # str: stderr | |
| 16 self.rc = rc # int: return code | |
| 17 self.dt = dt # float: run time | |
| 18 self.timed_out = timed_out # bool: did the test time out | |
| 19 | |
| 20 def describe_failure(self): | |
| 21 if self.timed_out: | |
| 22 return "Timeout" | |
| 23 lines = self.err.splitlines() | |
| 24 for line in lines: | |
| 25 # Skip the asm.js compilation success message. | |
| 26 if "Successfully compiled asm.js code" not in line: | |
| 27 return line | |
| 28 return "Unknown" | |
| 29 | |
| 30 class NullTestOutput: | |
| 31 """Variant of TestOutput that indicates a test was not run.""" | |
| 32 def __init__(self, test): | |
| 33 self.test = test | |
| 34 self.cmd = '' | |
| 35 self.out = '' | |
| 36 self.err = '' | |
| 37 self.rc = 0 | |
| 38 self.dt = 0.0 | |
| 39 self.timed_out = False | |
| 40 | |
| 41 class TestResult: | |
| 42 PASS = 'PASS' | |
| 43 FAIL = 'FAIL' | |
| 44 CRASH = 'CRASH' | |
| 45 | |
| 46 """Classified result from a test run.""" | |
| 47 def __init__(self, test, result, results): | |
| 48 self.test = test | |
| 49 self.result = result | |
| 50 self.results = results | |
| 51 | |
| 52 @classmethod | |
| 53 def from_output(cls, output): | |
| 54 test = output.test | |
| 55 result = None # str: overall result, see class-level variables | |
| 56 results = [] # (str,str) list: subtest results (pass/fail, message) | |
| 57 | |
| 58 out, rc = output.out, output.rc | |
| 59 | |
| 60 failures = 0 | |
| 61 passes = 0 | |
| 62 | |
| 63 expected_rcs = [] | |
| 64 if test.path.endswith('-n.js'): | |
| 65 expected_rcs.append(3) | |
| 66 | |
| 67 for line in out.split('\n'): | |
| 68 if line.startswith(' FAILED!'): | |
| 69 failures += 1 | |
| 70 msg = line[len(' FAILED! '):] | |
| 71 results.append((cls.FAIL, msg)) | |
| 72 elif line.startswith(' PASSED!'): | |
| 73 passes += 1 | |
| 74 msg = line[len(' PASSED! '):] | |
| 75 results.append((cls.PASS, msg)) | |
| 76 else: | |
| 77 m = re.match('--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE ((?:-|\\d)+) ---', line) | |
| 78 if m: | |
| 79 expected_rcs.append(int(m.group(1))) | |
| 80 | |
| 81 if rc and not rc in expected_rcs: | |
| 82 if rc == 3: | |
| 83 result = cls.FAIL | |
| 84 else: | |
| 85 result = cls.CRASH | |
| 86 else: | |
| 87 if (rc or passes > 0) and failures == 0: | |
| 88 result = cls.PASS | |
| 89 else: | |
| 90 result = cls.FAIL | |
| 91 | |
| 92 return cls(test, result, results) | |
| 93 | |
| 94 class ResultsSink: | |
| 95 def __init__(self, options, testcount): | |
| 96 self.options = options | |
| 97 self.fp = options.output_fp | |
| 98 | |
| 99 self.groups = {} | |
| 100 self.counts = {'PASS': 0, 'FAIL': 0, 'TIMEOUT': 0, 'SKIP': 0} | |
| 101 self.n = 0 | |
| 102 | |
| 103 if options.hide_progress: | |
| 104 self.pb = NullProgressBar() | |
| 105 else: | |
| 106 fmt = [ | |
| 107 {'value': 'PASS', 'color': 'green'}, | |
| 108 {'value': 'FAIL', 'color': 'red'}, | |
| 109 {'value': 'TIMEOUT', 'color': 'blue'}, | |
| 110 {'value': 'SKIP', 'color': 'brightgray'}, | |
| 111 ] | |
| 112 self.pb = ProgressBar(testcount, fmt) | |
| 113 | |
| 114 def push(self, output): | |
| 115 if output.timed_out: | |
| 116 self.counts['TIMEOUT'] += 1 | |
| 117 if isinstance(output, NullTestOutput): | |
| 118 if self.options.tinderbox: | |
| 119 self.print_tinderbox_result('TEST-KNOWN-FAIL', output.test.path, time=output.dt, skip=True) | |
| 120 self.counts['SKIP'] += 1 | |
| 121 self.n += 1 | |
| 122 else: | |
| 123 result = TestResult.from_output(output) | |
| 124 tup = (result.result, result.test.expect, result.test.random) | |
| 125 dev_label = self.LABELS[tup][1] | |
| 126 if output.timed_out: | |
| 127 dev_label = 'TIMEOUTS' | |
| 128 self.groups.setdefault(dev_label, []).append(result.test.path) | |
| 129 | |
| 130 show = self.options.show | |
| 131 if self.options.failed_only and dev_label not in ('REGRESSIONS', 'TIMEOUTS'): | |
| 132 show = False | |
| 133 if show: | |
| 134 self.pb.beginline() | |
| 135 | |
| 136 if show: | |
| 137 if self.options.show_output: | |
| 138 print >> self.fp, '## %s: rc = %d, run time = %f' % (output.test.path, output.rc, output.dt) | |
| 139 | |
| 140 if self.options.show_cmd: | |
| 141 print >> self.fp, escape_cmdline(output.cmd) | |
| 142 | |
| 143 if self.options.show_output: | |
| 144 self.fp.write(output.out) | |
| 145 self.fp.write(output.err) | |
| 146 | |
| 147 self.n += 1 | |
| 148 | |
| 149 if result.result == TestResult.PASS and not result.test.random: | |
| 150 self.counts['PASS'] += 1 | |
| 151 elif result.test.expect and not result.test.random: | |
| 152 self.counts['FAIL'] += 1 | |
| 153 else: | |
| 154 self.counts['SKIP'] += 1 | |
| 155 | |
| 156 if self.options.tinderbox: | |
| 157 if len(result.results) > 1: | |
| 158 for sub_ok, msg in result.results: | |
| 159 label = self.LABELS[(sub_ok, result.test.expect, result.test.random)][0] | |
| 160 if label == 'TEST-UNEXPECTED-PASS': | |
| 161 label = 'TEST-PASS (EXPECTED RANDOM)' | |
| 162 self.print_tinderbox_result(label, result.test.path, time=output.dt, message=msg) | |
| 163 self.print_tinderbox_result(self.LABELS[ | |
| 164 (result.result, result.test.expect, result.test.random)][0], | |
| 165 result.test.path, time=output.dt) | |
| 166 return | |
| 167 | |
| 168 if dev_label: | |
| 169 def singular(label): | |
| 170 return "FIXED" if label == "FIXES" else label[:-1] | |
| 171 self.pb.message("%s - %s" % (singular(dev_label), output.test.path)) | |
| 172 | |
| 173 self.pb.update(self.n, self.counts) | |
| 174 | |
| 175 def finish(self, completed): | |
| 176 self.pb.finish(completed) | |
| 177 if not self.options.tinderbox: | |
| 178 self.list(completed) | |
| 179 | |
| 180 # Conceptually, this maps (test result x test expection) to text labels. | |
| 181 # key is (result, expect, random) | |
| 182 # value is (tinderbox label, dev test category) | |
| 183 LABELS = { | |
| 184 (TestResult.CRASH, False, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'), | |
| 185 (TestResult.CRASH, False, True): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'), | |
| 186 (TestResult.CRASH, True, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'), | |
| 187 (TestResult.CRASH, True, True): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'), | |
| 188 | |
| 189 (TestResult.FAIL, False, False): ('TEST-KNOWN-FAIL', ''), | |
| 190 (TestResult.FAIL, False, True): ('TEST-KNOWN-FAIL (EXPECTED RANDOM)', ''), | |
| 191 (TestResult.FAIL, True, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'), | |
| 192 (TestResult.FAIL, True, True): ('TEST-KNOWN-FAIL (EXPECTED RANDOM)', ''), | |
| 193 | |
| 194 (TestResult.PASS, False, False): ('TEST-UNEXPECTED-PASS', 'FIXES'), | |
| 195 (TestResult.PASS, False, True): ('TEST-PASS (EXPECTED RANDOM)', ''), | |
| 196 (TestResult.PASS, True, False): ('TEST-PASS', ''), | |
| 197 (TestResult.PASS, True, True): ('TEST-PASS (EXPECTED RANDOM)', ''), | |
| 198 } | |
| 199 | |
| 200 def list(self, completed): | |
| 201 for label, paths in sorted(self.groups.items()): | |
| 202 if label == '': continue | |
| 203 | |
| 204 print label | |
| 205 for path in paths: | |
| 206 print ' %s'%path | |
| 207 | |
| 208 if self.options.failure_file: | |
| 209 failure_file = open(self.options.failure_file, 'w') | |
| 210 if not self.all_passed(): | |
| 211 if 'REGRESSIONS' in self.groups: | |
| 212 for path in self.groups['REGRESSIONS']: | |
| 213 print >> failure_file, path | |
| 214 if 'TIMEOUTS' in self.groups: | |
| 215 for path in self.groups['TIMEOUTS']: | |
| 216 print >> failure_file, path | |
| 217 failure_file.close() | |
| 218 | |
| 219 suffix = '' if completed else ' (partial run -- interrupted by user)' | |
| 220 if self.all_passed(): | |
| 221 print 'PASS' + suffix | |
| 222 else: | |
| 223 print 'FAIL' + suffix | |
| 224 | |
| 225 def all_passed(self): | |
| 226 return 'REGRESSIONS' not in self.groups and 'TIMEOUTS' not in self.groups | |
| 227 | |
| 228 def print_tinderbox_result(self, label, path, message=None, skip=False, time=None): | |
| 229 result = label | |
| 230 result += " | " + path | |
| 231 result += " |" + self.options.shell_args | |
| 232 if message: | |
| 233 result += " | " + message | |
| 234 if skip: | |
| 235 result += ' | (SKIP)' | |
| 236 if time > self.options.timeout: | |
| 237 result += ' | (TIMEOUT)' | |
| 238 print result | |
| 239 |
