js/src/tests/lib/results.py@129ffea94266
js/src/tests/lib/results.py
Sat, 03 Jan 2015 20:18:00 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Sat, 03 Jan 2015 20:18:00 +0100
- branch
- TOR_BUG_3246
- changeset 7
- 129ffea94266
- permissions
- -rw-r--r--
Conditionally enable double key logic according to:
private browsing mode or privacy.thirdparty.isolate preference and
implement in GetCookieStringCommon and FindCookie where it counts...
With some reservations of how to convince FindCookie users to test
condition and pass a nullptr when disabling double key logic.
1 import re
2 from progressbar import NullProgressBar, ProgressBar
3 import pipes
5 # subprocess.list2cmdline does not properly escape for sh-like shells
6 def escape_cmdline(args):
7 return ' '.join([ pipes.quote(a) for a in args ])
9 class TestOutput:
10 """Output from a test run."""
11 def __init__(self, test, cmd, out, err, rc, dt, timed_out):
12 self.test = test # Test
13 self.cmd = cmd # str: command line of test
14 self.out = out # str: stdout
15 self.err = err # str: stderr
16 self.rc = rc # int: return code
17 self.dt = dt # float: run time
18 self.timed_out = timed_out # bool: did the test time out
20 def describe_failure(self):
21 if self.timed_out:
22 return "Timeout"
23 lines = self.err.splitlines()
24 for line in lines:
25 # Skip the asm.js compilation success message.
26 if "Successfully compiled asm.js code" not in line:
27 return line
28 return "Unknown"
30 class NullTestOutput:
31 """Variant of TestOutput that indicates a test was not run."""
32 def __init__(self, test):
33 self.test = test
34 self.cmd = ''
35 self.out = ''
36 self.err = ''
37 self.rc = 0
38 self.dt = 0.0
39 self.timed_out = False
41 class TestResult:
42 PASS = 'PASS'
43 FAIL = 'FAIL'
44 CRASH = 'CRASH'
46 """Classified result from a test run."""
47 def __init__(self, test, result, results):
48 self.test = test
49 self.result = result
50 self.results = results
52 @classmethod
53 def from_output(cls, output):
54 test = output.test
55 result = None # str: overall result, see class-level variables
56 results = [] # (str,str) list: subtest results (pass/fail, message)
58 out, rc = output.out, output.rc
60 failures = 0
61 passes = 0
63 expected_rcs = []
64 if test.path.endswith('-n.js'):
65 expected_rcs.append(3)
67 for line in out.split('\n'):
68 if line.startswith(' FAILED!'):
69 failures += 1
70 msg = line[len(' FAILED! '):]
71 results.append((cls.FAIL, msg))
72 elif line.startswith(' PASSED!'):
73 passes += 1
74 msg = line[len(' PASSED! '):]
75 results.append((cls.PASS, msg))
76 else:
77 m = re.match('--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE ((?:-|\\d)+) ---', line)
78 if m:
79 expected_rcs.append(int(m.group(1)))
81 if rc and not rc in expected_rcs:
82 if rc == 3:
83 result = cls.FAIL
84 else:
85 result = cls.CRASH
86 else:
87 if (rc or passes > 0) and failures == 0:
88 result = cls.PASS
89 else:
90 result = cls.FAIL
92 return cls(test, result, results)
94 class ResultsSink:
95 def __init__(self, options, testcount):
96 self.options = options
97 self.fp = options.output_fp
99 self.groups = {}
100 self.counts = {'PASS': 0, 'FAIL': 0, 'TIMEOUT': 0, 'SKIP': 0}
101 self.n = 0
103 if options.hide_progress:
104 self.pb = NullProgressBar()
105 else:
106 fmt = [
107 {'value': 'PASS', 'color': 'green'},
108 {'value': 'FAIL', 'color': 'red'},
109 {'value': 'TIMEOUT', 'color': 'blue'},
110 {'value': 'SKIP', 'color': 'brightgray'},
111 ]
112 self.pb = ProgressBar(testcount, fmt)
114 def push(self, output):
115 if output.timed_out:
116 self.counts['TIMEOUT'] += 1
117 if isinstance(output, NullTestOutput):
118 if self.options.tinderbox:
119 self.print_tinderbox_result('TEST-KNOWN-FAIL', output.test.path, time=output.dt, skip=True)
120 self.counts['SKIP'] += 1
121 self.n += 1
122 else:
123 result = TestResult.from_output(output)
124 tup = (result.result, result.test.expect, result.test.random)
125 dev_label = self.LABELS[tup][1]
126 if output.timed_out:
127 dev_label = 'TIMEOUTS'
128 self.groups.setdefault(dev_label, []).append(result.test.path)
130 show = self.options.show
131 if self.options.failed_only and dev_label not in ('REGRESSIONS', 'TIMEOUTS'):
132 show = False
133 if show:
134 self.pb.beginline()
136 if show:
137 if self.options.show_output:
138 print >> self.fp, '## %s: rc = %d, run time = %f' % (output.test.path, output.rc, output.dt)
140 if self.options.show_cmd:
141 print >> self.fp, escape_cmdline(output.cmd)
143 if self.options.show_output:
144 self.fp.write(output.out)
145 self.fp.write(output.err)
147 self.n += 1
149 if result.result == TestResult.PASS and not result.test.random:
150 self.counts['PASS'] += 1
151 elif result.test.expect and not result.test.random:
152 self.counts['FAIL'] += 1
153 else:
154 self.counts['SKIP'] += 1
156 if self.options.tinderbox:
157 if len(result.results) > 1:
158 for sub_ok, msg in result.results:
159 label = self.LABELS[(sub_ok, result.test.expect, result.test.random)][0]
160 if label == 'TEST-UNEXPECTED-PASS':
161 label = 'TEST-PASS (EXPECTED RANDOM)'
162 self.print_tinderbox_result(label, result.test.path, time=output.dt, message=msg)
163 self.print_tinderbox_result(self.LABELS[
164 (result.result, result.test.expect, result.test.random)][0],
165 result.test.path, time=output.dt)
166 return
168 if dev_label:
169 def singular(label):
170 return "FIXED" if label == "FIXES" else label[:-1]
171 self.pb.message("%s - %s" % (singular(dev_label), output.test.path))
173 self.pb.update(self.n, self.counts)
175 def finish(self, completed):
176 self.pb.finish(completed)
177 if not self.options.tinderbox:
178 self.list(completed)
180 # Conceptually, this maps (test result x test expection) to text labels.
181 # key is (result, expect, random)
182 # value is (tinderbox label, dev test category)
183 LABELS = {
184 (TestResult.CRASH, False, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
185 (TestResult.CRASH, False, True): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
186 (TestResult.CRASH, True, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
187 (TestResult.CRASH, True, True): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
189 (TestResult.FAIL, False, False): ('TEST-KNOWN-FAIL', ''),
190 (TestResult.FAIL, False, True): ('TEST-KNOWN-FAIL (EXPECTED RANDOM)', ''),
191 (TestResult.FAIL, True, False): ('TEST-UNEXPECTED-FAIL', 'REGRESSIONS'),
192 (TestResult.FAIL, True, True): ('TEST-KNOWN-FAIL (EXPECTED RANDOM)', ''),
194 (TestResult.PASS, False, False): ('TEST-UNEXPECTED-PASS', 'FIXES'),
195 (TestResult.PASS, False, True): ('TEST-PASS (EXPECTED RANDOM)', ''),
196 (TestResult.PASS, True, False): ('TEST-PASS', ''),
197 (TestResult.PASS, True, True): ('TEST-PASS (EXPECTED RANDOM)', ''),
198 }
200 def list(self, completed):
201 for label, paths in sorted(self.groups.items()):
202 if label == '': continue
204 print label
205 for path in paths:
206 print ' %s'%path
208 if self.options.failure_file:
209 failure_file = open(self.options.failure_file, 'w')
210 if not self.all_passed():
211 if 'REGRESSIONS' in self.groups:
212 for path in self.groups['REGRESSIONS']:
213 print >> failure_file, path
214 if 'TIMEOUTS' in self.groups:
215 for path in self.groups['TIMEOUTS']:
216 print >> failure_file, path
217 failure_file.close()
219 suffix = '' if completed else ' (partial run -- interrupted by user)'
220 if self.all_passed():
221 print 'PASS' + suffix
222 else:
223 print 'FAIL' + suffix
225 def all_passed(self):
226 return 'REGRESSIONS' not in self.groups and 'TIMEOUTS' not in self.groups
228 def print_tinderbox_result(self, label, path, message=None, skip=False, time=None):
229 result = label
230 result += " | " + path
231 result += " |" + self.options.shell_args
232 if message:
233 result += " | " + message
234 if skip:
235 result += ' | (SKIP)'
236 if time > self.options.timeout:
237 result += ' | (TIMEOUT)'
238 print result
