michael@0: import re
michael@0: from progressbar import NullProgressBar, ProgressBar
michael@0: import pipes
michael@0: 
michael@0: # subprocess.list2cmdline does not properly escape for sh-like shells
michael@0: def escape_cmdline(args):
michael@0:     return ' '.join([ pipes.quote(a) for a in args ])
michael@0: 
michael@0: class TestOutput:
michael@0:     """Output from a test run."""
michael@0:     def __init__(self, test, cmd, out, err, rc, dt, timed_out):
michael@0:         self.test = test   # Test
michael@0:         self.cmd = cmd     # str:   command line of test
michael@0:         self.out = out     # str:   stdout
michael@0:         self.err = err     # str:   stderr
michael@0:         self.rc = rc       # int:   return code
michael@0:         self.dt = dt       # float: run time
michael@0:         self.timed_out = timed_out # bool: did the test time out
michael@0: 
michael@0:     def describe_failure(self):
michael@0:         if self.timed_out:
michael@0:             return "Timeout"
michael@0:         lines = self.err.splitlines()
michael@0:         for line in lines:
michael@0:             # Skip the asm.js compilation success message.
michael@0:             if "Successfully compiled asm.js code" not in line:
michael@0:                 return line
michael@0:         return "Unknown"
michael@0: 
michael@0: class NullTestOutput:
michael@0:     """Variant of TestOutput that indicates a test was not run."""
michael@0:     def __init__(self, test):
michael@0:         self.test = test
michael@0:         self.cmd = ''
michael@0:         self.out = ''
michael@0:         self.err = ''
michael@0:         self.rc = 0
michael@0:         self.dt = 0.0
michael@0:         self.timed_out = False
michael@0: 
michael@0: class TestResult:
michael@0:     PASS = 'PASS'
michael@0:     FAIL = 'FAIL'
michael@0:     CRASH = 'CRASH'
michael@0: 
michael@0:     """Classified result from a test run."""
michael@0:     def __init__(self, test, result, results):
michael@0:         self.test = test
michael@0:         self.result = result
michael@0:         self.results = results
michael@0: 
michael@0:     @classmethod
michael@0:     def from_output(cls, output):
michael@0:         test = output.test
michael@0:         result = None          # str:      overall result, see class-level variables
michael@0:         results = []           # (str,str) list: subtest results (pass/fail, message)
michael@0: 
michael@0:         out, rc = output.out, output.rc
michael@0: 
michael@0:         failures = 0
michael@0:         passes = 0
michael@0: 
michael@0:         expected_rcs = []
michael@0:         if test.path.endswith('-n.js'):
michael@0:             expected_rcs.append(3)
michael@0: 
michael@0:         for line in out.split('\n'):
michael@0:             if line.startswith(' FAILED!'):
michael@0:                 failures += 1
michael@0:                 msg = line[len(' FAILED! '):]
michael@0:                 results.append((cls.FAIL, msg))
michael@0:             elif line.startswith(' PASSED!'):
michael@0:                 passes += 1
michael@0:                 msg = line[len(' PASSED! '):]
michael@0:                 results.append((cls.PASS, msg))
michael@0:             else:
michael@0:                 m = re.match('--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE ((?:-|\\d)+) ---', line)
michael@0:                 if m:
michael@0:                     expected_rcs.append(int(m.group(1)))
michael@0: 
michael@0:         if rc and not rc in expected_rcs:
michael@0:             if rc == 3:
michael@0:                 result = cls.FAIL
michael@0:             else:
michael@0:                 result = cls.CRASH
michael@0:         else:
michael@0:             if (rc or passes > 0) and failures == 0:
michael@0:                 result = cls.PASS
michael@0:             else:
michael@0:                 result = cls.FAIL
michael@0: 
michael@0:         return cls(test, result, results)
michael@0: 
michael@0: class ResultsSink:
michael@0:     def __init__(self, options, testcount):
michael@0:         self.options = options
michael@0:         self.fp = options.output_fp
michael@0: 
michael@0:         self.groups = {}
michael@0:         self.counts = {'PASS': 0, 'FAIL': 0, 'TIMEOUT': 0, 'SKIP': 0}
michael@0:         self.n = 0
michael@0: 
michael@0:         if options.hide_progress:
michael@0:             self.pb = NullProgressBar()
michael@0:         else:
michael@0:             fmt = [
michael@0:                 {'value': 'PASS',    'color': 'green'},
michael@0:                 {'value': 'FAIL',    'color': 'red'},
michael@0:                 {'value': 'TIMEOUT', 'color': 'blue'},
michael@0:                 {'value': 'SKIP',    'color': 'brightgray'},
michael@0:             ]
michael@0:             self.pb = ProgressBar(testcount, fmt)
michael@0: 
michael@0:     def push(self, output):
michael@0:         if output.timed_out:
michael@0:             self.counts['TIMEOUT'] += 1
michael@0:         if isinstance(output, NullTestOutput):
michael@0:             if self.options.tinderbox:
michael@0:                 self.print_tinderbox_result('TEST-KNOWN-FAIL', output.test.path, time=output.dt, skip=True)
michael@0:             self.counts['SKIP'] += 1
michael@0:             self.n += 1
michael@0:         else:
michael@0:             result = TestResult.from_output(output)
michael@0:             tup = (result.result, result.test.expect, result.test.random)
michael@0:             dev_label = self.LABELS[tup][1]
michael@0:             if output.timed_out:
michael@0:                 dev_label = 'TIMEOUTS'
michael@0:             self.groups.setdefault(dev_label, []).append(result.test.path)
michael@0: 
michael@0:             show = self.options.show
michael@0:             if self.options.failed_only and dev_label not in ('REGRESSIONS', 'TIMEOUTS'):
michael@0:                 show = False
michael@0:             if show:
michael@0:                 self.pb.beginline()
michael@0: 
michael@0:             if show:
michael@0:                 if self.options.show_output:
michael@0:                     print >> self.fp, '## %s: rc = %d, run time = %f' % (output.test.path, output.rc, output.dt)
michael@0: 
michael@0:                 if self.options.show_cmd:
michael@0:                     print >> self.fp, escape_cmdline(output.cmd)
michael@0: 
michael@0:                 if self.options.show_output:
michael@0:                     self.fp.write(output.out)
michael@0:                     self.fp.write(output.err)
michael@0: 
michael@0:             self.n += 1
michael@0: 
michael@0:             if result.result == TestResult.PASS and not result.test.random:
michael@0:                 self.counts['PASS'] += 1
michael@0:             elif result.test.expect and not result.test.random:
michael@0:                 self.counts['FAIL'] += 1
michael@0:             else:
michael@0:                 self.counts['SKIP'] += 1
michael@0: 
michael@0:             if self.options.tinderbox:
michael@0:                 if len(result.results) > 1:
michael@0:                     for sub_ok, msg in result.results:
michael@0:                         label = self.LABELS[(sub_ok, result.test.expect, result.test.random)][0]
michael@0:                         if label == 'TEST-UNEXPECTED-PASS':
michael@0:                             label = 'TEST-PASS (EXPECTED RANDOM)'
michael@0:                         self.print_tinderbox_result(label, result.test.path, time=output.dt, message=msg)
michael@0:                 self.print_tinderbox_result(self.LABELS[
michael@0:                     (result.result, result.test.expect, result.test.random)][0],
michael@0:                     result.test.path, time=output.dt)
michael@0:                 return
michael@0: 
michael@0:             if dev_label:
michael@0:                 def singular(label):
michael@0:                     return "FIXED" if label == "FIXES" else label[:-1]
michael@0:                 self.pb.message("%s - %s" % (singular(dev_label), output.test.path))
michael@0: 
michael@0:         self.pb.update(self.n, self.counts)
michael@0: 
michael@0:     def finish(self, completed):
michael@0:         self.pb.finish(completed)
michael@0:         if not self.options.tinderbox:
michael@0:             self.list(completed)
michael@0: 
michael@0:     # Conceptually, this maps (test result x test expection) to text labels.
michael@0:     #      key   is (result, expect, random)
michael@0:     #      value is (tinderbox label, dev test category)
michael@0:     LABELS = {
michael@0:         (TestResult.CRASH, False, False): ('TEST-UNEXPECTED-FAIL',               'REGRESSIONS'),
michael@0:         (TestResult.CRASH, False, True):  ('TEST-UNEXPECTED-FAIL',               'REGRESSIONS'),
michael@0:         (TestResult.CRASH, True,  False): ('TEST-UNEXPECTED-FAIL',               'REGRESSIONS'),
michael@0:         (TestResult.CRASH, True,  True):  ('TEST-UNEXPECTED-FAIL',               'REGRESSIONS'),
michael@0: 
michael@0:         (TestResult.FAIL,  False, False): ('TEST-KNOWN-FAIL',                    ''),
michael@0:         (TestResult.FAIL,  False, True):  ('TEST-KNOWN-FAIL (EXPECTED RANDOM)',  ''),
michael@0:         (TestResult.FAIL,  True,  False): ('TEST-UNEXPECTED-FAIL',               'REGRESSIONS'),
michael@0:         (TestResult.FAIL,  True,  True):  ('TEST-KNOWN-FAIL (EXPECTED RANDOM)',  ''),
michael@0: 
michael@0:         (TestResult.PASS,  False, False): ('TEST-UNEXPECTED-PASS',               'FIXES'),
michael@0:         (TestResult.PASS,  False, True):  ('TEST-PASS (EXPECTED RANDOM)',        ''),
michael@0:         (TestResult.PASS,  True,  False): ('TEST-PASS',                          ''),
michael@0:         (TestResult.PASS,  True,  True):  ('TEST-PASS (EXPECTED RANDOM)',        ''),
michael@0:         }
michael@0: 
michael@0:     def list(self, completed):
michael@0:         for label, paths in sorted(self.groups.items()):
michael@0:             if label == '': continue
michael@0: 
michael@0:             print label
michael@0:             for path in paths:
michael@0:                 print '    %s'%path
michael@0: 
michael@0:         if self.options.failure_file:
michael@0:               failure_file = open(self.options.failure_file, 'w')
michael@0:               if not self.all_passed():
michael@0:                   if 'REGRESSIONS' in self.groups:
michael@0:                       for path in self.groups['REGRESSIONS']:
michael@0:                           print >> failure_file, path
michael@0:                   if 'TIMEOUTS' in self.groups:
michael@0:                       for path in self.groups['TIMEOUTS']:
michael@0:                           print >> failure_file, path
michael@0:               failure_file.close()
michael@0: 
michael@0:         suffix = '' if completed else ' (partial run -- interrupted by user)'
michael@0:         if self.all_passed():
michael@0:             print 'PASS' + suffix
michael@0:         else:
michael@0:             print 'FAIL' + suffix
michael@0: 
michael@0:     def all_passed(self):
michael@0:         return 'REGRESSIONS' not in self.groups and 'TIMEOUTS' not in self.groups
michael@0: 
michael@0:     def print_tinderbox_result(self, label, path, message=None, skip=False, time=None):
michael@0:         result = label
michael@0:         result += " | " + path
michael@0:         result += " |" + self.options.shell_args
michael@0:         if message:
michael@0:             result += " | " + message
michael@0:         if skip:
michael@0:             result += ' | (SKIP)'
michael@0:         if time > self.options.timeout:
michael@0:             result += ' | (TIMEOUT)'
michael@0:         print result
michael@0: