michael@0: # This Source Code Form is subject to the terms of the Mozilla Public michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/. michael@0: michael@0: # Works with python2.6 michael@0: michael@0: import datetime, os, re, sys, traceback michael@0: import math, string, copy, json michael@0: import subprocess michael@0: from subprocess import * michael@0: from operator import itemgetter michael@0: michael@0: class Test: michael@0: def __init__(self, path, name): michael@0: self.path = path michael@0: self.name = name michael@0: michael@0: @classmethod michael@0: def from_file(cls, path, name, options): michael@0: return cls(path, name) michael@0: michael@0: def find_tests(dir, substring = None): michael@0: ans = [] michael@0: for dirpath, dirnames, filenames in os.walk(dir): michael@0: if dirpath == '.': michael@0: continue michael@0: for filename in filenames: michael@0: if not filename.endswith('.js'): michael@0: continue michael@0: test = os.path.join(dirpath, filename) michael@0: if substring is None or substring in os.path.relpath(test, dir): michael@0: ans.append([test, filename]) michael@0: return ans michael@0: michael@0: def get_test_cmd(path): michael@0: return [ JS, '-f', path ] michael@0: michael@0: def avg(seq): michael@0: return sum(seq) / len(seq) michael@0: michael@0: def stddev(seq, mean): michael@0: diffs = ((float(item) - mean) ** 2 for item in seq) michael@0: return math.sqrt(sum(diffs) / len(seq)) michael@0: michael@0: def run_test(test): michael@0: env = os.environ.copy() michael@0: env['MOZ_GCTIMER'] = 'stderr' michael@0: cmd = get_test_cmd(test.path) michael@0: total = [] michael@0: mark = [] michael@0: sweep = [] michael@0: close_fds = sys.platform != 'win32' michael@0: p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=close_fds, env=env) michael@0: out, err = p.communicate() michael@0: out, err = out.decode(), err.decode() michael@0: michael@0: float_array = [float(_) for _ in err.split()] michael@0: michael@0: if len(float_array) == 0: michael@0: print('Error: No data from application. Configured with --enable-gctimer?') michael@0: sys.exit(1) michael@0: michael@0: for i, currItem in enumerate(float_array): michael@0: if (i % 3 == 0): michael@0: total.append(currItem) michael@0: else: michael@0: if (i % 3 == 1): michael@0: mark.append(currItem) michael@0: else: michael@0: sweep.append(currItem) michael@0: michael@0: return max(total), avg(total), max(mark), avg(mark), max(sweep), avg(sweep) michael@0: michael@0: def run_tests(tests, test_dir): michael@0: bench_map = {} michael@0: michael@0: try: michael@0: for i, test in enumerate(tests): michael@0: filename_str = '"%s"' % test.name michael@0: TMax, TAvg, MMax, MAvg, SMax, SAvg = run_test(test) michael@0: bench_map[test.name] = [TMax, TAvg, MMax, MAvg, SMax, SAvg] michael@0: fmt = '%20s: {"TMax": %4.1f, "TAvg": %4.1f, "MMax": %4.1f, "MAvg": %4.1f, "SMax": %4.1f, "SAvg": %4.1f}' michael@0: if (i != len(tests) - 1): michael@0: fmt += ',' michael@0: print(fmt %(filename_str ,TMax, TAvg, MMax, MAvg, SMax, MAvg)) michael@0: except KeyboardInterrupt: michael@0: print('fail') michael@0: michael@0: return dict((filename, dict(TMax=TMax, TAvg=TAvg, MMax=MMax, MAvg=MAvg, SMax=SMax, SAvg=SAvg)) michael@0: for filename, (TMax, TAvg, MMax, MAvg, SMax, SAvg) in bench_map.iteritems()) michael@0: michael@0: def compare(current, baseline): michael@0: percent_speedups = [] michael@0: for key, current_result in current.iteritems(): michael@0: try: michael@0: baseline_result = baseline[key] michael@0: except KeyError: michael@0: print key, 'missing from baseline' michael@0: continue michael@0: michael@0: val_getter = itemgetter('TMax', 'TAvg', 'MMax', 'MAvg', 'SMax', 'SAvg') michael@0: BTMax, BTAvg, BMMax, BMAvg, BSMax, BSAvg = val_getter(baseline_result) michael@0: CTMax, CTAvg, CMMax, CMAvg, CSMax, CSAvg = val_getter(current_result) michael@0: michael@0: fmt = '%30s: %s' michael@0: if CTAvg <= BTAvg: michael@0: speedup = (CTAvg / BTAvg - 1) * 100 michael@0: result = 'faster: %6.2f < baseline %6.2f (%+6.2f%%)' % \ michael@0: (CTAvg, BTAvg, speedup) michael@0: percent_speedups.append(speedup) michael@0: else: michael@0: slowdown = (CTAvg / BTAvg - 1) * 100 michael@0: result = 'SLOWER: %6.2f > baseline %6.2f (%+6.2f%%) ' % \ michael@0: (CTAvg, BTAvg, slowdown) michael@0: percent_speedups.append(slowdown) michael@0: print '%30s: %s' % (key, result) michael@0: if percent_speedups: michael@0: print 'Average speedup: %.2f%%' % avg(percent_speedups) michael@0: michael@0: if __name__ == '__main__': michael@0: script_path = os.path.abspath(__file__) michael@0: script_dir = os.path.dirname(script_path) michael@0: test_dir = os.path.join(script_dir, 'tests') michael@0: michael@0: from optparse import OptionParser michael@0: op = OptionParser(usage='%prog [options] JS_SHELL [TESTS]') michael@0: michael@0: op.add_option('-b', '--baseline', metavar='JSON_PATH', michael@0: dest='baseline_path', help='json file with baseline values to ' michael@0: 'compare against') michael@0: michael@0: (OPTIONS, args) = op.parse_args() michael@0: if len(args) < 1: michael@0: op.error('missing JS_SHELL argument') michael@0: # We need to make sure we are using backslashes on Windows. michael@0: JS, test_args = os.path.normpath(args[0]), args[1:] michael@0: michael@0: test_list = [] michael@0: bench_map = {} michael@0: michael@0: test_list = find_tests(test_dir) michael@0: michael@0: if not test_list: michael@0: print >> sys.stderr, "No tests found matching command line arguments." michael@0: sys.exit(0) michael@0: michael@0: test_list = [ Test.from_file(tst, name, OPTIONS) for tst, name in test_list ] michael@0: michael@0: try: michael@0: print("{") michael@0: bench_map = run_tests(test_list, test_dir) michael@0: print("}") michael@0: michael@0: except OSError: michael@0: if not os.path.exists(JS): michael@0: print >> sys.stderr, "JS shell argument: file does not exist: '%s'"%JS michael@0: sys.exit(1) michael@0: else: michael@0: raise michael@0: michael@0: if OPTIONS.baseline_path: michael@0: baseline_map = [] michael@0: fh = open(OPTIONS.baseline_path, 'r') michael@0: baseline_map = json.load(fh) michael@0: fh.close() michael@0: compare(current=bench_map, baseline=baseline_map)