1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/devtools/gc/gc-test.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,166 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +# Works with python2.6
1.9 +
1.10 +import datetime, os, re, sys, traceback
1.11 +import math, string, copy, json
1.12 +import subprocess
1.13 +from subprocess import *
1.14 +from operator import itemgetter
1.15 +
1.16 +class Test:
1.17 + def __init__(self, path, name):
1.18 + self.path = path
1.19 + self.name = name
1.20 +
1.21 + @classmethod
1.22 + def from_file(cls, path, name, options):
1.23 + return cls(path, name)
1.24 +
1.25 +def find_tests(dir, substring = None):
1.26 + ans = []
1.27 + for dirpath, dirnames, filenames in os.walk(dir):
1.28 + if dirpath == '.':
1.29 + continue
1.30 + for filename in filenames:
1.31 + if not filename.endswith('.js'):
1.32 + continue
1.33 + test = os.path.join(dirpath, filename)
1.34 + if substring is None or substring in os.path.relpath(test, dir):
1.35 + ans.append([test, filename])
1.36 + return ans
1.37 +
1.38 +def get_test_cmd(path):
1.39 + return [ JS, '-f', path ]
1.40 +
1.41 +def avg(seq):
1.42 + return sum(seq) / len(seq)
1.43 +
1.44 +def stddev(seq, mean):
1.45 + diffs = ((float(item) - mean) ** 2 for item in seq)
1.46 + return math.sqrt(sum(diffs) / len(seq))
1.47 +
1.48 +def run_test(test):
1.49 + env = os.environ.copy()
1.50 + env['MOZ_GCTIMER'] = 'stderr'
1.51 + cmd = get_test_cmd(test.path)
1.52 + total = []
1.53 + mark = []
1.54 + sweep = []
1.55 + close_fds = sys.platform != 'win32'
1.56 + p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=close_fds, env=env)
1.57 + out, err = p.communicate()
1.58 + out, err = out.decode(), err.decode()
1.59 +
1.60 + float_array = [float(_) for _ in err.split()]
1.61 +
1.62 + if len(float_array) == 0:
1.63 + print('Error: No data from application. Configured with --enable-gctimer?')
1.64 + sys.exit(1)
1.65 +
1.66 + for i, currItem in enumerate(float_array):
1.67 + if (i % 3 == 0):
1.68 + total.append(currItem)
1.69 + else:
1.70 + if (i % 3 == 1):
1.71 + mark.append(currItem)
1.72 + else:
1.73 + sweep.append(currItem)
1.74 +
1.75 + return max(total), avg(total), max(mark), avg(mark), max(sweep), avg(sweep)
1.76 +
1.77 +def run_tests(tests, test_dir):
1.78 + bench_map = {}
1.79 +
1.80 + try:
1.81 + for i, test in enumerate(tests):
1.82 + filename_str = '"%s"' % test.name
1.83 + TMax, TAvg, MMax, MAvg, SMax, SAvg = run_test(test)
1.84 + bench_map[test.name] = [TMax, TAvg, MMax, MAvg, SMax, SAvg]
1.85 + fmt = '%20s: {"TMax": %4.1f, "TAvg": %4.1f, "MMax": %4.1f, "MAvg": %4.1f, "SMax": %4.1f, "SAvg": %4.1f}'
1.86 + if (i != len(tests) - 1):
1.87 + fmt += ','
1.88 + print(fmt %(filename_str ,TMax, TAvg, MMax, MAvg, SMax, MAvg))
1.89 + except KeyboardInterrupt:
1.90 + print('fail')
1.91 +
1.92 + return dict((filename, dict(TMax=TMax, TAvg=TAvg, MMax=MMax, MAvg=MAvg, SMax=SMax, SAvg=SAvg))
1.93 + for filename, (TMax, TAvg, MMax, MAvg, SMax, SAvg) in bench_map.iteritems())
1.94 +
1.95 +def compare(current, baseline):
1.96 + percent_speedups = []
1.97 + for key, current_result in current.iteritems():
1.98 + try:
1.99 + baseline_result = baseline[key]
1.100 + except KeyError:
1.101 + print key, 'missing from baseline'
1.102 + continue
1.103 +
1.104 + val_getter = itemgetter('TMax', 'TAvg', 'MMax', 'MAvg', 'SMax', 'SAvg')
1.105 + BTMax, BTAvg, BMMax, BMAvg, BSMax, BSAvg = val_getter(baseline_result)
1.106 + CTMax, CTAvg, CMMax, CMAvg, CSMax, CSAvg = val_getter(current_result)
1.107 +
1.108 + fmt = '%30s: %s'
1.109 + if CTAvg <= BTAvg:
1.110 + speedup = (CTAvg / BTAvg - 1) * 100
1.111 + result = 'faster: %6.2f < baseline %6.2f (%+6.2f%%)' % \
1.112 + (CTAvg, BTAvg, speedup)
1.113 + percent_speedups.append(speedup)
1.114 + else:
1.115 + slowdown = (CTAvg / BTAvg - 1) * 100
1.116 + result = 'SLOWER: %6.2f > baseline %6.2f (%+6.2f%%) ' % \
1.117 + (CTAvg, BTAvg, slowdown)
1.118 + percent_speedups.append(slowdown)
1.119 + print '%30s: %s' % (key, result)
1.120 + if percent_speedups:
1.121 + print 'Average speedup: %.2f%%' % avg(percent_speedups)
1.122 +
1.123 +if __name__ == '__main__':
1.124 + script_path = os.path.abspath(__file__)
1.125 + script_dir = os.path.dirname(script_path)
1.126 + test_dir = os.path.join(script_dir, 'tests')
1.127 +
1.128 + from optparse import OptionParser
1.129 + op = OptionParser(usage='%prog [options] JS_SHELL [TESTS]')
1.130 +
1.131 + op.add_option('-b', '--baseline', metavar='JSON_PATH',
1.132 + dest='baseline_path', help='json file with baseline values to '
1.133 + 'compare against')
1.134 +
1.135 + (OPTIONS, args) = op.parse_args()
1.136 + if len(args) < 1:
1.137 + op.error('missing JS_SHELL argument')
1.138 + # We need to make sure we are using backslashes on Windows.
1.139 + JS, test_args = os.path.normpath(args[0]), args[1:]
1.140 +
1.141 + test_list = []
1.142 + bench_map = {}
1.143 +
1.144 + test_list = find_tests(test_dir)
1.145 +
1.146 + if not test_list:
1.147 + print >> sys.stderr, "No tests found matching command line arguments."
1.148 + sys.exit(0)
1.149 +
1.150 + test_list = [ Test.from_file(tst, name, OPTIONS) for tst, name in test_list ]
1.151 +
1.152 + try:
1.153 + print("{")
1.154 + bench_map = run_tests(test_list, test_dir)
1.155 + print("}")
1.156 +
1.157 + except OSError:
1.158 + if not os.path.exists(JS):
1.159 + print >> sys.stderr, "JS shell argument: file does not exist: '%s'"%JS
1.160 + sys.exit(1)
1.161 + else:
1.162 + raise
1.163 +
1.164 + if OPTIONS.baseline_path:
1.165 + baseline_map = []
1.166 + fh = open(OPTIONS.baseline_path, 'r')
1.167 + baseline_map = json.load(fh)
1.168 + fh.close()
1.169 + compare(current=bench_map, baseline=baseline_map)
