1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/tests/compare_bench.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,73 @@
1.4 +#!/usr/bin/env python2.4
1.5 +"""usage: %progname candidate_path baseline_path
1.6 +"""
1.7 +
1.8 +import json
1.9 +import optparse
1.10 +from contextlib import nested
1.11 +from operator import itemgetter
1.12 +
1.13 +
1.14 +def avg(seq):
1.15 + return sum(seq) / len(seq)
1.16 +
1.17 +
1.18 +def compare(current, baseline):
1.19 + percent_speedups = []
1.20 + for key, current_result in current.iteritems():
1.21 + try:
1.22 + baseline_result = baseline[key]
1.23 + except KeyError:
1.24 + print key, 'missing from baseline'
1.25 + continue
1.26 + val_getter = itemgetter('average_ms', 'stddev_ms')
1.27 + base_avg, base_stddev = val_getter(baseline_result)
1.28 + current_avg, current_stddev = val_getter(current_result)
1.29 + t_best, t_worst = current_avg - current_stddev, current_avg + current_stddev
1.30 + base_t_best, base_t_worst = base_avg - base_stddev, base_avg + base_stddev
1.31 + fmt = '%30s: %s'
1.32 + if t_worst < base_t_best: # Worst takes less time (better) than baseline's best.
1.33 + speedup = -((t_worst - base_t_best) / base_t_best) * 100
1.34 + result = 'faster: %6.2fms < baseline %6.2fms (%+6.2f%%)' % \
1.35 + (t_worst, base_t_best, speedup)
1.36 + percent_speedups.append(speedup)
1.37 + elif t_best > base_t_worst: # Best takes more time (worse) than baseline's worst.
1.38 + slowdown = -((t_best - base_t_worst) / base_t_worst) * 100
1.39 + result = 'SLOWER: %6.2fms > baseline %6.2fms (%+6.2f%%) ' % \
1.40 + (t_best, base_t_worst, slowdown)
1.41 + percent_speedups.append(slowdown)
1.42 + else:
1.43 + result = 'Meh.'
1.44 + print '%30s: %s' % (key, result)
1.45 + if percent_speedups:
1.46 + print 'Average speedup: %.2f%%' % avg(percent_speedups)
1.47 +
1.48 +
1.49 +def compare_immediate(current_map, baseline_path):
1.50 + baseline_file = open(baseline_path)
1.51 + baseline_map = json.load(baseline_file)
1.52 + baseline_file.close()
1.53 + compare(current_map, baseline_map)
1.54 +
1.55 +
1.56 +def main(candidate_path, baseline_path):
1.57 + candidate_file, baseline_file = open(candidate_path), open(baseline_path)
1.58 + candidate = json.load(candidate_file)
1.59 + baseline = json.load(baseline_file)
1.60 + compare(candidate, baseline)
1.61 + candidate_file.close()
1.62 + baseline_file.close()
1.63 +
1.64 +
1.65 +if __name__ == '__main__':
1.66 + parser = optparse.OptionParser(usage=__doc__.strip())
1.67 + options, args = parser.parse_args()
1.68 + try:
1.69 + candidate_path = args.pop(0)
1.70 + except IndexError:
1.71 + parser.error('A JSON filepath to compare against baseline is required')
1.72 + try:
1.73 + baseline_path = args.pop(0)
1.74 + except IndexError:
1.75 + parser.error('A JSON filepath for baseline is required')
1.76 + main(candidate_path, baseline_path)
