michael@0: #!/usr/bin/env python2.4
michael@0: """usage: %progname candidate_path baseline_path
michael@0: """
michael@0: 
michael@0: import json
michael@0: import optparse
michael@0: from contextlib import nested
michael@0: from operator import itemgetter
michael@0: 
michael@0: 
michael@0: def avg(seq):
michael@0:     return sum(seq) / len(seq)
michael@0: 
michael@0: 
michael@0: def compare(current, baseline):
michael@0:     percent_speedups = []
michael@0:     for key, current_result in current.iteritems():
michael@0:         try:
michael@0:             baseline_result = baseline[key]
michael@0:         except KeyError:
michael@0:             print key, 'missing from baseline'
michael@0:             continue
michael@0:         val_getter = itemgetter('average_ms', 'stddev_ms')
michael@0:         base_avg, base_stddev = val_getter(baseline_result)
michael@0:         current_avg, current_stddev = val_getter(current_result)
michael@0:         t_best, t_worst = current_avg - current_stddev, current_avg + current_stddev
michael@0:         base_t_best, base_t_worst = base_avg - base_stddev, base_avg + base_stddev
michael@0:         fmt = '%30s: %s'
michael@0:         if t_worst < base_t_best: # Worst takes less time (better) than baseline's best.
michael@0:             speedup = -((t_worst - base_t_best) / base_t_best) * 100
michael@0:             result = 'faster: %6.2fms < baseline %6.2fms (%+6.2f%%)' % \
michael@0:                     (t_worst, base_t_best, speedup)
michael@0:             percent_speedups.append(speedup)
michael@0:         elif t_best > base_t_worst: # Best takes more time (worse) than baseline's worst.
michael@0:             slowdown = -((t_best - base_t_worst) / base_t_worst) * 100
michael@0:             result = 'SLOWER: %6.2fms > baseline %6.2fms (%+6.2f%%) ' % \
michael@0:                     (t_best, base_t_worst, slowdown)
michael@0:             percent_speedups.append(slowdown)
michael@0:         else:
michael@0:             result = 'Meh.'
michael@0:         print '%30s: %s' % (key, result)
michael@0:     if percent_speedups:
michael@0:         print 'Average speedup: %.2f%%' % avg(percent_speedups)
michael@0: 
michael@0: 
michael@0: def compare_immediate(current_map, baseline_path):
michael@0:     baseline_file = open(baseline_path)
michael@0:     baseline_map = json.load(baseline_file)
michael@0:     baseline_file.close()
michael@0:     compare(current_map, baseline_map)
michael@0: 
michael@0: 
michael@0: def main(candidate_path, baseline_path):
michael@0:     candidate_file, baseline_file = open(candidate_path), open(baseline_path)
michael@0:     candidate = json.load(candidate_file)
michael@0:     baseline = json.load(baseline_file)
michael@0:     compare(candidate, baseline)
michael@0:     candidate_file.close()
michael@0:     baseline_file.close()
michael@0: 
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     parser = optparse.OptionParser(usage=__doc__.strip())
michael@0:     options, args = parser.parse_args()
michael@0:     try:
michael@0:         candidate_path = args.pop(0)
michael@0:     except IndexError:
michael@0:         parser.error('A JSON filepath to compare against baseline is required')
michael@0:     try:
michael@0:         baseline_path = args.pop(0)
michael@0:     except IndexError:
michael@0:         parser.error('A JSON filepath for baseline is required')
michael@0:     main(candidate_path, baseline_path)