michael@0: #!/usr/bin/env python
michael@0: 
michael@0: """%prog [options] shellpath dirpath
michael@0: 
michael@0: Pulls performance data on parsing via the js shell.
michael@0: Displays the average number of milliseconds it took to parse each file.
michael@0: 
michael@0: For comparison, something apparently approximating a t-test is performed:
michael@0: "Faster" means that:
michael@0: 
michael@0:     t_baseline_goodrun = (t_baseline_avg - t_baseline_stddev)
michael@0:     t_current_badrun = (t_current_avg + t_current_stddev) 
michael@0:     t_current_badrun < t_baseline_goodrun
michael@0: 
michael@0: Effectively, a bad run from the current data is better than a good run from the
michael@0: baseline data, we're probably faster. A similar computation is used for
michael@0: determining the "slower" designation.
michael@0: 
michael@0: Arguments:
michael@0:   shellpath             executable JavaScript shell
michael@0:   dirpath               directory filled with parsilicious js files
michael@0: """
michael@0: 
michael@0: import math
michael@0: import optparse
michael@0: import os
michael@0: import subprocess as subp
michael@0: import sys
michael@0: from string import Template
michael@0: 
michael@0: try:
michael@0:     import compare_bench
michael@0: except ImportError:
michael@0:     compare_bench = None
michael@0: 
michael@0: 
michael@0: _DIR = os.path.dirname(__file__)
michael@0: JS_CODE_TEMPLATE = Template("""
michael@0: if (typeof snarf !== 'undefined') read = snarf
michael@0: var contents = read("$filepath");
michael@0: for (var i = 0; i < $warmup_run_count; i++)
michael@0:     parse(contents);
michael@0: var results = [];
michael@0: for (var i = 0; i < $real_run_count; i++) {
michael@0:     var start = new Date();
michael@0:     parse(contents);
michael@0:     var end = new Date();
michael@0:     results.push(end - start);
michael@0: }
michael@0: print(results);
michael@0: """)
michael@0: 
michael@0: 
michael@0: def gen_filepaths(dirpath, target_ext='.js'):
michael@0:     for filename in os.listdir(dirpath):
michael@0:         if filename.endswith(target_ext):
michael@0:             yield os.path.join(dirpath, filename)
michael@0: 
michael@0: 
michael@0: def avg(seq):
michael@0:     return sum(seq) / len(seq)
michael@0: 
michael@0: 
michael@0: def stddev(seq, mean):
michael@0:     diffs = ((float(item) - mean) ** 2 for item in seq)
michael@0:     return math.sqrt(sum(diffs) / len(seq))
michael@0: 
michael@0: 
michael@0: def bench(shellpath, filepath, warmup_runs, counted_runs, stfu=False):
michael@0:     """Return a list of milliseconds for the counted runs."""
michael@0:     assert '"' not in filepath
michael@0:     code = JS_CODE_TEMPLATE.substitute(filepath=filepath,
michael@0:             warmup_run_count=warmup_runs, real_run_count=counted_runs)
michael@0:     proc = subp.Popen([shellpath, '-e', code], stdout=subp.PIPE)
michael@0:     stdout, _ = proc.communicate()
michael@0:     milliseconds = [float(val) for val in stdout.split(',')]
michael@0:     mean = avg(milliseconds)
michael@0:     sigma = stddev(milliseconds, mean)
michael@0:     if not stfu:
michael@0:         print 'Runs:', [int(ms) for ms in milliseconds]
michael@0:         print 'Mean:', mean
michael@0:         print 'Stddev: %.2f (%.2f%% of mean)' % (sigma, sigma / mean * 100)
michael@0:     return mean, sigma
michael@0: 
michael@0: 
michael@0: def parsemark(filepaths, fbench, stfu=False):
michael@0:     """:param fbench: fbench(filename) -> float"""
michael@0:     bench_map = {} # {filename: (avg, stddev)}
michael@0:     for filepath in filepaths:
michael@0:         filename = os.path.split(filepath)[-1]
michael@0:         if not stfu:
michael@0:             print 'Parsemarking %s...' % filename
michael@0:         bench_map[filename] = fbench(filepath)
michael@0:     print '{'
michael@0:     for i, (filename, (avg, stddev)) in enumerate(bench_map.iteritems()):
michael@0:         assert '"' not in filename
michael@0:         fmt = '    %30s: {"average_ms": %6.2f, "stddev_ms": %6.2f}'
michael@0:         if i != len(bench_map) - 1:
michael@0:             fmt += ','
michael@0:         filename_str = '"%s"' % filename
michael@0:         print fmt % (filename_str, avg, stddev)
michael@0:     print '}'
michael@0:     return dict((filename, dict(average_ms=avg, stddev_ms=stddev))
michael@0:             for filename, (avg, stddev) in bench_map.iteritems())
michael@0: 
michael@0: 
michael@0: def main():
michael@0:     parser = optparse.OptionParser(usage=__doc__.strip())
michael@0:     parser.add_option('-w', '--warmup-runs', metavar='COUNT', type=int,
michael@0:             default=5, help='used to minimize test instability [%default]')
michael@0:     parser.add_option('-c', '--counted-runs', metavar='COUNT', type=int,
michael@0:             default=50, help='timed data runs that count towards the average [%default]')
michael@0:     parser.add_option('-s', '--shell', metavar='PATH', help='explicit shell '
michael@0:             'location; when omitted, will look in likely places')
michael@0:     parser.add_option('-b', '--baseline', metavar='JSON_PATH',
michael@0:             dest='baseline_path', help='json file with baseline values to '
michael@0:             'compare against')
michael@0:     parser.add_option('-q', '--quiet', dest='stfu', action='store_true',
michael@0:             default=False, help='only print JSON to stdout [%default]')
michael@0:     options, args = parser.parse_args()
michael@0:     try:
michael@0:         shellpath = args.pop(0)
michael@0:     except IndexError:
michael@0:         parser.print_help()
michael@0:         print
michael@0:         print >> sys.stderr, 'error: shellpath required'
michael@0:         return -1
michael@0:     try:
michael@0:         dirpath = args.pop(0)
michael@0:     except IndexError:
michael@0:         parser.print_help()
michael@0:         print
michael@0:         print >> sys.stderr, 'error: dirpath required'
michael@0:         return -1
michael@0:     if not shellpath or not os.path.exists(shellpath):
michael@0:         print >> sys.stderr, 'error: could not find shell:', shellpath
michael@0:         return -1
michael@0:     if options.baseline_path:
michael@0:         if not os.path.isfile(options.baseline_path):
michael@0:             print >> sys.stderr, 'error: baseline file does not exist'
michael@0:             return -1
michael@0:         if not compare_bench:
michael@0:             print >> sys.stderr, 'error: JSON support is missing, cannot compare benchmarks'
michael@0:             return -1
michael@0:     benchfile = lambda filepath: bench(shellpath, filepath,
michael@0:             options.warmup_runs, options.counted_runs, stfu=options.stfu)
michael@0:     bench_map = parsemark(gen_filepaths(dirpath), benchfile, options.stfu)
michael@0:     if options.baseline_path:
michael@0:         compare_bench.compare_immediate(bench_map, options.baseline_path)
michael@0:     return 0
michael@0: 
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     sys.exit(main())