michael@0: // Copyright 2012 the V8 project authors. All rights reserved.
michael@0: // Redistribution and use in source and binary forms, with or without
michael@0: // modification, are permitted provided that the following conditions are
michael@0: // met:
michael@0: //
michael@0: //     * Redistributions of source code must retain the above copyright
michael@0: //       notice, this list of conditions and the following disclaimer.
michael@0: //     * Redistributions in binary form must reproduce the above
michael@0: //       copyright notice, this list of conditions and the following
michael@0: //       disclaimer in the documentation and/or other materials provided
michael@0: //       with the distribution.
michael@0: //     * Neither the name of Google Inc. nor the names of its
michael@0: //       contributors may be used to endorse or promote products derived
michael@0: //       from this software without specific prior written permission.
michael@0: //
michael@0: // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
michael@0: // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
michael@0: // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
michael@0: // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
michael@0: // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
michael@0: // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
michael@0: // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
michael@0: // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
michael@0: // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
michael@0: // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
michael@0: // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
michael@0: 
michael@0: 
michael@0: // Simple framework for running the benchmark suites and
michael@0: // computing a score based on the timing measurements.
michael@0: 
michael@0: 
michael@0: // A benchmark has a name (string) and a function that will be run to
michael@0: // do the performance measurement. The optional setup and tearDown
michael@0: // arguments are functions that will be invoked before and after
michael@0: // running the benchmark, but the running time of these functions will
michael@0: // not be accounted for in the benchmark score.
michael@0: function Benchmark(name, run, setup, tearDown) {
michael@0:   this.name = name;
michael@0:   this.run = run;
michael@0:   this.Setup = setup ? setup : function() { };
michael@0:   this.TearDown = tearDown ? tearDown : function() { };
michael@0: }
michael@0: 
michael@0: 
michael@0: // Benchmark results hold the benchmark and the measured time used to
michael@0: // run the benchmark. The benchmark score is computed later once a
michael@0: // full benchmark suite has run to completion.
michael@0: function BenchmarkResult(benchmark, time) {
michael@0:   this.benchmark = benchmark;
michael@0:   this.time = time;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Automatically convert results to numbers. Used by the geometric
michael@0: // mean computation.
michael@0: BenchmarkResult.prototype.valueOf = function() {
michael@0:   return this.time;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Suites of benchmarks consist of a name and the set of benchmarks in
michael@0: // addition to the reference timing that the final score will be based
michael@0: // on. This way, all scores are relative to a reference run and higher
michael@0: // scores implies better performance.
michael@0: function BenchmarkSuite(name, reference, benchmarks) {
michael@0:   this.name = name;
michael@0:   this.reference = reference;
michael@0:   this.benchmarks = benchmarks;
michael@0:   BenchmarkSuite.suites.push(this);
michael@0: }
michael@0: 
michael@0: 
michael@0: // Keep track of all declared benchmark suites.
michael@0: BenchmarkSuite.suites = [];
michael@0: 
michael@0: 
michael@0: // Scores are not comparable across versions. Bump the version if
michael@0: // you're making changes that will affect that scores, e.g. if you add
michael@0: // a new benchmark or change an existing one.
michael@0: BenchmarkSuite.version = '7';
michael@0: 
michael@0: 
michael@0: // To make the benchmark results predictable, we replace Math.random
michael@0: // with a 100% deterministic alternative.
michael@0: Math.random = (function() {
michael@0:   var seed = 49734321;
michael@0:   return function() {
michael@0:     // Robert Jenkins' 32 bit integer hash function.
michael@0:     seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
michael@0:     seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
michael@0:     seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
michael@0:     seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
michael@0:     seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
michael@0:     seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
michael@0:     return (seed & 0xfffffff) / 0x10000000;
michael@0:   };
michael@0: })();
michael@0: 
michael@0: 
michael@0: // Runs all registered benchmark suites and optionally yields between
michael@0: // each individual benchmark to avoid running for too long in the
michael@0: // context of browsers. Once done, the final score is reported to the
michael@0: // runner.
michael@0: BenchmarkSuite.RunSuites = function(runner) {
michael@0:   var continuation = null;
michael@0:   var suites = BenchmarkSuite.suites;
michael@0:   var length = suites.length;
michael@0:   BenchmarkSuite.scores = [];
michael@0:   var index = 0;
michael@0:   function RunStep() {
michael@0:     while (continuation || index < length) {
michael@0:       if (continuation) {
michael@0:         continuation = continuation();
michael@0:       } else {
michael@0:         var suite = suites[index++];
michael@0:         if (runner.NotifyStart) runner.NotifyStart(suite.name);
michael@0:         continuation = suite.RunStep(runner);
michael@0:       }
michael@0:       if (continuation && typeof window != 'undefined' && window.setTimeout) {
michael@0:         window.setTimeout(RunStep, 25);
michael@0:         return;
michael@0:       }
michael@0:     }
michael@0:     if (runner.NotifyScore) {
michael@0:       var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores);
michael@0:       var formatted = BenchmarkSuite.FormatScore(100 * score);
michael@0:       runner.NotifyScore(formatted);
michael@0:     }
michael@0:   }
michael@0:   RunStep();
michael@0: }
michael@0: 
michael@0: 
michael@0: // Counts the total number of registered benchmarks. Useful for
michael@0: // showing progress as a percentage.
michael@0: BenchmarkSuite.CountBenchmarks = function() {
michael@0:   var result = 0;
michael@0:   var suites = BenchmarkSuite.suites;
michael@0:   for (var i = 0; i < suites.length; i++) {
michael@0:     result += suites[i].benchmarks.length;
michael@0:   }
michael@0:   return result;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Computes the geometric mean of a set of numbers.
michael@0: BenchmarkSuite.GeometricMean = function(numbers) {
michael@0:   var log = 0;
michael@0:   for (var i = 0; i < numbers.length; i++) {
michael@0:     log += Math.log(numbers[i]);
michael@0:   }
michael@0:   return Math.pow(Math.E, log / numbers.length);
michael@0: }
michael@0: 
michael@0: 
michael@0: // Converts a score value to a string with at least three significant
michael@0: // digits.
michael@0: BenchmarkSuite.FormatScore = function(value) {
michael@0:   if (value > 100) {
michael@0:     return value.toFixed(0);
michael@0:   } else {
michael@0:     return value.toPrecision(3);
michael@0:   }
michael@0: }
michael@0: 
michael@0: // Notifies the runner that we're done running a single benchmark in
michael@0: // the benchmark suite. This can be useful to report progress.
michael@0: BenchmarkSuite.prototype.NotifyStep = function(result) {
michael@0:   this.results.push(result);
michael@0:   if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name);
michael@0: }
michael@0: 
michael@0: 
michael@0: // Notifies the runner that we're done with running a suite and that
michael@0: // we have a result which can be reported to the user if needed.
michael@0: BenchmarkSuite.prototype.NotifyResult = function() {
michael@0:   var mean = BenchmarkSuite.GeometricMean(this.results);
michael@0:   var score = this.reference / mean;
michael@0:   BenchmarkSuite.scores.push(score);
michael@0:   if (this.runner.NotifyResult) {
michael@0:     var formatted = BenchmarkSuite.FormatScore(100 * score);
michael@0:     this.runner.NotifyResult(this.name, formatted);
michael@0:   }
michael@0: }
michael@0: 
michael@0: 
michael@0: // Notifies the runner that running a benchmark resulted in an error.
michael@0: BenchmarkSuite.prototype.NotifyError = function(error) {
michael@0:   if (this.runner.NotifyError) {
michael@0:     this.runner.NotifyError(this.name, error);
michael@0:   }
michael@0:   if (this.runner.NotifyStep) {
michael@0:     this.runner.NotifyStep(this.name);
michael@0:   }
michael@0: }
michael@0: 
michael@0: 
michael@0: // Runs a single benchmark for at least a second and computes the
michael@0: // average time it takes to run a single iteration.
michael@0: BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) {
michael@0:   function Measure(data) {
michael@0:     var elapsed = 0;
michael@0:     var start = new Date();
michael@0:     for (var n = 0; elapsed < 1000; n++) {
michael@0:       benchmark.run();
michael@0:       elapsed = new Date() - start;
michael@0:     }
michael@0:     if (data != null) {
michael@0:       data.runs += n;
michael@0:       data.elapsed += elapsed;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   if (data == null) {
michael@0:     // Measure the benchmark once for warm up and throw the result
michael@0:     // away. Return a fresh data object.
michael@0:     Measure(null);
michael@0:     return { runs: 0, elapsed: 0 };
michael@0:   } else {
michael@0:     Measure(data);
michael@0:     // If we've run too few iterations, we continue for another second.
michael@0:     if (data.runs < 32) return data;
michael@0:     var usec = (data.elapsed * 1000) / data.runs;
michael@0:     this.NotifyStep(new BenchmarkResult(benchmark, usec));
michael@0:     return null;
michael@0:   }
michael@0: }
michael@0: 
michael@0: 
michael@0: // This function starts running a suite, but stops between each
michael@0: // individual benchmark in the suite and returns a continuation
michael@0: // function which can be invoked to run the next benchmark. Once the
michael@0: // last benchmark has been executed, null is returned.
michael@0: BenchmarkSuite.prototype.RunStep = function(runner) {
michael@0:   this.results = [];
michael@0:   this.runner = runner;
michael@0:   var length = this.benchmarks.length;
michael@0:   var index = 0;
michael@0:   var suite = this;
michael@0:   var data;
michael@0: 
michael@0:   // Run the setup, the actual benchmark, and the tear down in three
michael@0:   // separate steps to allow the framework to yield between any of the
michael@0:   // steps.
michael@0: 
michael@0:   function RunNextSetup() {
michael@0:     if (index < length) {
michael@0:       try {
michael@0:         suite.benchmarks[index].Setup();
michael@0:       } catch (e) {
michael@0:         suite.NotifyError(e);
michael@0:         return null;
michael@0:       }
michael@0:       return RunNextBenchmark;
michael@0:     }
michael@0:     suite.NotifyResult();
michael@0:     return null;
michael@0:   }
michael@0: 
michael@0:   function RunNextBenchmark() {
michael@0:     try {
michael@0:       data = suite.RunSingleBenchmark(suite.benchmarks[index], data);
michael@0:     } catch (e) {
michael@0:       suite.NotifyError(e);
michael@0:       return null;
michael@0:     }
michael@0:     // If data is null, we're done with this benchmark.
michael@0:     return (data == null) ? RunNextTearDown : RunNextBenchmark();
michael@0:   }
michael@0: 
michael@0:   function RunNextTearDown() {
michael@0:     try {
michael@0:       suite.benchmarks[index++].TearDown();
michael@0:     } catch (e) {
michael@0:       suite.NotifyError(e);
michael@0:       return null;
michael@0:     }
michael@0:     return RunNextSetup;
michael@0:   }
michael@0: 
michael@0:   // Start out running the setup.
michael@0:   return RunNextSetup();
michael@0: }