|
1 // Copyright 2012 the V8 project authors. All rights reserved. |
|
2 // Redistribution and use in source and binary forms, with or without |
|
3 // modification, are permitted provided that the following conditions are |
|
4 // met: |
|
5 // |
|
6 // * Redistributions of source code must retain the above copyright |
|
7 // notice, this list of conditions and the following disclaimer. |
|
8 // * Redistributions in binary form must reproduce the above |
|
9 // copyright notice, this list of conditions and the following |
|
10 // disclaimer in the documentation and/or other materials provided |
|
11 // with the distribution. |
|
12 // * Neither the name of Google Inc. nor the names of its |
|
13 // contributors may be used to endorse or promote products derived |
|
14 // from this software without specific prior written permission. |
|
15 // |
|
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
17 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
18 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
19 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
20 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
21 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
22 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
23 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
24 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
25 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
26 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
27 |
|
28 |
|
29 // Simple framework for running the benchmark suites and |
|
30 // computing a score based on the timing measurements. |
|
31 |
|
32 |
|
33 // A benchmark has a name (string) and a function that will be run to |
|
34 // do the performance measurement. The optional setup and tearDown |
|
35 // arguments are functions that will be invoked before and after |
|
36 // running the benchmark, but the running time of these functions will |
|
37 // not be accounted for in the benchmark score. |
|
38 function Benchmark(name, run, setup, tearDown) { |
|
39 this.name = name; |
|
40 this.run = run; |
|
41 this.Setup = setup ? setup : function() { }; |
|
42 this.TearDown = tearDown ? tearDown : function() { }; |
|
43 } |
|
44 |
|
45 |
|
46 // Benchmark results hold the benchmark and the measured time used to |
|
47 // run the benchmark. The benchmark score is computed later once a |
|
48 // full benchmark suite has run to completion. |
|
49 function BenchmarkResult(benchmark, time) { |
|
50 this.benchmark = benchmark; |
|
51 this.time = time; |
|
52 } |
|
53 |
|
54 |
|
55 // Automatically convert results to numbers. Used by the geometric |
|
56 // mean computation. |
|
57 BenchmarkResult.prototype.valueOf = function() { |
|
58 return this.time; |
|
59 } |
|
60 |
|
61 |
|
62 // Suites of benchmarks consist of a name and the set of benchmarks in |
|
63 // addition to the reference timing that the final score will be based |
|
64 // on. This way, all scores are relative to a reference run and higher |
|
65 // scores implies better performance. |
|
66 function BenchmarkSuite(name, reference, benchmarks) { |
|
67 this.name = name; |
|
68 this.reference = reference; |
|
69 this.benchmarks = benchmarks; |
|
70 BenchmarkSuite.suites.push(this); |
|
71 } |
|
72 |
|
73 |
|
74 // Keep track of all declared benchmark suites. |
|
75 BenchmarkSuite.suites = []; |
|
76 |
|
77 |
|
78 // Scores are not comparable across versions. Bump the version if |
|
79 // you're making changes that will affect that scores, e.g. if you add |
|
80 // a new benchmark or change an existing one. |
|
81 BenchmarkSuite.version = '7'; |
|
82 |
|
83 |
|
84 // To make the benchmark results predictable, we replace Math.random |
|
85 // with a 100% deterministic alternative. |
|
86 Math.random = (function() { |
|
87 var seed = 49734321; |
|
88 return function() { |
|
89 // Robert Jenkins' 32 bit integer hash function. |
|
90 seed = ((seed + 0x7ed55d16) + (seed << 12)) & 0xffffffff; |
|
91 seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff; |
|
92 seed = ((seed + 0x165667b1) + (seed << 5)) & 0xffffffff; |
|
93 seed = ((seed + 0xd3a2646c) ^ (seed << 9)) & 0xffffffff; |
|
94 seed = ((seed + 0xfd7046c5) + (seed << 3)) & 0xffffffff; |
|
95 seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff; |
|
96 return (seed & 0xfffffff) / 0x10000000; |
|
97 }; |
|
98 })(); |
|
99 |
|
100 |
|
101 // Runs all registered benchmark suites and optionally yields between |
|
102 // each individual benchmark to avoid running for too long in the |
|
103 // context of browsers. Once done, the final score is reported to the |
|
104 // runner. |
|
105 BenchmarkSuite.RunSuites = function(runner) { |
|
106 var continuation = null; |
|
107 var suites = BenchmarkSuite.suites; |
|
108 var length = suites.length; |
|
109 BenchmarkSuite.scores = []; |
|
110 var index = 0; |
|
111 function RunStep() { |
|
112 while (continuation || index < length) { |
|
113 if (continuation) { |
|
114 continuation = continuation(); |
|
115 } else { |
|
116 var suite = suites[index++]; |
|
117 if (runner.NotifyStart) runner.NotifyStart(suite.name); |
|
118 continuation = suite.RunStep(runner); |
|
119 } |
|
120 if (continuation && typeof window != 'undefined' && window.setTimeout) { |
|
121 window.setTimeout(RunStep, 25); |
|
122 return; |
|
123 } |
|
124 } |
|
125 if (runner.NotifyScore) { |
|
126 var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores); |
|
127 var formatted = BenchmarkSuite.FormatScore(100 * score); |
|
128 runner.NotifyScore(formatted); |
|
129 } |
|
130 } |
|
131 RunStep(); |
|
132 } |
|
133 |
|
134 |
|
135 // Counts the total number of registered benchmarks. Useful for |
|
136 // showing progress as a percentage. |
|
137 BenchmarkSuite.CountBenchmarks = function() { |
|
138 var result = 0; |
|
139 var suites = BenchmarkSuite.suites; |
|
140 for (var i = 0; i < suites.length; i++) { |
|
141 result += suites[i].benchmarks.length; |
|
142 } |
|
143 return result; |
|
144 } |
|
145 |
|
146 |
|
147 // Computes the geometric mean of a set of numbers. |
|
148 BenchmarkSuite.GeometricMean = function(numbers) { |
|
149 var log = 0; |
|
150 for (var i = 0; i < numbers.length; i++) { |
|
151 log += Math.log(numbers[i]); |
|
152 } |
|
153 return Math.pow(Math.E, log / numbers.length); |
|
154 } |
|
155 |
|
156 |
|
157 // Converts a score value to a string with at least three significant |
|
158 // digits. |
|
159 BenchmarkSuite.FormatScore = function(value) { |
|
160 if (value > 100) { |
|
161 return value.toFixed(0); |
|
162 } else { |
|
163 return value.toPrecision(3); |
|
164 } |
|
165 } |
|
166 |
|
167 // Notifies the runner that we're done running a single benchmark in |
|
168 // the benchmark suite. This can be useful to report progress. |
|
169 BenchmarkSuite.prototype.NotifyStep = function(result) { |
|
170 this.results.push(result); |
|
171 if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name); |
|
172 } |
|
173 |
|
174 |
|
175 // Notifies the runner that we're done with running a suite and that |
|
176 // we have a result which can be reported to the user if needed. |
|
177 BenchmarkSuite.prototype.NotifyResult = function() { |
|
178 var mean = BenchmarkSuite.GeometricMean(this.results); |
|
179 var score = this.reference / mean; |
|
180 BenchmarkSuite.scores.push(score); |
|
181 if (this.runner.NotifyResult) { |
|
182 var formatted = BenchmarkSuite.FormatScore(100 * score); |
|
183 this.runner.NotifyResult(this.name, formatted); |
|
184 } |
|
185 } |
|
186 |
|
187 |
|
188 // Notifies the runner that running a benchmark resulted in an error. |
|
189 BenchmarkSuite.prototype.NotifyError = function(error) { |
|
190 if (this.runner.NotifyError) { |
|
191 this.runner.NotifyError(this.name, error); |
|
192 } |
|
193 if (this.runner.NotifyStep) { |
|
194 this.runner.NotifyStep(this.name); |
|
195 } |
|
196 } |
|
197 |
|
198 |
|
199 // Runs a single benchmark for at least a second and computes the |
|
200 // average time it takes to run a single iteration. |
|
201 BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) { |
|
202 function Measure(data) { |
|
203 var elapsed = 0; |
|
204 var start = new Date(); |
|
205 for (var n = 0; elapsed < 1000; n++) { |
|
206 benchmark.run(); |
|
207 elapsed = new Date() - start; |
|
208 } |
|
209 if (data != null) { |
|
210 data.runs += n; |
|
211 data.elapsed += elapsed; |
|
212 } |
|
213 } |
|
214 |
|
215 if (data == null) { |
|
216 // Measure the benchmark once for warm up and throw the result |
|
217 // away. Return a fresh data object. |
|
218 Measure(null); |
|
219 return { runs: 0, elapsed: 0 }; |
|
220 } else { |
|
221 Measure(data); |
|
222 // If we've run too few iterations, we continue for another second. |
|
223 if (data.runs < 32) return data; |
|
224 var usec = (data.elapsed * 1000) / data.runs; |
|
225 this.NotifyStep(new BenchmarkResult(benchmark, usec)); |
|
226 return null; |
|
227 } |
|
228 } |
|
229 |
|
230 |
|
231 // This function starts running a suite, but stops between each |
|
232 // individual benchmark in the suite and returns a continuation |
|
233 // function which can be invoked to run the next benchmark. Once the |
|
234 // last benchmark has been executed, null is returned. |
|
235 BenchmarkSuite.prototype.RunStep = function(runner) { |
|
236 this.results = []; |
|
237 this.runner = runner; |
|
238 var length = this.benchmarks.length; |
|
239 var index = 0; |
|
240 var suite = this; |
|
241 var data; |
|
242 |
|
243 // Run the setup, the actual benchmark, and the tear down in three |
|
244 // separate steps to allow the framework to yield between any of the |
|
245 // steps. |
|
246 |
|
247 function RunNextSetup() { |
|
248 if (index < length) { |
|
249 try { |
|
250 suite.benchmarks[index].Setup(); |
|
251 } catch (e) { |
|
252 suite.NotifyError(e); |
|
253 return null; |
|
254 } |
|
255 return RunNextBenchmark; |
|
256 } |
|
257 suite.NotifyResult(); |
|
258 return null; |
|
259 } |
|
260 |
|
261 function RunNextBenchmark() { |
|
262 try { |
|
263 data = suite.RunSingleBenchmark(suite.benchmarks[index], data); |
|
264 } catch (e) { |
|
265 suite.NotifyError(e); |
|
266 return null; |
|
267 } |
|
268 // If data is null, we're done with this benchmark. |
|
269 return (data == null) ? RunNextTearDown : RunNextBenchmark(); |
|
270 } |
|
271 |
|
272 function RunNextTearDown() { |
|
273 try { |
|
274 suite.benchmarks[index++].TearDown(); |
|
275 } catch (e) { |
|
276 suite.NotifyError(e); |
|
277 return null; |
|
278 } |
|
279 return RunNextSetup; |
|
280 } |
|
281 |
|
282 // Start out running the setup. |
|
283 return RunNextSetup(); |
|
284 } |