|
1 /* This Source Code Form is subject to the terms of the Mozilla Public |
|
2 * License, v. 2.0. If a copy of the MPL was not distributed with this |
|
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
|
4 |
|
5 "use strict"; |
|
6 |
|
7 module.metadata = { |
|
8 "stability": "deprecated" |
|
9 }; |
|
10 |
|
11 const memory = require('./memory'); |
|
12 var timer = require("../timers"); |
|
13 var cfxArgs = require("@test/options"); |
|
14 |
|
15 exports.findAndRunTests = function findAndRunTests(options) { |
|
16 var TestFinder = require("./unit-test-finder").TestFinder; |
|
17 var finder = new TestFinder({ |
|
18 filter: options.filter, |
|
19 testInProcess: options.testInProcess, |
|
20 testOutOfProcess: options.testOutOfProcess |
|
21 }); |
|
22 var runner = new TestRunner({fs: options.fs}); |
|
23 finder.findTests( |
|
24 function (tests) { |
|
25 runner.startMany({tests: tests, |
|
26 stopOnError: options.stopOnError, |
|
27 onDone: options.onDone}); |
|
28 }); |
|
29 }; |
|
30 |
|
31 var TestRunner = exports.TestRunner = function TestRunner(options) { |
|
32 if (options) { |
|
33 this.fs = options.fs; |
|
34 } |
|
35 this.console = (options && "console" in options) ? options.console : console; |
|
36 memory.track(this); |
|
37 this.passed = 0; |
|
38 this.failed = 0; |
|
39 this.testRunSummary = []; |
|
40 this.expectFailNesting = 0; |
|
41 }; |
|
42 |
|
43 TestRunner.prototype = { |
|
44 toString: function toString() "[object TestRunner]", |
|
45 |
|
46 DEFAULT_PAUSE_TIMEOUT: 5*60000, |
|
47 PAUSE_DELAY: 500, |
|
48 |
|
49 _logTestFailed: function _logTestFailed(why) { |
|
50 if (!(why in this.test.errors)) |
|
51 this.test.errors[why] = 0; |
|
52 this.test.errors[why]++; |
|
53 }, |
|
54 |
|
55 pass: function pass(message) { |
|
56 if(!this.expectFailure) { |
|
57 if ("testMessage" in this.console) |
|
58 this.console.testMessage(true, true, this.test.name, message); |
|
59 else |
|
60 this.console.info("pass:", message); |
|
61 this.passed++; |
|
62 this.test.passed++; |
|
63 } |
|
64 else { |
|
65 this.expectFailure = false; |
|
66 this._logTestFailed("failure"); |
|
67 if ("testMessage" in this.console) { |
|
68 this.console.testMessage(true, false, this.test.name, message); |
|
69 } |
|
70 else { |
|
71 this.console.error("fail:", 'Failure Expected: ' + message) |
|
72 this.console.trace(); |
|
73 } |
|
74 this.failed++; |
|
75 this.test.failed++; |
|
76 } |
|
77 }, |
|
78 |
|
79 fail: function fail(message) { |
|
80 if(!this.expectFailure) { |
|
81 this._logTestFailed("failure"); |
|
82 if ("testMessage" in this.console) { |
|
83 this.console.testMessage(false, false, this.test.name, message); |
|
84 } |
|
85 else { |
|
86 this.console.error("fail:", message) |
|
87 this.console.trace(); |
|
88 } |
|
89 this.failed++; |
|
90 this.test.failed++; |
|
91 } |
|
92 else { |
|
93 this.expectFailure = false; |
|
94 if ("testMessage" in this.console) |
|
95 this.console.testMessage(false, true, this.test.name, message); |
|
96 else |
|
97 this.console.info("pass:", message); |
|
98 this.passed++; |
|
99 this.test.passed++; |
|
100 } |
|
101 }, |
|
102 |
|
103 expectFail: function(callback) { |
|
104 this.expectFailure = true; |
|
105 callback(); |
|
106 this.expectFailure = false; |
|
107 }, |
|
108 |
|
109 exception: function exception(e) { |
|
110 this._logTestFailed("exception"); |
|
111 if (cfxArgs.parseable) |
|
112 this.console.print("TEST-UNEXPECTED-FAIL | " + this.test.name + " | " + e + "\n"); |
|
113 this.console.exception(e); |
|
114 this.failed++; |
|
115 this.test.failed++; |
|
116 }, |
|
117 |
|
118 assertMatches: function assertMatches(string, regexp, message) { |
|
119 if (regexp.test(string)) { |
|
120 if (!message) |
|
121 message = uneval(string) + " matches " + uneval(regexp); |
|
122 this.pass(message); |
|
123 } else { |
|
124 var no = uneval(string) + " doesn't match " + uneval(regexp); |
|
125 if (!message) |
|
126 message = no; |
|
127 else |
|
128 message = message + " (" + no + ")"; |
|
129 this.fail(message); |
|
130 } |
|
131 }, |
|
132 |
|
133 assertRaises: function assertRaises(func, predicate, message) { |
|
134 try { |
|
135 func(); |
|
136 if (message) |
|
137 this.fail(message + " (no exception thrown)"); |
|
138 else |
|
139 this.fail("function failed to throw exception"); |
|
140 } catch (e) { |
|
141 var errorMessage; |
|
142 if (typeof(e) == "string") |
|
143 errorMessage = e; |
|
144 else |
|
145 errorMessage = e.message; |
|
146 if (typeof(predicate) == "string") |
|
147 this.assertEqual(errorMessage, predicate, message); |
|
148 else |
|
149 this.assertMatches(errorMessage, predicate, message); |
|
150 } |
|
151 }, |
|
152 |
|
153 assert: function assert(a, message) { |
|
154 if (!a) { |
|
155 if (!message) |
|
156 message = "assertion failed, value is " + a; |
|
157 this.fail(message); |
|
158 } else |
|
159 this.pass(message || "assertion successful"); |
|
160 }, |
|
161 |
|
162 assertNotEqual: function assertNotEqual(a, b, message) { |
|
163 if (a != b) { |
|
164 if (!message) |
|
165 message = "a != b != " + uneval(a); |
|
166 this.pass(message); |
|
167 } else { |
|
168 var equality = uneval(a) + " == " + uneval(b); |
|
169 if (!message) |
|
170 message = equality; |
|
171 else |
|
172 message += " (" + equality + ")"; |
|
173 this.fail(message); |
|
174 } |
|
175 }, |
|
176 |
|
177 assertEqual: function assertEqual(a, b, message) { |
|
178 if (a == b) { |
|
179 if (!message) |
|
180 message = "a == b == " + uneval(a); |
|
181 this.pass(message); |
|
182 } else { |
|
183 var inequality = uneval(a) + " != " + uneval(b); |
|
184 if (!message) |
|
185 message = inequality; |
|
186 else |
|
187 message += " (" + inequality + ")"; |
|
188 this.fail(message); |
|
189 } |
|
190 }, |
|
191 |
|
192 assertNotStrictEqual: function assertNotStrictEqual(a, b, message) { |
|
193 if (a !== b) { |
|
194 if (!message) |
|
195 message = "a !== b !== " + uneval(a); |
|
196 this.pass(message); |
|
197 } else { |
|
198 var equality = uneval(a) + " === " + uneval(b); |
|
199 if (!message) |
|
200 message = equality; |
|
201 else |
|
202 message += " (" + equality + ")"; |
|
203 this.fail(message); |
|
204 } |
|
205 }, |
|
206 |
|
207 assertStrictEqual: function assertStrictEqual(a, b, message) { |
|
208 if (a === b) { |
|
209 if (!message) |
|
210 message = "a === b === " + uneval(a); |
|
211 this.pass(message); |
|
212 } else { |
|
213 var inequality = uneval(a) + " !== " + uneval(b); |
|
214 if (!message) |
|
215 message = inequality; |
|
216 else |
|
217 message += " (" + inequality + ")"; |
|
218 this.fail(message); |
|
219 } |
|
220 }, |
|
221 |
|
222 assertFunction: function assertFunction(a, message) { |
|
223 this.assertStrictEqual('function', typeof a, message); |
|
224 }, |
|
225 |
|
226 assertUndefined: function(a, message) { |
|
227 this.assertStrictEqual('undefined', typeof a, message); |
|
228 }, |
|
229 |
|
230 assertNotUndefined: function(a, message) { |
|
231 this.assertNotStrictEqual('undefined', typeof a, message); |
|
232 }, |
|
233 |
|
234 assertNull: function(a, message) { |
|
235 this.assertStrictEqual(null, a, message); |
|
236 }, |
|
237 |
|
238 assertNotNull: function(a, message) { |
|
239 this.assertNotStrictEqual(null, a, message); |
|
240 }, |
|
241 |
|
242 assertObject: function(a, message) { |
|
243 this.assertStrictEqual('[object Object]', Object.prototype.toString.apply(a), message); |
|
244 }, |
|
245 |
|
246 assertString: function(a, message) { |
|
247 this.assertStrictEqual('[object String]', Object.prototype.toString.apply(a), message); |
|
248 }, |
|
249 |
|
250 assertArray: function(a, message) { |
|
251 this.assertStrictEqual('[object Array]', Object.prototype.toString.apply(a), message); |
|
252 }, |
|
253 |
|
254 assertNumber: function(a, message) { |
|
255 this.assertStrictEqual('[object Number]', Object.prototype.toString.apply(a), message); |
|
256 }, |
|
257 |
|
258 done: function done() { |
|
259 if (!this.isDone) { |
|
260 this.isDone = true; |
|
261 if(this.test.teardown) { |
|
262 this.test.teardown(this); |
|
263 } |
|
264 if (this.waitTimeout !== null) { |
|
265 timer.clearTimeout(this.waitTimeout); |
|
266 this.waitTimeout = null; |
|
267 } |
|
268 // Do not leave any callback set when calling to `waitUntil` |
|
269 this.waitUntilCallback = null; |
|
270 if (this.test.passed == 0 && this.test.failed == 0) { |
|
271 this._logTestFailed("empty test"); |
|
272 if ("testMessage" in this.console) { |
|
273 this.console.testMessage(false, false, this.test.name, "Empty test"); |
|
274 } |
|
275 else { |
|
276 this.console.error("fail:", "Empty test") |
|
277 } |
|
278 this.failed++; |
|
279 this.test.failed++; |
|
280 } |
|
281 |
|
282 this.testRunSummary.push({ |
|
283 name: this.test.name, |
|
284 passed: this.test.passed, |
|
285 failed: this.test.failed, |
|
286 errors: [error for (error in this.test.errors)].join(", ") |
|
287 }); |
|
288 |
|
289 if (this.onDone !== null) { |
|
290 var onDone = this.onDone; |
|
291 var self = this; |
|
292 this.onDone = null; |
|
293 timer.setTimeout(function() { onDone(self); }, 0); |
|
294 } |
|
295 } |
|
296 }, |
|
297 |
|
298 // Set of assertion functions to wait for an assertion to become true |
|
299 // These functions take the same arguments as the TestRunner.assert* methods. |
|
300 waitUntil: function waitUntil() { |
|
301 return this._waitUntil(this.assert, arguments); |
|
302 }, |
|
303 |
|
304 waitUntilNotEqual: function waitUntilNotEqual() { |
|
305 return this._waitUntil(this.assertNotEqual, arguments); |
|
306 }, |
|
307 |
|
308 waitUntilEqual: function waitUntilEqual() { |
|
309 return this._waitUntil(this.assertEqual, arguments); |
|
310 }, |
|
311 |
|
312 waitUntilMatches: function waitUntilMatches() { |
|
313 return this._waitUntil(this.assertMatches, arguments); |
|
314 }, |
|
315 |
|
316 /** |
|
317 * Internal function that waits for an assertion to become true. |
|
318 * @param {Function} assertionMethod |
|
319 * Reference to a TestRunner assertion method like test.assert, |
|
320 * test.assertEqual, ... |
|
321 * @param {Array} args |
|
322 * List of arguments to give to the previous assertion method. |
|
323 * All functions in this list are going to be called to retrieve current |
|
324 * assertion values. |
|
325 */ |
|
326 _waitUntil: function waitUntil(assertionMethod, args) { |
|
327 let count = 0; |
|
328 let maxCount = this.DEFAULT_PAUSE_TIMEOUT / this.PAUSE_DELAY; |
|
329 |
|
330 // We need to ensure that test is asynchronous |
|
331 if (!this.waitTimeout) |
|
332 this.waitUntilDone(this.DEFAULT_PAUSE_TIMEOUT); |
|
333 |
|
334 let callback = null; |
|
335 let finished = false; |
|
336 |
|
337 let test = this; |
|
338 |
|
339 // capture a traceback before we go async. |
|
340 let traceback = require("../console/traceback"); |
|
341 let stack = traceback.get(); |
|
342 stack.splice(-2, 2); |
|
343 let currentWaitStack = traceback.format(stack); |
|
344 let timeout = null; |
|
345 |
|
346 function loop(stopIt) { |
|
347 timeout = null; |
|
348 |
|
349 // Build a mockup object to fake TestRunner API and intercept calls to |
|
350 // pass and fail methods, in order to retrieve nice error messages |
|
351 // and assertion result |
|
352 let mock = { |
|
353 pass: function (msg) { |
|
354 test.pass(msg); |
|
355 test.waitUntilCallback = null; |
|
356 if (callback && !stopIt) |
|
357 callback(); |
|
358 finished = true; |
|
359 }, |
|
360 fail: function (msg) { |
|
361 // If we are called on test timeout, we stop the loop |
|
362 // and print which test keeps failing: |
|
363 if (stopIt) { |
|
364 test.console.error("test assertion never became true:\n", |
|
365 msg + "\n", |
|
366 currentWaitStack); |
|
367 if (timeout) |
|
368 timer.clearTimeout(timeout); |
|
369 return; |
|
370 } |
|
371 timeout = timer.setTimeout(loop, test.PAUSE_DELAY); |
|
372 } |
|
373 }; |
|
374 |
|
375 // Automatically call args closures in order to build arguments for |
|
376 // assertion function |
|
377 let appliedArgs = []; |
|
378 for (let i = 0, l = args.length; i < l; i++) { |
|
379 let a = args[i]; |
|
380 if (typeof a == "function") { |
|
381 try { |
|
382 a = a(); |
|
383 } |
|
384 catch(e) { |
|
385 test.fail("Exception when calling asynchronous assertion: " + e + |
|
386 "\n" + e.stack); |
|
387 finished = true; |
|
388 return; |
|
389 } |
|
390 } |
|
391 appliedArgs.push(a); |
|
392 } |
|
393 |
|
394 // Finally call assertion function with current assertion values |
|
395 assertionMethod.apply(mock, appliedArgs); |
|
396 } |
|
397 loop(); |
|
398 this.waitUntilCallback = loop; |
|
399 |
|
400 // Return an object with `then` method, to offer a way to execute |
|
401 // some code when the assertion passed or failed |
|
402 return { |
|
403 then: function (c) { |
|
404 callback = c; |
|
405 |
|
406 // In case of immediate positive result, we need to execute callback |
|
407 // immediately here: |
|
408 if (finished) |
|
409 callback(); |
|
410 } |
|
411 }; |
|
412 }, |
|
413 |
|
414 waitUntilDone: function waitUntilDone(ms) { |
|
415 if (ms === undefined) |
|
416 ms = this.DEFAULT_PAUSE_TIMEOUT; |
|
417 |
|
418 var self = this; |
|
419 |
|
420 function tiredOfWaiting() { |
|
421 self._logTestFailed("timed out"); |
|
422 if ("testMessage" in self.console) { |
|
423 self.console.testMessage(false, false, self.test.name, "Timed out"); |
|
424 } |
|
425 else { |
|
426 self.console.error("fail:", "Timed out") |
|
427 } |
|
428 if (self.waitUntilCallback) { |
|
429 self.waitUntilCallback(true); |
|
430 self.waitUntilCallback = null; |
|
431 } |
|
432 self.failed++; |
|
433 self.test.failed++; |
|
434 self.done(); |
|
435 } |
|
436 |
|
437 // We may already have registered a timeout callback |
|
438 if (this.waitTimeout) |
|
439 timer.clearTimeout(this.waitTimeout); |
|
440 |
|
441 this.waitTimeout = timer.setTimeout(tiredOfWaiting, ms); |
|
442 }, |
|
443 |
|
444 startMany: function startMany(options) { |
|
445 function runNextTest(self) { |
|
446 var test = options.tests.shift(); |
|
447 if (options.stopOnError && self.test && self.test.failed) { |
|
448 self.console.error("aborted: test failed and --stop-on-error was specified"); |
|
449 options.onDone(self); |
|
450 } else if (test) { |
|
451 self.start({test: test, onDone: runNextTest}); |
|
452 } else { |
|
453 options.onDone(self); |
|
454 } |
|
455 } |
|
456 runNextTest(this); |
|
457 }, |
|
458 |
|
459 start: function start(options) { |
|
460 this.test = options.test; |
|
461 this.test.passed = 0; |
|
462 this.test.failed = 0; |
|
463 this.test.errors = {}; |
|
464 |
|
465 this.isDone = false; |
|
466 this.onDone = function(self) { |
|
467 if (cfxArgs.parseable) |
|
468 self.console.print("TEST-END | " + self.test.name + "\n"); |
|
469 options.onDone(self); |
|
470 } |
|
471 this.waitTimeout = null; |
|
472 |
|
473 try { |
|
474 if (cfxArgs.parseable) |
|
475 this.console.print("TEST-START | " + this.test.name + "\n"); |
|
476 else |
|
477 this.console.info("executing '" + this.test.name + "'"); |
|
478 |
|
479 if(this.test.setup) { |
|
480 this.test.setup(this); |
|
481 } |
|
482 this.test.testFunction(this); |
|
483 } catch (e) { |
|
484 this.exception(e); |
|
485 } |
|
486 if (this.waitTimeout === null) |
|
487 this.done(); |
|
488 } |
|
489 }; |