michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: "use strict";
michael@0: 
michael@0: module.metadata = {
michael@0:   "stability": "deprecated"
michael@0: };
michael@0: 
michael@0: const memory = require('./memory');
michael@0: var timer = require("../timers");
michael@0: var cfxArgs = require("@test/options");
michael@0: 
michael@0: exports.findAndRunTests = function findAndRunTests(options) {
michael@0:   var TestFinder = require("./unit-test-finder").TestFinder;
michael@0:   var finder = new TestFinder({
michael@0:     filter: options.filter,
michael@0:     testInProcess: options.testInProcess,
michael@0:     testOutOfProcess: options.testOutOfProcess
michael@0:   });
michael@0:   var runner = new TestRunner({fs: options.fs});
michael@0:   finder.findTests(
michael@0:     function (tests) {
michael@0:       runner.startMany({tests: tests,
michael@0:                         stopOnError: options.stopOnError,
michael@0:                         onDone: options.onDone});
michael@0:     });
michael@0: };
michael@0: 
michael@0: var TestRunner = exports.TestRunner = function TestRunner(options) {
michael@0:   if (options) {
michael@0:     this.fs = options.fs;
michael@0:   }
michael@0:   this.console = (options && "console" in options) ? options.console : console;
michael@0:   memory.track(this);
michael@0:   this.passed = 0;
michael@0:   this.failed = 0;
michael@0:   this.testRunSummary = [];
michael@0:   this.expectFailNesting = 0;
michael@0: };
michael@0: 
michael@0: TestRunner.prototype = {
michael@0:   toString: function toString() "[object TestRunner]",
michael@0: 
michael@0:   DEFAULT_PAUSE_TIMEOUT: 5*60000,
michael@0:   PAUSE_DELAY: 500,
michael@0: 
michael@0:   _logTestFailed: function _logTestFailed(why) {
michael@0:     if (!(why in this.test.errors))
michael@0:       this.test.errors[why] = 0;
michael@0:     this.test.errors[why]++;
michael@0:   },
michael@0: 
michael@0:   pass: function pass(message) {
michael@0:     if(!this.expectFailure) {
michael@0:       if ("testMessage" in this.console)
michael@0:         this.console.testMessage(true, true, this.test.name, message);
michael@0:       else
michael@0:         this.console.info("pass:", message);
michael@0:       this.passed++;
michael@0:       this.test.passed++;
michael@0:     }
michael@0:     else {
michael@0:       this.expectFailure = false;
michael@0:       this._logTestFailed("failure");
michael@0:       if ("testMessage" in this.console) {
michael@0:         this.console.testMessage(true, false, this.test.name, message);
michael@0:       }
michael@0:       else {
michael@0:         this.console.error("fail:", 'Failure Expected: ' + message)
michael@0:         this.console.trace();
michael@0:       }
michael@0:       this.failed++;
michael@0:       this.test.failed++;
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   fail: function fail(message) {
michael@0:     if(!this.expectFailure) {
michael@0:       this._logTestFailed("failure");
michael@0:       if ("testMessage" in this.console) {
michael@0:         this.console.testMessage(false, false, this.test.name, message);
michael@0:       }
michael@0:       else {
michael@0:         this.console.error("fail:", message)
michael@0:         this.console.trace();
michael@0:       }
michael@0:       this.failed++;
michael@0:       this.test.failed++;
michael@0:     }
michael@0:     else {
michael@0:       this.expectFailure = false;
michael@0:       if ("testMessage" in this.console)
michael@0:         this.console.testMessage(false, true, this.test.name, message);
michael@0:       else
michael@0:         this.console.info("pass:", message);
michael@0:       this.passed++;
michael@0:       this.test.passed++;
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   expectFail: function(callback) {
michael@0:     this.expectFailure = true;
michael@0:     callback();
michael@0:     this.expectFailure = false;
michael@0:   },
michael@0: 
michael@0:   exception: function exception(e) {
michael@0:     this._logTestFailed("exception");
michael@0:     if (cfxArgs.parseable)
michael@0:       this.console.print("TEST-UNEXPECTED-FAIL | " + this.test.name + " | " + e + "\n");
michael@0:     this.console.exception(e);
michael@0:     this.failed++;
michael@0:     this.test.failed++;
michael@0:   },
michael@0: 
michael@0:   assertMatches: function assertMatches(string, regexp, message) {
michael@0:     if (regexp.test(string)) {
michael@0:       if (!message)
michael@0:         message = uneval(string) + " matches " + uneval(regexp);
michael@0:       this.pass(message);
michael@0:     } else {
michael@0:       var no = uneval(string) + " doesn't match " + uneval(regexp);
michael@0:       if (!message)
michael@0:         message = no;
michael@0:       else
michael@0:         message = message + " (" + no + ")";
michael@0:       this.fail(message);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   assertRaises: function assertRaises(func, predicate, message) {
michael@0:     try {
michael@0:       func();
michael@0:       if (message)
michael@0:         this.fail(message + " (no exception thrown)");
michael@0:       else
michael@0:         this.fail("function failed to throw exception");
michael@0:     } catch (e) {
michael@0:       var errorMessage;
michael@0:       if (typeof(e) == "string")
michael@0:         errorMessage = e;
michael@0:       else
michael@0:         errorMessage = e.message;
michael@0:       if (typeof(predicate) == "string")
michael@0:         this.assertEqual(errorMessage, predicate, message);
michael@0:       else
michael@0:         this.assertMatches(errorMessage, predicate, message);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   assert: function assert(a, message) {
michael@0:     if (!a) {
michael@0:       if (!message)
michael@0:         message = "assertion failed, value is " + a;
michael@0:       this.fail(message);
michael@0:     } else
michael@0:       this.pass(message || "assertion successful");
michael@0:   },
michael@0: 
michael@0:   assertNotEqual: function assertNotEqual(a, b, message) {
michael@0:     if (a != b) {
michael@0:       if (!message)
michael@0:         message = "a != b != " + uneval(a);
michael@0:       this.pass(message);
michael@0:     } else {
michael@0:       var equality = uneval(a) + " == " + uneval(b);
michael@0:       if (!message)
michael@0:         message = equality;
michael@0:       else
michael@0:         message += " (" + equality + ")";
michael@0:       this.fail(message);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   assertEqual: function assertEqual(a, b, message) {
michael@0:     if (a == b) {
michael@0:       if (!message)
michael@0:         message = "a == b == " + uneval(a);
michael@0:       this.pass(message);
michael@0:     } else {
michael@0:       var inequality = uneval(a) + " != " + uneval(b);
michael@0:       if (!message)
michael@0:         message = inequality;
michael@0:       else
michael@0:         message += " (" + inequality + ")";
michael@0:       this.fail(message);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   assertNotStrictEqual: function assertNotStrictEqual(a, b, message) {
michael@0:     if (a !== b) {
michael@0:       if (!message)
michael@0:         message = "a !== b !== " + uneval(a);
michael@0:       this.pass(message);
michael@0:     } else {
michael@0:       var equality = uneval(a) + " === " + uneval(b);
michael@0:       if (!message)
michael@0:         message = equality;
michael@0:       else
michael@0:         message += " (" + equality + ")";
michael@0:       this.fail(message);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   assertStrictEqual: function assertStrictEqual(a, b, message) {
michael@0:     if (a === b) {
michael@0:       if (!message)
michael@0:         message = "a === b === " + uneval(a);
michael@0:       this.pass(message);
michael@0:     } else {
michael@0:       var inequality = uneval(a) + " !== " + uneval(b);
michael@0:       if (!message)
michael@0:         message = inequality;
michael@0:       else
michael@0:         message += " (" + inequality + ")";
michael@0:       this.fail(message);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   assertFunction: function assertFunction(a, message) {
michael@0:     this.assertStrictEqual('function', typeof a, message);
michael@0:   },
michael@0: 
michael@0:   assertUndefined: function(a, message) {
michael@0:     this.assertStrictEqual('undefined', typeof a, message);
michael@0:   },
michael@0: 
michael@0:   assertNotUndefined: function(a, message) {
michael@0:     this.assertNotStrictEqual('undefined', typeof a, message);
michael@0:   },
michael@0: 
michael@0:   assertNull: function(a, message) {
michael@0:     this.assertStrictEqual(null, a, message);
michael@0:   },
michael@0: 
michael@0:   assertNotNull: function(a, message) {
michael@0:     this.assertNotStrictEqual(null, a, message);
michael@0:   },
michael@0: 
michael@0:   assertObject: function(a, message) {
michael@0:     this.assertStrictEqual('[object Object]', Object.prototype.toString.apply(a), message);
michael@0:   },
michael@0: 
michael@0:   assertString: function(a, message) {
michael@0:     this.assertStrictEqual('[object String]', Object.prototype.toString.apply(a), message);
michael@0:   },
michael@0: 
michael@0:   assertArray: function(a, message) {
michael@0:     this.assertStrictEqual('[object Array]', Object.prototype.toString.apply(a), message);
michael@0:   },
michael@0:   
michael@0:   assertNumber: function(a, message) {
michael@0:     this.assertStrictEqual('[object Number]', Object.prototype.toString.apply(a), message);                
michael@0:   },
michael@0: 
michael@0:   done: function done() {
michael@0:     if (!this.isDone) {
michael@0:       this.isDone = true;
michael@0:       if(this.test.teardown) {
michael@0:         this.test.teardown(this);
michael@0:       }
michael@0:       if (this.waitTimeout !== null) {
michael@0:         timer.clearTimeout(this.waitTimeout);
michael@0:         this.waitTimeout = null;
michael@0:       }
michael@0:       // Do not leave any callback set when calling to `waitUntil`
michael@0:       this.waitUntilCallback = null;
michael@0:       if (this.test.passed == 0 && this.test.failed == 0) {
michael@0:         this._logTestFailed("empty test");
michael@0:         if ("testMessage" in this.console) {
michael@0:           this.console.testMessage(false, false, this.test.name, "Empty test");
michael@0:         }
michael@0:         else {
michael@0:           this.console.error("fail:", "Empty test")
michael@0:         }
michael@0:         this.failed++;
michael@0:         this.test.failed++;
michael@0:       }
michael@0:       
michael@0:       this.testRunSummary.push({
michael@0:         name: this.test.name,
michael@0:         passed: this.test.passed,
michael@0:         failed: this.test.failed,
michael@0:         errors: [error for (error in this.test.errors)].join(", ")
michael@0:       });
michael@0:       
michael@0:       if (this.onDone !== null) {
michael@0:         var onDone = this.onDone;
michael@0:         var self = this;
michael@0:         this.onDone = null;
michael@0:         timer.setTimeout(function() { onDone(self); }, 0);
michael@0:       }
michael@0:     }
michael@0:   },
michael@0:   
michael@0:   // Set of assertion functions to wait for an assertion to become true
michael@0:   // These functions take the same arguments as the TestRunner.assert* methods.
michael@0:   waitUntil: function waitUntil() {
michael@0:     return this._waitUntil(this.assert, arguments);
michael@0:   },
michael@0:   
michael@0:   waitUntilNotEqual: function waitUntilNotEqual() {
michael@0:     return this._waitUntil(this.assertNotEqual, arguments);
michael@0:   },
michael@0:   
michael@0:   waitUntilEqual: function waitUntilEqual() {
michael@0:     return this._waitUntil(this.assertEqual, arguments);
michael@0:   },
michael@0:   
michael@0:   waitUntilMatches: function waitUntilMatches() {
michael@0:     return this._waitUntil(this.assertMatches, arguments);
michael@0:   },
michael@0:   
michael@0:   /**
michael@0:    * Internal function that waits for an assertion to become true.
michael@0:    * @param {Function} assertionMethod
michael@0:    *    Reference to a TestRunner assertion method like test.assert, 
michael@0:    *    test.assertEqual, ...
michael@0:    * @param {Array} args
michael@0:    *    List of arguments to give to the previous assertion method. 
michael@0:    *    All functions in this list are going to be called to retrieve current
michael@0:    *    assertion values.
michael@0:    */
michael@0:   _waitUntil: function waitUntil(assertionMethod, args) {
michael@0:     let count = 0;
michael@0:     let maxCount = this.DEFAULT_PAUSE_TIMEOUT / this.PAUSE_DELAY;
michael@0: 
michael@0:     // We need to ensure that test is asynchronous
michael@0:     if (!this.waitTimeout)
michael@0:       this.waitUntilDone(this.DEFAULT_PAUSE_TIMEOUT);
michael@0: 
michael@0:     let callback = null;
michael@0:     let finished = false;
michael@0:     
michael@0:     let test = this;
michael@0: 
michael@0:     // capture a traceback before we go async.
michael@0:     let traceback = require("../console/traceback");
michael@0:     let stack = traceback.get();
michael@0:     stack.splice(-2, 2);
michael@0:     let currentWaitStack = traceback.format(stack);
michael@0:     let timeout = null;
michael@0: 
michael@0:     function loop(stopIt) {
michael@0:       timeout = null;
michael@0: 
michael@0:       // Build a mockup object to fake TestRunner API and intercept calls to
michael@0:       // pass and fail methods, in order to retrieve nice error messages
michael@0:       // and assertion result
michael@0:       let mock = {
michael@0:         pass: function (msg) {
michael@0:           test.pass(msg);
michael@0:           test.waitUntilCallback = null;
michael@0:           if (callback && !stopIt)
michael@0:             callback();
michael@0:           finished = true;
michael@0:         },
michael@0:         fail: function (msg) {
michael@0:           // If we are called on test timeout, we stop the loop
michael@0:           // and print which test keeps failing:
michael@0:           if (stopIt) {
michael@0:             test.console.error("test assertion never became true:\n",
michael@0:                                msg + "\n",
michael@0:                                currentWaitStack);
michael@0:             if (timeout)
michael@0:               timer.clearTimeout(timeout);
michael@0:             return;
michael@0:           }
michael@0:           timeout = timer.setTimeout(loop, test.PAUSE_DELAY);
michael@0:         }
michael@0:       };
michael@0:       
michael@0:       // Automatically call args closures in order to build arguments for 
michael@0:       // assertion function
michael@0:       let appliedArgs = [];
michael@0:       for (let i = 0, l = args.length; i < l; i++) {
michael@0:         let a = args[i];
michael@0:         if (typeof a == "function") {
michael@0:           try {
michael@0:             a = a();
michael@0:           }
michael@0:           catch(e) {
michael@0:             test.fail("Exception when calling asynchronous assertion: " + e +
michael@0:                       "\n" + e.stack);
michael@0:             finished = true;
michael@0:             return;
michael@0:           }
michael@0:         }
michael@0:         appliedArgs.push(a);
michael@0:       }
michael@0:       
michael@0:       // Finally call assertion function with current assertion values
michael@0:       assertionMethod.apply(mock, appliedArgs);
michael@0:     }
michael@0:     loop();
michael@0:     this.waitUntilCallback = loop;
michael@0:     
michael@0:     // Return an object with `then` method, to offer a way to execute 
michael@0:     // some code when the assertion passed or failed
michael@0:     return {
michael@0:       then: function (c) {
michael@0:         callback = c;
michael@0:         
michael@0:         // In case of immediate positive result, we need to execute callback
michael@0:         // immediately here:
michael@0:         if (finished)
michael@0:           callback();
michael@0:       }
michael@0:     };
michael@0:   },
michael@0:   
michael@0:   waitUntilDone: function waitUntilDone(ms) {
michael@0:     if (ms === undefined)
michael@0:       ms = this.DEFAULT_PAUSE_TIMEOUT;
michael@0: 
michael@0:     var self = this;
michael@0: 
michael@0:     function tiredOfWaiting() {
michael@0:       self._logTestFailed("timed out");
michael@0:       if ("testMessage" in self.console) {
michael@0:         self.console.testMessage(false, false, self.test.name, "Timed out");
michael@0:       }
michael@0:       else {
michael@0:         self.console.error("fail:", "Timed out")
michael@0:       }
michael@0:       if (self.waitUntilCallback) {
michael@0:         self.waitUntilCallback(true);
michael@0:         self.waitUntilCallback = null;
michael@0:       }
michael@0:       self.failed++;
michael@0:       self.test.failed++;
michael@0:       self.done();
michael@0:     }
michael@0: 
michael@0:     // We may already have registered a timeout callback
michael@0:     if (this.waitTimeout)
michael@0:       timer.clearTimeout(this.waitTimeout);
michael@0: 
michael@0:     this.waitTimeout = timer.setTimeout(tiredOfWaiting, ms);
michael@0:   },
michael@0: 
michael@0:   startMany: function startMany(options) {
michael@0:     function runNextTest(self) {
michael@0:       var test = options.tests.shift();
michael@0:       if (options.stopOnError && self.test && self.test.failed) {
michael@0:         self.console.error("aborted: test failed and --stop-on-error was specified");
michael@0:         options.onDone(self);
michael@0:       } else if (test) {
michael@0:         self.start({test: test, onDone: runNextTest});
michael@0:       } else {
michael@0:         options.onDone(self);
michael@0:       }
michael@0:     }
michael@0:     runNextTest(this);
michael@0:   },
michael@0: 
michael@0:   start: function start(options) {
michael@0:     this.test = options.test;
michael@0:     this.test.passed = 0;
michael@0:     this.test.failed = 0;
michael@0:     this.test.errors = {};
michael@0: 
michael@0:     this.isDone = false;
michael@0:     this.onDone = function(self) {
michael@0:       if (cfxArgs.parseable)
michael@0:         self.console.print("TEST-END | " + self.test.name + "\n");
michael@0:       options.onDone(self);
michael@0:     }
michael@0:     this.waitTimeout = null;
michael@0: 
michael@0:     try {
michael@0:       if (cfxArgs.parseable)
michael@0:         this.console.print("TEST-START | " + this.test.name + "\n");
michael@0:       else
michael@0:         this.console.info("executing '" + this.test.name + "'");
michael@0: 
michael@0:       if(this.test.setup) {
michael@0:         this.test.setup(this);
michael@0:       }
michael@0:       this.test.testFunction(this);
michael@0:     } catch (e) {
michael@0:       this.exception(e);
michael@0:     }
michael@0:     if (this.waitTimeout === null)
michael@0:       this.done();
michael@0:   }
michael@0: };