michael@0: // Copyright (c) 2009 The Chromium Authors. All rights reserved.
michael@0: // Use of this source code is governed by a BSD-style license that can be
michael@0: // found in the LICENSE file.
michael@0: 
michael@0: // This is a test harness for running javascript tests in the browser.
michael@0: // The only identifier exposed by this harness is WebGLTestHarnessModule.
michael@0: //
michael@0: // To use it make an HTML page with an iframe. Then call the harness like this
michael@0: //
michael@0: //    function reportResults(type, msg, success) {
michael@0: //      ...
michael@0: //      return true;
michael@0: //    }
michael@0: //
michael@0: //    var fileListURL = '00_test_list.txt';
michael@0: //    var testHarness = new WebGLTestHarnessModule.TestHarness(
michael@0: //        iframe,
michael@0: //        fileListURL,
michael@0: //        reportResults);
michael@0: //
michael@0: // The harness will load the fileListURL and parse it for the URLs, one URL
michael@0: // per line. URLs should be on the same domain and at the same folder level
michael@0: // or below the main html file.  If any URL ends in .txt it will be parsed
michael@0: // as well so you can nest .txt files. URLs inside a .txt file should be
michael@0: // relative to that text file.
michael@0: //
michael@0: // During startup, for each page found the reportFunction will be called with
michael@0: // WebGLTestHarnessModule.TestHarness.reportType.ADD_PAGE and msg will be
michael@0: // the URL of the test.
michael@0: //
michael@0: // Each test is required to call testHarness.reportResults. This is most easily
michael@0: // accomplished by storing that value on the main window with
michael@0: //
michael@0: //     window.webglTestHarness = testHarness
michael@0: //
michael@0: // and then adding these to functions to your tests.
michael@0: //
michael@0: //     function reportTestResultsToHarness(success, msg) {
michael@0: //       if (window.parent.webglTestHarness) {
michael@0: //         window.parent.webglTestHarness.reportResults(success, msg);
michael@0: //       }
michael@0: //     }
michael@0: //
michael@0: //     function notifyFinishedToHarness() {
michael@0: //       if (window.parent.webglTestHarness) {
michael@0: //         window.parent.webglTestHarness.notifyFinished();
michael@0: //       }
michael@0: //     }
michael@0: //
michael@0: // This way your tests will still run without the harness and you can use
michael@0: // any testing framework you want.
michael@0: //
michael@0: // Each test should call reportTestResultsToHarness with true for success if it
michael@0: // succeeded and false if it fail followed and any message it wants to
michael@0: // associate with the test. If your testing framework supports checking for
michael@0: // timeout you can call it with success equal to undefined in that case.
michael@0: //
michael@0: // To run the tests, call testHarness.runTests();
michael@0: //
michael@0: // For each test run, before the page is loaded the reportFunction will be
michael@0: // called with WebGLTestHarnessModule.TestHarness.reportType.START_PAGE and msg
michael@0: // will be the URL of the test. You may return false if you want the test to be
michael@0: // skipped.
michael@0: //
michael@0: // For each test completed the reportFunction will be called with
michael@0: // with WebGLTestHarnessModule.TestHarness.reportType.TEST_RESULT,
michael@0: // success = true on success, false on failure, undefined on timeout
michael@0: // and msg is any message the test choose to pass on.
michael@0: //
michael@0: // When all the tests on the page have finished your page must call
michael@0: // notifyFinishedToHarness.  If notifyFinishedToHarness is not called
michael@0: // the harness will assume the test timed out.
michael@0: //
michael@0: // When all the tests on a page have finished OR the page as timed out the
michael@0: // reportFunction will be called with
michael@0: // WebGLTestHarnessModule.TestHarness.reportType.FINISH_PAGE
michael@0: // where success = true if the page has completed or undefined if the page timed
michael@0: // out.
michael@0: //
michael@0: // Finally, when all the tests have completed the reportFunction will be called
michael@0: // with WebGLTestHarnessModule.TestHarness.reportType.FINISHED_ALL_TESTS.
michael@0: //
michael@0: 
michael@0: WebGLTestHarnessModule = function() {
michael@0: 
michael@0: /**
michael@0:  * Wrapped logging function.
michael@0:  */
michael@0: var log = function(msg) {
michael@0:   if (window.console && window.console.log) {
michael@0:     window.console.log(msg);
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Loads text from an external file. This function is synchronous.
michael@0:  * @param {string} url The url of the external file.
michael@0:  * @param {!function(bool, string): void} callback that is sent a bool for
michael@0:  *     success and the string.
michael@0:  */
michael@0: var loadTextFileAsynchronous = function(url, callback) {
michael@0:   log ("loading: " + url);
michael@0:   var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
michael@0:   var request;
michael@0:   if (window.XMLHttpRequest) {
michael@0:     request = new XMLHttpRequest();
michael@0:     if (request.overrideMimeType) {
michael@0:       request.overrideMimeType('text/plain');
michael@0:     }
michael@0:   } else {
michael@0:     throw 'XMLHttpRequest is disabled';
michael@0:   }
michael@0:   try {
michael@0:     request.open('GET', url, true);
michael@0:     request.onreadystatechange = function() {
michael@0:       if (request.readyState == 4) {
michael@0:         var text = '';
michael@0:         // HTTP reports success with a 200 status. The file protocol reports
michael@0:         // success with zero. HTTP does not use zero as a status code (they
michael@0:         // start at 100).
michael@0:         // https://developer.mozilla.org/En/Using_XMLHttpRequest
michael@0:         var success = request.status == 200 || request.status == 0;
michael@0:         if (success) {
michael@0:           text = request.responseText;
michael@0:         }
michael@0:         log("loaded: " + url);
michael@0:         callback(success, text);
michael@0:       }
michael@0:     };
michael@0:     request.send(null);
michael@0:   } catch (e) {
michael@0:     log("failed to load: " + url);
michael@0:     callback(false, '');
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Compare version strings.
michael@0:  */
michael@0: var greaterThanOrEqualToVersion = function(have, want) {
michael@0:   have = have.split(" ")[0].split(".");
michael@0:   want = want.split(" ")[0].split(".");
michael@0: 
michael@0:   //have 1.2.3   want  1.1
michael@0:   //have 1.1.1   want  1.1
michael@0:   //have 1.0.9   want  1.1
michael@0:   //have 1.1     want  1.1.1
michael@0: 
michael@0:   for (var ii = 0; ii < want.length; ++ii) {
michael@0:     var wantNum = parseInt(want[ii]);
michael@0:     var haveNum = have[ii] ? parseInt(have[ii]) : 0
michael@0:     if (haveNum < wantNum) {
michael@0:       return false;
michael@0:     }
michael@0:   }
michael@0:   return true;
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Reads a file, recursively adding files referenced inside.
michael@0:  *
michael@0:  * Each line of URL is parsed, comments starting with '#' or ';'
michael@0:  * or '//' are stripped.
michael@0:  *
michael@0:  * arguments beginning with -- are extracted
michael@0:  *
michael@0:  * lines that end in .txt are recursively scanned for more files
michael@0:  * other lines are added to the list of files.
michael@0:  *
michael@0:  * @param {string} url The url of the file to read.
michael@0:  * @param {void function(boolean, !Array.<string>)} callback.
michael@0:  *      Callback that is called with true for success and an
michael@0:  *      array of filenames.
michael@0:  * @param {Object} options. Optional options
michael@0:  *
michael@0:  * Options:
michael@0:  *    version: {string} The version of the conformance test.
michael@0:  *    Tests with the argument --min-version <version> will
michael@0:  *    be ignored version is less then <version>
michael@0:  *
michael@0:  */
michael@0: var getFileList = function(url, callback, options) {
michael@0:   var files = [];
michael@0: 
michael@0:   var copyObject = function(obj) {
michael@0:     return JSON.parse(JSON.stringify(obj));
michael@0:   };
michael@0: 
michael@0:   var toCamelCase = function(str) {
michael@0:     return str.replace(/-([a-z])/g, function (g) { return g[1].toUpperCase() });
michael@0:   };
michael@0: 
michael@0:   var globalOptions = copyObject(options);
michael@0:   globalOptions.defaultVersion = "1.0";
michael@0: 
michael@0:   var getFileListImpl = function(prefix, line, hierarchicalOptions, callback) {
michael@0:     var files = [];
michael@0: 
michael@0:     var args = line.split(/\s+/);
michael@0:     var nonOptions = [];
michael@0:     var useTest = true;
michael@0:     var testOptions = {};
michael@0:     for (var jj = 0; jj < args.length; ++jj) {
michael@0:       var arg = args[jj];
michael@0:       if (arg[0] == '-') {
michael@0:         if (arg[1] != '-') {
michael@0:           throw ("bad option at in " + url + ":" + (ii + 1) + ": " + str);
michael@0:         }
michael@0:         var option = arg.substring(2);
michael@0:         switch (option) {
michael@0:           case 'min-version':
michael@0:             ++jj;
michael@0:             testOptions[toCamelCase(option)] = args[jj];
michael@0:             break;
michael@0:           default:
michael@0:             throw ("bad unknown option '" + option + "' at in " + url + ":" + (ii + 1) + ": " + str);
michael@0:         }
michael@0:       } else {
michael@0:         nonOptions.push(arg);
michael@0:       }
michael@0:     }
michael@0:     var url = prefix + nonOptions.join(" ");
michael@0: 
michael@0:     if (url.substr(url.length - 4) != '.txt') {
michael@0:       var minVersion = testOptions.minVersion;
michael@0:       if (!minVersion) {
michael@0:         minVersion = hierarchicalOptions.defaultVersion;
michael@0:       }
michael@0: 
michael@0:       if (globalOptions.minVersion) {
michael@0:         useTest = greaterThanOrEqualToVersion(minVersion, globalOptions.minVersion);
michael@0:       } else {
michael@0:         useTest = greaterThanOrEqualToVersion(globalOptions.version, minVersion);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (!useTest) {
michael@0:       callback(true, []);
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     if (url.substr(url.length - 4) == '.txt') {
michael@0:       // If a version was explicity specified pass it down.
michael@0:       if (testOptions.minVersion) {
michael@0:         hierarchicalOptions.defaultVersion = testOptions.minVersion;
michael@0:       }
michael@0:       loadTextFileAsynchronous(url, function() {
michael@0:         return function(success, text) {
michael@0:           if (!success) {
michael@0:             callback(false, '');
michael@0:             return;
michael@0:           }
michael@0:           var lines = text.split('\n');
michael@0:           var prefix = '';
michael@0:           var lastSlash = url.lastIndexOf('/');
michael@0:           if (lastSlash >= 0) {
michael@0:             prefix = url.substr(0, lastSlash + 1);
michael@0:           }
michael@0:           var fail = false;
michael@0:           var count = 1;
michael@0:           var index = 0;
michael@0:           for (var ii = 0; ii < lines.length; ++ii) {
michael@0:             var str = lines[ii].replace(/^\s\s*/, '').replace(/\s\s*$/, '');
michael@0:             if (str.length > 4 &&
michael@0:                 str[0] != '#' &&
michael@0:                 str[0] != ";" &&
michael@0:                 str.substr(0, 2) != "//") {
michael@0:               ++count;
michael@0:               getFileListImpl(prefix, str, copyObject(hierarchicalOptions), function(index) {
michael@0:                 return function(success, new_files) {
michael@0:                   log("got files: " + new_files.length);
michael@0:                   if (success) {
michael@0:                     files[index] = new_files;
michael@0:                   }
michael@0:                   finish(success);
michael@0:                 };
michael@0:               }(index++));
michael@0:             }
michael@0:           }
michael@0:           finish(true);
michael@0: 
michael@0:           function finish(success) {
michael@0:             if (!success) {
michael@0:               fail = true;
michael@0:             }
michael@0:             --count;
michael@0:             log("count: " + count);
michael@0:             if (!count) {
michael@0:               callback(!fail, files);
michael@0:             }
michael@0:           }
michael@0:         }
michael@0:       }());
michael@0:     } else {
michael@0:       files.push(url);
michael@0:       callback(true, files);
michael@0:     }
michael@0:   };
michael@0: 
michael@0:   getFileListImpl('', url, globalOptions, function(success, files) {
michael@0:     // flatten
michael@0:     var flat = [];
michael@0:     flatten(files);
michael@0:     function flatten(files) {
michael@0:       for (var ii = 0; ii < files.length; ++ii) {
michael@0:         var value = files[ii];
michael@0:         if (typeof(value) == "string") {
michael@0:           flat.push(value);
michael@0:         } else {
michael@0:           flatten(value);
michael@0:         }
michael@0:       }
michael@0:     }
michael@0:     callback(success, flat);
michael@0:   });
michael@0: };
michael@0: 
michael@0: var TestFile = function(url) {
michael@0:   this.url = url;
michael@0: };
michael@0: 
michael@0: var TestHarness = function(iframe, filelistUrl, reportFunc, options) {
michael@0:   this.window = window;
michael@0:   this.iframe = iframe;
michael@0:   this.reportFunc = reportFunc;
michael@0:   this.timeoutDelay = 20000;
michael@0:   this.files = [];
michael@0: 
michael@0:   var that = this;
michael@0:   getFileList(filelistUrl, function() {
michael@0:     return function(success, files) {
michael@0:       that.addFiles_(success, files);
michael@0:     };
michael@0:   }(), options);
michael@0: 
michael@0: };
michael@0: 
michael@0: TestHarness.reportType = {
michael@0:   ADD_PAGE: 1,
michael@0:   READY: 2,
michael@0:   START_PAGE: 3,
michael@0:   TEST_RESULT: 4,
michael@0:   FINISH_PAGE: 5,
michael@0:   FINISHED_ALL_TESTS: 6
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.addFiles_ = function(success, files) {
michael@0:   if (!success) {
michael@0:     this.reportFunc(
michael@0:         TestHarness.reportType.FINISHED_ALL_TESTS,
michael@0:         'Unable to load tests. Are you running locally?\n' +
michael@0:         'You need to run from a server or configure your\n' +
michael@0:         'browser to allow access to local files (not recommended).\n\n' +
michael@0:         'Note: An easy way to run from a server:\n\n' +
michael@0:         '\tcd path_to_tests\n' +
michael@0:         '\tpython -m SimpleHTTPServer\n\n' +
michael@0:         'then point your browser to ' +
michael@0:           '<a href="http://localhost:8000/webgl-conformance-tests.html">' +
michael@0:           'http://localhost:8000/webgl-conformance-tests.html</a>',
michael@0:         false)
michael@0:     return;
michael@0:   }
michael@0:   log("total files: " + files.length);
michael@0:   for (var ii = 0; ii < files.length; ++ii) {
michael@0:     log("" + ii + ": " + files[ii]);
michael@0:     this.files.push(new TestFile(files[ii]));
michael@0:     this.reportFunc(TestHarness.reportType.ADD_PAGE, files[ii], undefined);
michael@0:   }
michael@0:   this.reportFunc(TestHarness.reportType.READY, undefined, undefined);
michael@0: }
michael@0: 
michael@0: TestHarness.prototype.runTests = function(opt_start, opt_count) {
michael@0:   var count = opt_count || this.files.length;
michael@0:   this.nextFileIndex = opt_start || 0;
michael@0:   this.lastFileIndex = this.nextFileIndex + count;
michael@0:   this.startNextFile();
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.setTimeout = function() {
michael@0:   var that = this;
michael@0:   this.timeoutId = this.window.setTimeout(function() {
michael@0:       that.timeout();
michael@0:     }, this.timeoutDelay);
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.clearTimeout = function() {
michael@0:   this.window.clearTimeout(this.timeoutId);
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.startNextFile = function() {
michael@0:   if (this.nextFileIndex >= this.lastFileIndex) {
michael@0:     log("done");
michael@0:     this.reportFunc(TestHarness.reportType.FINISHED_ALL_TESTS,
michael@0:                     '', true);
michael@0:   } else {
michael@0:     this.currentFile = this.files[this.nextFileIndex++];
michael@0:     log("loading: " + this.currentFile.url);
michael@0:     if (this.reportFunc(TestHarness.reportType.START_PAGE,
michael@0:                         this.currentFile.url, undefined)) {
michael@0:       this.iframe.src = this.currentFile.url;
michael@0:       this.setTimeout();
michael@0:     } else {
michael@0:       this.reportResults(false, "skipped");
michael@0:       this.notifyFinished();
michael@0:     }
michael@0:   }
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.reportResults = function (success, msg) {
michael@0:   this.clearTimeout();
michael@0:   log(success ? "PASS" : "FAIL", msg);
michael@0:   this.reportFunc(TestHarness.reportType.TEST_RESULT, msg, success);
michael@0:   // For each result we get, reset the timeout
michael@0:   this.setTimeout();
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.notifyFinished = function () {
michael@0:   this.clearTimeout();
michael@0:   var url = this.currentFile ? this.currentFile.url : 'unknown';
michael@0:   log(url + ": finished");
michael@0:   this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, true);
michael@0:   this.startNextFile();
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.timeout = function() {
michael@0:   this.clearTimeout();
michael@0:   var url = this.currentFile ? this.currentFile.url : 'unknown';
michael@0:   log(url + ": timeout");
michael@0:   this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, undefined);
michael@0:   this.startNextFile();
michael@0: };
michael@0: 
michael@0: TestHarness.prototype.setTimeoutDelay = function(x) {
michael@0:   this.timeoutDelay = x;
michael@0: };
michael@0: 
michael@0: return {
michael@0:     'TestHarness': TestHarness
michael@0:   };
michael@0: 
michael@0: }();
michael@0: 
michael@0: 
michael@0: