1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/dom/imptests/testharnessreport.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,299 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +var W3CTest = {
1.9 + /**
1.10 + * Dictionary mapping a test URL to either the string "all", which means that
1.11 + * all tests in this file are expected to fail, or a dictionary mapping test
1.12 + * names to either the boolean |true|, or the string "debug". The former
1.13 + * means that this test is expected to fail in all builds, and the latter
1.14 + * that it is only expected to fail in debug builds.
1.15 + */
1.16 + "expectedFailures": {},
1.17 +
1.18 + /**
1.19 + * If set to true, we will dump the test failures to the console.
1.20 + */
1.21 + "dumpFailures": false,
1.22 +
1.23 + /**
1.24 + * If dumpFailures is true, this holds a structure like necessary for
1.25 + * expectedFailures, for ease of updating the expectations.
1.26 + */
1.27 + "failures": {},
1.28 +
1.29 + /**
1.30 + * List of test results, needed by TestRunner to update the UI.
1.31 + */
1.32 + "tests": [],
1.33 +
1.34 + /**
1.35 + * Number of unlogged passes, to stop buildbot from truncating the log.
1.36 + * We will print a message every MAX_COLLAPSED_MESSAGES passes.
1.37 + */
1.38 + "collapsedMessages": 0,
1.39 + "MAX_COLLAPSED_MESSAGES": 100,
1.40 +
1.41 + /**
1.42 + * Reference to the TestRunner object in the parent frame.
1.43 + */
1.44 + "runner": parent === this ? null : parent.TestRunner || parent.wrappedJSObject.TestRunner,
1.45 +
1.46 + /**
1.47 + * Prefixes for the error logging. Indexed first by int(todo) and second by
1.48 + * int(result).
1.49 + */
1.50 + "prefixes": [
1.51 + ["TEST-UNEXPECTED-FAIL", "TEST-PASS"],
1.52 + ["TEST-KNOWN-FAIL", "TEST-UNEXPECTED-PASS"]
1.53 + ],
1.54 +
1.55 + /**
1.56 + * Prefix of the path to parent of the the failures directory.
1.57 + */
1.58 + "pathprefix": "/tests/dom/imptests/",
1.59 +
1.60 + /**
1.61 + * Returns the URL of the current test, relative to the root W3C tests
1.62 + * directory. Used as a key into the expectedFailures dictionary.
1.63 + */
1.64 + "getPath": function() {
1.65 + var url = this.getURL();
1.66 + if (!url.startsWith(this.pathprefix)) {
1.67 + return "";
1.68 + }
1.69 + return url.substring(this.pathprefix.length);
1.70 + },
1.71 +
1.72 + /**
1.73 + * Returns the root-relative URL of the current test.
1.74 + */
1.75 + "getURL": function() {
1.76 + return this.runner ? this.runner.currentTestURL : location.pathname;
1.77 + },
1.78 +
1.79 + /**
1.80 + * Report the results in the tests array.
1.81 + */
1.82 + "reportResults": function() {
1.83 + var element = function element(aLocalName) {
1.84 + var xhtmlNS = "http://www.w3.org/1999/xhtml";
1.85 + return document.createElementNS(xhtmlNS, aLocalName);
1.86 + };
1.87 +
1.88 + var stylesheet = element("link");
1.89 + stylesheet.setAttribute("rel", "stylesheet");
1.90 + stylesheet.setAttribute("href", "/resources/testharness.css");
1.91 + var heads = document.getElementsByTagName("head");
1.92 + if (heads.length) {
1.93 + heads[0].appendChild(stylesheet);
1.94 + }
1.95 +
1.96 + var log = document.getElementById("log");
1.97 + if (!log) {
1.98 + return;
1.99 + }
1.100 + var section = log.appendChild(element("section"));
1.101 + section.id = "summary";
1.102 + section.appendChild(element("h2")).textContent = "Details";
1.103 +
1.104 + var table = section.appendChild(element("table"));
1.105 + table.id = "results";
1.106 +
1.107 + var tr = table.appendChild(element("thead")).appendChild(element("tr"));
1.108 + for (var header of ["Result", "Test Name", "Message"]) {
1.109 + tr.appendChild(element("th")).textContent = header;
1.110 + }
1.111 + var statuses = [
1.112 + ["Unexpected Fail", "Pass"],
1.113 + ["Known Fail", "Unexpected Pass"]
1.114 + ];
1.115 + var tbody = table.appendChild(element("tbody"));
1.116 + for (var test of this.tests) {
1.117 + tr = tbody.appendChild(element("tr"));
1.118 + tr.className = (test.result === !test.todo ? "pass" : "fail");
1.119 + tr.appendChild(element("td")).textContent =
1.120 + statuses[+test.todo][+test.result];
1.121 + tr.appendChild(element("td")).textContent = test.name;
1.122 + tr.appendChild(element("td")).textContent = test.message;
1.123 + }
1.124 + },
1.125 +
1.126 + /**
1.127 + * Returns a printable message based on aTest's 'name' and 'message'
1.128 + * properties.
1.129 + */
1.130 + "formatTestMessage": function(aTest) {
1.131 + return aTest.name + (aTest.message ? ": " + aTest.message : "");
1.132 + },
1.133 +
1.134 + /**
1.135 + * Lets the test runner know about a test result.
1.136 + */
1.137 + "_log": function(test) {
1.138 + var url = this.getURL();
1.139 + var msg = this.prefixes[+test.todo][+test.result] + " | ";
1.140 + if (url) {
1.141 + msg += url;
1.142 + }
1.143 + msg += " | " + this.formatTestMessage(test);
1.144 + if (this.runner) {
1.145 + this.runner[(test.result === !test.todo) ? "log" : "error"](msg);
1.146 + } else {
1.147 + dump(msg + "\n");
1.148 + }
1.149 + },
1.150 +
1.151 + /**
1.152 + * Logs a message about collapsed messages (if any), and resets the counter.
1.153 + */
1.154 + "_logCollapsedMessages": function() {
1.155 + if (this.collapsedMessages) {
1.156 + this._log({
1.157 + "name": document.title,
1.158 + "result": true,
1.159 + "todo": false,
1.160 + "message": "Elided " + this.collapsedMessages + " passes or known failures."
1.161 + });
1.162 + }
1.163 + this.collapsedMessages = 0;
1.164 + },
1.165 +
1.166 + /**
1.167 + * Maybe logs a result, eliding up to MAX_COLLAPSED_MESSAGES consecutive
1.168 + * passes.
1.169 + */
1.170 + "_maybeLog": function(test) {
1.171 + var success = (test.result === !test.todo);
1.172 + if (success && ++this.collapsedMessages < this.MAX_COLLAPSED_MESSAGES) {
1.173 + return;
1.174 + }
1.175 + this._logCollapsedMessages();
1.176 + this._log(test);
1.177 + },
1.178 +
1.179 + /**
1.180 + * Reports a test result. The argument is an object with the following
1.181 + * properties:
1.182 + *
1.183 + * o message (string): message to be reported
1.184 + * o result (boolean): whether this test failed
1.185 + * o todo (boolean): whether this test is expected to fail
1.186 + */
1.187 + "report": function(test) {
1.188 + this.tests.push(test);
1.189 + this._maybeLog(test);
1.190 + },
1.191 +
1.192 + /**
1.193 + * Returns true if this test is expected to fail, and false otherwise.
1.194 + */
1.195 + "_todo": function(test) {
1.196 + if (this.expectedFailures === "all") {
1.197 + return true;
1.198 + }
1.199 + var value = this.expectedFailures[test.name];
1.200 + return value === true || (value === "debug" && !!SpecialPowers.isDebugBuild);
1.201 + },
1.202 +
1.203 + /**
1.204 + * Callback function for testharness.js. Called when one test in a file
1.205 + * finishes.
1.206 + */
1.207 + "result": function(test) {
1.208 + var url = this.getPath();
1.209 + this.report({
1.210 + "name": test.name,
1.211 + "message": test.message || "",
1.212 + "result": test.status === test.PASS,
1.213 + "todo": this._todo(test)
1.214 + });
1.215 + if (this.dumpFailures && test.status !== test.PASS) {
1.216 + this.failures[test.name] = true;
1.217 + }
1.218 + },
1.219 +
1.220 + /**
1.221 + * Callback function for testharness.js. Called when the entire test file
1.222 + * finishes.
1.223 + */
1.224 + "finish": function(tests, status) {
1.225 + var url = this.getPath();
1.226 + this.report({
1.227 + "name": "Finished test",
1.228 + "message": "Status: " + status.status,
1.229 + "result": status.status === status.OK,
1.230 + "todo":
1.231 + url in this.expectedFailures &&
1.232 + this.expectedFailures[url] === "error"
1.233 + });
1.234 +
1.235 + this._logCollapsedMessages();
1.236 +
1.237 + if (this.dumpFailures) {
1.238 + dump("@@@ @@@ Failures\n");
1.239 + dump(url + "@@@" + JSON.stringify(this.failures) + "\n");
1.240 + }
1.241 + if (this.runner) {
1.242 + this.runner.testFinished(this.tests.map(function(aTest) {
1.243 + return {
1.244 + "message": this.formatTestMessage(aTest),
1.245 + "result": aTest.result,
1.246 + "todo": aTest.todo
1.247 + };
1.248 + }, this));
1.249 + } else {
1.250 + this.reportResults();
1.251 + }
1.252 + },
1.253 +
1.254 + /**
1.255 + * Log an unexpected failure. Intended to be used from harness code, not
1.256 + * from tests.
1.257 + */
1.258 + "logFailure": function(aTestName, aMessage) {
1.259 + this.report({
1.260 + "name": aTestName,
1.261 + "message": aMessage,
1.262 + "result": false,
1.263 + "todo": false
1.264 + });
1.265 + },
1.266 +
1.267 + /**
1.268 + * Timeout the current test. Intended to be used from harness code, not
1.269 + * from tests.
1.270 + */
1.271 + "timeout": function() {
1.272 + this.logFailure("Timeout", "Test runner timed us out.");
1.273 + timeout();
1.274 + }
1.275 +};
1.276 +(function() {
1.277 + try {
1.278 + var path = W3CTest.getPath();
1.279 + if (path) {
1.280 + // Get expected fails. If there aren't any, there will be a 404, which is
1.281 + // fine. Anything else is unexpected.
1.282 + var url = W3CTest.pathprefix + "failures/" + path + ".json";
1.283 + var request = new XMLHttpRequest();
1.284 + request.open("GET", url, false);
1.285 + request.send();
1.286 + if (request.status === 200) {
1.287 + W3CTest.expectedFailures = JSON.parse(request.responseText);
1.288 + } else if (request.status !== 404) {
1.289 + W3CTest.logFailure("Fetching failures file", "Request status was " + request.status);
1.290 + }
1.291 + }
1.292 +
1.293 + add_result_callback(W3CTest.result.bind(W3CTest));
1.294 + add_completion_callback(W3CTest.finish.bind(W3CTest));
1.295 + setup({
1.296 + "output": false,
1.297 + "explicit_timeout": true
1.298 + });
1.299 + } catch (e) {
1.300 + W3CTest.logFailure("Harness setup", "Unexpected exception: " + e);
1.301 + }
1.302 +})();
