1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/addon-sdk/source/lib/sdk/test/harness.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,627 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +"use strict";
1.8 +
1.9 +module.metadata = {
1.10 + "stability": "experimental"
1.11 +};
1.12 +
1.13 +const { Cc, Ci, Cu } = require("chrome");
1.14 +const { Loader } = require('./loader');
1.15 +const { serializeStack, parseStack } = require("toolkit/loader");
1.16 +const { setTimeout } = require('../timers');
1.17 +const { PlainTextConsole } = require("../console/plain-text");
1.18 +const { when: unload } = require("../system/unload");
1.19 +const { format, fromException } = require("../console/traceback");
1.20 +const system = require("../system");
1.21 +const memory = require('../deprecated/memory');
1.22 +const { gc: gcPromise } = require('./memory');
1.23 +const { defer } = require('../core/promise');
1.24 +
1.25 +// Trick manifest builder to make it think we need these modules ?
1.26 +const unit = require("../deprecated/unit-test");
1.27 +const test = require("../../test");
1.28 +const url = require("../url");
1.29 +
1.30 +function emptyPromise() {
1.31 + let { promise, resolve } = defer();
1.32 + resolve();
1.33 + return promise;
1.34 +}
1.35 +
1.36 +var cService = Cc['@mozilla.org/consoleservice;1'].getService()
1.37 + .QueryInterface(Ci.nsIConsoleService);
1.38 +
1.39 +// The console used to log messages
1.40 +var testConsole;
1.41 +
1.42 +// Cuddlefish loader in which we load and execute tests.
1.43 +var loader;
1.44 +
1.45 +// Function to call when we're done running tests.
1.46 +var onDone;
1.47 +
1.48 +// Function to print text to a console, w/o CR at the end.
1.49 +var print;
1.50 +
1.51 +// How many more times to run all tests.
1.52 +var iterationsLeft;
1.53 +
1.54 +// Whether to report memory profiling information.
1.55 +var profileMemory;
1.56 +
1.57 +// Whether we should stop as soon as a test reports a failure.
1.58 +var stopOnError;
1.59 +
1.60 +// Function to call to retrieve a list of tests to execute
1.61 +var findAndRunTests;
1.62 +
1.63 +// Combined information from all test runs.
1.64 +var results = {
1.65 + passed: 0,
1.66 + failed: 0,
1.67 + testRuns: []
1.68 +};
1.69 +
1.70 +// A list of the compartments and windows loaded after startup
1.71 +var startLeaks;
1.72 +
1.73 +// JSON serialization of last memory usage stats; we keep it stringified
1.74 +// so we don't actually change the memory usage stats (in terms of objects)
1.75 +// of the JSRuntime we're profiling.
1.76 +var lastMemoryUsage;
1.77 +
1.78 +function analyzeRawProfilingData(data) {
1.79 + var graph = data.graph;
1.80 + var shapes = {};
1.81 +
1.82 + // Convert keys in the graph from strings to ints.
1.83 + // TODO: Can we get rid of this ridiculousness?
1.84 + var newGraph = {};
1.85 + for (id in graph) {
1.86 + newGraph[parseInt(id)] = graph[id];
1.87 + }
1.88 + graph = newGraph;
1.89 +
1.90 + var modules = 0;
1.91 + var moduleIds = [];
1.92 + var moduleObjs = {UNKNOWN: 0};
1.93 + for (let name in data.namedObjects) {
1.94 + moduleObjs[name] = 0;
1.95 + moduleIds[data.namedObjects[name]] = name;
1.96 + modules++;
1.97 + }
1.98 +
1.99 + var count = 0;
1.100 + for (id in graph) {
1.101 + var parent = graph[id].parent;
1.102 + while (parent) {
1.103 + if (parent in moduleIds) {
1.104 + var name = moduleIds[parent];
1.105 + moduleObjs[name]++;
1.106 + break;
1.107 + }
1.108 + if (!(parent in graph)) {
1.109 + moduleObjs.UNKNOWN++;
1.110 + break;
1.111 + }
1.112 + parent = graph[parent].parent;
1.113 + }
1.114 + count++;
1.115 + }
1.116 +
1.117 + print("\nobject count is " + count + " in " + modules + " modules" +
1.118 + " (" + data.totalObjectCount + " across entire JS runtime)\n");
1.119 + if (lastMemoryUsage) {
1.120 + var last = JSON.parse(lastMemoryUsage);
1.121 + var diff = {
1.122 + moduleObjs: dictDiff(last.moduleObjs, moduleObjs),
1.123 + totalObjectClasses: dictDiff(last.totalObjectClasses,
1.124 + data.totalObjectClasses)
1.125 + };
1.126 +
1.127 + for (let name in diff.moduleObjs)
1.128 + print(" " + diff.moduleObjs[name] + " in " + name + "\n");
1.129 + for (let name in diff.totalObjectClasses)
1.130 + print(" " + diff.totalObjectClasses[name] + " instances of " +
1.131 + name + "\n");
1.132 + }
1.133 + lastMemoryUsage = JSON.stringify(
1.134 + {moduleObjs: moduleObjs,
1.135 + totalObjectClasses: data.totalObjectClasses}
1.136 + );
1.137 +}
1.138 +
1.139 +function dictDiff(last, curr) {
1.140 + var diff = {};
1.141 +
1.142 + for (let name in last) {
1.143 + var result = (curr[name] || 0) - last[name];
1.144 + if (result)
1.145 + diff[name] = (result > 0 ? "+" : "") + result;
1.146 + }
1.147 + for (let name in curr) {
1.148 + var result = curr[name] - (last[name] || 0);
1.149 + if (result)
1.150 + diff[name] = (result > 0 ? "+" : "") + result;
1.151 + }
1.152 + return diff;
1.153 +}
1.154 +
1.155 +function reportMemoryUsage() {
1.156 + if (!profileMemory) {
1.157 + return emptyPromise();
1.158 + }
1.159 +
1.160 + return gcPromise().then((function () {
1.161 + var mgr = Cc["@mozilla.org/memory-reporter-manager;1"]
1.162 + .getService(Ci.nsIMemoryReporterManager);
1.163 + let count = 0;
1.164 + function logReporter(process, path, kind, units, amount, description) {
1.165 + print(((++count == 1) ? "\n" : "") + description + ": " + amount + "\n");
1.166 + }
1.167 + mgr.getReportsForThisProcess(logReporter, null);
1.168 +
1.169 + var weakrefs = [info.weakref.get()
1.170 + for each (info in memory.getObjects())];
1.171 + weakrefs = [weakref for each (weakref in weakrefs) if (weakref)];
1.172 + print("Tracked memory objects in testing sandbox: " + weakrefs.length + "\n");
1.173 + }));
1.174 +}
1.175 +
1.176 +var gWeakrefInfo;
1.177 +
1.178 +function checkMemory() {
1.179 + return gcPromise().then(_ => {
1.180 + let leaks = getPotentialLeaks();
1.181 +
1.182 + let compartmentURLs = Object.keys(leaks.compartments).filter(function(url) {
1.183 + return !(url in startLeaks.compartments);
1.184 + });
1.185 +
1.186 + let windowURLs = Object.keys(leaks.windows).filter(function(url) {
1.187 + return !(url in startLeaks.windows);
1.188 + });
1.189 +
1.190 + for (let url of compartmentURLs)
1.191 + console.warn("LEAKED", leaks.compartments[url]);
1.192 +
1.193 + for (let url of windowURLs)
1.194 + console.warn("LEAKED", leaks.windows[url]);
1.195 + }).then(showResults);
1.196 +}
1.197 +
1.198 +function showResults() {
1.199 + let { promise, resolve } = defer();
1.200 +
1.201 + if (gWeakrefInfo) {
1.202 + gWeakrefInfo.forEach(
1.203 + function(info) {
1.204 + var ref = info.weakref.get();
1.205 + if (ref !== null) {
1.206 + var data = ref.__url__ ? ref.__url__ : ref;
1.207 + var warning = data == "[object Object]"
1.208 + ? "[object " + data.constructor.name + "(" +
1.209 + [p for (p in data)].join(", ") + ")]"
1.210 + : data;
1.211 + console.warn("LEAK", warning, info.bin);
1.212 + }
1.213 + }
1.214 + );
1.215 + }
1.216 +
1.217 + onDone(results);
1.218 +
1.219 + resolve();
1.220 + return promise;
1.221 +}
1.222 +
1.223 +function cleanup() {
1.224 + let coverObject = {};
1.225 + try {
1.226 + for (let name in loader.modules)
1.227 + memory.track(loader.modules[name],
1.228 + "module global scope: " + name);
1.229 + memory.track(loader, "Cuddlefish Loader");
1.230 +
1.231 + if (profileMemory) {
1.232 + gWeakrefInfo = [{ weakref: info.weakref, bin: info.bin }
1.233 + for each (info in memory.getObjects())];
1.234 + }
1.235 +
1.236 + loader.unload();
1.237 +
1.238 + if (loader.globals.console.errorsLogged && !results.failed) {
1.239 + results.failed++;
1.240 + console.error("warnings and/or errors were logged.");
1.241 + }
1.242 +
1.243 + if (consoleListener.errorsLogged && !results.failed) {
1.244 + console.warn(consoleListener.errorsLogged + " " +
1.245 + "warnings or errors were logged to the " +
1.246 + "platform's nsIConsoleService, which could " +
1.247 + "be of no consequence; however, they could also " +
1.248 + "be indicative of aberrant behavior.");
1.249 + }
1.250 +
1.251 + // read the code coverage object, if it exists, from CoverJS-moz
1.252 + if (typeof loader.globals.global == "object") {
1.253 + coverObject = loader.globals.global['__$coverObject'] || {};
1.254 + }
1.255 +
1.256 + consoleListener.errorsLogged = 0;
1.257 + loader = null;
1.258 +
1.259 + memory.gc();
1.260 + }
1.261 + catch (e) {
1.262 + results.failed++;
1.263 + console.error("unload.send() threw an exception.");
1.264 + console.exception(e);
1.265 + };
1.266 +
1.267 + setTimeout(require('@test/options').checkMemory ? checkMemory : showResults, 1);
1.268 +
1.269 + // dump the coverobject
1.270 + if (Object.keys(coverObject).length){
1.271 + const self = require('sdk/self');
1.272 + const {pathFor} = require("sdk/system");
1.273 + let file = require('sdk/io/file');
1.274 + const {env} = require('sdk/system/environment');
1.275 + console.log("CWD:", env.PWD);
1.276 + let out = file.join(env.PWD,'coverstats-'+self.id+'.json');
1.277 + console.log('coverstats:', out);
1.278 + let outfh = file.open(out,'w');
1.279 + outfh.write(JSON.stringify(coverObject,null,2));
1.280 + outfh.flush();
1.281 + outfh.close();
1.282 + }
1.283 +}
1.284 +
1.285 +function getPotentialLeaks() {
1.286 + memory.gc();
1.287 +
1.288 + // Things we can assume are part of the platform and so aren't leaks
1.289 + let WHITELIST_BASE_URLS = [
1.290 + "chrome://",
1.291 + "resource:///",
1.292 + "resource://app/",
1.293 + "resource://gre/",
1.294 + "resource://gre-resources/",
1.295 + "resource://pdf.js/",
1.296 + "resource://pdf.js.components/",
1.297 + "resource://services-common/",
1.298 + "resource://services-crypto/",
1.299 + "resource://services-sync/"
1.300 + ];
1.301 +
1.302 + let ioService = Cc["@mozilla.org/network/io-service;1"].
1.303 + getService(Ci.nsIIOService);
1.304 + let uri = ioService.newURI("chrome://global/content/", "UTF-8", null);
1.305 + let chromeReg = Cc["@mozilla.org/chrome/chrome-registry;1"].
1.306 + getService(Ci.nsIChromeRegistry);
1.307 + uri = chromeReg.convertChromeURL(uri);
1.308 + let spec = uri.spec;
1.309 + let pos = spec.indexOf("!/");
1.310 + WHITELIST_BASE_URLS.push(spec.substring(0, pos + 2));
1.311 +
1.312 + let zoneRegExp = new RegExp("^explicit/js-non-window/zones/zone[^/]+/compartment\\((.+)\\)");
1.313 + let compartmentRegexp = new RegExp("^explicit/js-non-window/compartments/non-window-global/compartment\\((.+)\\)/");
1.314 + let compartmentDetails = new RegExp("^([^,]+)(?:, (.+?))?(?: \\(from: (.*)\\))?$");
1.315 + let windowRegexp = new RegExp("^explicit/window-objects/top\\((.*)\\)/active");
1.316 + let windowDetails = new RegExp("^(.*), id=.*$");
1.317 +
1.318 + function isPossibleLeak(item) {
1.319 + if (!item.location)
1.320 + return false;
1.321 +
1.322 + for (let whitelist of WHITELIST_BASE_URLS) {
1.323 + if (item.location.substring(0, whitelist.length) == whitelist)
1.324 + return false;
1.325 + }
1.326 +
1.327 + return true;
1.328 + }
1.329 +
1.330 + let compartments = {};
1.331 + let windows = {};
1.332 + function logReporter(process, path, kind, units, amount, description) {
1.333 + let matches;
1.334 +
1.335 + if ((matches = compartmentRegexp.exec(path)) || (matches = zoneRegExp.exec(path))) {
1.336 + if (matches[1] in compartments)
1.337 + return;
1.338 +
1.339 + let details = compartmentDetails.exec(matches[1]);
1.340 + if (!details) {
1.341 + console.error("Unable to parse compartment detail " + matches[1]);
1.342 + return;
1.343 + }
1.344 +
1.345 + let item = {
1.346 + path: matches[1],
1.347 + principal: details[1],
1.348 + location: details[2] ? details[2].replace("\\", "/", "g") : undefined,
1.349 + source: details[3] ? details[3].split(" -> ").reverse() : undefined,
1.350 + toString: function() this.location
1.351 + };
1.352 +
1.353 + if (!isPossibleLeak(item))
1.354 + return;
1.355 +
1.356 + compartments[matches[1]] = item;
1.357 + return;
1.358 + }
1.359 +
1.360 + if (matches = windowRegexp.exec(path)) {
1.361 + if (matches[1] in windows)
1.362 + return;
1.363 +
1.364 + let details = windowDetails.exec(matches[1]);
1.365 + if (!details) {
1.366 + console.error("Unable to parse window detail " + matches[1]);
1.367 + return;
1.368 + }
1.369 +
1.370 + let item = {
1.371 + path: matches[1],
1.372 + location: details[1].replace("\\", "/", "g"),
1.373 + source: [details[1].replace("\\", "/", "g")],
1.374 + toString: function() this.location
1.375 + };
1.376 +
1.377 + if (!isPossibleLeak(item))
1.378 + return;
1.379 +
1.380 + windows[matches[1]] = item;
1.381 + }
1.382 + }
1.383 +
1.384 + Cc["@mozilla.org/memory-reporter-manager;1"]
1.385 + .getService(Ci.nsIMemoryReporterManager)
1.386 + .getReportsForThisProcess(logReporter, null);
1.387 +
1.388 + return { compartments: compartments, windows: windows };
1.389 +}
1.390 +
1.391 +function nextIteration(tests) {
1.392 + if (tests) {
1.393 + results.passed += tests.passed;
1.394 + results.failed += tests.failed;
1.395 +
1.396 + reportMemoryUsage().then(_ => {
1.397 + let testRun = [];
1.398 + for each (let test in tests.testRunSummary) {
1.399 + let testCopy = {};
1.400 + for (let info in test) {
1.401 + testCopy[info] = test[info];
1.402 + }
1.403 + testRun.push(testCopy);
1.404 + }
1.405 +
1.406 + results.testRuns.push(testRun);
1.407 + iterationsLeft--;
1.408 +
1.409 + checkForEnd();
1.410 + })
1.411 + }
1.412 + else {
1.413 + checkForEnd();
1.414 + }
1.415 +}
1.416 +
1.417 +function checkForEnd() {
1.418 + if (iterationsLeft && (!stopOnError || results.failed == 0)) {
1.419 + // Pass the loader which has a hooked console that doesn't dispatch
1.420 + // errors to the JS console and avoid firing false alarm in our
1.421 + // console listener
1.422 + findAndRunTests(loader, nextIteration);
1.423 + }
1.424 + else {
1.425 + setTimeout(cleanup, 0);
1.426 + }
1.427 +}
1.428 +
1.429 +var POINTLESS_ERRORS = [
1.430 + 'Invalid chrome URI:',
1.431 + 'OpenGL LayerManager Initialized Succesfully.',
1.432 + '[JavaScript Error: "TelemetryStopwatch:',
1.433 + 'reference to undefined property',
1.434 + '[JavaScript Error: "The character encoding of the HTML document was ' +
1.435 + 'not declared.',
1.436 + '[Javascript Warning: "Error: Failed to preserve wrapper of wrapped ' +
1.437 + 'native weak map key',
1.438 + '[JavaScript Warning: "Duplicate resource declaration for',
1.439 + 'file: "chrome://browser/content/',
1.440 + 'file: "chrome://global/content/',
1.441 + '[JavaScript Warning: "The character encoding of a framed document was ' +
1.442 + 'not declared.'
1.443 +];
1.444 +
1.445 +var consoleListener = {
1.446 + errorsLogged: 0,
1.447 + observe: function(object) {
1.448 + if (!(object instanceof Ci.nsIScriptError))
1.449 + return;
1.450 + this.errorsLogged++;
1.451 + var message = object.QueryInterface(Ci.nsIConsoleMessage).message;
1.452 + var pointless = [err for each (err in POINTLESS_ERRORS)
1.453 + if (message.indexOf(err) >= 0)];
1.454 + if (pointless.length == 0 && message)
1.455 + testConsole.log(message);
1.456 + }
1.457 +};
1.458 +
1.459 +function TestRunnerConsole(base, options) {
1.460 + this.__proto__ = {
1.461 + errorsLogged: 0,
1.462 + warn: function warn() {
1.463 + this.errorsLogged++;
1.464 + base.warn.apply(base, arguments);
1.465 + },
1.466 + error: function error() {
1.467 + this.errorsLogged++;
1.468 + base.error.apply(base, arguments);
1.469 + },
1.470 + info: function info(first) {
1.471 + if (options.verbose)
1.472 + base.info.apply(base, arguments);
1.473 + else
1.474 + if (first == "pass:")
1.475 + print(".");
1.476 + },
1.477 + __proto__: base
1.478 + };
1.479 +}
1.480 +
1.481 +function stringify(arg) {
1.482 + try {
1.483 + return String(arg);
1.484 + }
1.485 + catch(ex) {
1.486 + return "<toString() error>";
1.487 + }
1.488 +}
1.489 +
1.490 +function stringifyArgs(args) {
1.491 + return Array.map(args, stringify).join(" ");
1.492 +}
1.493 +
1.494 +function TestRunnerTinderboxConsole(base, options) {
1.495 + this.base = base;
1.496 + this.print = options.print;
1.497 + this.verbose = options.verbose;
1.498 + this.errorsLogged = 0;
1.499 +
1.500 + // Binding all the public methods to an instance so that they can be used
1.501 + // as callback / listener functions straightaway.
1.502 + this.log = this.log.bind(this);
1.503 + this.info = this.info.bind(this);
1.504 + this.warn = this.warn.bind(this);
1.505 + this.error = this.error.bind(this);
1.506 + this.debug = this.debug.bind(this);
1.507 + this.exception = this.exception.bind(this);
1.508 + this.trace = this.trace.bind(this);
1.509 +};
1.510 +
1.511 +TestRunnerTinderboxConsole.prototype = {
1.512 + testMessage: function testMessage(pass, expected, test, message) {
1.513 + let type = "TEST-";
1.514 + if (expected) {
1.515 + if (pass)
1.516 + type += "PASS";
1.517 + else
1.518 + type += "KNOWN-FAIL";
1.519 + }
1.520 + else {
1.521 + this.errorsLogged++;
1.522 + if (pass)
1.523 + type += "UNEXPECTED-PASS";
1.524 + else
1.525 + type += "UNEXPECTED-FAIL";
1.526 + }
1.527 +
1.528 + this.print(type + " | " + test + " | " + message + "\n");
1.529 + if (!expected)
1.530 + this.trace();
1.531 + },
1.532 +
1.533 + log: function log() {
1.534 + this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
1.535 + },
1.536 +
1.537 + info: function info(first) {
1.538 + this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
1.539 + },
1.540 +
1.541 + warn: function warn() {
1.542 + this.errorsLogged++;
1.543 + this.print("TEST-UNEXPECTED-FAIL | " + stringifyArgs(arguments) + "\n");
1.544 + },
1.545 +
1.546 + error: function error() {
1.547 + this.errorsLogged++;
1.548 + this.print("TEST-UNEXPECTED-FAIL | " + stringifyArgs(arguments) + "\n");
1.549 + this.base.error.apply(this.base, arguments);
1.550 + },
1.551 +
1.552 + debug: function debug() {
1.553 + this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
1.554 + },
1.555 +
1.556 + exception: function exception(e) {
1.557 + this.print("An exception occurred.\n" +
1.558 + require("../console/traceback").format(e) + "\n" + e + "\n");
1.559 + },
1.560 +
1.561 + trace: function trace() {
1.562 + var traceback = require("../console/traceback");
1.563 + var stack = traceback.get();
1.564 + stack.splice(-1, 1);
1.565 + this.print("TEST-INFO | " + stringify(traceback.format(stack)) + "\n");
1.566 + }
1.567 +};
1.568 +
1.569 +var runTests = exports.runTests = function runTests(options) {
1.570 + iterationsLeft = options.iterations;
1.571 + profileMemory = options.profileMemory;
1.572 + stopOnError = options.stopOnError;
1.573 + onDone = options.onDone;
1.574 + print = options.print;
1.575 + findAndRunTests = options.findAndRunTests;
1.576 +
1.577 + try {
1.578 + cService.registerListener(consoleListener);
1.579 + print("Running tests on " + system.name + " " + system.version +
1.580 + "/Gecko " + system.platformVersion + " (" +
1.581 + system.id + ") under " +
1.582 + system.platform + "/" + system.architecture + ".\n");
1.583 +
1.584 + if (options.parseable)
1.585 + testConsole = new TestRunnerTinderboxConsole(new PlainTextConsole(), options);
1.586 + else
1.587 + testConsole = new TestRunnerConsole(new PlainTextConsole(), options);
1.588 +
1.589 + loader = Loader(module, {
1.590 + console: testConsole,
1.591 + global: {} // useful for storing things like coverage testing.
1.592 + });
1.593 +
1.594 + // Load these before getting initial leak stats as they will still be in
1.595 + // memory when we check later
1.596 + require("../deprecated/unit-test");
1.597 + require("../deprecated/unit-test-finder");
1.598 + startLeaks = getPotentialLeaks();
1.599 +
1.600 + nextIteration();
1.601 + } catch (e) {
1.602 + let frames = fromException(e).reverse().reduce(function(frames, frame) {
1.603 + if (frame.fileName.split("/").pop() === "unit-test-finder.js")
1.604 + frames.done = true
1.605 + if (!frames.done) frames.push(frame)
1.606 +
1.607 + return frames
1.608 + }, [])
1.609 +
1.610 + let prototype = typeof(e) === "object" ? e.constructor.prototype :
1.611 + Error.prototype;
1.612 + let stack = serializeStack(frames.reverse());
1.613 +
1.614 + let error = Object.create(prototype, {
1.615 + message: { value: e.message, writable: true, configurable: true },
1.616 + fileName: { value: e.fileName, writable: true, configurable: true },
1.617 + lineNumber: { value: e.lineNumber, writable: true, configurable: true },
1.618 + stack: { value: stack, writable: true, configurable: true },
1.619 + toString: { value: function() String(e), writable: true, configurable: true },
1.620 + });
1.621 +
1.622 + print("Error: " + error + " \n " + format(error));
1.623 + onDone({passed: 0, failed: 1});
1.624 + }
1.625 +};
1.626 +
1.627 +unload(function() {
1.628 + cService.unregisterListener(consoleListener);
1.629 +});
1.630 +
