1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/toolkit/components/telemetry/tests/unit/test_TelemetryPing.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,503 @@
1.4 +/* Any copyright is dedicated to the Public Domain.
1.5 + http://creativecommons.org/publicdomain/zero/1.0/
1.6 +*/
1.7 +/* This testcase triggers two telemetry pings.
1.8 + *
1.9 + * Telemetry code keeps histograms of past telemetry pings. The first
1.10 + * ping populates these histograms. One of those histograms is then
1.11 + * checked in the second request.
1.12 + */
1.13 +
1.14 +const Cc = Components.classes;
1.15 +const Ci = Components.interfaces;
1.16 +const Cu = Components.utils;
1.17 +const Cr = Components.results;
1.18 +
1.19 +Cu.import("resource://testing-common/httpd.js", this);
1.20 +Cu.import("resource://gre/modules/Services.jsm");
1.21 +Cu.import("resource://gre/modules/LightweightThemeManager.jsm", this);
1.22 +Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
1.23 +Cu.import("resource://gre/modules/TelemetryPing.jsm", this);
1.24 +Cu.import("resource://gre/modules/TelemetryFile.jsm", this);
1.25 +Cu.import("resource://gre/modules/Task.jsm", this);
1.26 +Cu.import("resource://gre/modules/Promise.jsm", this);
1.27 +
1.28 +const IGNORE_HISTOGRAM = "test::ignore_me";
1.29 +const IGNORE_HISTOGRAM_TO_CLONE = "MEMORY_HEAP_ALLOCATED";
1.30 +const IGNORE_CLONED_HISTOGRAM = "test::ignore_me_also";
1.31 +const ADDON_NAME = "Telemetry test addon";
1.32 +const ADDON_HISTOGRAM = "addon-histogram";
1.33 +// Add some unicode characters here to ensure that sending them works correctly.
1.34 +const FLASH_VERSION = "\u201c1.1.1.1\u201d";
1.35 +const SHUTDOWN_TIME = 10000;
1.36 +const FAILED_PROFILE_LOCK_ATTEMPTS = 2;
1.37 +
1.38 +// Constants from prio.h for nsIFileOutputStream.init
1.39 +const PR_WRONLY = 0x2;
1.40 +const PR_CREATE_FILE = 0x8;
1.41 +const PR_TRUNCATE = 0x20;
1.42 +const RW_OWNER = 0600;
1.43 +
1.44 +const NUMBER_OF_THREADS_TO_LAUNCH = 30;
1.45 +let gNumberOfThreadsLaunched = 0;
1.46 +
1.47 +const Telemetry = Cc["@mozilla.org/base/telemetry;1"].getService(Ci.nsITelemetry);
1.48 +
1.49 +let gHttpServer = new HttpServer();
1.50 +let gServerStarted = false;
1.51 +let gRequestIterator = null;
1.52 +
1.53 +function sendPing () {
1.54 + TelemetryPing.gatherStartup();
1.55 + if (gServerStarted) {
1.56 + return TelemetryPing.testPing("http://localhost:" + gHttpServer.identity.primaryPort);
1.57 + } else {
1.58 + return TelemetryPing.testPing("http://doesnotexist");
1.59 + }
1.60 +}
1.61 +
1.62 +function wrapWithExceptionHandler(f) {
1.63 + function wrapper(...args) {
1.64 + try {
1.65 + f(...args);
1.66 + } catch (ex if typeof(ex) == 'object') {
1.67 + dump("Caught exception: " + ex.message + "\n");
1.68 + dump(ex.stack);
1.69 + do_test_finished();
1.70 + }
1.71 + }
1.72 + return wrapper;
1.73 +}
1.74 +
1.75 +function registerPingHandler(handler) {
1.76 + gHttpServer.registerPrefixHandler("/submit/telemetry/",
1.77 + wrapWithExceptionHandler(handler));
1.78 +}
1.79 +
1.80 +function setupTestData() {
1.81 + Telemetry.newHistogram(IGNORE_HISTOGRAM, "never", 1, 2, 3, Telemetry.HISTOGRAM_BOOLEAN);
1.82 + Telemetry.histogramFrom(IGNORE_CLONED_HISTOGRAM, IGNORE_HISTOGRAM_TO_CLONE);
1.83 + Services.startup.interrupted = true;
1.84 + Telemetry.registerAddonHistogram(ADDON_NAME, ADDON_HISTOGRAM, 1, 5, 6,
1.85 + Telemetry.HISTOGRAM_LINEAR);
1.86 + h1 = Telemetry.getAddonHistogram(ADDON_NAME, ADDON_HISTOGRAM);
1.87 + h1.add(1);
1.88 +}
1.89 +
1.90 +function getSavedHistogramsFile(basename) {
1.91 + let tmpDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
1.92 + let histogramsFile = tmpDir.clone();
1.93 + histogramsFile.append(basename);
1.94 + if (histogramsFile.exists()) {
1.95 + histogramsFile.remove(true);
1.96 + }
1.97 + do_register_cleanup(function () {
1.98 + try {
1.99 + histogramsFile.remove(true);
1.100 + } catch (e) {
1.101 + }
1.102 + });
1.103 + return histogramsFile;
1.104 +}
1.105 +
1.106 +function decodeRequestPayload(request) {
1.107 + let s = request.bodyInputStream;
1.108 + let payload = null;
1.109 + let decoder = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON)
1.110 +
1.111 + if (request.getHeader("content-encoding") == "gzip") {
1.112 + let observer = {
1.113 + buffer: "",
1.114 + onStreamComplete: function(loader, context, status, length, result) {
1.115 + this.buffer = String.fromCharCode.apply(this, result);
1.116 + }
1.117 + };
1.118 +
1.119 + let scs = Cc["@mozilla.org/streamConverters;1"]
1.120 + .getService(Ci.nsIStreamConverterService);
1.121 + let listener = Cc["@mozilla.org/network/stream-loader;1"]
1.122 + .createInstance(Ci.nsIStreamLoader);
1.123 + listener.init(observer);
1.124 + let converter = scs.asyncConvertData("gzip", "uncompressed",
1.125 + listener, null);
1.126 + converter.onStartRequest(null, null);
1.127 + converter.onDataAvailable(null, null, s, 0, s.available());
1.128 + converter.onStopRequest(null, null, null);
1.129 + let unicodeConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
1.130 + .createInstance(Ci.nsIScriptableUnicodeConverter);
1.131 + unicodeConverter.charset = "UTF-8";
1.132 + let utf8string = unicodeConverter.ConvertToUnicode(observer.buffer);
1.133 + utf8string += unicodeConverter.Finish();
1.134 + payload = decoder.decode(utf8string);
1.135 + } else {
1.136 + payload = decoder.decodeFromStream(s, s.available());
1.137 + }
1.138 +
1.139 + return payload;
1.140 +}
1.141 +
1.142 +function checkPayloadInfo(payload, reason) {
1.143 + // get rid of the non-deterministic field
1.144 + const expected_info = {
1.145 + OS: "XPCShell",
1.146 + appID: "xpcshell@tests.mozilla.org",
1.147 + appVersion: "1",
1.148 + appName: "XPCShell",
1.149 + appBuildID: "2007010101",
1.150 + platformBuildID: "2007010101",
1.151 + flashVersion: FLASH_VERSION
1.152 + };
1.153 +
1.154 + for (let f in expected_info) {
1.155 + do_check_eq(payload.info[f], expected_info[f]);
1.156 + }
1.157 +
1.158 + do_check_eq(payload.info.reason, reason);
1.159 + do_check_true("appUpdateChannel" in payload.info);
1.160 + do_check_true("locale" in payload.info);
1.161 + do_check_true("revision" in payload.info);
1.162 + do_check_true(payload.info.revision.startsWith("http"));
1.163 +
1.164 + try {
1.165 + // If we've not got nsIGfxInfoDebug, then this will throw and stop us doing
1.166 + // this test.
1.167 + let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfoDebug);
1.168 + let isWindows = ("@mozilla.org/windows-registry-key;1" in Components.classes);
1.169 + let isOSX = ("nsILocalFileMac" in Components.interfaces);
1.170 +
1.171 + if (isWindows || isOSX) {
1.172 + do_check_true("adapterVendorID" in payload.info);
1.173 + do_check_true("adapterDeviceID" in payload.info);
1.174 + }
1.175 + }
1.176 + catch (x) {
1.177 + }
1.178 +}
1.179 +
1.180 +function checkPayload(request, reason, successfulPings) {
1.181 + let payload = decodeRequestPayload(request);
1.182 + // Take off ["","submit","telemetry"].
1.183 + let pathComponents = request.path.split("/").slice(3);
1.184 +
1.185 + checkPayloadInfo(payload, reason);
1.186 + do_check_eq(reason, pathComponents[1]);
1.187 + do_check_eq(request.getHeader("content-type"), "application/json; charset=UTF-8");
1.188 + do_check_true(payload.simpleMeasurements.uptime >= 0);
1.189 + do_check_true(payload.simpleMeasurements.startupInterrupted === 1);
1.190 + do_check_eq(payload.simpleMeasurements.shutdownDuration, SHUTDOWN_TIME);
1.191 + do_check_eq(payload.simpleMeasurements.savedPings, 1);
1.192 + do_check_true("maximalNumberOfConcurrentThreads" in payload.simpleMeasurements);
1.193 + do_check_true(payload.simpleMeasurements.maximalNumberOfConcurrentThreads >= gNumberOfThreadsLaunched);
1.194 +
1.195 + do_check_eq(payload.simpleMeasurements.failedProfileLockCount,
1.196 + FAILED_PROFILE_LOCK_ATTEMPTS);
1.197 + let profileDirectory = Services.dirsvc.get("ProfD", Ci.nsIFile);
1.198 + let failedProfileLocksFile = profileDirectory.clone();
1.199 + failedProfileLocksFile.append("Telemetry.FailedProfileLocks.txt");
1.200 + do_check_true(!failedProfileLocksFile.exists());
1.201 +
1.202 +
1.203 + let isWindows = ("@mozilla.org/windows-registry-key;1" in Components.classes);
1.204 + if (isWindows) {
1.205 + do_check_true(payload.simpleMeasurements.startupSessionRestoreReadBytes > 0);
1.206 + do_check_true(payload.simpleMeasurements.startupSessionRestoreWriteBytes > 0);
1.207 + }
1.208 +
1.209 + const TELEMETRY_PING = "TELEMETRY_PING";
1.210 + const TELEMETRY_SUCCESS = "TELEMETRY_SUCCESS";
1.211 + const TELEMETRY_TEST_FLAG = "TELEMETRY_TEST_FLAG";
1.212 + const READ_SAVED_PING_SUCCESS = "READ_SAVED_PING_SUCCESS";
1.213 + do_check_true(TELEMETRY_PING in payload.histograms);
1.214 + do_check_true(READ_SAVED_PING_SUCCESS in payload.histograms);
1.215 + let rh = Telemetry.registeredHistograms([]);
1.216 + for (let name of rh) {
1.217 + if (/SQLITE/.test(name) && name in payload.histograms) {
1.218 + do_check_true(("STARTUP_" + name) in payload.histograms);
1.219 + }
1.220 + }
1.221 + do_check_false(IGNORE_HISTOGRAM in payload.histograms);
1.222 + do_check_false(IGNORE_CLONED_HISTOGRAM in payload.histograms);
1.223 +
1.224 + // Flag histograms should automagically spring to life.
1.225 + const expected_flag = {
1.226 + range: [1, 2],
1.227 + bucket_count: 3,
1.228 + histogram_type: 3,
1.229 + values: {0:1, 1:0},
1.230 + sum: 0,
1.231 + sum_squares_lo: 0,
1.232 + sum_squares_hi: 0
1.233 + };
1.234 + let flag = payload.histograms[TELEMETRY_TEST_FLAG];
1.235 + do_check_eq(uneval(flag), uneval(expected_flag));
1.236 +
1.237 + // There should be one successful report from the previous telemetry ping.
1.238 + const expected_tc = {
1.239 + range: [1, 2],
1.240 + bucket_count: 3,
1.241 + histogram_type: 2,
1.242 + values: {0:1, 1:successfulPings, 2:0},
1.243 + sum: successfulPings,
1.244 + sum_squares_lo: successfulPings,
1.245 + sum_squares_hi: 0
1.246 + };
1.247 + let tc = payload.histograms[TELEMETRY_SUCCESS];
1.248 + do_check_eq(uneval(tc), uneval(expected_tc));
1.249 +
1.250 + let h = payload.histograms[READ_SAVED_PING_SUCCESS];
1.251 + do_check_eq(h.values[0], 1);
1.252 +
1.253 + // The ping should include data from memory reporters. We can't check that
1.254 + // this data is correct, because we can't control the values returned by the
1.255 + // memory reporters. But we can at least check that the data is there.
1.256 + //
1.257 + // It's important to check for the presence of reporters with a mix of units,
1.258 + // because TelemetryPing has separate logic for each one. But we can't
1.259 + // currently check UNITS_COUNT_CUMULATIVE or UNITS_PERCENTAGE because
1.260 + // Telemetry doesn't touch a memory reporter with these units that's
1.261 + // available on all platforms.
1.262 +
1.263 + do_check_true('MEMORY_JS_GC_HEAP' in payload.histograms); // UNITS_BYTES
1.264 + do_check_true('MEMORY_JS_COMPARTMENTS_SYSTEM' in payload.histograms); // UNITS_COUNT
1.265 +
1.266 + // We should have included addon histograms.
1.267 + do_check_true("addonHistograms" in payload);
1.268 + do_check_true(ADDON_NAME in payload.addonHistograms);
1.269 + do_check_true(ADDON_HISTOGRAM in payload.addonHistograms[ADDON_NAME]);
1.270 +
1.271 + do_check_true(("mainThread" in payload.slowSQL) &&
1.272 + ("otherThreads" in payload.slowSQL));
1.273 +}
1.274 +
1.275 +function dummyTheme(id) {
1.276 + return {
1.277 + id: id,
1.278 + name: Math.random().toString(),
1.279 + headerURL: "http://lwttest.invalid/a.png",
1.280 + footerURL: "http://lwttest.invalid/b.png",
1.281 + textcolor: Math.random().toString(),
1.282 + accentcolor: Math.random().toString()
1.283 + };
1.284 +}
1.285 +
1.286 +// A fake plugin host for testing flash version telemetry
1.287 +let PluginHost = {
1.288 + getPluginTags: function(countRef) {
1.289 + let plugins = [{name: "Shockwave Flash", version: FLASH_VERSION}];
1.290 + countRef.value = plugins.length;
1.291 + return plugins;
1.292 + },
1.293 +
1.294 + QueryInterface: function(iid) {
1.295 + if (iid.equals(Ci.nsIPluginHost)
1.296 + || iid.equals(Ci.nsISupports))
1.297 + return this;
1.298 +
1.299 + throw Components.results.NS_ERROR_NO_INTERFACE;
1.300 + }
1.301 +}
1.302 +
1.303 +let PluginHostFactory = {
1.304 + createInstance: function (outer, iid) {
1.305 + if (outer != null)
1.306 + throw Components.results.NS_ERROR_NO_AGGREGATION;
1.307 + return PluginHost.QueryInterface(iid);
1.308 + }
1.309 +};
1.310 +
1.311 +const PLUGINHOST_CONTRACTID = "@mozilla.org/plugin/host;1";
1.312 +const PLUGINHOST_CID = Components.ID("{2329e6ea-1f15-4cbe-9ded-6e98e842de0e}");
1.313 +
1.314 +function registerFakePluginHost() {
1.315 + let registrar = Components.manager.QueryInterface(Ci.nsIComponentRegistrar);
1.316 + registrar.registerFactory(PLUGINHOST_CID, "Fake Plugin Host",
1.317 + PLUGINHOST_CONTRACTID, PluginHostFactory);
1.318 +}
1.319 +
1.320 +function writeStringToFile(file, contents) {
1.321 + let ostream = Cc["@mozilla.org/network/safe-file-output-stream;1"]
1.322 + .createInstance(Ci.nsIFileOutputStream);
1.323 + ostream.init(file, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE,
1.324 + RW_OWNER, ostream.DEFER_OPEN);
1.325 + ostream.write(contents, contents.length);
1.326 + ostream.QueryInterface(Ci.nsISafeOutputStream).finish();
1.327 + ostream.close();
1.328 +}
1.329 +
1.330 +function write_fake_shutdown_file() {
1.331 + let profileDirectory = Services.dirsvc.get("ProfD", Ci.nsIFile);
1.332 + let file = profileDirectory.clone();
1.333 + file.append("Telemetry.ShutdownTime.txt");
1.334 + let contents = "" + SHUTDOWN_TIME;
1.335 + writeStringToFile(file, contents);
1.336 +}
1.337 +
1.338 +function write_fake_failedprofilelocks_file() {
1.339 + let profileDirectory = Services.dirsvc.get("ProfD", Ci.nsIFile);
1.340 + let file = profileDirectory.clone();
1.341 + file.append("Telemetry.FailedProfileLocks.txt");
1.342 + let contents = "" + FAILED_PROFILE_LOCK_ATTEMPTS;
1.343 + writeStringToFile(file, contents);
1.344 +}
1.345 +
1.346 +function run_test() {
1.347 + do_test_pending();
1.348 + try {
1.349 + let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfoDebug);
1.350 + gfxInfo.spoofVendorID("0xabcd");
1.351 + gfxInfo.spoofDeviceID("0x1234");
1.352 + } catch (x) {
1.353 + // If we can't test gfxInfo, that's fine, we'll note it later.
1.354 + }
1.355 +
1.356 + // Addon manager needs a profile directory
1.357 + do_get_profile();
1.358 + createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
1.359 +
1.360 + // Make it look like we've previously failed to lock a profile a couple times.
1.361 + write_fake_failedprofilelocks_file();
1.362 +
1.363 + // Make it look like we've shutdown before.
1.364 + write_fake_shutdown_file();
1.365 +
1.366 + let currentMaxNumberOfThreads = Telemetry.maximalNumberOfConcurrentThreads;
1.367 + do_check_true(currentMaxNumberOfThreads > 0);
1.368 +
1.369 + // Try to augment the maximal number of threads currently launched
1.370 + let threads = [];
1.371 + try {
1.372 + for (let i = 0; i < currentMaxNumberOfThreads + 10; ++i) {
1.373 + threads.push(Services.tm.newThread(0));
1.374 + }
1.375 + } catch (ex) {
1.376 + // If memory is too low, it is possible that not all threads will be launched.
1.377 + }
1.378 + gNumberOfThreadsLaunched = threads.length;
1.379 +
1.380 + do_check_true(Telemetry.maximalNumberOfConcurrentThreads >= gNumberOfThreadsLaunched);
1.381 +
1.382 + do_register_cleanup(function() {
1.383 + threads.forEach(function(thread) {
1.384 + thread.shutdown();
1.385 + });
1.386 + });
1.387 +
1.388 + Telemetry.asyncFetchTelemetryData(wrapWithExceptionHandler(actualTest));
1.389 +}
1.390 +
1.391 +function actualTest() {
1.392 + // try to make LightweightThemeManager do stuff
1.393 + let gInternalManager = Cc["@mozilla.org/addons/integration;1"]
1.394 + .getService(Ci.nsIObserver)
1.395 + .QueryInterface(Ci.nsITimerCallback);
1.396 +
1.397 + gInternalManager.observe(null, "addons-startup", null);
1.398 + LightweightThemeManager.currentTheme = dummyTheme("1234");
1.399 +
1.400 + // fake plugin host for consistent flash version data
1.401 + registerFakePluginHost();
1.402 +
1.403 + run_next_test();
1.404 +}
1.405 +
1.406 +// Ensure that not overwriting an existing file fails silently
1.407 +add_task(function* test_overwritePing() {
1.408 + let ping = {slug: "foo"}
1.409 + yield TelemetryFile.savePing(ping, true);
1.410 + yield TelemetryFile.savePing(ping, false);
1.411 + yield TelemetryFile.cleanupPingFile(ping);
1.412 +});
1.413 +
1.414 +// Ensures that expired histograms are not part of the payload.
1.415 +add_task(function* test_expiredHistogram() {
1.416 + let histogram_id = "FOOBAR";
1.417 + let dummy = Telemetry.newHistogram(histogram_id, "30", 1, 2, 3, Telemetry.HISTOGRAM_EXPONENTIAL);
1.418 +
1.419 + dummy.add(1);
1.420 +
1.421 + do_check_eq(TelemetryPing.getPayload()["histograms"][histogram_id], undefined);
1.422 + do_check_eq(TelemetryPing.getPayload()["histograms"]["TELEMETRY_TEST_EXPIRED"], undefined);
1.423 +});
1.424 +
1.425 +// Checks that an invalid histogram file is deleted if TelemetryFile fails to parse it.
1.426 +add_task(function* test_runInvalidJSON() {
1.427 + let histogramsFile = getSavedHistogramsFile("invalid-histograms.dat");
1.428 +
1.429 + writeStringToFile(histogramsFile, "this.is.invalid.JSON");
1.430 + do_check_true(histogramsFile.exists());
1.431 +
1.432 + yield TelemetryPing.testLoadHistograms(histogramsFile);
1.433 + do_check_false(histogramsFile.exists());
1.434 +});
1.435 +
1.436 +// Sends a ping to a non existing server.
1.437 +add_task(function* test_noServerPing() {
1.438 + yield sendPing();
1.439 +});
1.440 +
1.441 +// Checks that a sent ping is correctly received by a dummy http server.
1.442 +add_task(function* test_simplePing() {
1.443 + gHttpServer.start(-1);
1.444 + gServerStarted = true;
1.445 + gRequestIterator = Iterator(new Request());
1.446 +
1.447 + yield sendPing();
1.448 + decodeRequestPayload(yield gRequestIterator.next());
1.449 +});
1.450 +
1.451 +// Saves the current session histograms, reloads them, perfoms a ping
1.452 +// and checks that the dummy http server received both the previously
1.453 +// saved histograms and the new ones.
1.454 +add_task(function* test_saveLoadPing() {
1.455 + let histogramsFile = getSavedHistogramsFile("saved-histograms.dat");
1.456 +
1.457 + setupTestData();
1.458 + yield TelemetryPing.testSaveHistograms(histogramsFile);
1.459 + yield TelemetryPing.testLoadHistograms(histogramsFile);
1.460 + yield sendPing();
1.461 + checkPayload((yield gRequestIterator.next()), "test-ping", 1);
1.462 + checkPayload((yield gRequestIterator.next()), "saved-session", 1);
1.463 +});
1.464 +
1.465 +// Checks that an expired histogram file is deleted when loaded.
1.466 +add_task(function* test_runOldPingFile() {
1.467 + let histogramsFile = getSavedHistogramsFile("old-histograms.dat");
1.468 +
1.469 + yield TelemetryPing.testSaveHistograms(histogramsFile);
1.470 + do_check_true(histogramsFile.exists());
1.471 + let mtime = histogramsFile.lastModifiedTime;
1.472 + histogramsFile.lastModifiedTime = mtime - (14 * 24 * 60 * 60 * 1000 + 60000); // 14 days, 1m
1.473 +
1.474 + yield TelemetryPing.testLoadHistograms(histogramsFile);
1.475 + do_check_false(histogramsFile.exists());
1.476 +});
1.477 +
1.478 +add_task(function* stopServer(){
1.479 + gHttpServer.stop(do_test_finished);
1.480 +});
1.481 +
1.482 +// An iterable sequence of http requests
1.483 +function Request() {
1.484 + let defers = [];
1.485 + let current = 0;
1.486 +
1.487 + function RequestIterator() {}
1.488 +
1.489 + // Returns a promise that resolves to the next http request
1.490 + RequestIterator.prototype.next = function() {
1.491 + let deferred = defers[current++];
1.492 + return deferred.promise;
1.493 + }
1.494 +
1.495 + this.__iterator__ = function(){
1.496 + return new RequestIterator();
1.497 + }
1.498 +
1.499 + registerPingHandler((request, response) => {
1.500 + let deferred = defers[defers.length - 1];
1.501 + defers.push(Promise.defer());
1.502 + deferred.resolve(request);
1.503 + });
1.504 +
1.505 + defers.push(Promise.defer());
1.506 +}
