michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: /**
michael@0:  * Managing safe shutdown of asynchronous services.
michael@0:  *
michael@0:  * Firefox shutdown is composed of phases that take place
michael@0:  * sequentially. Typically, each shutdown phase removes some
michael@0:  * capabilities from the application. For instance, at the end of
michael@0:  * phase profileBeforeChange, no service is permitted to write to the
michael@0:  * profile directory (with the exception of Telemetry). Consequently,
michael@0:  * if any service has requested I/O to the profile directory before or
michael@0:  * during phase profileBeforeChange, the system must be informed that
michael@0:  * these requests need to be completed before the end of phase
michael@0:  * profileBeforeChange. Failing to inform the system of this
michael@0:  * requirement can (and has been known to) cause data loss.
michael@0:  *
michael@0:  * Example: At some point during shutdown, the Add-On Manager needs to
michael@0:  * ensure that all add-ons have safely written their data to disk,
michael@0:  * before writing its own data. Since the data is saved to the
michael@0:  * profile, this must be completed during phase profileBeforeChange.
michael@0:  *
michael@0:  * AsyncShutdown.profileBeforeChange.addBlocker(
michael@0:  *   "Add-on manager: shutting down",
michael@0:  *   function condition() {
michael@0:  *     // Do things.
michael@0:  *     // Perform I/O that must take place during phase profile-before-change
michael@0:  *     return promise;
michael@0:  *   }
michael@0:  * });
michael@0:  *
michael@0:  * In this example, function |condition| will be called at some point
michael@0:  * during phase profileBeforeChange and phase profileBeforeChange
michael@0:  * itself is guaranteed to not terminate until |promise| is either
michael@0:  * resolved or rejected.
michael@0:  */
michael@0: 
michael@0: "use strict";
michael@0: 
michael@0: const Cu = Components.utils;
michael@0: const Cc = Components.classes;
michael@0: const Ci = Components.interfaces;
michael@0: Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
michael@0: Cu.import("resource://gre/modules/Services.jsm", this);
michael@0: 
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "Promise",
michael@0:   "resource://gre/modules/Promise.jsm");
michael@0: XPCOMUtils.defineLazyServiceGetter(this, "gDebug",
michael@0:   "@mozilla.org/xpcom/debug;1", "nsIDebug");
michael@0: Object.defineProperty(this, "gCrashReporter", {
michael@0:   get: function() {
michael@0:     delete this.gCrashReporter;
michael@0:     try {
michael@0:       let reporter = Cc["@mozilla.org/xre/app-info;1"].
michael@0:             getService(Ci.nsICrashReporter);
michael@0:       return this.gCrashReporter = reporter;
michael@0:     } catch (ex) {
michael@0:       return this.gCrashReporter = null;
michael@0:     }
michael@0:   },
michael@0:   configurable: true
michael@0: });
michael@0: 
michael@0: // Display timeout warnings after 10 seconds
michael@0: const DELAY_WARNING_MS = 10 * 1000;
michael@0: 
michael@0: 
michael@0: // Crash the process if shutdown is really too long
michael@0: // (allowing for sleep).
michael@0: const PREF_DELAY_CRASH_MS = "toolkit.asyncshutdown.crash_timeout";
michael@0: let DELAY_CRASH_MS = 60 * 1000; // One minute
michael@0: try {
michael@0:   DELAY_CRASH_MS = Services.prefs.getIntPref(PREF_DELAY_CRASH_MS);
michael@0: } catch (ex) {
michael@0:   // Ignore errors
michael@0: }
michael@0: Services.prefs.addObserver(PREF_DELAY_CRASH_MS, function() {
michael@0:   DELAY_CRASH_MS = Services.prefs.getIntPref(PREF_DELAY_CRASH_MS);
michael@0: }, false);
michael@0: 
michael@0: 
michael@0: /**
michael@0:  * Display a warning.
michael@0:  *
michael@0:  * As this code is generally used during shutdown, there are chances
michael@0:  * that the UX will not be available to display warnings on the
michael@0:  * console. We therefore use dump() rather than Cu.reportError().
michael@0:  */
michael@0: function log(msg, prefix = "", error = null) {
michael@0:   dump(prefix + msg + "\n");
michael@0:   if (error) {
michael@0:     dump(prefix + error + "\n");
michael@0:     if (typeof error == "object" && "stack" in error) {
michael@0:       dump(prefix + error.stack + "\n");
michael@0:     }
michael@0:   }
michael@0: }
michael@0: function warn(msg, error = null) {
michael@0:   return log(msg, "WARNING: ", error);
michael@0: }
michael@0: function err(msg, error = null) {
michael@0:   return log(msg, "ERROR: ", error);
michael@0: }
michael@0: 
michael@0: // Utility function designed to get the current state of execution
michael@0: // of a blocker.
michael@0: // We are a little paranoid here to ensure that in case of evaluation
michael@0: // error we do not block the AsyncShutdown.
michael@0: function safeGetState(state) {
michael@0:   if (!state) {
michael@0:     return "(none)";
michael@0:   }
michael@0:   let data, string;
michael@0:   try {
michael@0:     // Evaluate state(), normalize the result into something that we can
michael@0:     // safely stringify or upload.
michael@0:     string = JSON.stringify(state());
michael@0:     data = JSON.parse(string);
michael@0:     // Simplify the rest of the code by ensuring that we can simply
michael@0:     // concatenate the result to a message.
michael@0:     if (data && typeof data == "object") {
michael@0:       data.toString = function() {
michael@0:         return string;
michael@0:       };
michael@0:     }
michael@0:     return data;
michael@0:   } catch (ex) {
michael@0:     if (string) {
michael@0:       return string;
michael@0:     }
michael@0:     try {
michael@0:       return "Error getting state: " + ex + " at " + ex.stack;
michael@0:     } catch (ex2) {
michael@0:       return "Error getting state but could not display error";
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Countdown for a given duration, skipping beats if the computer is too busy,
michael@0:  * sleeping or otherwise unavailable.
michael@0:  *
michael@0:  * @param {number} delay An approximate delay to wait in milliseconds (rounded
michael@0:  * up to the closest second).
michael@0:  *
michael@0:  * @return Deferred
michael@0:  */
michael@0: function looseTimer(delay) {
michael@0:   let DELAY_BEAT = 1000;
michael@0:   let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
michael@0:   let beats = Math.ceil(delay / DELAY_BEAT);
michael@0:   let deferred = Promise.defer();
michael@0:   timer.initWithCallback(function() {
michael@0:     if (beats <= 0) {
michael@0:       deferred.resolve();
michael@0:     }
michael@0:     --beats;
michael@0:   }, DELAY_BEAT, Ci.nsITimer.TYPE_REPEATING_PRECISE_CAN_SKIP);
michael@0:   // Ensure that the timer is both canceled once we are done with it
michael@0:   // and not garbage-collected until then.
michael@0:   deferred.promise.then(() => timer.cancel(), () => timer.cancel());
michael@0:   return deferred;
michael@0: }
michael@0: 
michael@0: this.EXPORTED_SYMBOLS = ["AsyncShutdown"];
michael@0: 
michael@0: /**
michael@0:  * {string} topic -> phase
michael@0:  */
michael@0: let gPhases = new Map();
michael@0: 
michael@0: this.AsyncShutdown = {
michael@0:   /**
michael@0:    * Access function getPhase. For testing purposes only.
michael@0:    */
michael@0:   get _getPhase() {
michael@0:     let accepted = false;
michael@0:     try {
michael@0:       accepted = Services.prefs.getBoolPref("toolkit.asyncshutdown.testing");
michael@0:     } catch (ex) {
michael@0:       // Ignore errors
michael@0:     }
michael@0:     if (accepted) {
michael@0:       return getPhase;
michael@0:     }
michael@0:     return undefined;
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Register a new phase.
michael@0:  *
michael@0:  * @param {string} topic The notification topic for this Phase.
michael@0:  * @see {https://developer.mozilla.org/en-US/docs/Observer_Notifications}
michael@0:  */
michael@0: function getPhase(topic) {
michael@0:   let phase = gPhases.get(topic);
michael@0:   if (phase) {
michael@0:     return phase;
michael@0:   }
michael@0:   let spinner = new Spinner(topic);
michael@0:   phase = Object.freeze({
michael@0:     /**
michael@0:      * Register a blocker for the completion of a phase.
michael@0:      *
michael@0:      * @param {string} name The human-readable name of the blocker. Used
michael@0:      * for debugging/error reporting. Please make sure that the name
michael@0:      * respects the following model: "Some Service: some action in progress" -
michael@0:      * for instance "OS.File: flushing all pending I/O";
michael@0:      * @param {function|promise|*} condition A condition blocking the
michael@0:      * completion of the phase. Generally, this is a function
michael@0:      * returning a promise. This function is evaluated during the
michael@0:      * phase and the phase is guaranteed to not terminate until the
michael@0:      * resulting promise is either resolved or rejected. If
michael@0:      * |condition| is not a function but another value |v|, it behaves
michael@0:      * as if it were a function returning |v|.
michael@0:      * @param {function*} state Optionally, a function returning
michael@0:      * information about the current state of the blocker as an
michael@0:      * object. Used for providing more details when logging errors or
michael@0:      * crashing.
michael@0:      *
michael@0:      * Examples:
michael@0:      * AsyncShutdown.profileBeforeChange.addBlocker("Module: just a promise",
michael@0:      *      promise); // profileBeforeChange will not complete until
michael@0:      *                // promise is resolved or rejected
michael@0:      *
michael@0:      * AsyncShutdown.profileBeforeChange.addBlocker("Module: a callback",
michael@0:      *     function callback() {
michael@0:      *       // ...
michael@0:      *       // Execute this code during profileBeforeChange
michael@0:      *       return promise;
michael@0:      *       // profileBeforeChange will not complete until promise
michael@0:      *       // is resolved or rejected
michael@0:      * });
michael@0:      *
michael@0:      * AsyncShutdown.profileBeforeChange.addBlocker("Module: trivial callback",
michael@0:      *     function callback() {
michael@0:      *       // ...
michael@0:      *       // Execute this code during profileBeforeChange
michael@0:      *       // No specific guarantee about completion of profileBeforeChange
michael@0:      * });
michael@0:      *
michael@0:      */
michael@0:     addBlocker: function(name, condition, state = null) {
michael@0:       if (typeof name != "string") {
michael@0:         throw new TypeError("Expected a human-readable name as first argument");
michael@0:       }
michael@0:       if (state && typeof state != "function") {
michael@0:         throw new TypeError("Expected nothing or a function as third argument");
michael@0:       }
michael@0:       spinner.addBlocker({name: name, condition: condition, state: state});
michael@0:     }
michael@0:   });
michael@0:   gPhases.set(topic, phase);
michael@0:   return phase;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Utility class used to spin the event loop until all blockers for a
michael@0:  * Phase are satisfied.
michael@0:  *
michael@0:  * @param {string} topic The xpcom notification for that phase.
michael@0:  */
michael@0: function Spinner(topic) {
michael@0:   this._topic = topic;
michael@0:   this._conditions = new Set(); // set to |null| once it is too late to register
michael@0:   Services.obs.addObserver(this, topic, false);
michael@0: }
michael@0: 
michael@0: Spinner.prototype = {
michael@0:   /**
michael@0:    * Register a new condition for this phase.
michael@0:    *
michael@0:    * @param {object} condition A Condition that must be fulfilled before
michael@0:    * we complete this Phase.
michael@0:    * Must contain fields:
michael@0:    * - {string} name The human-readable name of the condition. Used
michael@0:    * for debugging/error reporting.
michael@0:    * - {function} action An action that needs to be completed
michael@0:    * before we proceed to the next runstate. If |action| returns a promise,
michael@0:    * we wait until the promise is resolved/rejected before proceeding
michael@0:    * to the next runstate.
michael@0:    */
michael@0:   addBlocker: function(condition) {
michael@0:     if (!this._conditions) {
michael@0:       throw new Error("Phase " + this._topic +
michael@0:                       " has already begun, it is too late to register" +
michael@0:                       " completion condition '" + condition.name + "'.");
michael@0:     }
michael@0:     this._conditions.add(condition);
michael@0:   },
michael@0: 
michael@0:   observe: function() {
michael@0:     let topic = this._topic;
michael@0:     Services.obs.removeObserver(this, topic);
michael@0: 
michael@0:     let conditions = this._conditions;
michael@0:     this._conditions = null; // Too late to register
michael@0: 
michael@0:     if (conditions.size == 0) {
michael@0:       // No need to spin anything
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     // The promises for which we are waiting.
michael@0:     let allPromises = [];
michael@0: 
michael@0:     // Information to determine and report to the user which conditions
michael@0:     // are not satisfied yet.
michael@0:     let allMonitors = [];
michael@0: 
michael@0:     for (let {condition, name, state} of conditions) {
michael@0:       // Gather all completion conditions
michael@0: 
michael@0:       try {
michael@0:         if (typeof condition == "function") {
michael@0:           // Normalize |condition| to the result of the function.
michael@0:           try {
michael@0:             condition = condition(topic);
michael@0:           } catch (ex) {
michael@0:             condition = Promise.reject(ex);
michael@0:           }
michael@0:         }
michael@0:         // Normalize to a promise. Of course, if |condition| was not a
michael@0:         // promise in the first place (in particular if the above
michael@0:         // function returned |undefined| or failed), that new promise
michael@0:         // isn't going to be terribly interesting, but it will behave
michael@0:         // as a promise.
michael@0:         condition = Promise.resolve(condition);
michael@0: 
michael@0:         // If the promise takes too long to be resolved/rejected,
michael@0:         // we need to notify the user.
michael@0:         //
michael@0:         // If it takes way too long, we need to crash.
michael@0: 
michael@0:         let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
michael@0:         timer.initWithCallback(function() {
michael@0:           let msg = "A phase completion condition is" +
michael@0:             " taking too long to complete." +
michael@0:             " Condition: " + monitor.name +
michael@0:             " Phase: " + topic +
michael@0:             " State: " + safeGetState(state);
michael@0:           warn(msg);
michael@0:         }, DELAY_WARNING_MS, Ci.nsITimer.TYPE_ONE_SHOT);
michael@0: 
michael@0:         let monitor = {
michael@0:           isFrozen: true,
michael@0:           name: name,
michael@0:           state: state
michael@0:         };
michael@0:         condition = condition.then(function onSuccess() {
michael@0:             timer.cancel(); // As a side-effect, this prevents |timer| from
michael@0:                             // being garbage-collected too early.
michael@0:             monitor.isFrozen = false;
michael@0:           }, function onError(error) {
michael@0:             timer.cancel();
michael@0:             let msg = "A completion condition encountered an error" +
michael@0:                 " while we were spinning the event loop." +
michael@0:                 " Condition: " + name +
michael@0:                 " Phase: " + topic +
michael@0:                 " State: " + safeGetState(state);
michael@0:             warn(msg, error);
michael@0:             monitor.isFrozen = false;
michael@0:         });
michael@0:         allMonitors.push(monitor);
michael@0:         allPromises.push(condition);
michael@0: 
michael@0:       } catch (error) {
michael@0:           let msg = "A completion condition encountered an error" +
michael@0:                 " while we were initializing the phase." +
michael@0:                 " Condition: " + name +
michael@0:                 " Phase: " + topic +
michael@0:                 " State: " + safeGetState(state);
michael@0:           warn(msg, error);
michael@0:       }
michael@0: 
michael@0:     }
michael@0:     conditions = null;
michael@0: 
michael@0:     let promise = Promise.all(allPromises);
michael@0:     allPromises = null;
michael@0: 
michael@0:     promise = promise.then(null, function onError(error) {
michael@0:       // I don't think that this can happen.
michael@0:       // However, let's be overcautious with async/shutdown error reporting.
michael@0:       let msg = "An uncaught error appeared while completing the phase." +
michael@0:             " Phase: " + topic;
michael@0:       warn(msg, error);
michael@0:     });
michael@0: 
michael@0:     let satisfied = false; // |true| once we have satisfied all conditions
michael@0: 
michael@0:     // If after DELAY_CRASH_MS (approximately one minute, adjusted to take
michael@0:     // into account sleep and otherwise busy computer) we have not finished
michael@0:     // this shutdown phase, we assume that the shutdown is somehow frozen,
michael@0:     // presumably deadlocked. At this stage, the only thing we can do to
michael@0:     // avoid leaving the user's computer in an unstable (and battery-sucking)
michael@0:     // situation is report the issue and crash.
michael@0:     let timeToCrash = looseTimer(DELAY_CRASH_MS);
michael@0:     timeToCrash.promise.then(
michael@0:       function onTimeout() {
michael@0:         // Report the problem as best as we can, then crash.
michael@0:         let frozen = [];
michael@0:         let states = [];
michael@0:         for (let {name, isFrozen, state} of allMonitors) {
michael@0:           if (isFrozen) {
michael@0:             frozen.push({name: name, state: safeGetState(state)});
michael@0:           }
michael@0:         }
michael@0: 
michael@0:         let msg = "At least one completion condition failed to complete" +
michael@0:               " within a reasonable amount of time. Causing a crash to" +
michael@0:               " ensure that we do not leave the user with an unresponsive" +
michael@0:               " process draining resources." +
michael@0:               " Conditions: " + JSON.stringify(frozen) +
michael@0:               " Phase: " + topic;
michael@0:         err(msg);
michael@0:         if (gCrashReporter && gCrashReporter.enabled) {
michael@0:           let data = {
michael@0:             phase: topic,
michael@0:             conditions: frozen
michael@0:           };
michael@0:           gCrashReporter.annotateCrashReport("AsyncShutdownTimeout",
michael@0:             JSON.stringify(data));
michael@0:         } else {
michael@0:           warn("No crash reporter available");
michael@0:         }
michael@0: 
michael@0:         let error = new Error();
michael@0:         gDebug.abort(error.fileName, error.lineNumber + 1);
michael@0:       },
michael@0:       function onSatisfied() {
michael@0:         // The promise has been rejected, which means that we have satisfied
michael@0:         // all completion conditions.
michael@0:       });
michael@0: 
michael@0:     promise = promise.then(function() {
michael@0:       satisfied = true;
michael@0:       timeToCrash.reject();
michael@0:     }/* No error is possible here*/);
michael@0: 
michael@0:     // Now, spin the event loop
michael@0:     let thread = Services.tm.mainThread;
michael@0:     while(!satisfied) {
michael@0:       thread.processNextEvent(true);
michael@0:     }
michael@0:   }
michael@0: };
michael@0: 
michael@0: 
michael@0: // List of well-known runstates
michael@0: // Ideally, runstates should be registered from the component that decides
michael@0: // when they start/stop. For compatibility with existing startup/shutdown
michael@0: // mechanisms, we register a few runstates here.
michael@0: 
michael@0: this.AsyncShutdown.profileChangeTeardown = getPhase("profile-change-teardown");
michael@0: this.AsyncShutdown.profileBeforeChange = getPhase("profile-before-change");
michael@0: this.AsyncShutdown.sendTelemetry = getPhase("profile-before-change2");
michael@0: this.AsyncShutdown.webWorkersShutdown = getPhase("web-workers-shutdown");
michael@0: Object.freeze(this.AsyncShutdown);