michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: "use strict";
michael@0: 
michael@0: const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
michael@0: 
michael@0: Cu.import("resource://gre/modules/Log.jsm", this);
michael@0: Cu.import("resource://gre/modules/osfile.jsm", this)
michael@0: Cu.import("resource://gre/modules/Promise.jsm", this);
michael@0: Cu.import("resource://gre/modules/Services.jsm", this);
michael@0: Cu.import("resource://gre/modules/Task.jsm", this);
michael@0: Cu.import("resource://gre/modules/Timer.jsm", this);
michael@0: Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
michael@0: Cu.import("resource://services-common/utils.js", this);
michael@0: 
michael@0: this.EXPORTED_SYMBOLS = [
michael@0:   "CrashManager",
michael@0: ];
michael@0: 
michael@0: /**
michael@0:  * How long to wait after application startup before crash event files are
michael@0:  * automatically aggregated.
michael@0:  *
michael@0:  * We defer aggregation for performance reasons, as we don't want too many
michael@0:  * services competing for I/O immediately after startup.
michael@0:  */
michael@0: const AGGREGATE_STARTUP_DELAY_MS = 57000;
michael@0: 
michael@0: const MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000;
michael@0: 
michael@0: // Converts Date to days since UNIX epoch.
michael@0: // This was copied from /services/metrics.storage.jsm. The implementation
michael@0: // does not account for leap seconds.
michael@0: function dateToDays(date) {
michael@0:   return Math.floor(date.getTime() / MILLISECONDS_IN_DAY);
michael@0: }
michael@0: 
michael@0: 
michael@0: /**
michael@0:  * A gateway to crash-related data.
michael@0:  *
michael@0:  * This type is generic and can be instantiated any number of times.
michael@0:  * However, most applications will typically only have one instance
michael@0:  * instantiated and that instance will point to profile and user appdata
michael@0:  * directories.
michael@0:  *
michael@0:  * Instances are created by passing an object with properties.
michael@0:  * Recognized properties are:
michael@0:  *
michael@0:  *   pendingDumpsDir (string) (required)
michael@0:  *     Where dump files that haven't been uploaded are located.
michael@0:  *
michael@0:  *   submittedDumpsDir (string) (required)
michael@0:  *     Where records of uploaded dumps are located.
michael@0:  *
michael@0:  *   eventsDirs (array)
michael@0:  *     Directories (defined as strings) where events files are written. This
michael@0:  *     instance will collects events from files in the directories specified.
michael@0:  *
michael@0:  *   storeDir (string)
michael@0:  *     Directory we will use for our data store. This instance will write
michael@0:  *     data files into the directory specified.
michael@0:  *
michael@0:  *   telemetryStoreSizeKey (string)
michael@0:  *     Telemetry histogram to report store size under.
michael@0:  */
michael@0: this.CrashManager = function (options) {
michael@0:   for (let k of ["pendingDumpsDir", "submittedDumpsDir", "eventsDirs",
michael@0:     "storeDir"]) {
michael@0:     if (!(k in options)) {
michael@0:       throw new Error("Required key not present in options: " + k);
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   this._log = Log.repository.getLogger("Crashes.CrashManager");
michael@0: 
michael@0:   for (let k in options) {
michael@0:     let v = options[k];
michael@0: 
michael@0:     switch (k) {
michael@0:       case "pendingDumpsDir":
michael@0:         this._pendingDumpsDir = v;
michael@0:         break;
michael@0: 
michael@0:       case "submittedDumpsDir":
michael@0:         this._submittedDumpsDir = v;
michael@0:         break;
michael@0: 
michael@0:       case "eventsDirs":
michael@0:         this._eventsDirs = v;
michael@0:         break;
michael@0: 
michael@0:       case "storeDir":
michael@0:         this._storeDir = v;
michael@0:         break;
michael@0: 
michael@0:       case "telemetryStoreSizeKey":
michael@0:         this._telemetryStoreSizeKey = v;
michael@0:         break;
michael@0: 
michael@0:       default:
michael@0:         throw new Error("Unknown property in options: " + k);
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Promise for in-progress aggregation operation. We store it on the
michael@0:   // object so it can be returned for in-progress operations.
michael@0:   this._aggregatePromise = null;
michael@0: 
michael@0:   // The CrashStore currently attached to this object.
michael@0:   this._store = null;
michael@0: 
michael@0:   // The timer controlling the expiration of the CrashStore instance.
michael@0:   this._storeTimer = null;
michael@0: 
michael@0:   // This is a semaphore that prevents the store from being freed by our
michael@0:   // timer-based resource freeing mechanism.
michael@0:   this._storeProtectedCount = 0;
michael@0: };
michael@0: 
michael@0: this.CrashManager.prototype = Object.freeze({
michael@0:   DUMP_REGEX: /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.dmp$/i,
michael@0:   SUBMITTED_REGEX: /^bp-(?:hr-)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.txt$/i,
michael@0:   ALL_REGEX: /^(.*)$/,
michael@0: 
michael@0:   // How long the store object should persist in memory before being
michael@0:   // automatically garbage collected.
michael@0:   STORE_EXPIRATION_MS: 60 * 1000,
michael@0: 
michael@0:   // Number of days after which a crash with no activity will get purged.
michael@0:   PURGE_OLDER_THAN_DAYS: 180,
michael@0: 
michael@0:   // The following are return codes for individual event file processing.
michael@0:   // File processed OK.
michael@0:   EVENT_FILE_SUCCESS: "ok",
michael@0:   // The event appears to be malformed.
michael@0:   EVENT_FILE_ERROR_MALFORMED: "malformed",
michael@0:   // The type of event is unknown.
michael@0:   EVENT_FILE_ERROR_UNKNOWN_EVENT: "unknown-event",
michael@0: 
michael@0:   /**
michael@0:    * Obtain a list of all dumps pending upload.
michael@0:    *
michael@0:    * The returned value is a promise that resolves to an array of objects
michael@0:    * on success. Each element in the array has the following properties:
michael@0:    *
michael@0:    *   id (string)
michael@0:    *      The ID of the crash (a UUID).
michael@0:    *
michael@0:    *   path (string)
michael@0:    *      The filename of the crash (<UUID.dmp>)
michael@0:    *
michael@0:    *   date (Date)
michael@0:    *      When this dump was created
michael@0:    *
michael@0:    * The returned arry is sorted by the modified time of the file backing
michael@0:    * the entry, oldest to newest.
michael@0:    *
michael@0:    * @return Promise<Array>
michael@0:    */
michael@0:   pendingDumps: function () {
michael@0:     return this._getDirectoryEntries(this._pendingDumpsDir, this.DUMP_REGEX);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain a list of all dump files corresponding to submitted crashes.
michael@0:    *
michael@0:    * The returned value is a promise that resolves to an Array of
michael@0:    * objects. Each object has the following properties:
michael@0:    *
michael@0:    *   path (string)
michael@0:    *     The path of the file this entry comes from.
michael@0:    *
michael@0:    *   id (string)
michael@0:    *     The crash UUID.
michael@0:    *
michael@0:    *   date (Date)
michael@0:    *     The (estimated) date this crash was submitted.
michael@0:    *
michael@0:    * The returned array is sorted by the modified time of the file backing
michael@0:    * the entry, oldest to newest.
michael@0:    *
michael@0:    * @return Promise<Array>
michael@0:    */
michael@0:   submittedDumps: function () {
michael@0:     return this._getDirectoryEntries(this._submittedDumpsDir,
michael@0:                                      this.SUBMITTED_REGEX);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Aggregates "loose" events files into the unified "database."
michael@0:    *
michael@0:    * This function should be called periodically to collect metadata from
michael@0:    * all events files into the central data store maintained by this manager.
michael@0:    *
michael@0:    * Once events have been stored in the backing store the corresponding
michael@0:    * source files are deleted.
michael@0:    *
michael@0:    * Only one aggregation operation is allowed to occur at a time. If this
michael@0:    * is called when an existing aggregation is in progress, the promise for
michael@0:    * the original call will be returned.
michael@0:    *
michael@0:    * @return promise<int> The number of event files that were examined.
michael@0:    */
michael@0:   aggregateEventsFiles: function () {
michael@0:     if (this._aggregatePromise) {
michael@0:       return this._aggregatePromise;
michael@0:     }
michael@0: 
michael@0:     return this._aggregatePromise = Task.spawn(function* () {
michael@0:       if (this._aggregatePromise) {
michael@0:         return this._aggregatePromise;
michael@0:       }
michael@0: 
michael@0:       try {
michael@0:         let unprocessedFiles = yield this._getUnprocessedEventsFiles();
michael@0: 
michael@0:         let deletePaths = [];
michael@0:         let needsSave = false;
michael@0: 
michael@0:         this._storeProtectedCount++;
michael@0:         for (let entry of unprocessedFiles) {
michael@0:           try {
michael@0:             let result = yield this._processEventFile(entry);
michael@0: 
michael@0:             switch (result) {
michael@0:               case this.EVENT_FILE_SUCCESS:
michael@0:                 needsSave = true;
michael@0:                 // Fall through.
michael@0: 
michael@0:               case this.EVENT_FILE_ERROR_MALFORMED:
michael@0:                 deletePaths.push(entry.path);
michael@0:                 break;
michael@0: 
michael@0:               case this.EVENT_FILE_ERROR_UNKNOWN_EVENT:
michael@0:                 break;
michael@0: 
michael@0:               default:
michael@0:                 Cu.reportError("Unhandled crash event file return code. Please " +
michael@0:                                "file a bug: " + result);
michael@0:             }
michael@0:           } catch (ex if ex instanceof OS.File.Error) {
michael@0:             this._log.warn("I/O error reading " + entry.path + ": " +
michael@0:                            CommonUtils.exceptionStr(ex));
michael@0:           } catch (ex) {
michael@0:             // We should never encounter an exception. This likely represents
michael@0:             // a coding error because all errors should be detected and
michael@0:             // converted to return codes.
michael@0:             //
michael@0:             // If we get here, report the error and delete the source file
michael@0:             // so we don't see it again.
michael@0:             Cu.reportError("Exception when processing crash event file: " +
michael@0:                            CommonUtils.exceptionStr(ex));
michael@0:             deletePaths.push(entry.path);
michael@0:           }
michael@0:         }
michael@0: 
michael@0:         if (needsSave) {
michael@0:           let store = yield this._getStore();
michael@0:           yield store.save();
michael@0:         }
michael@0: 
michael@0:         for (let path of deletePaths) {
michael@0:           try {
michael@0:             yield OS.File.remove(path);
michael@0:           } catch (ex) {
michael@0:             this._log.warn("Error removing event file (" + path + "): " +
michael@0:                            CommonUtils.exceptionStr(ex));
michael@0:           }
michael@0:         }
michael@0: 
michael@0:         return unprocessedFiles.length;
michael@0: 
michael@0:       } finally {
michael@0:         this._aggregatePromise = false;
michael@0:         this._storeProtectedCount--;
michael@0:       }
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Prune old crash data.
michael@0:    *
michael@0:    * @param date
michael@0:    *        (Date) The cutoff point for pruning. Crashes without data newer
michael@0:    *        than this will be pruned.
michael@0:    */
michael@0:   pruneOldCrashes: function (date) {
michael@0:     return Task.spawn(function* () {
michael@0:       let store = yield this._getStore();
michael@0:       store.pruneOldCrashes(date);
michael@0:       yield store.save();
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Run tasks that should be periodically performed.
michael@0:    */
michael@0:   runMaintenanceTasks: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       yield this.aggregateEventsFiles();
michael@0: 
michael@0:       let offset = this.PURGE_OLDER_THAN_DAYS * MILLISECONDS_IN_DAY;
michael@0:       yield this.pruneOldCrashes(new Date(Date.now() - offset));
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Schedule maintenance tasks for some point in the future.
michael@0:    *
michael@0:    * @param delay
michael@0:    *        (integer) Delay in milliseconds when maintenance should occur.
michael@0:    */
michael@0:   scheduleMaintenance: function (delay) {
michael@0:     let deferred = Promise.defer();
michael@0: 
michael@0:     setTimeout(() => {
michael@0:       this.runMaintenanceTasks().then(deferred.resolve, deferred.reject);
michael@0:     }, delay);
michael@0: 
michael@0:     return deferred.promise;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain the paths of all unprocessed events files.
michael@0:    *
michael@0:    * The promise-resolved array is sorted by file mtime, oldest to newest.
michael@0:    */
michael@0:   _getUnprocessedEventsFiles: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       let entries = [];
michael@0: 
michael@0:       for (let dir of this._eventsDirs) {
michael@0:         for (let e of yield this._getDirectoryEntries(dir, this.ALL_REGEX)) {
michael@0:           entries.push(e);
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       entries.sort((a, b) => { return a.date - b.date; });
michael@0: 
michael@0:       return entries;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   // See docs/crash-events.rst for the file format specification.
michael@0:   _processEventFile: function (entry) {
michael@0:     return Task.spawn(function* () {
michael@0:       let data = yield OS.File.read(entry.path);
michael@0:       let store = yield this._getStore();
michael@0: 
michael@0:       let decoder = new TextDecoder();
michael@0:       data = decoder.decode(data);
michael@0: 
michael@0:       let type, time, payload;
michael@0:       let start = 0;
michael@0:       for (let i = 0; i < 2; i++) {
michael@0:         let index = data.indexOf("\n", start);
michael@0:         if (index == -1) {
michael@0:           return this.EVENT_FILE_ERROR_MALFORMED;
michael@0:         }
michael@0: 
michael@0:         let sub = data.substring(start, index);
michael@0:         switch (i) {
michael@0:           case 0:
michael@0:             type = sub;
michael@0:             break;
michael@0:           case 1:
michael@0:             time = sub;
michael@0:             try {
michael@0:               time = parseInt(time, 10);
michael@0:             } catch (ex) {
michael@0:               return this.EVENT_FILE_ERROR_MALFORMED;
michael@0:             }
michael@0:         }
michael@0: 
michael@0:         start = index + 1;
michael@0:       }
michael@0:       let date = new Date(time * 1000);
michael@0:       let payload = data.substring(start);
michael@0: 
michael@0:       return this._handleEventFilePayload(store, entry, type, date, payload);
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   _handleEventFilePayload: function (store, entry, type, date, payload) {
michael@0:       // The payload types and formats are documented in docs/crash-events.rst.
michael@0:       // Do not change the format of an existing type. Instead, invent a new
michael@0:       // type.
michael@0: 
michael@0:       let eventMap = {
michael@0:         "crash.main.1": "addMainProcessCrash",
michael@0:         "crash.plugin.1": "addPluginCrash",
michael@0:         "hang.plugin.1": "addPluginHang",
michael@0:       };
michael@0: 
michael@0:       if (type in eventMap) {
michael@0:         let lines = payload.split("\n");
michael@0:         if (lines.length > 1) {
michael@0:           this._log.warn("Multiple lines unexpected in payload for " +
michael@0:                          entry.path);
michael@0:           return this.EVENT_FILE_ERROR_MALFORMED;
michael@0:         }
michael@0: 
michael@0:         store[eventMap[type]](payload, date);
michael@0:         return this.EVENT_FILE_SUCCESS;
michael@0:       }
michael@0: 
michael@0:       // DO NOT ADD NEW TYPES WITHOUT DOCUMENTING!
michael@0: 
michael@0:       return this.EVENT_FILE_ERROR_UNKNOWN_EVENT;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * The resolved promise is an array of objects with the properties:
michael@0:    *
michael@0:    *   path -- String filename
michael@0:    *   id -- regexp.match()[1] (likely the crash ID)
michael@0:    *   date -- Date mtime of the file
michael@0:    */
michael@0:   _getDirectoryEntries: function (path, re) {
michael@0:     return Task.spawn(function* () {
michael@0:       try {
michael@0:         yield OS.File.stat(path);
michael@0:       } catch (ex if ex instanceof OS.File.Error && ex.becauseNoSuchFile) {
michael@0:           return [];
michael@0:       }
michael@0: 
michael@0:       let it = new OS.File.DirectoryIterator(path);
michael@0:       let entries = [];
michael@0: 
michael@0:       try {
michael@0:         yield it.forEach((entry, index, it) => {
michael@0:           if (entry.isDir) {
michael@0:             return;
michael@0:           }
michael@0: 
michael@0:           let match = re.exec(entry.name);
michael@0:           if (!match) {
michael@0:             return;
michael@0:           }
michael@0: 
michael@0:           return OS.File.stat(entry.path).then((info) => {
michael@0:             entries.push({
michael@0:               path: entry.path,
michael@0:               id: match[1],
michael@0:               date: info.lastModificationDate,
michael@0:             });
michael@0:           });
michael@0:         });
michael@0:       } finally {
michael@0:         it.close();
michael@0:       }
michael@0: 
michael@0:       entries.sort((a, b) => { return a.date - b.date; });
michael@0: 
michael@0:       return entries;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   _getStore: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       if (!this._store) {
michael@0:         yield OS.File.makeDir(this._storeDir, {
michael@0:           ignoreExisting: true,
michael@0:           unixMode: OS.Constants.libc.S_IRWXU,
michael@0:         });
michael@0: 
michael@0:         let store = new CrashStore(this._storeDir, this._telemetryStoreSizeKey);
michael@0:         yield store.load();
michael@0: 
michael@0:         this._store = store;
michael@0:         this._storeTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
michael@0:       }
michael@0: 
michael@0:       // The application can go long periods without interacting with the
michael@0:       // store. Since the store takes up resources, we automatically "free"
michael@0:       // the store after inactivity so resources can be returned to the system.
michael@0:       // We do this via a timer and a mechanism that tracks when the store
michael@0:       // is being accessed.
michael@0:       this._storeTimer.cancel();
michael@0: 
michael@0:       // This callback frees resources from the store unless the store
michael@0:       // is protected from freeing by some other process.
michael@0:       let timerCB = function () {
michael@0:         if (this._storeProtectedCount) {
michael@0:           this._storeTimer.initWithCallback(timerCB, this.STORE_EXPIRATION_MS,
michael@0:                                             this._storeTimer.TYPE_ONE_SHOT);
michael@0:           return;
michael@0:         }
michael@0: 
michael@0:         // We kill the reference that we hold. GC will kill it later. If
michael@0:         // someone else holds a reference, that will prevent GC until that
michael@0:         // reference is gone.
michael@0:         this._store = null;
michael@0:         this._storeTimer = null;
michael@0:       }.bind(this);
michael@0: 
michael@0:       this._storeTimer.initWithCallback(timerCB, this.STORE_EXPIRATION_MS,
michael@0:                                         this._storeTimer.TYPE_ONE_SHOT);
michael@0: 
michael@0:       return this._store;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain information about all known crashes.
michael@0:    *
michael@0:    * Returns an array of CrashRecord instances. Instances are read-only.
michael@0:    */
michael@0:   getCrashes: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       let store = yield this._getStore();
michael@0: 
michael@0:       return store.crashes;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   getCrashCountsByDay: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       let store = yield this._getStore();
michael@0: 
michael@0:       return store._countsByDay;
michael@0:     }.bind(this));
michael@0:   },
michael@0: });
michael@0: 
michael@0: let gCrashManager;
michael@0: 
michael@0: /**
michael@0:  * Interface to storage of crash data.
michael@0:  *
michael@0:  * This type handles storage of crash metadata. It exists as a separate type
michael@0:  * from the crash manager for performance reasons: since all crash metadata
michael@0:  * needs to be loaded into memory for access, we wish to easily dispose of all
michael@0:  * associated memory when this data is no longer needed. Having an isolated
michael@0:  * object whose references can easily be lost faciliates that simple disposal.
michael@0:  *
michael@0:  * When metadata is updated, the caller must explicitly persist the changes
michael@0:  * to disk. This prevents excessive I/O during updates.
michael@0:  *
michael@0:  * The store has a mechanism for ensuring it doesn't grow too large. A ceiling
michael@0:  * is placed on the number of daily events that can occur for events that can
michael@0:  * occur with relatively high frequency, notably plugin crashes and hangs
michael@0:  * (plugins can enter cycles where they repeatedly crash). If we've reached
michael@0:  * the high water mark and new data arrives, it's silently dropped.
michael@0:  * However, the count of actual events is always preserved. This allows
michael@0:  * us to report on the severity of problems beyond the storage threshold.
michael@0:  *
michael@0:  * Main process crashes are excluded from limits because they are both
michael@0:  * important and should be rare.
michael@0:  *
michael@0:  * @param storeDir (string)
michael@0:  *        Directory the store should be located in.
michael@0:  * @param telemetrySizeKey (string)
michael@0:  *        The telemetry histogram that should be used to store the size
michael@0:  *        of the data file.
michael@0:  */
michael@0: function CrashStore(storeDir, telemetrySizeKey) {
michael@0:   this._storeDir = storeDir;
michael@0:   this._telemetrySizeKey = telemetrySizeKey;
michael@0: 
michael@0:   this._storePath = OS.Path.join(storeDir, "store.json.mozlz4");
michael@0: 
michael@0:   // Holds the read data from disk.
michael@0:   this._data = null;
michael@0: 
michael@0:   // Maps days since UNIX epoch to a Map of event types to counts.
michael@0:   // This data structure is populated when the JSON file is loaded
michael@0:   // and is also updated when new events are added.
michael@0:   this._countsByDay = new Map();
michael@0: }
michael@0: 
michael@0: CrashStore.prototype = Object.freeze({
michael@0:   // A crash that occurred in the main process.
michael@0:   TYPE_MAIN_CRASH: "main-crash",
michael@0: 
michael@0:   // A crash in a plugin process.
michael@0:   TYPE_PLUGIN_CRASH: "plugin-crash",
michael@0: 
michael@0:   // A hang in a plugin process.
michael@0:   TYPE_PLUGIN_HANG: "plugin-hang",
michael@0: 
michael@0:   // Maximum number of events to store per day. This establishes a
michael@0:   // ceiling on the per-type/per-day records that will be stored.
michael@0:   HIGH_WATER_DAILY_THRESHOLD: 100,
michael@0: 
michael@0:   /**
michael@0:    * Load data from disk.
michael@0:    *
michael@0:    * @return Promise
michael@0:    */
michael@0:   load: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       // Loading replaces data. So reset data structures.
michael@0:       this._data = {
michael@0:         v: 1,
michael@0:         crashes: new Map(),
michael@0:         corruptDate: null,
michael@0:       };
michael@0:       this._countsByDay = new Map();
michael@0: 
michael@0:       try {
michael@0:         let decoder = new TextDecoder();
michael@0:         let data = yield OS.File.read(this._storePath, {compression: "lz4"});
michael@0:         data = JSON.parse(decoder.decode(data));
michael@0: 
michael@0:         if (data.corruptDate) {
michael@0:           this._data.corruptDate = new Date(data.corruptDate);
michael@0:         }
michael@0: 
michael@0:         // actualCounts is used to validate that the derived counts by
michael@0:         // days stored in the payload matches up to actual data.
michael@0:         let actualCounts = new Map();
michael@0: 
michael@0:         for (let id in data.crashes) {
michael@0:           let crash = data.crashes[id];
michael@0:           let denormalized = this._denormalize(crash);
michael@0: 
michael@0:           this._data.crashes.set(id, denormalized);
michael@0: 
michael@0:           let key = dateToDays(denormalized.crashDate) + "-" + denormalized.type;
michael@0:           actualCounts.set(key, (actualCounts.get(key) || 0) + 1);
michael@0:         }
michael@0: 
michael@0:         // The validation in this loop is arguably not necessary. We perform
michael@0:         // it as a defense against unknown bugs.
michael@0:         for (let dayKey in data.countsByDay) {
michael@0:           let day = parseInt(dayKey, 10);
michael@0:           for (let type in data.countsByDay[day]) {
michael@0:             this._ensureCountsForDay(day);
michael@0: 
michael@0:             let count = data.countsByDay[day][type];
michael@0:             let key = day + "-" + type;
michael@0: 
michael@0:             // If the payload says we have data for a given day but we
michael@0:             // don't, the payload is wrong. Ignore it.
michael@0:             if (!actualCounts.has(key)) {
michael@0:               continue;
michael@0:             }
michael@0: 
michael@0:             // If we encountered more data in the payload than what the
michael@0:             // data structure says, use the proper value.
michael@0:             count = Math.max(count, actualCounts.get(key));
michael@0: 
michael@0:             this._countsByDay.get(day).set(type, count);
michael@0:           }
michael@0:         }
michael@0:       } catch (ex if ex instanceof OS.File.Error && ex.becauseNoSuchFile) {
michael@0:         // Missing files (first use) are allowed.
michael@0:       } catch (ex) {
michael@0:         // If we can't load for any reason, mark a corrupt date in the instance
michael@0:         // and swallow the error.
michael@0:         //
michael@0:         // The marking of a corrupted file is intentionally not persisted to
michael@0:         // disk yet. Instead, we wait until the next save(). This is to give
michael@0:         // non-permanent failures the opportunity to recover on their own.
michael@0:         this._data.corruptDate = new Date();
michael@0:       }
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Save data to disk.
michael@0:    *
michael@0:    * @return Promise<null>
michael@0:    */
michael@0:   save: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       if (!this._data) {
michael@0:         return;
michael@0:       }
michael@0: 
michael@0:       let normalized = {
michael@0:         // The version should be incremented whenever the format
michael@0:         // changes.
michael@0:         v: 1,
michael@0:         // Maps crash IDs to objects defining the crash.
michael@0:         crashes: {},
michael@0:         // Maps days since UNIX epoch to objects mapping event types to
michael@0:         // counts. This is a mirror of this._countsByDay. e.g.
michael@0:         // {
michael@0:         //    15000: {
michael@0:         //        "main-crash": 2,
michael@0:         //        "plugin-crash": 1
michael@0:         //    }
michael@0:         // }
michael@0:         countsByDay: {},
michael@0: 
michael@0:         // When the store was last corrupted.
michael@0:         corruptDate: null,
michael@0:       };
michael@0: 
michael@0:       if (this._data.corruptDate) {
michael@0:         normalized.corruptDate = this._data.corruptDate.getTime();
michael@0:       }
michael@0: 
michael@0:       for (let [id, crash] of this._data.crashes) {
michael@0:         let c = this._normalize(crash);
michael@0:         normalized.crashes[id] = c;
michael@0:       }
michael@0: 
michael@0:       for (let [day, m] of this._countsByDay) {
michael@0:         normalized.countsByDay[day] = {};
michael@0:         for (let [type, count] of m) {
michael@0:           normalized.countsByDay[day][type] = count;
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       let encoder = new TextEncoder();
michael@0:       let data = encoder.encode(JSON.stringify(normalized));
michael@0:       let size = yield OS.File.writeAtomic(this._storePath, data, {
michael@0:                                            tmpPath: this._storePath + ".tmp",
michael@0:                                            compression: "lz4"});
michael@0:       if (this._telemetrySizeKey) {
michael@0:         Services.telemetry.getHistogramById(this._telemetrySizeKey).add(size);
michael@0:       }
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Normalize an object into one fit for serialization.
michael@0:    *
michael@0:    * This function along with _denormalize() serve to hack around the
michael@0:    * default handling of Date JSON serialization because Date serialization
michael@0:    * is undefined by JSON.
michael@0:    *
michael@0:    * Fields ending with "Date" are assumed to contain Date instances.
michael@0:    * We convert these to milliseconds since epoch on output and back to
michael@0:    * Date on input.
michael@0:    */
michael@0:   _normalize: function (o) {
michael@0:     let normalized = {};
michael@0: 
michael@0:     for (let k in o) {
michael@0:       let v = o[k];
michael@0:       if (v && k.endsWith("Date")) {
michael@0:         normalized[k] = v.getTime();
michael@0:       } else {
michael@0:         normalized[k] = v;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return normalized;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Convert a serialized object back to its native form.
michael@0:    */
michael@0:   _denormalize: function (o) {
michael@0:     let n = {};
michael@0: 
michael@0:     for (let k in o) {
michael@0:       let v = o[k];
michael@0:       if (v && k.endsWith("Date")) {
michael@0:         n[k] = new Date(parseInt(v, 10));
michael@0:       } else {
michael@0:         n[k] = v;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return n;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Prune old crash data.
michael@0:    *
michael@0:    * Crashes without recent activity are pruned from the store so the
michael@0:    * size of the store is not unbounded. If there is activity on a crash,
michael@0:    * that activity will keep the crash and all its data around for longer.
michael@0:    *
michael@0:    * @param date
michael@0:    *        (Date) The cutoff at which data will be pruned. If an entry
michael@0:    *        doesn't have data newer than this, it will be pruned.
michael@0:    */
michael@0:   pruneOldCrashes: function (date) {
michael@0:     for (let crash of this.crashes) {
michael@0:       let newest = crash.newestDate;
michael@0:       if (!newest || newest.getTime() < date.getTime()) {
michael@0:         this._data.crashes.delete(crash.id);
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Date the store was last corrupted and required a reset.
michael@0:    *
michael@0:    * May be null (no corruption has ever occurred) or a Date instance.
michael@0:    */
michael@0:   get corruptDate() {
michael@0:     return this._data.corruptDate;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * The number of distinct crashes tracked.
michael@0:    */
michael@0:   get crashesCount() {
michael@0:     return this._data.crashes.size;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * All crashes tracked.
michael@0:    *
michael@0:    * This is an array of CrashRecord.
michael@0:    */
michael@0:   get crashes() {
michael@0:     let crashes = [];
michael@0:     for (let [id, crash] of this._data.crashes) {
michael@0:       crashes.push(new CrashRecord(crash));
michael@0:     }
michael@0: 
michael@0:     return crashes;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain a particular crash from its ID.
michael@0:    *
michael@0:    * A CrashRecord will be returned if the crash exists. null will be returned
michael@0:    * if the crash is unknown.
michael@0:    */
michael@0:   getCrash: function (id) {
michael@0:     for (let crash of this.crashes) {
michael@0:       if (crash.id == id) {
michael@0:         return crash;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return null;
michael@0:   },
michael@0: 
michael@0:   _ensureCountsForDay: function (day) {
michael@0:     if (!this._countsByDay.has(day)) {
michael@0:       this._countsByDay.set(day, new Map());
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Ensure the crash record is present in storage.
michael@0:    *
michael@0:    * Returns the crash record if we're allowed to store it or null
michael@0:    * if we've hit the high water mark.
michael@0:    *
michael@0:    * @param id
michael@0:    *        (string) The crash ID.
michael@0:    * @param type
michael@0:    *        (string) One of the this.TYPE_* constants describing the crash type.
michael@0:    * @param date
michael@0:    *        (Date) When this crash occurred.
michael@0:    *
michael@0:    * @return null | object crash record
michael@0:    */
michael@0:   _ensureCrashRecord: function (id, type, date) {
michael@0:     let day = dateToDays(date);
michael@0:     this._ensureCountsForDay(day);
michael@0: 
michael@0:     let count = (this._countsByDay.get(day).get(type) || 0) + 1;
michael@0:     this._countsByDay.get(day).set(type, count);
michael@0: 
michael@0:     if (count > this.HIGH_WATER_DAILY_THRESHOLD && type != this.TYPE_MAIN_CRASH) {
michael@0:       return null;
michael@0:     }
michael@0: 
michael@0:     if (!this._data.crashes.has(id)) {
michael@0:       this._data.crashes.set(id, {
michael@0:         id: id,
michael@0:         type: type,
michael@0:         crashDate: date,
michael@0:       });
michael@0:     }
michael@0: 
michael@0:     let crash = this._data.crashes.get(id);
michael@0:     crash.type = type;
michael@0:     crash.date = date;
michael@0: 
michael@0:     return crash;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Record the occurrence of a crash in the main process.
michael@0:    *
michael@0:    * @param id (string) Crash ID. Likely a UUID.
michael@0:    * @param date (Date) When the crash occurred.
michael@0:    */
michael@0:   addMainProcessCrash: function (id, date) {
michael@0:     this._ensureCrashRecord(id, this.TYPE_MAIN_CRASH, date);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Record the occurrence of a crash in a plugin process.
michael@0:    *
michael@0:    * @param id (string) Crash ID. Likely a UUID.
michael@0:    * @param date (Date) When the crash occurred.
michael@0:    */
michael@0:   addPluginCrash: function (id, date) {
michael@0:     this._ensureCrashRecord(id, this.TYPE_PLUGIN_CRASH, date);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Record the occurrence of a hang in a plugin process.
michael@0:    *
michael@0:    * @param id (string) Crash ID. Likely a UUID.
michael@0:    * @param date (Date) When the hang was reported.
michael@0:    */
michael@0:   addPluginHang: function (id, date) {
michael@0:     this._ensureCrashRecord(id, this.TYPE_PLUGIN_HANG, date);
michael@0:   },
michael@0: 
michael@0:   get mainProcessCrashes() {
michael@0:     let crashes = [];
michael@0:     for (let crash of this.crashes) {
michael@0:       if (crash.isMainProcessCrash) {
michael@0:         crashes.push(crash);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return crashes;
michael@0:   },
michael@0: 
michael@0:   get pluginCrashes() {
michael@0:     let crashes = [];
michael@0:     for (let crash of this.crashes) {
michael@0:       if (crash.isPluginCrash) {
michael@0:         crashes.push(crash);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return crashes;
michael@0:   },
michael@0: 
michael@0:   get pluginHangs() {
michael@0:     let crashes = [];
michael@0:     for (let crash of this.crashes) {
michael@0:       if (crash.isPluginHang) {
michael@0:         crashes.push(crash);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return crashes;
michael@0:   },
michael@0: });
michael@0: 
michael@0: /**
michael@0:  * Represents an individual crash with metadata.
michael@0:  *
michael@0:  * This is a wrapper around the low-level anonymous JS objects that define
michael@0:  * crashes. It exposes a consistent and helpful API.
michael@0:  *
michael@0:  * Instances of this type should only be constructured inside this module,
michael@0:  * not externally. The constructor is not considered a public API.
michael@0:  *
michael@0:  * @param o (object)
michael@0:  *        The crash's entry from the CrashStore.
michael@0:  */
michael@0: function CrashRecord(o) {
michael@0:   this._o = o;
michael@0: }
michael@0: 
michael@0: CrashRecord.prototype = Object.freeze({
michael@0:   get id() {
michael@0:     return this._o.id;
michael@0:   },
michael@0: 
michael@0:   get crashDate() {
michael@0:     return this._o.crashDate;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain the newest date in this record.
michael@0:    *
michael@0:    * This is a convenience getter. The returned value is used to determine when
michael@0:    * to expire a record.
michael@0:    */
michael@0:   get newestDate() {
michael@0:     // We currently only have 1 date, so this is easy.
michael@0:     return this._o.crashDate;
michael@0:   },
michael@0: 
michael@0:   get oldestDate() {
michael@0:     return this._o.crashDate;
michael@0:   },
michael@0: 
michael@0:   get type() {
michael@0:     return this._o.type;
michael@0:   },
michael@0: 
michael@0:   get isMainProcessCrash() {
michael@0:     return this._o.type == CrashStore.prototype.TYPE_MAIN_CRASH;
michael@0:   },
michael@0: 
michael@0:   get isPluginCrash() {
michael@0:     return this._o.type == CrashStore.prototype.TYPE_PLUGIN_CRASH;
michael@0:   },
michael@0: 
michael@0:   get isPluginHang() {
michael@0:     return this._o.type == CrashStore.prototype.TYPE_PLUGIN_HANG;
michael@0:   },
michael@0: });
michael@0: 
michael@0: /**
michael@0:  * Obtain the global CrashManager instance used by the running application.
michael@0:  *
michael@0:  * CrashManager is likely only ever instantiated once per application lifetime.
michael@0:  * The main reason it's implemented as a reusable type is to facilitate testing.
michael@0:  */
michael@0: XPCOMUtils.defineLazyGetter(this.CrashManager, "Singleton", function () {
michael@0:   if (gCrashManager) {
michael@0:     return gCrashManager;
michael@0:   }
michael@0: 
michael@0:   let crPath = OS.Path.join(OS.Constants.Path.userApplicationDataDir,
michael@0:                             "Crash Reports");
michael@0:   let storePath = OS.Path.join(OS.Constants.Path.profileDir, "crashes");
michael@0: 
michael@0:   gCrashManager = new CrashManager({
michael@0:     pendingDumpsDir: OS.Path.join(crPath, "pending"),
michael@0:     submittedDumpsDir: OS.Path.join(crPath, "submitted"),
michael@0:     eventsDirs: [OS.Path.join(crPath, "events"), OS.Path.join(storePath, "events")],
michael@0:     storeDir: storePath,
michael@0:     telemetryStoreSizeKey: "CRASH_STORE_COMPRESSED_BYTES",
michael@0:   });
michael@0: 
michael@0:   // Automatically aggregate event files shortly after startup. This
michael@0:   // ensures it happens with some frequency.
michael@0:   //
michael@0:   // There are performance considerations here. While this is doing
michael@0:   // work and could negatively impact performance, the amount of work
michael@0:   // is kept small per run by periodically aggregating event files.
michael@0:   // Furthermore, well-behaving installs should not have much work
michael@0:   // here to do. If there is a lot of work, that install has bigger
michael@0:   // issues beyond reduced performance near startup.
michael@0:   gCrashManager.scheduleMaintenance(AGGREGATE_STARTUP_DELAY_MS);
michael@0: 
michael@0:   return gCrashManager;
michael@0: });