The Tor Browser: services/healthreport/healthreporter.jsm@97036ab72558 (annotated)

services/healthreport/healthreporter.jsm@97036ab72558 (annotated)

services/healthreport/healthreporter.jsm

Tue, 06 Jan 2015 21:39:09 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Tue, 06 Jan 2015 21:39:09 +0100
branch: TOR_BUG_9701
changeset 8: 97036ab72558
permissions: -rw-r--r--

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 "use strict";
 #ifndef MERGED_COMPARTMENT
 this.EXPORTED_SYMBOLS = ["HealthReporter"];
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
 Cu.import("resource://gre/modules/Metrics.jsm");
 Cu.import("resource://services-common/async.js");
 Cu.import("resource://services-common/bagheeraclient.js");
 #endif
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://services-common/utils.js");
 Cu.import("resource://gre/modules/Promise.jsm");
 Cu.import("resource://gre/modules/osfile.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/TelemetryStopwatch.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "UpdateChannel",
                                   "resource://gre/modules/UpdateChannel.jsm");
 // Oldest year to allow in date preferences. This module was implemented in
 // 2012 and no dates older than that should be encountered.
 const OLDEST_ALLOWED_YEAR = 2012;
 const DAYS_IN_PAYLOAD = 180;
 const DEFAULT_DATABASE_NAME = "healthreport.sqlite";
 const TELEMETRY_INIT = "HEALTHREPORT_INIT_MS";
 const TELEMETRY_INIT_FIRSTRUN = "HEALTHREPORT_INIT_FIRSTRUN_MS";
 const TELEMETRY_DB_OPEN = "HEALTHREPORT_DB_OPEN_MS";
 const TELEMETRY_DB_OPEN_FIRSTRUN = "HEALTHREPORT_DB_OPEN_FIRSTRUN_MS";
 const TELEMETRY_GENERATE_PAYLOAD = "HEALTHREPORT_GENERATE_JSON_PAYLOAD_MS";
 const TELEMETRY_JSON_PAYLOAD_SERIALIZE = "HEALTHREPORT_JSON_PAYLOAD_SERIALIZE_MS";
 const TELEMETRY_PAYLOAD_SIZE_UNCOMPRESSED = "HEALTHREPORT_PAYLOAD_UNCOMPRESSED_BYTES";
 const TELEMETRY_PAYLOAD_SIZE_COMPRESSED = "HEALTHREPORT_PAYLOAD_COMPRESSED_BYTES";
 const TELEMETRY_UPLOAD = "HEALTHREPORT_UPLOAD_MS";
 const TELEMETRY_SHUTDOWN_DELAY = "HEALTHREPORT_SHUTDOWN_DELAY_MS";
 const TELEMETRY_COLLECT_CONSTANT = "HEALTHREPORT_COLLECT_CONSTANT_DATA_MS";
 const TELEMETRY_COLLECT_DAILY = "HEALTHREPORT_COLLECT_DAILY_MS";
 const TELEMETRY_SHUTDOWN = "HEALTHREPORT_SHUTDOWN_MS";
 const TELEMETRY_COLLECT_CHECKPOINT = "HEALTHREPORT_POST_COLLECT_CHECKPOINT_MS";
 /**
  * Helper type to assist with management of Health Reporter state.
  *
  * Instances are not meant to be created outside of a HealthReporter instance.
  *
  * There are two types of IDs associated with clients.
  *
  * Since the beginning of FHR, there has existed a per-upload ID: a UUID is
  * generated at upload time and associated with the state before upload starts.
  * That same upload includes a request to delete all other upload IDs known by
  * the client.
  *
  * Per-upload IDs had the unintended side-effect of creating "orphaned"
  * records/upload IDs on the server. So, a stable client identifer has been
  * introduced. This client identifier is generated when it's missing and sent
  * as part of every upload.
  *
  * There is a high chance we may remove upload IDs in the future.
  */
 function HealthReporterState(reporter) {
   this._reporter = reporter;
   let profD = OS.Constants.Path.profileDir;
   if (!profD || !profD.length) {
     throw new Error("Could not obtain profile directory. OS.File not " +
                     "initialized properly?");
   }
   this._log = reporter._log;
   this._stateDir = OS.Path.join(profD, "healthreport");
   // To facilitate testing.
   let leaf = reporter._stateLeaf || "state.json";
   this._filename = OS.Path.join(this._stateDir, leaf);
   this._log.debug("Storing state in " + this._filename);
   this._s = null;
 }
 HealthReporterState.prototype = Object.freeze({
   /**
    * Persistent string identifier associated with this client.
    */
   get clientID() {
     return this._s.clientID;
   },
   /**
    * The version associated with the client ID.
    */
   get clientIDVersion() {
     return this._s.clientIDVersion;
   },
   get lastPingDate() {
     return new Date(this._s.lastPingTime);
   },
   get lastSubmitID() {
     return this._s.remoteIDs[0];
   },
   get remoteIDs() {
     return this._s.remoteIDs;
   },
   get _lastPayloadPath() {
     return OS.Path.join(this._stateDir, "lastpayload.json");
   },
   init: function () {
     return Task.spawn(function init() {
       try {
         OS.File.makeDir(this._stateDir);
       } catch (ex if ex instanceof OS.FileError) {
         if (!ex.becauseExists) {
           throw ex;
         }
       }
       let resetObjectState = function () {
         this._s = {
           // The payload version. This is bumped whenever there is a
           // backwards-incompatible change.
           v: 1,
           // The persistent client identifier.
           clientID: CommonUtils.generateUUID(),
           // Denotes the mechanism used to generate the client identifier.
           // 1: Random UUID.
           clientIDVersion: 1,
           // Upload IDs that might be on the server.
           remoteIDs: [],
           // When we last performed an uploaded.
           lastPingTime: 0,
           // Tracks whether we removed an outdated payload.
           removedOutdatedLastpayload: false,
         };
       }.bind(this);
       try {
         this._s = yield CommonUtils.readJSON(this._filename);
       } catch (ex if ex instanceof OS.File.Error) {
         if (!ex.becauseNoSuchFile) {
           throw ex;
         }
         this._log.warn("Saved state file does not exist.");
         resetObjectState();
       } catch (ex) {
         this._log.error("Exception when reading state from disk: " +
                         CommonUtils.exceptionStr(ex));
         resetObjectState();
         // Don't save in case it goes away on next run.
       }
       if (typeof(this._s) != "object") {
         this._log.warn("Read state is not an object. Resetting state.");
         resetObjectState();
         yield this.save();
       }
       if (this._s.v != 1) {
         this._log.warn("Unknown version in state file: " + this._s.v);
         resetObjectState();
         // We explicitly don't save here in the hopes an application re-upgrade
         // comes along and fixes us.
       }
       let regen = false;
       if (!this._s.clientID) {
         this._log.warn("No client ID stored. Generating random ID.");
         regen = true;
       }
       if (typeof(this._s.clientID) != "string") {
         this._log.warn("Client ID is not a string. Regenerating.");
         regen = true;
       }
       if (regen) {
         this._s.clientID = CommonUtils.generateUUID();
         this._s.clientIDVersion = 1;
         yield this.save();
       }
       // Always look for preferences. This ensures that downgrades followed
       // by reupgrades don't result in excessive data loss.
       for (let promise of this._migratePrefs()) {
         yield promise;
       }
     }.bind(this));
   },
   save: function () {
     this._log.info("Writing state file: " + this._filename);
     return CommonUtils.writeJSON(this._s, this._filename);
   },
   addRemoteID: function (id) {
     this._log.warn("Recording new remote ID: " + id);
     this._s.remoteIDs.push(id);
     return this.save();
   },
   removeRemoteID: function (id) {
     return this.removeRemoteIDs(id ? [id] : []);
   },
   removeRemoteIDs: function (ids) {
     if (!ids || !ids.length) {
       this._log.warn("No IDs passed for removal.");
       return Promise.resolve();
     }
     this._log.warn("Removing documents from remote ID list: " + ids);
     let filtered = this._s.remoteIDs.filter((x) => ids.indexOf(x) === -1);
     if (filtered.length == this._s.remoteIDs.length) {
       return Promise.resolve();
     }
     this._s.remoteIDs = filtered;
     return this.save();
   },
   setLastPingDate: function (date) {
     this._s.lastPingTime = date.getTime();
     return this.save();
   },
   updateLastPingAndRemoveRemoteID: function (date, id) {
     return this.updateLastPingAndRemoveRemoteIDs(date, id ? [id] : []);
   },
   updateLastPingAndRemoveRemoteIDs: function (date, ids) {
     if (!ids) {
       return this.setLastPingDate(date);
     }
     this._log.info("Recording last ping time and deleted remote document.");
     this._s.lastPingTime = date.getTime();
     return this.removeRemoteIDs(ids);
   },
   /**
    * Reset the client ID to something else.
    *
    * This fails if remote IDs are stored because changing the client ID
    * while there is remote data will create orphaned records.
    */
   resetClientID: function () {
     if (this.remoteIDs.length) {
       throw new Error("Cannot reset client ID while remote IDs are stored.");
     }
     this._log.warn("Resetting client ID.");
     this._s.clientID = CommonUtils.generateUUID();
     this._s.clientIDVersion = 1;
     return this.save();
   },
   _migratePrefs: function () {
     let prefs = this._reporter._prefs;
     let lastID = prefs.get("lastSubmitID", null);
     let lastPingDate = CommonUtils.getDatePref(prefs, "lastPingTime",
 , this._log, OLDEST_ALLOWED_YEAR);
     // If we have state from prefs, migrate and save it to a file then clear
     // out old prefs.
     if (lastID || (lastPingDate && lastPingDate.getTime() > 0)) {
       this._log.warn("Migrating saved state from preferences.");
       if (lastID) {
         this._log.info("Migrating last saved ID: " + lastID);
         this._s.remoteIDs.push(lastID);
       }
       let ourLast = this.lastPingDate;
       if (lastPingDate && lastPingDate.getTime() > ourLast.getTime()) {
         this._log.info("Migrating last ping time: " + lastPingDate);
         this._s.lastPingTime = lastPingDate.getTime();
       }
       yield this.save();
       prefs.reset(["lastSubmitID", "lastPingTime"]);
     } else {
       this._log.warn("No prefs data found.");
     }
   },
 });
 /**
  * This is the abstract base class of `HealthReporter`. It exists so that
  * we can sanely divide work on platforms where control of Firefox Health
  * Report is outside of Gecko (e.g., Android).
  */
 function AbstractHealthReporter(branch, policy, sessionRecorder) {
   if (!branch.endsWith(".")) {
     throw new Error("Branch must end with a period (.): " + branch);
   }
   if (!policy) {
     throw new Error("Must provide policy to HealthReporter constructor.");
   }
   this._log = Log.repository.getLogger("Services.HealthReport.HealthReporter");
   this._log.info("Initializing health reporter instance against " + branch);
   this._branch = branch;
   this._prefs = new Preferences(branch);
   this._policy = policy;
   this.sessionRecorder = sessionRecorder;
   this._dbName = this._prefs.get("dbName") || DEFAULT_DATABASE_NAME;
   this._storage = null;
   this._storageInProgress = false;
   this._providerManager = null;
   this._providerManagerInProgress = false;
   this._initializeStarted = false;
   this._initialized = false;
   this._initializeHadError = false;
   this._initializedDeferred = Promise.defer();
   this._shutdownRequested = false;
   this._shutdownInitiated = false;
   this._shutdownComplete = false;
   this._shutdownCompleteCallback = null;
   this._errors = [];
   this._lastDailyDate = null;
   // Yes, this will probably run concurrently with remaining constructor work.
   let hasFirstRun = this._prefs.get("service.firstRun", false);
   this._initHistogram = hasFirstRun ? TELEMETRY_INIT : TELEMETRY_INIT_FIRSTRUN;
   this._dbOpenHistogram = hasFirstRun ? TELEMETRY_DB_OPEN : TELEMETRY_DB_OPEN_FIRSTRUN;
 }
 AbstractHealthReporter.prototype = Object.freeze({
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
   /**
    * Whether the service is fully initialized and running.
    *
    * If this is false, it is not safe to call most functions.
    */
   get initialized() {
     return this._initialized;
   },
   /**
    * Initialize the instance.
    *
    * This must be called once after object construction or the instance is
    * useless.
    */
   init: function () {
     if (this._initializeStarted) {
       throw new Error("We have already started initialization.");
     }
     this._initializeStarted = true;
     TelemetryStopwatch.start(this._initHistogram, this);
     this._initializeState().then(this._onStateInitialized.bind(this),
                                  this._onInitError.bind(this));
     return this.onInit();
   },
   //----------------------------------------------------
   // SERVICE CONTROL FUNCTIONS
   //
   // You shouldn't need to call any of these externally.
   //----------------------------------------------------
   _onInitError: function (error) {
     TelemetryStopwatch.cancel(this._initHistogram, this);
     TelemetryStopwatch.cancel(this._dbOpenHistogram, this);
     delete this._initHistogram;
     delete this._dbOpenHistogram;
     this._recordError("Error during initialization", error);
     this._initializeHadError = true;
     this._initiateShutdown();
     this._initializedDeferred.reject(error);
     // FUTURE consider poisoning prototype's functions so calls fail with a
     // useful error message.
   },
   _initializeState: function () {
     return Promise.resolve();
   },
   _onStateInitialized: function () {
     return Task.spawn(function onStateInitialized () {
       try {
         if (!this._state._s.removedOutdatedLastpayload) {
           yield this._deleteOldLastPayload();
           this._state._s.removedOutdatedLastpayload = true;
           // Normally we should save this to a file but it directly conflicts with
           //  the "application re-upgrade" decision in HealthReporterState::init()
           //  which specifically does not save the state to a file.
         }
       } catch (ex) {
         this._log.error("Error deleting last payload: " +
                         CommonUtils.exceptionStr(ex));
       }
       // As soon as we have could storage, we need to register cleanup or
       // else bad things happen on shutdown.
       Services.obs.addObserver(this, "quit-application", false);
       Services.obs.addObserver(this, "profile-before-change", false);
       this._storageInProgress = true;
       TelemetryStopwatch.start(this._dbOpenHistogram, this);
       Metrics.Storage(this._dbName).then(this._onStorageCreated.bind(this),
                                          this._onInitError.bind(this));
     }.bind(this));
   },
   /**
    * Removes the outdated lastpaylaod.json and lastpayload.json.tmp files
    * @see Bug #867902
    * @return a promise for when all the files have been deleted
    */
   _deleteOldLastPayload: function () {
     let paths = [this._state._lastPayloadPath, this._state._lastPayloadPath + ".tmp"];
     return Task.spawn(function removeAllFiles () {
       for (let path of paths) {
         try {
           OS.File.remove(path);
         } catch (ex) {
           if (!ex.becauseNoSuchFile) {
             this._log.error("Exception when removing outdated payload files: " +
                             CommonUtils.exceptionStr(ex));
           }
         }
       }
     }.bind(this));
   },
   // Called when storage has been opened.
   _onStorageCreated: function (storage) {
     TelemetryStopwatch.finish(this._dbOpenHistogram, this);
     delete this._dbOpenHistogram;
     this._log.info("Storage initialized.");
     this._storage = storage;
     this._storageInProgress = false;
     if (this._shutdownRequested) {
       this._initiateShutdown();
       return;
     }
     Task.spawn(this._initializeProviderManager.bind(this))
         .then(this._onProviderManagerInitialized.bind(this),
               this._onInitError.bind(this));
   },
   _initializeProviderManager: function () {
     if (this._collector) {
       throw new Error("Provider manager has already been initialized.");
     }
     this._log.info("Initializing provider manager.");
     this._providerManager = new Metrics.ProviderManager(this._storage);
     this._providerManager.onProviderError = this._recordError.bind(this);
     this._providerManager.onProviderInit = this._initProvider.bind(this);
     this._providerManagerInProgress = true;
     let catString = this._prefs.get("service.providerCategories") || "";
     if (catString.length) {
       for (let category of catString.split(",")) {
         yield this._providerManager.registerProvidersFromCategoryManager(category);
       }
     }
   },
   _onProviderManagerInitialized: function () {
     TelemetryStopwatch.finish(this._initHistogram, this);
     delete this._initHistogram;
     this._log.debug("Provider manager initialized.");
     this._providerManagerInProgress = false;
     if (this._shutdownRequested) {
       this._initiateShutdown();
       return;
     }
     this._log.info("HealthReporter started.");
     this._initialized = true;
     Services.obs.addObserver(this, "idle-daily", false);
     // If upload is not enabled, ensure daily collection works. If upload
     // is enabled, this will be performed as part of upload.
     //
     // This is important because it ensures about:healthreport contains
     // longitudinal data even if upload is disabled. Having about:healthreport
     // provide useful info even if upload is disabled was a core launch
     // requirement.
     //
     // We do not catch changes to the backing pref. So, if the session lasts
     // many days, we may fail to collect. However, most sessions are short and
     // this code will likely be refactored as part of splitting up policy to
     // serve Android. So, meh.
     if (!this._policy.healthReportUploadEnabled) {
       this._log.info("Upload not enabled. Scheduling daily collection.");
       // Since the timer manager is a singleton and there could be multiple
       // HealthReporter instances, we need to encode a unique identifier in
       // the timer ID.
       try {
         let timerName = this._branch.replace(".", "-", "g") + "lastDailyCollection";
         let tm = Cc["@mozilla.org/updates/timer-manager;1"]
                    .getService(Ci.nsIUpdateTimerManager);
         tm.registerTimer(timerName, this.collectMeasurements.bind(this),
 * 60 * 60);
       } catch (ex) {
         this._log.error("Error registering collection timer: " +
                         CommonUtils.exceptionStr(ex));
       }
     }
     // Clean up caches and reduce memory usage.
     this._storage.compact();
     this._initializedDeferred.resolve(this);
   },
   // nsIObserver to handle shutdown.
   observe: function (subject, topic, data) {
     switch (topic) {
       case "quit-application":
         Services.obs.removeObserver(this, "quit-application");
         this._initiateShutdown();
         break;
       case "profile-before-change":
         Services.obs.removeObserver(this, "profile-before-change");
         this._waitForShutdown();
         break;
       case "idle-daily":
         this._performDailyMaintenance();
         break;
     }
   },
   _initiateShutdown: function () {
     // Ensure we only begin the main shutdown sequence once.
     if (this._shutdownInitiated) {
       this._log.warn("Shutdown has already been initiated. No-op.");
       return;
     }
     this._log.info("Request to shut down.");
     this._initialized = false;
     this._shutdownRequested = true;
     if (this._initializeHadError) {
       this._log.warn("Initialization had error. Shutting down immediately.");
     } else {
       if (this._providerManagerInProcess) {
         this._log.warn("Provider manager is in progress of initializing. " +
                        "Waiting to finish.");
         return;
       }
       // If storage is in the process of initializing, we need to wait for it
       // to finish before continuing. The initialization process will call us
       // again once storage has initialized.
       if (this._storageInProgress) {
         this._log.warn("Storage is in progress of initializing. Waiting to finish.");
         return;
       }
     }
     this._log.warn("Initiating main shutdown procedure.");
     // Everything from here must only be performed once or else race conditions
     // could occur.
     TelemetryStopwatch.start(TELEMETRY_SHUTDOWN, this);
     this._shutdownInitiated = true;
     // We may not have registered the observer yet. If not, this will
     // throw.
     try {
       Services.obs.removeObserver(this, "idle-daily");
     } catch (ex) { }
     if (this._providerManager) {
       let onShutdown = this._onProviderManagerShutdown.bind(this);
       Task.spawn(this._shutdownProviderManager.bind(this))
           .then(onShutdown, onShutdown);
       return;
     }
     this._log.warn("Don't have provider manager. Proceeding to storage shutdown.");
     this._shutdownStorage();
   },
   _shutdownProviderManager: function () {
     this._log.info("Shutting down provider manager.");
     for (let provider of this._providerManager.providers) {
       try {
         yield provider.shutdown();
       } catch (ex) {
         this._log.warn("Error when shutting down provider: " +
                        CommonUtils.exceptionStr(ex));
       }
     }
   },
   _onProviderManagerShutdown: function () {
     this._log.info("Provider manager shut down.");
     this._providerManager = null;
     this._shutdownStorage();
   },
   _shutdownStorage: function () {
     if (!this._storage) {
       this._onShutdownComplete();
     }
     this._log.info("Shutting down storage.");
     let onClose = this._onStorageClose.bind(this);
     this._storage.close().then(onClose, onClose);
   },
   _onStorageClose: function (error) {
     this._log.info("Storage has been closed.");
     if (error) {
       this._log.warn("Error when closing storage: " +
                      CommonUtils.exceptionStr(error));
     }
     this._storage = null;
     this._onShutdownComplete();
   },
   _onShutdownComplete: function () {
     this._log.warn("Shutdown complete.");
     this._shutdownComplete = true;
     TelemetryStopwatch.finish(TELEMETRY_SHUTDOWN, this);
     if (this._shutdownCompleteCallback) {
       this._shutdownCompleteCallback();
     }
   },
   _waitForShutdown: function () {
     if (this._shutdownComplete) {
       return;
     }
     TelemetryStopwatch.start(TELEMETRY_SHUTDOWN_DELAY, this);
     try {
       this._shutdownCompleteCallback = Async.makeSpinningCallback();
       this._shutdownCompleteCallback.wait();
       this._shutdownCompleteCallback = null;
     } finally {
       TelemetryStopwatch.finish(TELEMETRY_SHUTDOWN_DELAY, this);
     }
   },
   /**
    * Convenience method to shut down the instance.
    *
    * This should *not* be called outside of tests.
    */
   _shutdown: function () {
     this._initiateShutdown();
     this._waitForShutdown();
   },
   /**
    * Return a promise that is resolved once the service has been initialized.
    */
   onInit: function () {
     if (this._initializeHadError) {
       throw new Error("Service failed to initialize.");
     }
     if (this._initialized) {
       return CommonUtils.laterTickResolvingPromise(this);
     }
     return this._initializedDeferred.promise;
   },
   _performDailyMaintenance: function () {
     this._log.info("Request to perform daily maintenance.");
     if (!this._initialized) {
       return;
     }
     let now = new Date();
     let cutoff = new Date(now.getTime() - MILLISECONDS_PER_DAY * (DAYS_IN_PAYLOAD - 1));
     // The operation is enqueued and put in a transaction by the storage module.
     this._storage.pruneDataBefore(cutoff);
   },
   //--------------------
   // Provider Management
   //--------------------
   /**
    * Obtain a provider from its name.
    *
    * This will only return providers that are currently initialized. If
    * a provider is lazy initialized (like pull-only providers) this
    * will likely not return anything.
    */
   getProvider: function (name) {
     if (!this._providerManager) {
       return null;
     }
     return this._providerManager.getProvider(name);
   },
   _initProvider: function (provider) {
     provider.healthReporter = this;
   },
   /**
    * Record an exception for reporting in the payload.
    *
    * A side effect is the exception is logged.
    *
    * Note that callers need to be extra sensitive about ensuring personal
    * or otherwise private details do not leak into this. All of the user data
    * on the stack in FHR code should be limited to data we were collecting with
    * the intent to submit. So, it is covered under the user's consent to use
    * the feature.
    *
    * @param message
    *        (string) Human readable message describing error.
    * @param ex
    *        (Error) The error that should be captured.
    */
   _recordError: function (message, ex) {
     let recordMessage = message;
     let logMessage = message;
     if (ex) {
       recordMessage += ": " + CommonUtils.exceptionStr(ex);
       logMessage += ": " + CommonUtils.exceptionStr(ex);
     }
     // Scrub out potentially identifying information from strings that could
     // make the payload.
     let appData = Services.dirsvc.get("UAppData", Ci.nsIFile);
     let profile = Services.dirsvc.get("ProfD", Ci.nsIFile);
     let appDataURI = Services.io.newFileURI(appData);
     let profileURI = Services.io.newFileURI(profile);
     // Order of operation is important here. We do the URI before the path version
     // because the path may be a subset of the URI. We also have to check for the case
     // where UAppData is underneath the profile directory (or vice-versa) so we
     // don't substitute incomplete strings.
     function replace(uri, path, thing) {
       // Try is because .spec can throw on invalid URI.
       try {
         recordMessage = recordMessage.replace(uri.spec, '<' + thing + 'URI>', 'g');
       } catch (ex) { }
       recordMessage = recordMessage.replace(path, '<' + thing + 'Path>', 'g');
     }
     if (appData.path.contains(profile.path)) {
       replace(appDataURI, appData.path, 'AppData');
       replace(profileURI, profile.path, 'Profile');
     } else {
       replace(profileURI, profile.path, 'Profile');
       replace(appDataURI, appData.path, 'AppData');
     }
     this._log.warn(logMessage);
     this._errors.push(recordMessage);
   },
   /**
    * Collect all measurements for all registered providers.
    */
   collectMeasurements: function () {
     if (!this._initialized) {
       return Promise.reject(new Error("Not initialized."));
     }
     return Task.spawn(function doCollection() {
       yield this._providerManager.ensurePullOnlyProvidersRegistered();
       try {
         TelemetryStopwatch.start(TELEMETRY_COLLECT_CONSTANT, this);
         yield this._providerManager.collectConstantData();
         TelemetryStopwatch.finish(TELEMETRY_COLLECT_CONSTANT, this);
       } catch (ex) {
         TelemetryStopwatch.cancel(TELEMETRY_COLLECT_CONSTANT, this);
         this._log.warn("Error collecting constant data: " +
                        CommonUtils.exceptionStr(ex));
       }
       // Daily data is collected if it hasn't yet been collected this
       // application session or if it has been more than a day since the
       // last collection. This means that providers could see many calls to
       // collectDailyData per calendar day. However, this collection API
       // makes no guarantees about limits. The alternative would involve
       // recording state. The simpler implementation prevails for now.
       if (!this._lastDailyDate ||
           Date.now() - this._lastDailyDate > MILLISECONDS_PER_DAY) {
         try {
           TelemetryStopwatch.start(TELEMETRY_COLLECT_DAILY, this);
           this._lastDailyDate = new Date();
           yield this._providerManager.collectDailyData();
           TelemetryStopwatch.finish(TELEMETRY_COLLECT_DAILY, this);
         } catch (ex) {
           TelemetryStopwatch.cancel(TELEMETRY_COLLECT_DAILY, this);
           this._log.warn("Error collecting daily data from providers: " +
                          CommonUtils.exceptionStr(ex));
         }
       }
       yield this._providerManager.ensurePullOnlyProvidersUnregistered();
       // Flush gathered data to disk. This will incur an fsync. But, if
       // there is ever a time we want to persist data to disk, it's
       // after a massive collection.
       try {
         TelemetryStopwatch.start(TELEMETRY_COLLECT_CHECKPOINT, this);
         yield this._storage.checkpoint();
         TelemetryStopwatch.finish(TELEMETRY_COLLECT_CHECKPOINT, this);
       } catch (ex) {
         TelemetryStopwatch.cancel(TELEMETRY_COLLECT_CHECKPOINT, this);
         throw ex;
       }
       throw new Task.Result();
     }.bind(this));
   },
   /**
    * Helper function to perform data collection and obtain the JSON payload.
    *
    * If you are looking for an up-to-date snapshot of FHR data that pulls in
    * new data since the last upload, this is how you should obtain it.
    *
    * @param asObject
    *        (bool) Whether to resolve an object or JSON-encoded string of that
    *        object (the default).
    *
    * @return Promise<Object | string>
    */
   collectAndObtainJSONPayload: function (asObject=false) {
     if (!this._initialized) {
       return Promise.reject(new Error("Not initialized."));
     }
     return Task.spawn(function collectAndObtain() {
       yield this._storage.setAutoCheckpoint(0);
       yield this._providerManager.ensurePullOnlyProvidersRegistered();
       let payload;
       let error;
       try {
         yield this.collectMeasurements();
         payload = yield this.getJSONPayload(asObject);
       } catch (ex) {
         error = ex;
         this._collectException("Error collecting and/or retrieving JSON payload",
                                ex);
       } finally {
         yield this._providerManager.ensurePullOnlyProvidersUnregistered();
         yield this._storage.setAutoCheckpoint(1);
         if (error) {
           throw error;
         }
       }
       // We hold off throwing to ensure that behavior between finally
       // and generators and throwing is sane.
       throw new Task.Result(payload);
     }.bind(this));
   },
   /**
    * Obtain the JSON payload for currently-collected data.
    *
    * The payload only contains data that has been recorded to FHR. Some
    * providers may have newer data available. If you want to ensure you
    * have all available data, call `collectAndObtainJSONPayload`
    * instead.
    *
    * @param asObject
    *        (bool) Whether to return an object or JSON encoding of that
    *        object (the default).
    *
    * @return Promise<string|object>
    */
   getJSONPayload: function (asObject=false) {
     TelemetryStopwatch.start(TELEMETRY_GENERATE_PAYLOAD, this);
     let deferred = Promise.defer();
     Task.spawn(this._getJSONPayload.bind(this, this._now(), asObject)).then(
       function onResult(result) {
         TelemetryStopwatch.finish(TELEMETRY_GENERATE_PAYLOAD, this);
         deferred.resolve(result);
       }.bind(this),
       function onError(error) {
         TelemetryStopwatch.cancel(TELEMETRY_GENERATE_PAYLOAD, this);
         deferred.reject(error);
       }.bind(this)
     );
     return deferred.promise;
   },
   _getJSONPayload: function (now, asObject=false) {
     let pingDateString = this._formatDate(now);
     this._log.info("Producing JSON payload for " + pingDateString);
     // May not be present if we are generating as a result of init error.
     if (this._providerManager) {
       yield this._providerManager.ensurePullOnlyProvidersRegistered();
     }
     let o = {
       version: 2,
       clientID: this._state.clientID,
       clientIDVersion: this._state.clientIDVersion,
       thisPingDate: pingDateString,
       geckoAppInfo: this.obtainAppInfo(this._log),
       data: {last: {}, days: {}},
     };
     let outputDataDays = o.data.days;
     // Guard here in case we don't track this (e.g., on Android).
     let lastPingDate = this.lastPingDate;
     if (lastPingDate && lastPingDate.getTime() > 0) {
       o.lastPingDate = this._formatDate(lastPingDate);
     }
     // We can still generate a payload even if we're not initialized.
     // This is to facilitate error upload on init failure.
     if (this._initialized) {
       for (let provider of this._providerManager.providers) {
         let providerName = provider.name;
         let providerEntry = {
           measurements: {},
         };
         // Measurement name to recorded version.
         let lastVersions = {};
         // Day string to mapping of measurement name to recorded version.
         let dayVersions = {};
         for (let [measurementKey, measurement] of provider.measurements) {
           let name = providerName + "." + measurement.name;
           let version = measurement.version;
           let serializer;
           try {
             // The measurement is responsible for returning a serializer which
             // is aware of the measurement version.
             serializer = measurement.serializer(measurement.SERIALIZE_JSON);
           } catch (ex) {
             this._recordError("Error obtaining serializer for measurement: " +
                               name, ex);
             continue;
           }
           let data;
           try {
             data = yield measurement.getValues();
           } catch (ex) {
             this._recordError("Error obtaining data for measurement: " + name,
                               ex);
             continue;
           }
           if (data.singular.size) {
             try {
               let serialized = serializer.singular(data.singular);
               if (serialized) {
                 // Only replace the existing data if there is no data or if our
                 // version is newer than the old one.
                 if (!(name in o.data.last) || version > lastVersions[name]) {
                   o.data.last[name] = serialized;
                   lastVersions[name] = version;
                 }
               }
             } catch (ex) {
               this._recordError("Error serializing singular data: " + name,
                                 ex);
               continue;
             }
           }
           let dataDays = data.days;
           for (let i = 0; i < DAYS_IN_PAYLOAD; i++) {
             let date = new Date(now.getTime() - i * MILLISECONDS_PER_DAY);
             if (!dataDays.hasDay(date)) {
               continue;
             }
             let dateFormatted = this._formatDate(date);
             try {
               let serialized = serializer.daily(dataDays.getDay(date));
               if (!serialized) {
                 continue;
               }
               if (!(dateFormatted in outputDataDays)) {
                 outputDataDays[dateFormatted] = {};
               }
               // This needs to be separate because dayVersions is provider
               // specific and gets blown away in a loop while outputDataDays
               // is persistent.
               if (!(dateFormatted in dayVersions)) {
                 dayVersions[dateFormatted] = {};
               }
               if (!(name in outputDataDays[dateFormatted]) ||
                   version > dayVersions[dateFormatted][name]) {
                 outputDataDays[dateFormatted][name] = serialized;
                 dayVersions[dateFormatted][name] = version;
               }
             } catch (ex) {
               this._recordError("Error populating data for day: " + name, ex);
               continue;
             }
           }
         }
       }
     } else {
       o.notInitialized = 1;
       this._log.warn("Not initialized. Sending report with only error info.");
     }
     if (this._errors.length) {
       o.errors = this._errors.slice(0, 20);
     }
     if (this._initialized) {
       this._storage.compact();
     }
     if (!asObject) {
       TelemetryStopwatch.start(TELEMETRY_JSON_PAYLOAD_SERIALIZE, this);
       o = JSON.stringify(o);
       TelemetryStopwatch.finish(TELEMETRY_JSON_PAYLOAD_SERIALIZE, this);
     }
     if (this._providerManager) {
       yield this._providerManager.ensurePullOnlyProvidersUnregistered();
     }
     throw new Task.Result(o);
   },
   _now: function _now() {
     return new Date();
   },
   // These are stolen from AppInfoProvider.
   appInfoVersion: 1,
   appInfoFields: {
     // From nsIXULAppInfo.
     vendor: "vendor",
     name: "name",
     id: "ID",
     version: "version",
     appBuildID: "appBuildID",
     platformVersion: "platformVersion",
     platformBuildID: "platformBuildID",
     // From nsIXULRuntime.
     os: "OS",
     xpcomabi: "XPCOMABI",
   },
   /**
    * Statically return a bundle of app info data, a subset of that produced by
    * AppInfoProvider._populateConstants. This allows us to more usefully handle
    * payloads that, due to error, contain no data.
    *
    * Returns a very sparse object if Services.appinfo is unavailable.
    */
   obtainAppInfo: function () {
     let out = {"_v": this.appInfoVersion};
     try {
       let ai = Services.appinfo;
       for (let [k, v] in Iterator(this.appInfoFields)) {
         out[k] = ai[v];
       }
     } catch (ex) {
       this._log.warn("Could not obtain Services.appinfo: " +
                      CommonUtils.exceptionStr(ex));
     }
     try {
       out["updateChannel"] = UpdateChannel.get();
     } catch (ex) {
       this._log.warn("Could not obtain update channel: " +
                      CommonUtils.exceptionStr(ex));
     }
     return out;
   },
 });
 /**
  * HealthReporter and its abstract superclass coordinate collection and
  * submission of health report metrics.
  *
  * This is the main type for Firefox Health Report on desktop. It glues all the
  * lower-level components (such as collection and submission) together.
  *
  * An instance of this type is created as an XPCOM service. See
  * DataReportingService.js and
  * DataReporting.manifest/HealthReportComponents.manifest.
  *
  * It is theoretically possible to have multiple instances of this running
  * in the application. For example, this type may one day handle submission
  * of telemetry data as well. However, there is some moderate coupling between
  * this type and *the* Firefox Health Report (e.g., the policy). This could
  * be abstracted if needed.
  *
  * Note that `AbstractHealthReporter` exists to allow for Firefox Health Report
  * to be more easily implemented on platforms where a separate controlling
  * layer is responsible for payload upload and deletion.
  *
  * IMPLEMENTATION NOTES
  * ====================
  *
  * These notes apply to the combination of `HealthReporter` and
  * `AbstractHealthReporter`.
  *
  * Initialization and shutdown are somewhat complicated and worth explaining
  * in extra detail.
  *
  * The complexity is driven by the requirements of SQLite connection management.
  * Once you have a SQLite connection, it isn't enough to just let the
  * application shut down. If there is an open connection or if there are
  * outstanding SQL statements come XPCOM shutdown time, Storage will assert.
  * On debug builds you will crash. On release builds you will get a shutdown
  * hang. This must be avoided!
  *
  * During initialization, the second we create a SQLite connection (via
  * Metrics.Storage) we register observers for application shutdown. The
  * "quit-application" notification initiates our shutdown procedure. The
  * subsequent "profile-do-change" notification ensures it has completed.
  *
  * The handler for "profile-do-change" may result in event loop spinning. This
  * is because of race conditions between our shutdown code and application
  * shutdown.
  *
  * All of our shutdown routines are async. There is the potential that these
  * async functions will not complete before XPCOM shutdown. If they don't
  * finish in time, we could get assertions in Storage. Our solution is to
  * initiate storage early in the shutdown cycle ("quit-application").
  * Hopefully all the async operations have completed by the time we reach
  * "profile-do-change." If so, great. If not, we spin the event loop until
  * they have completed, avoiding potential race conditions.
  *
  * @param branch
  *        (string) The preferences branch to use for state storage. The value
  *        must end with a period (.).
  *
  * @param policy
  *        (HealthReportPolicy) Policy driving execution of HealthReporter.
  */
 this.HealthReporter = function (branch, policy, sessionRecorder, stateLeaf=null) {
   this._stateLeaf = stateLeaf;
   this._uploadInProgress = false;
   AbstractHealthReporter.call(this, branch, policy, sessionRecorder);
   if (!this.serverURI) {
     throw new Error("No server URI defined. Did you forget to define the pref?");
   }
   if (!this.serverNamespace) {
     throw new Error("No server namespace defined. Did you forget a pref?");
   }
   this._state = new HealthReporterState(this);
 }
 this.HealthReporter.prototype = Object.freeze({
   __proto__: AbstractHealthReporter.prototype,
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
   get lastSubmitID() {
     return this._state.lastSubmitID;
   },
   /**
    * When we last successfully submitted data to the server.
    *
    * This is sent as part of the upload. This is redundant with similar data
    * in the policy because we like the modules to be loosely coupled and the
    * similar data in the policy is only used for forensic purposes.
    */
   get lastPingDate() {
     return this._state.lastPingDate;
   },
   /**
    * The base URI of the document server to which to submit data.
    *
    * This is typically a Bagheera server instance. It is the URI up to but not
    * including the version prefix. e.g. https://data.metrics.mozilla.com/
    */
   get serverURI() {
     return this._prefs.get("documentServerURI", null);
   },
   set serverURI(value) {
     if (!value) {
       throw new Error("serverURI must have a value.");
     }
     if (typeof(value) != "string") {
       throw new Error("serverURI must be a string: " + value);
     }
     this._prefs.set("documentServerURI", value);
   },
   /**
    * The namespace on the document server to which we will be submitting data.
    */
   get serverNamespace() {
     return this._prefs.get("documentServerNamespace", "metrics");
   },
   set serverNamespace(value) {
     if (!value) {
       throw new Error("serverNamespace must have a value.");
     }
     if (typeof(value) != "string") {
       throw new Error("serverNamespace must be a string: " + value);
     }
     this._prefs.set("documentServerNamespace", value);
   },
   /**
    * Whether this instance will upload data to a server.
    */
   get willUploadData() {
     return this._policy.dataSubmissionPolicyAccepted &&
            this._policy.healthReportUploadEnabled;
   },
   /**
    * Whether remote data is currently stored.
    *
    * @return bool
    */
   haveRemoteData: function () {
     return !!this._state.lastSubmitID;
   },
   /**
    * Called to initiate a data upload.
    *
    * The passed argument is a `DataSubmissionRequest` from policy.jsm.
    */
   requestDataUpload: function (request) {
     if (!this._initialized) {
       return Promise.reject(new Error("Not initialized."));
     }
     return Task.spawn(function doUpload() {
       yield this._providerManager.ensurePullOnlyProvidersRegistered();
       try {
         yield this.collectMeasurements();
         try {
           yield this._uploadData(request);
         } catch (ex) {
           this._onSubmitDataRequestFailure(ex);
         }
       } finally {
         yield this._providerManager.ensurePullOnlyProvidersUnregistered();
       }
     }.bind(this));
   },
   /**
    * Request that server data be deleted.
    *
    * If deletion is scheduled to occur immediately, a promise will be returned
    * that will be fulfilled when the deletion attempt finishes. Otherwise,
    * callers should poll haveRemoteData() to determine when remote data is
    * deleted.
    */
   requestDeleteRemoteData: function (reason) {
     if (!this.haveRemoteData()) {
       return;
     }
     return this._policy.deleteRemoteData(reason);
   },
   _initializeState: function() {
     return this._state.init();
   },
   /**
    * Override default handler to incur an upload describing the error.
    */
   _onInitError: function (error) {
     // Need to capture this before we call the parent else it's always
     // set.
     let inShutdown = this._shutdownRequested;
     let result;
     try {
       result = AbstractHealthReporter.prototype._onInitError.call(this, error);
     } catch (ex) {
       this._log.error("Error when calling _onInitError: " +
                       CommonUtils.exceptionStr(ex));
     }
     // This bypasses a lot of the checks in policy, such as respect for
     // backoff. We should arguably not do this. However, reporting
     // startup errors is important. And, they should not occur with much
     // frequency in the wild. So, it shouldn't be too big of a deal.
     if (!inShutdown &&
         this._policy.ensureNotifyResponse(new Date()) &&
         this._policy.healthReportUploadEnabled) {
       // We don't care about what happens to this request. It's best
       // effort.
       let request = {
         onNoDataAvailable: function () {},
         onSubmissionSuccess: function () {},
         onSubmissionFailureSoft: function () {},
         onSubmissionFailureHard: function () {},
         onUploadInProgress: function () {},
       };
       this._uploadData(request);
     }
     return result;
   },
   _onBagheeraResult: function (request, isDelete, date, result) {
     this._log.debug("Received Bagheera result.");
     return Task.spawn(function onBagheeraResult() {
       let hrProvider = this.getProvider("org.mozilla.healthreport");
       if (!result.transportSuccess) {
         // The built-in provider may not be initialized if this instance failed
         // to initialize fully.
         if (hrProvider && !isDelete) {
           hrProvider.recordEvent("uploadTransportFailure", date);
         }
         request.onSubmissionFailureSoft("Network transport error.");
         throw new Task.Result(false);
       }
       if (!result.serverSuccess) {
         if (hrProvider && !isDelete) {
           hrProvider.recordEvent("uploadServerFailure", date);
         }
         request.onSubmissionFailureHard("Server failure.");
         throw new Task.Result(false);
       }
       if (hrProvider && !isDelete) {
         hrProvider.recordEvent("uploadSuccess", date);
       }
       if (isDelete) {
         this._log.warn("Marking delete as successful.");
         yield this._state.removeRemoteIDs([result.id]);
       } else {
         this._log.warn("Marking upload as successful.");
         yield this._state.updateLastPingAndRemoveRemoteIDs(date, result.deleteIDs);
       }
       request.onSubmissionSuccess(this._now());
       throw new Task.Result(true);
     }.bind(this));
   },
   _onSubmitDataRequestFailure: function (error) {
     this._log.error("Error processing request to submit data: " +
                     CommonUtils.exceptionStr(error));
   },
   _formatDate: function (date) {
     // Why, oh, why doesn't JS have a strftime() equivalent?
     return date.toISOString().substr(0, 10);
   },
   _uploadData: function (request) {
     // Under ideal circumstances, clients should never race to this
     // function. However, server logs have observed behavior where
     // racing to this function could be a cause. So, this lock was
     // instituted.
     if (this._uploadInProgress) {
       this._log.warn("Upload requested but upload already in progress.");
       let provider = this.getProvider("org.mozilla.healthreport");
       let promise = provider.recordEvent("uploadAlreadyInProgress");
       request.onUploadInProgress("Upload already in progress.");
       return promise;
     }
     let id = CommonUtils.generateUUID();
     this._log.info("Uploading data to server: " + this.serverURI + " " +
                    this.serverNamespace + ":" + id);
     let client = new BagheeraClient(this.serverURI);
     let now = this._now();
     return Task.spawn(function doUpload() {
       try {
         // The test for upload locking monkeypatches getJSONPayload.
         // If the next two lines change, be sure to verify the test is
         // accurate!
         this._uploadInProgress = true;
         let payload = yield this.getJSONPayload();
         let histogram = Services.telemetry.getHistogramById(TELEMETRY_PAYLOAD_SIZE_UNCOMPRESSED);
         histogram.add(payload.length);
         let lastID = this.lastSubmitID;
         yield this._state.addRemoteID(id);
         let hrProvider = this.getProvider("org.mozilla.healthreport");
         if (hrProvider) {
           let event = lastID ? "continuationUploadAttempt"
                              : "firstDocumentUploadAttempt";
           hrProvider.recordEvent(event, now);
         }
         TelemetryStopwatch.start(TELEMETRY_UPLOAD, this);
         let result;
         try {
           let options = {
             deleteIDs: this._state.remoteIDs.filter((x) => { return x != id; }),
             telemetryCompressed: TELEMETRY_PAYLOAD_SIZE_COMPRESSED,
           };
           result = yield client.uploadJSON(this.serverNamespace, id, payload,
                                            options);
           TelemetryStopwatch.finish(TELEMETRY_UPLOAD, this);
         } catch (ex) {
           TelemetryStopwatch.cancel(TELEMETRY_UPLOAD, this);
           if (hrProvider) {
             hrProvider.recordEvent("uploadClientFailure", now);
           }
           throw ex;
         }
         yield this._onBagheeraResult(request, false, now, result);
       } finally {
         this._uploadInProgress = false;
       }
     }.bind(this));
   },
   /**
    * Request deletion of remote data.
    *
    * @param request
    *        (DataSubmissionRequest) Tracks progress of this request.
    */
   deleteRemoteData: function (request) {
     if (!this._state.lastSubmitID) {
       this._log.info("Received request to delete remote data but no data stored.");
       request.onNoDataAvailable();
       return;
     }
     this._log.warn("Deleting remote data.");
     let client = new BagheeraClient(this.serverURI);
     return Task.spawn(function* doDelete() {
       try {
         let result = yield client.deleteDocument(this.serverNamespace,
                                                  this.lastSubmitID);
         yield this._onBagheeraResult(request, true, this._now(), result);
       } catch (ex) {
         this._log.error("Error processing request to delete data: " +
                         CommonUtils.exceptionStr(error));
       } finally {
         // If we don't have any remote documents left, nuke the ID.
         // This is done for privacy reasons. Why preserve the ID if we
         // don't need to?
         if (!this.haveRemoteData()) {
           yield this._state.resetClientID();
         }
       }
     }.bind(this));
   },
 });

The Tor Browser / annotate

services/healthreport/healthreporter.jsm@97036ab72558 (annotated)

services/healthreport/healthreporter.jsm