The Tor Browser: services/metrics/storage.jsm@b8a032363ba2 (annotated)

services/metrics/storage.jsm@b8a032363ba2 (annotated)

services/metrics/storage.jsm

Thu, 22 Jan 2015 13:21:57 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 22 Jan 2015 13:21:57 +0100
branch: TOR_BUG_9701
changeset 15: b8a032363ba2
permissions: -rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 "use strict";
 #ifndef MERGED_COMPARTMENT
 this.EXPORTED_SYMBOLS = [
   "DailyValues",
   "MetricsStorageBackend",
   "dateToDays",
   "daysToDate",
 ];
 const {utils: Cu} = Components;
 const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
 #endif
 Cu.import("resource://gre/modules/Promise.jsm");
 Cu.import("resource://gre/modules/Sqlite.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://services-common/utils.js");
 // These do not account for leap seconds. Meh.
 function dateToDays(date) {
   return Math.floor(date.getTime() / MILLISECONDS_PER_DAY);
 }
 function daysToDate(days) {
   return new Date(days * MILLISECONDS_PER_DAY);
 }
 /**
  * Represents a collection of per-day values.
  *
  * This is a proxy around a Map which can transparently round Date instances to
  * their appropriate key.
  *
  * This emulates Map by providing .size and iterator support. Note that keys
  * from the iterator are Date instances corresponding to midnight of the start
  * of the day. get(), has(), and set() are modeled as getDay(), hasDay(), and
  * setDay(), respectively.
  *
  * All days are defined in terms of UTC (as opposed to local time).
  */
 this.DailyValues = function () {
   this._days = new Map();
 };
 DailyValues.prototype = Object.freeze({
   __iterator__: function () {
     for (let [k, v] of this._days) {
       yield [daysToDate(k), v];
     }
   },
   get size() {
     return this._days.size;
   },
   hasDay: function (date) {
     return this._days.has(dateToDays(date));
   },
   getDay: function (date) {
     return this._days.get(dateToDays(date));
   },
   setDay: function (date, value) {
     this._days.set(dateToDays(date), value);
   },
   appendValue: function (date, value) {
     let key = dateToDays(date);
     if (this._days.has(key)) {
       return this._days.get(key).push(value);
     }
     this._days.set(key, [value]);
   },
 });
 /**
  * DATABASE INFO
  * =============
  *
  * We use a SQLite database as the backend for persistent storage of metrics
  * data.
  *
  * Every piece of recorded data is associated with a measurement. A measurement
  * is an entity with a name and version. Each measurement is associated with a
  * named provider.
  *
  * When the metrics system is initialized, we ask providers (the entities that
  * emit data) to configure the database for storage of their data. They tell
  * storage what their requirements are. For example, they'll register
  * named daily counters associated with specific measurements.
  *
  * Recorded data is stored differently depending on the requirements for
  * storing it. We have facilities for storing the following classes of data:
  *
  *  1) Counts of event/field occurrences aggregated by day.
  *  2) Discrete values of fields aggregated by day.
  *  3) Discrete values of fields aggregated by day max 1 per day (last write
  *     wins).
  *  4) Discrete values of fields max 1 (last write wins).
  *
  * Most data is aggregated per day mainly for privacy reasons. This does throw
  * away potentially useful data. But, it's not currently used, so there is no
  * need to keep the granular information.
  *
  * Database Schema
  * ---------------
  *
  * This database contains the following tables:
  *
  *   providers -- Maps provider string name to an internal ID.
  *   provider_state -- Holds opaque persisted state for providers.
  *   measurements -- Holds the set of known measurements (name, version,
  *     provider tuples).
  *   types -- The data types that can be stored in measurements/fields.
  *   fields -- Describes entities that occur within measurements.
  *   daily_counters -- Holds daily-aggregated counts of events. Each row is
  *     associated with a field and a day.
  *   daily_discrete_numeric -- Holds numeric values for fields grouped by day.
  *     Each row contains a discrete value associated with a field that occurred
  *     on a specific day. There can be multiple rows per field per day.
  *   daily_discrete_text -- Holds text values for fields grouped by day. Each
  *     row contains a discrete value associated with a field that occurred on a
  *     specific day.
  *   daily_last_numeric -- Holds numeric values where the last encountered
  *     value for a given day is retained.
  *   daily_last_text -- Like daily_last_numeric except for text values.
  *   last_numeric -- Holds the most recent value for a numeric field.
  *   last_text -- Like last_numeric except for text fields.
  *
  * Notes
  * -----
  *
  * It is tempting to use SQLite's julianday() function to store days that
  * things happened. However, a Julian Day begins at *noon* in 4714 B.C. This
  * results in weird half day offsets from UNIX time. So, we instead store
  * number of days since UNIX epoch, not Julian.
  */
 /**
  * All of our SQL statements are stored in a central mapping so they can easily
  * be audited for security, perf, etc.
  */
 const SQL = {
   // Create the providers table.
   createProvidersTable: "\
 CREATE TABLE providers (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   name TEXT, \
   UNIQUE (name) \
 )",
   createProviderStateTable: "\
 CREATE TABLE provider_state (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   provider_id INTEGER, \
   name TEXT, \
   VALUE TEXT, \
   UNIQUE (provider_id, name), \
   FOREIGN KEY (provider_id) REFERENCES providers(id) ON DELETE CASCADE\
 )",
   createProviderStateProviderIndex: "\
 CREATE INDEX i_provider_state_provider_id ON provider_state (provider_id)",
   createMeasurementsTable: "\
 CREATE TABLE measurements (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   provider_id INTEGER, \
   name TEXT, \
   version INTEGER, \
   UNIQUE (provider_id, name, version), \
   FOREIGN KEY (provider_id) REFERENCES providers(id) ON DELETE CASCADE\
 )",
   createMeasurementsProviderIndex: "\
 CREATE INDEX i_measurements_provider_id ON measurements (provider_id)",
   createMeasurementsView: "\
 CREATE VIEW v_measurements AS \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version \
   FROM providers, measurements \
   WHERE \
     measurements.provider_id = providers.id",
   createTypesTable: "\
 CREATE TABLE types (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   name TEXT, \
   UNIQUE (name)\
 )",
   createFieldsTable: "\
 CREATE TABLE fields (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   measurement_id INTEGER, \
   name TEXT, \
   value_type INTEGER , \
   UNIQUE (measurement_id, name), \
   FOREIGN KEY (measurement_id) REFERENCES measurements(id) ON DELETE CASCADE \
   FOREIGN KEY (value_type) REFERENCES types(id) ON DELETE CASCADE \
 )",
   createFieldsMeasurementIndex: "\
 CREATE INDEX i_fields_measurement_id ON fields (measurement_id)",
   createFieldsView: "\
 CREATE VIEW v_fields AS \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     types.id AS type_id, \
     types.name AS type_name \
   FROM providers, measurements, fields, types \
   WHERE \
     fields.measurement_id = measurements.id \
     AND measurements.provider_id = providers.id \
     AND fields.value_type = types.id",
   createDailyCountersTable: "\
 CREATE TABLE daily_counters (\
   field_id INTEGER, \
   day INTEGER, \
   value INTEGER, \
   UNIQUE(field_id, day), \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createDailyCountersFieldIndex: "\
 CREATE INDEX i_daily_counters_field_id ON daily_counters (field_id)",
   createDailyCountersDayIndex: "\
 CREATE INDEX i_daily_counters_day ON daily_counters (day)",
   createDailyCountersView: "\
 CREATE VIEW v_daily_counters AS SELECT \
   providers.id AS provider_id, \
   providers.name AS provider_name, \
   measurements.id AS measurement_id, \
   measurements.name AS measurement_name, \
   measurements.version AS measurement_version, \
   fields.id AS field_id, \
   fields.name AS field_name, \
   daily_counters.day AS day, \
   daily_counters.value AS value \
 FROM providers, measurements, fields, daily_counters \
 WHERE \
   daily_counters.field_id = fields.id \
   AND fields.measurement_id = measurements.id \
   AND measurements.provider_id = providers.id",
   createDailyDiscreteNumericsTable: "\
 CREATE TABLE daily_discrete_numeric (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   field_id INTEGER, \
   day INTEGER, \
   value INTEGER, \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createDailyDiscreteNumericsFieldIndex: "\
 CREATE INDEX i_daily_discrete_numeric_field_id \
 ON daily_discrete_numeric (field_id)",
   createDailyDiscreteNumericsDayIndex: "\
 CREATE INDEX i_daily_discrete_numeric_day \
 ON daily_discrete_numeric (day)",
   createDailyDiscreteTextTable: "\
 CREATE TABLE daily_discrete_text (\
   id INTEGER PRIMARY KEY AUTOINCREMENT, \
   field_id INTEGER, \
   day INTEGER, \
   value TEXT, \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createDailyDiscreteTextFieldIndex: "\
 CREATE INDEX i_daily_discrete_text_field_id \
 ON daily_discrete_text (field_id)",
   createDailyDiscreteTextDayIndex: "\
 CREATE INDEX i_daily_discrete_text_day \
 ON daily_discrete_text (day)",
   createDailyDiscreteView: "\
 CREATE VIEW v_daily_discrete AS \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     daily_discrete_numeric.id AS value_id, \
     daily_discrete_numeric.day AS day, \
     daily_discrete_numeric.value AS value, \
     \"numeric\" AS value_type \
     FROM providers, measurements, fields, daily_discrete_numeric \
     WHERE \
       daily_discrete_numeric.field_id = fields.id \
       AND fields.measurement_id = measurements.id \
       AND measurements.provider_id = providers.id \
   UNION ALL \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     daily_discrete_text.id AS value_id, \
     daily_discrete_text.day AS day, \
     daily_discrete_text.value AS value, \
     \"text\" AS value_type \
     FROM providers, measurements, fields, daily_discrete_text \
     WHERE \
       daily_discrete_text.field_id = fields.id \
       AND fields.measurement_id = measurements.id \
       AND measurements.provider_id = providers.id \
   ORDER BY day ASC, value_id ASC",
   createLastNumericTable: "\
 CREATE TABLE last_numeric (\
   field_id INTEGER PRIMARY KEY, \
   day INTEGER, \
   value NUMERIC, \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createLastTextTable: "\
 CREATE TABLE last_text (\
   field_id INTEGER PRIMARY KEY, \
   day INTEGER, \
   value TEXT, \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createLastView: "\
 CREATE VIEW v_last AS \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     last_numeric.day AS day, \
     last_numeric.value AS value, \
     \"numeric\" AS value_type \
     FROM providers, measurements, fields, last_numeric \
     WHERE \
       last_numeric.field_id = fields.id \
       AND fields.measurement_id = measurements.id \
       AND measurements.provider_id = providers.id \
   UNION ALL \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     last_text.day AS day, \
     last_text.value AS value, \
     \"text\" AS value_type \
     FROM providers, measurements, fields, last_text \
     WHERE \
       last_text.field_id = fields.id \
       AND fields.measurement_id = measurements.id \
       AND measurements.provider_id = providers.id",
   createDailyLastNumericTable: "\
 CREATE TABLE daily_last_numeric (\
   field_id INTEGER, \
   day INTEGER, \
   value NUMERIC, \
   UNIQUE (field_id, day) \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createDailyLastNumericFieldIndex: "\
 CREATE INDEX i_daily_last_numeric_field_id ON daily_last_numeric (field_id)",
   createDailyLastNumericDayIndex: "\
 CREATE INDEX i_daily_last_numeric_day ON daily_last_numeric (day)",
   createDailyLastTextTable: "\
 CREATE TABLE daily_last_text (\
   field_id INTEGER, \
   day INTEGER, \
   value TEXT, \
   UNIQUE (field_id, day) \
   FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
 )",
   createDailyLastTextFieldIndex: "\
 CREATE INDEX i_daily_last_text_field_id ON daily_last_text (field_id)",
   createDailyLastTextDayIndex: "\
 CREATE INDEX i_daily_last_text_day ON daily_last_text (day)",
   createDailyLastView: "\
 CREATE VIEW v_daily_last AS \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     daily_last_numeric.day AS day, \
     daily_last_numeric.value AS value, \
     \"numeric\" as value_type \
     FROM providers, measurements, fields, daily_last_numeric \
     WHERE \
       daily_last_numeric.field_id = fields.id \
       AND fields.measurement_id = measurements.id \
       AND measurements.provider_id = providers.id \
   UNION ALL \
   SELECT \
     providers.id AS provider_id, \
     providers.name AS provider_name, \
     measurements.id AS measurement_id, \
     measurements.name AS measurement_name, \
     measurements.version AS measurement_version, \
     fields.id AS field_id, \
     fields.name AS field_name, \
     daily_last_text.day AS day, \
     daily_last_text.value AS value, \
     \"text\" as value_type \
     FROM providers, measurements, fields, daily_last_text \
     WHERE \
       daily_last_text.field_id = fields.id \
       AND fields.measurement_id = measurements.id \
       AND measurements.provider_id = providers.id",
   // Mutation.
   addProvider: "INSERT INTO providers (name) VALUES (:provider)",
   setProviderState: "\
 INSERT OR REPLACE INTO provider_state \
   (provider_id, name, value) \
   VALUES (:provider_id, :name, :value)",
   addMeasurement: "\
 INSERT INTO measurements (provider_id, name, version) \
   VALUES (:provider_id, :measurement, :version)",
   addType: "INSERT INTO types (name) VALUES (:name)",
   addField: "\
 INSERT INTO fields (measurement_id, name, value_type) \
   VALUES (:measurement_id, :field, :value_type)",
   incrementDailyCounterFromFieldID: "\
 INSERT OR REPLACE INTO daily_counters VALUES (\
   :field_id, \
   :days, \
   COALESCE(\
     (SELECT value FROM daily_counters WHERE \
       field_id = :field_id AND day = :days \
     ), \
 \
   ) + :by)",
   deleteLastNumericFromFieldID: "\
 DELETE FROM last_numeric WHERE field_id = :field_id",
   deleteLastTextFromFieldID: "\
 DELETE FROM last_text WHERE field_id = :field_id",
   setLastNumeric: "\
 INSERT OR REPLACE INTO last_numeric VALUES (:field_id, :days, :value)",
   setLastText: "\
 INSERT OR REPLACE INTO last_text VALUES (:field_id, :days, :value)",
   setDailyLastNumeric: "\
 INSERT OR REPLACE INTO daily_last_numeric VALUES (:field_id, :days, :value)",
   setDailyLastText: "\
 INSERT OR REPLACE INTO daily_last_text VALUES (:field_id, :days, :value)",
   addDailyDiscreteNumeric: "\
 INSERT INTO daily_discrete_numeric \
 (field_id, day, value) VALUES (:field_id, :days, :value)",
   addDailyDiscreteText: "\
 INSERT INTO daily_discrete_text \
 (field_id, day, value) VALUES (:field_id, :days, :value)",
   pruneOldDailyCounters: "DELETE FROM daily_counters WHERE day < :days",
   pruneOldDailyDiscreteNumeric: "DELETE FROM daily_discrete_numeric WHERE day < :days",
   pruneOldDailyDiscreteText: "DELETE FROM daily_discrete_text WHERE day < :days",
   pruneOldDailyLastNumeric: "DELETE FROM daily_last_numeric WHERE day < :days",
   pruneOldDailyLastText: "DELETE FROM daily_last_text WHERE day < :days",
   pruneOldLastNumeric: "DELETE FROM last_numeric WHERE day < :days",
   pruneOldLastText: "DELETE FROM last_text WHERE day < :days",
   // Retrieval.
   getProviderID: "SELECT id FROM providers WHERE name = :provider",
   getProviders: "SELECT id, name FROM providers",
   getProviderStateWithName: "\
 SELECT value FROM provider_state \
   WHERE provider_id = :provider_id \
   AND name = :name",
   getMeasurements: "SELECT * FROM v_measurements",
   getMeasurementID: "\
 SELECT id FROM measurements \
   WHERE provider_id = :provider_id \
     AND name = :measurement \
     AND version = :version",
   getFieldID: "\
 SELECT id FROM fields \
   WHERE measurement_id = :measurement_id \
     AND name = :field \
     AND value_type = :value_type \
 ",
   getTypes: "SELECT * FROM types",
   getTypeID: "SELECT id FROM types WHERE name = :name",
   getDailyCounterCountsFromFieldID: "\
 SELECT day, value FROM daily_counters \
   WHERE field_id = :field_id \
   ORDER BY day ASC",
   getDailyCounterCountFromFieldID: "\
 SELECT value FROM daily_counters \
   WHERE field_id = :field_id \
     AND day = :days",
   getMeasurementDailyCounters: "\
 SELECT field_name, day, value FROM v_daily_counters \
 WHERE measurement_id = :measurement_id",
   getFieldInfo: "SELECT * FROM v_fields",
   getLastNumericFromFieldID: "\
 SELECT day, value FROM last_numeric WHERE field_id = :field_id",
   getLastTextFromFieldID: "\
 SELECT day, value FROM last_text WHERE field_id = :field_id",
   getMeasurementLastValues: "\
 SELECT field_name, day, value FROM v_last \
 WHERE measurement_id = :measurement_id",
   getDailyDiscreteNumericFromFieldID: "\
 SELECT day, value FROM daily_discrete_numeric \
   WHERE field_id = :field_id \
   ORDER BY day ASC, id ASC",
   getDailyDiscreteNumericFromFieldIDAndDay: "\
 SELECT day, value FROM daily_discrete_numeric \
   WHERE field_id = :field_id AND day = :days \
   ORDER BY id ASC",
   getDailyDiscreteTextFromFieldID: "\
 SELECT day, value FROM daily_discrete_text \
   WHERE field_id = :field_id \
   ORDER BY day ASC, id ASC",
   getDailyDiscreteTextFromFieldIDAndDay: "\
 SELECT day, value FROM daily_discrete_text \
   WHERE field_id = :field_id AND day = :days \
   ORDER BY id ASC",
   getMeasurementDailyDiscreteValues: "\
 SELECT field_name, day, value_id, value FROM v_daily_discrete \
 WHERE measurement_id = :measurement_id \
 ORDER BY day ASC, value_id ASC",
   getDailyLastNumericFromFieldID: "\
 SELECT day, value FROM daily_last_numeric \
   WHERE field_id = :field_id \
   ORDER BY day ASC",
   getDailyLastNumericFromFieldIDAndDay: "\
 SELECT day, value FROM daily_last_numeric \
   WHERE field_id = :field_id AND day = :days",
   getDailyLastTextFromFieldID: "\
 SELECT day, value FROM daily_last_text \
   WHERE field_id = :field_id \
   ORDER BY day ASC",
   getDailyLastTextFromFieldIDAndDay: "\
 SELECT day, value FROM daily_last_text \
   WHERE field_id = :field_id AND day = :days",
   getMeasurementDailyLastValues: "\
 SELECT field_name, day, value FROM v_daily_last \
 WHERE measurement_id = :measurement_id",
 };
 function dailyKeyFromDate(date) {
   let year = String(date.getUTCFullYear());
   let month = String(date.getUTCMonth() + 1);
   let day = String(date.getUTCDate());
   if (month.length < 2) {
     month = "0" + month;
   }
   if (day.length < 2) {
     day = "0" + day;
   }
   return year + "-" + month + "-" + day;
 }
 /**
  * Create a new backend instance bound to a SQLite database at the given path.
  *
  * This returns a promise that will resolve to a `MetricsStorageSqliteBackend`
  * instance. The resolved instance will be initialized and ready for use.
  *
  * Very few consumers have a need to call this. Instead, a higher-level entity
  * likely calls this and sets up the database connection for a service or
  * singleton.
  */
 this.MetricsStorageBackend = function (path) {
   return Task.spawn(function initTask() {
     let connection = yield Sqlite.openConnection({
       path: path,
       // There should only be one connection per database, so we disable this
       // for perf reasons.
       sharedMemoryCache: false,
     });
     // If we fail initializing the storage object, we need to close the
     // database connection or else Storage will assert on shutdown.
     let storage;
     try {
       storage = new MetricsStorageSqliteBackend(connection);
       yield storage._init();
     } catch (ex) {
       yield connection.close();
       throw ex;
     }
     throw new Task.Result(storage);
   });
 };
 /**
  * Manages storage of metrics data in a SQLite database.
  *
  * This is the main type used for interfacing with the database.
  *
  * Instances of this should be obtained by calling MetricsStorageConnection().
  *
  * The current implementation will not work if the database is mutated by
  * multiple connections because of the way we cache primary keys.
  *
  * FUTURE enforce 1 read/write connection per database limit.
  */
 function MetricsStorageSqliteBackend(connection) {
   this._log = Log.repository.getLogger("Services.Metrics.MetricsStorage");
   this._connection = connection;
   this._enabledWALCheckpointPages = null;
   // Integer IDs to string name.
   this._typesByID = new Map();
   // String name to integer IDs.
   this._typesByName = new Map();
   // Maps provider names to integer IDs.
   this._providerIDs = new Map();
   // Maps :-delimited strings of [provider name, name, version] to integer IDs.
   this._measurementsByInfo = new Map();
   // Integer IDs to Arrays of [provider name, name, version].
   this._measurementsByID = new Map();
   // Integer IDs to Arrays of [measurement id, field name, value name]
   this._fieldsByID = new Map();
   // Maps :-delimited strings of [measurement id, field name] to integer ID.
   this._fieldsByInfo = new Map();
   // Maps measurement ID to sets of field IDs.
   this._fieldsByMeasurement = new Map();
   this._queuedOperations = [];
   this._queuedInProgress = false;
 }
 MetricsStorageSqliteBackend.prototype = Object.freeze({
   // Max size (in kibibytes) the WAL log is allowed to grow to before it is
   // checkpointed.
   //
   // This was first deployed in bug 848136. We want a value large enough
   // that we aren't checkpointing all the time. However, we want it
   // small enough so we don't have to read so much when we open the
   // database.
   MAX_WAL_SIZE_KB: 512,
   FIELD_DAILY_COUNTER: "daily-counter",
   FIELD_DAILY_DISCRETE_NUMERIC: "daily-discrete-numeric",
   FIELD_DAILY_DISCRETE_TEXT: "daily-discrete-text",
   FIELD_DAILY_LAST_NUMERIC: "daily-last-numeric",
   FIELD_DAILY_LAST_TEXT: "daily-last-text",
   FIELD_LAST_NUMERIC: "last-numeric",
   FIELD_LAST_TEXT: "last-text",
   _BUILTIN_TYPES: [
     "FIELD_DAILY_COUNTER",
     "FIELD_DAILY_DISCRETE_NUMERIC",
     "FIELD_DAILY_DISCRETE_TEXT",
     "FIELD_DAILY_LAST_NUMERIC",
     "FIELD_DAILY_LAST_TEXT",
     "FIELD_LAST_NUMERIC",
     "FIELD_LAST_TEXT",
   ],
   // Statements that are used to create the initial DB schema.
   _SCHEMA_STATEMENTS: [
     "createProvidersTable",
     "createProviderStateTable",
     "createProviderStateProviderIndex",
     "createMeasurementsTable",
     "createMeasurementsProviderIndex",
     "createMeasurementsView",
     "createTypesTable",
     "createFieldsTable",
     "createFieldsMeasurementIndex",
     "createFieldsView",
     "createDailyCountersTable",
     "createDailyCountersFieldIndex",
     "createDailyCountersDayIndex",
     "createDailyCountersView",
     "createDailyDiscreteNumericsTable",
     "createDailyDiscreteNumericsFieldIndex",
     "createDailyDiscreteNumericsDayIndex",
     "createDailyDiscreteTextTable",
     "createDailyDiscreteTextFieldIndex",
     "createDailyDiscreteTextDayIndex",
     "createDailyDiscreteView",
     "createDailyLastNumericTable",
     "createDailyLastNumericFieldIndex",
     "createDailyLastNumericDayIndex",
     "createDailyLastTextTable",
     "createDailyLastTextFieldIndex",
     "createDailyLastTextDayIndex",
     "createDailyLastView",
     "createLastNumericTable",
     "createLastTextTable",
     "createLastView",
   ],
   // Statements that are used to prune old data.
   _PRUNE_STATEMENTS: [
     "pruneOldDailyCounters",
     "pruneOldDailyDiscreteNumeric",
     "pruneOldDailyDiscreteText",
     "pruneOldDailyLastNumeric",
     "pruneOldDailyLastText",
     "pruneOldLastNumeric",
     "pruneOldLastText",
   ],
   /**
    * Close the database connection.
    *
    * This should be called on all instances or the SQLite layer may complain
    * loudly. After this has been called, the connection cannot be used.
    *
    * @return Promise<>
    */
   close: function () {
     return Task.spawn(function doClose() {
       // There is some light magic involved here. First, we enqueue an
       // operation to ensure that all pending operations have the opportunity
       // to execute. We additionally execute a SQL operation. Due to the FIFO
       // execution order of issued statements, this will cause us to wait on
       // any outstanding statements before closing.
       try {
         yield this.enqueueOperation(function dummyOperation() {
           return this._connection.execute("SELECT 1");
         }.bind(this));
       } catch (ex) {}
       try {
         yield this._connection.close();
       } finally {
         this._connection = null;
       }
     }.bind(this));
   },
   /**
    * Whether a provider is known to exist.
    *
    * @param provider
    *        (string) Name of the provider.
    */
   hasProvider: function (provider) {
     return this._providerIDs.has(provider);
   },
   /**
    * Whether a measurement is known to exist.
    *
    * @param provider
    *        (string) Name of the provider.
    * @param name
    *        (string) Name of the measurement.
    * @param version
    *        (Number) Integer measurement version.
    */
   hasMeasurement: function (provider, name, version) {
     return this._measurementsByInfo.has([provider, name, version].join(":"));
   },
   /**
    * Whether a named field exists in a measurement.
    *
    * @param measurementID
    *        (Number) The integer primary key of the measurement.
    * @param field
    *        (string) The name of the field to look for.
    */
   hasFieldFromMeasurement: function (measurementID, field) {
     return this._fieldsByInfo.has([measurementID, field].join(":"));
   },
   /**
    * Whether a field is known.
    *
    * @param provider
    *        (string) Name of the provider having the field.
    * @param measurement
    *        (string) Name of the measurement in the provider having the field.
    * @param field
    *        (string) Name of the field in the measurement.
    */
   hasField: function (provider, measurement, version, field) {
     let key = [provider, measurement, version].join(":");
     let measurementID = this._measurementsByInfo.get(key);
     if (!measurementID) {
       return false;
     }
     return this.hasFieldFromMeasurement(measurementID, field);
   },
   /**
    * Look up the integer primary key of a provider.
    *
    * @param provider
    *        (string) Name of the provider.
    */
   providerID: function (provider) {
     return this._providerIDs.get(provider);
   },
   /**
    * Look up the integer primary key of a measurement.
    *
    * @param provider
    *        (string) Name of the provider.
    * @param measurement
    *        (string) Name of the measurement.
    * @param version
    *        (Number) Integer version of the measurement.
    */
   measurementID: function (provider, measurement, version) {
     return this._measurementsByInfo.get([provider, measurement, version].join(":"));
   },
   fieldIDFromMeasurement: function (measurementID, field) {
     return this._fieldsByInfo.get([measurementID, field].join(":"));
   },
   fieldID: function (provider, measurement, version, field) {
     let measurementID = this.measurementID(provider, measurement, version);
     if (!measurementID) {
       return null;
     }
     return this.fieldIDFromMeasurement(measurementID, field);
   },
   measurementHasAnyDailyCounterFields: function (measurementID) {
     return this.measurementHasAnyFieldsOfTypes(measurementID,
                                                [this.FIELD_DAILY_COUNTER]);
   },
   measurementHasAnyLastFields: function (measurementID) {
     return this.measurementHasAnyFieldsOfTypes(measurementID,
                                                [this.FIELD_LAST_NUMERIC,
                                                 this.FIELD_LAST_TEXT]);
   },
   measurementHasAnyDailyLastFields: function (measurementID) {
     return this.measurementHasAnyFieldsOfTypes(measurementID,
                                                [this.FIELD_DAILY_LAST_NUMERIC,
                                                 this.FIELD_DAILY_LAST_TEXT]);
   },
   measurementHasAnyDailyDiscreteFields: function (measurementID) {
     return this.measurementHasAnyFieldsOfTypes(measurementID,
                                                [this.FIELD_DAILY_DISCRETE_NUMERIC,
                                                 this.FIELD_DAILY_DISCRETE_TEXT]);
   },
   measurementHasAnyFieldsOfTypes: function (measurementID, types) {
     if (!this._fieldsByMeasurement.has(measurementID)) {
       return false;
     }
     let fieldIDs = this._fieldsByMeasurement.get(measurementID);
     for (let fieldID of fieldIDs) {
       let fieldType = this._fieldsByID.get(fieldID)[2];
       if (types.indexOf(fieldType) != -1) {
         return true;
       }
     }
     return false;
   },
   /**
    * Register a measurement with the backend.
    *
    * Measurements must be registered before storage can be allocated to them.
    *
    * A measurement consists of a string name and integer version attached
    * to a named provider.
    *
    * This returns a promise that resolves to the storage ID for this
    * measurement.
    *
    * If the measurement is not known to exist, it is registered with storage.
    * If the measurement has already been registered, this is effectively a
    * no-op (that still returns a promise resolving to the storage ID).
    *
    * @param provider
    *        (string) Name of the provider this measurement belongs to.
    * @param name
    *        (string) Name of this measurement.
    * @param version
    *        (Number) Integer version of this measurement.
    */
   registerMeasurement: function (provider, name, version) {
     if (this.hasMeasurement(provider, name, version)) {
       return CommonUtils.laterTickResolvingPromise(
         this.measurementID(provider, name, version));
     }
     // Registrations might not be safe to perform in parallel with provider
     // operations. So, we queue them.
     let self = this;
     return this.enqueueOperation(function createMeasurementOperation() {
       return Task.spawn(function createMeasurement() {
         let providerID = self._providerIDs.get(provider);
         if (!providerID) {
           yield self._connection.executeCached(SQL.addProvider, {provider: provider});
           let rows = yield self._connection.executeCached(SQL.getProviderID,
                                                           {provider: provider});
           providerID = rows[0].getResultByIndex(0);
           self._providerIDs.set(provider, providerID);
         }
         let params = {
           provider_id: providerID,
           measurement: name,
           version: version,
         };
         yield self._connection.executeCached(SQL.addMeasurement, params);
         let rows = yield self._connection.executeCached(SQL.getMeasurementID, params);
         let measurementID = rows[0].getResultByIndex(0);
         self._measurementsByInfo.set([provider, name, version].join(":"), measurementID);
         self._measurementsByID.set(measurementID, [provider, name, version]);
         self._fieldsByMeasurement.set(measurementID, new Set());
         throw new Task.Result(measurementID);
       });
     });
   },
   /**
    * Register a field with the backend.
    *
    * Fields are what recorded pieces of data are primarily associated with.
    *
    * Fields are associated with measurements. Measurements must be registered
    * via `registerMeasurement` before fields can be registered. This is
    * enforced by this function requiring the database primary key of the
    * measurement as an argument.
    *
    * @param measurementID
    *        (Number) Integer primary key of measurement this field belongs to.
    * @param field
    *        (string) Name of this field.
    * @param valueType
    *        (string) Type name of this field. Must be a registered type. Is
    *        likely one of the FIELD_ constants on this type.
    *
    * @return Promise<integer>
    */
   registerField: function (measurementID, field, valueType) {
     if (!valueType) {
       throw new Error("Value type must be defined.");
     }
     if (!this._measurementsByID.has(measurementID)) {
       throw new Error("Measurement not known: " + measurementID);
     }
     if (!this._typesByName.has(valueType)) {
       throw new Error("Unknown value type: " + valueType);
     }
     let typeID = this._typesByName.get(valueType);
     if (!typeID) {
       throw new Error("Undefined type: " + valueType);
     }
     if (this.hasFieldFromMeasurement(measurementID, field)) {
       let id = this.fieldIDFromMeasurement(measurementID, field);
       let existingType = this._fieldsByID.get(id)[2];
       if (valueType != existingType) {
         throw new Error("Field already defined with different type: " + existingType);
       }
       return CommonUtils.laterTickResolvingPromise(
         this.fieldIDFromMeasurement(measurementID, field));
     }
     let self = this;
     return Task.spawn(function createField() {
       let params = {
         measurement_id: measurementID,
         field: field,
         value_type: typeID,
       };
       yield self._connection.executeCached(SQL.addField, params);
       let rows = yield self._connection.executeCached(SQL.getFieldID, params);
       let fieldID = rows[0].getResultByIndex(0);
       self._fieldsByID.set(fieldID, [measurementID, field, valueType]);
       self._fieldsByInfo.set([measurementID, field].join(":"), fieldID);
       self._fieldsByMeasurement.get(measurementID).add(fieldID);
       throw new Task.Result(fieldID);
     });
   },
   /**
    * Initializes this instance with the database.
    *
    * This performs 2 major roles:
    *
    *   1) Set up database schema (creates tables).
    *   2) Synchronize database with local instance.
    */
   _init: function() {
     let self = this;
     return Task.spawn(function initTask() {
       // 0. Database file and connection configuration.
       // This should never fail. But, we assume the default of 1024 in case it
       // does.
       let rows = yield self._connection.execute("PRAGMA page_size");
       let pageSize = 1024;
       if (rows.length) {
         pageSize = rows[0].getResultByIndex(0);
       }
       self._log.debug("Page size is " + pageSize);
       // Ensure temp tables are stored in memory, not on disk.
       yield self._connection.execute("PRAGMA temp_store=MEMORY");
       let journalMode;
       rows = yield self._connection.execute("PRAGMA journal_mode=WAL");
       if (rows.length) {
         journalMode = rows[0].getResultByIndex(0);
       }
       self._log.info("Journal mode is " + journalMode);
       if (journalMode == "wal") {
         self._enabledWALCheckpointPages =
           Math.ceil(self.MAX_WAL_SIZE_KB * 1024 / pageSize);
         self._log.info("WAL auto checkpoint pages: " +
                        self._enabledWALCheckpointPages);
         // We disable auto checkpoint during initialization to make it
         // quicker.
         yield self.setAutoCheckpoint(0);
       } else {
         if (journalMode != "truncate") {
          // Fall back to truncate (which is faster than delete).
           yield self._connection.execute("PRAGMA journal_mode=TRUNCATE");
         }
         // And always use full synchronous mode to reduce possibility for data
         // loss.
         yield self._connection.execute("PRAGMA synchronous=FULL");
       }
       let doCheckpoint = false;
       // 1. Create the schema.
       yield self._connection.executeTransaction(function ensureSchema(conn) {
         let schema = yield conn.getSchemaVersion();
         if (schema == 0) {
           self._log.info("Creating database schema.");
           for (let k of self._SCHEMA_STATEMENTS) {
             yield self._connection.execute(SQL[k]);
           }
           yield self._connection.setSchemaVersion(1);
           doCheckpoint = true;
         } else if (schema != 1) {
           throw new Error("Unknown database schema: " + schema);
         } else {
           self._log.debug("Database schema up to date.");
         }
       });
       // 2. Retrieve existing types.
       yield self._connection.execute(SQL.getTypes, null, function onRow(row) {
         let id = row.getResultByName("id");
         let name = row.getResultByName("name");
         self._typesByID.set(id, name);
         self._typesByName.set(name, id);
       });
       // 3. Populate built-in types with database.
       let missingTypes = [];
       for (let type of self._BUILTIN_TYPES) {
         type = self[type];
         if (self._typesByName.has(type)) {
           continue;
         }
         missingTypes.push(type);
       }
       // Don't perform DB transaction unless there is work to do.
       if (missingTypes.length) {
         yield self._connection.executeTransaction(function populateBuiltinTypes() {
           for (let type of missingTypes) {
             let params = {name: type};
             yield self._connection.executeCached(SQL.addType, params);
             let rows = yield self._connection.executeCached(SQL.getTypeID, params);
             let id = rows[0].getResultByIndex(0);
             self._typesByID.set(id, type);
             self._typesByName.set(type, id);
           }
         });
         doCheckpoint = true;
       }
       // 4. Obtain measurement info.
       yield self._connection.execute(SQL.getMeasurements, null, function onRow(row) {
         let providerID = row.getResultByName("provider_id");
         let providerName = row.getResultByName("provider_name");
         let measurementID = row.getResultByName("measurement_id");
         let measurementName = row.getResultByName("measurement_name");
         let measurementVersion = row.getResultByName("measurement_version");
         self._providerIDs.set(providerName, providerID);
         let info = [providerName, measurementName, measurementVersion].join(":");
         self._measurementsByInfo.set(info, measurementID);
         self._measurementsByID.set(measurementID, info);
         self._fieldsByMeasurement.set(measurementID, new Set());
       });
       // 5. Obtain field info.
       yield self._connection.execute(SQL.getFieldInfo, null, function onRow(row) {
         let measurementID = row.getResultByName("measurement_id");
         let fieldID = row.getResultByName("field_id");
         let fieldName = row.getResultByName("field_name");
         let typeName = row.getResultByName("type_name");
         self._fieldsByID.set(fieldID, [measurementID, fieldName, typeName]);
         self._fieldsByInfo.set([measurementID, fieldName].join(":"), fieldID);
         self._fieldsByMeasurement.get(measurementID).add(fieldID);
       });
       // Perform a checkpoint after initialization (if needed) and
       // enable auto checkpoint during regular operation.
       if (doCheckpoint) {
         yield self.checkpoint();
       }
       yield self.setAutoCheckpoint(1);
     });
   },
   /**
    * Prune all data from earlier than the specified date.
    *
    * Data stored on days before the specified Date will be permanently
    * deleted.
    *
    * This returns a promise that will be resolved when data has been deleted.
    *
    * @param date
    *        (Date) Old data threshold.
    * @return Promise<>
    */
   pruneDataBefore: function (date) {
     let statements = this._PRUNE_STATEMENTS;
     let self = this;
     return this.enqueueOperation(function doPrune() {
       return self._connection.executeTransaction(function prune(conn) {
         let days = dateToDays(date);
         let params = {days: days};
         for (let name of statements) {
           yield conn.execute(SQL[name], params);
         }
       });
     });
   },
   /**
    * Reduce memory usage as much as possible.
    *
    * This returns a promise that will be resolved on completion.
    *
    * @return Promise<>
    */
   compact: function () {
     let self = this;
     return this.enqueueOperation(function doCompact() {
       self._connection.discardCachedStatements();
       return self._connection.shrinkMemory();
     });
   },
   /**
    * Checkpoint writes requiring flush to disk.
    *
    * This is called to persist queued and non-flushed writes to disk.
    * It will force an fsync, so it is expensive and should be used
    * sparingly.
    */
   checkpoint: function () {
     if (!this._enabledWALCheckpointPages) {
       return CommonUtils.laterTickResolvingPromise();
     }
     return this.enqueueOperation(function checkpoint() {
       this._log.info("Performing manual WAL checkpoint.");
       return this._connection.execute("PRAGMA wal_checkpoint");
     }.bind(this));
   },
   setAutoCheckpoint: function (on) {
     // If we aren't in WAL mode, wal_autocheckpoint won't do anything so
     // we no-op.
     if (!this._enabledWALCheckpointPages) {
       return CommonUtils.laterTickResolvingPromise();
     }
     let val = on ? this._enabledWALCheckpointPages : 0;
     return this.enqueueOperation(function setWALCheckpoint() {
       this._log.info("Setting WAL auto checkpoint to " + val);
       return this._connection.execute("PRAGMA wal_autocheckpoint=" + val);
     }.bind(this));
   },
   /**
    * Ensure a field ID matches a specified type.
    *
    * This is called internally as part of adding values to ensure that
    * the type of a field matches the operation being performed.
    */
   _ensureFieldType: function (id, type) {
     let info = this._fieldsByID.get(id);
     if (!info || !Array.isArray(info)) {
       throw new Error("Unknown field ID: " + id);
     }
     if (type != info[2]) {
       throw new Error("Field type does not match the expected for this " +
                       "operation. Actual: " + info[2] + "; Expected: " +
                       type);
     }
   },
   /**
    * Enqueue a storage operation to be performed when the database is ready.
    *
    * The primary use case of this function is to prevent potentially
    * conflicting storage operations from being performed in parallel. By
    * calling this function, passed storage operations will be serially
    * executed, avoiding potential order of operation issues.
    *
    * The passed argument is a function that will perform storage operations.
    * The function should return a promise that will be resolved when all
    * storage operations have been completed.
    *
    * The passed function may be executed immediately. If there are already
    * queued operations, it will be appended to the queue and executed after all
    * before it have finished.
    *
    * This function returns a promise that will be resolved or rejected with
    * the same value that the function's promise was resolved or rejected with.
    *
    * @param func
    *        (function) Function performing storage interactions.
    * @return Promise<>
    */
   enqueueOperation: function (func) {
     if (typeof(func) != "function") {
       throw new Error("enqueueOperation expects a function. Got: " + typeof(func));
     }
     this._log.trace("Enqueueing operation.");
     let deferred = Promise.defer();
     this._queuedOperations.push([func, deferred]);
     if (this._queuedOperations.length == 1) {
       this._popAndPerformQueuedOperation();
     }
     return deferred.promise;
   },
   /**
    * Enqueue a function to be performed as a transaction.
    *
    * The passed function should be a generator suitable for calling with
    * `executeTransaction` from the SQLite connection.
    */
   enqueueTransaction: function (func, type) {
     return this.enqueueOperation(
       this._connection.executeTransaction.bind(this._connection, func, type)
     );
   },
   _popAndPerformQueuedOperation: function () {
     if (!this._queuedOperations.length || this._queuedInProgress) {
       return;
     }
     this._log.trace("Performing queued operation.");
     let [func, deferred] = this._queuedOperations.shift();
     let promise;
     try {
       this._queuedInProgress = true;
       promise = func();
     } catch (ex) {
       this._log.warn("Queued operation threw during execution: " +
                      CommonUtils.exceptionStr(ex));
       this._queuedInProgress = false;
       deferred.reject(ex);
       this._popAndPerformQueuedOperation();
       return;
     }
     if (!promise || typeof(promise.then) != "function") {
       let msg = "Queued operation did not return a promise: " + func;
       this._log.warn(msg);
       this._queuedInProgress = false;
       deferred.reject(new Error(msg));
       this._popAndPerformQueuedOperation();
       return;
     }
     promise.then(
       function onSuccess(result) {
         this._log.trace("Queued operation completed.");
         this._queuedInProgress = false;
         deferred.resolve(result);
         this._popAndPerformQueuedOperation();
       }.bind(this),
       function onError(error) {
         this._log.warn("Failure when performing queued operation: " +
                        CommonUtils.exceptionStr(error));
         this._queuedInProgress = false;
         deferred.reject(error);
         this._popAndPerformQueuedOperation();
       }.bind(this)
     );
   },
   /**
    * Obtain all values associated with a measurement.
    *
    * This returns a promise that resolves to an object. The keys of the object
    * are:
    *
    *   days -- DailyValues where the values are Maps of field name to data
    *     structures. The data structures could be simple (string or number) or
    *     Arrays if the field type allows multiple values per day.
    *
    *   singular -- Map of field names to values. This holds all fields that
    *     don't have a temporal component.
    *
    * @param id
    *        (Number) Primary key of measurement whose values to retrieve.
    */
   getMeasurementValues: function (id) {
     let deferred = Promise.defer();
     let days = new DailyValues();
     let singular = new Map();
     let self = this;
     this.enqueueOperation(function enqueuedGetMeasurementValues() {
       return Task.spawn(function fetchMeasurementValues() {
         function handleResult(data) {
           for (let [field, values] of data) {
             for (let [day, value] of Iterator(values)) {
               if (!days.hasDay(day)) {
                 days.setDay(day, new Map());
               }
               days.getDay(day).set(field, value);
             }
           }
         }
         if (self.measurementHasAnyDailyCounterFields(id)) {
           let counters = yield self.getMeasurementDailyCountersFromMeasurementID(id);
           handleResult(counters);
         }
         if (self.measurementHasAnyDailyLastFields(id)) {
           let dailyLast = yield self.getMeasurementDailyLastValuesFromMeasurementID(id);
           handleResult(dailyLast);
         }
         if (self.measurementHasAnyDailyDiscreteFields(id)) {
           let dailyDiscrete = yield self.getMeasurementDailyDiscreteValuesFromMeasurementID(id);
           handleResult(dailyDiscrete);
         }
         if (self.measurementHasAnyLastFields(id)) {
           let last = yield self.getMeasurementLastValuesFromMeasurementID(id);
           for (let [field, value] of last) {
             singular.set(field, value);
           }
         }
       });
     }).then(function onSuccess() {
       deferred.resolve({singular: singular, days: days});
     }, function onError(error) {
       deferred.reject(error);
     });
     return deferred.promise;
   },
   //---------------------------------------------------------------------------
   // Low-level storage operations
   //
   // These will be performed immediately (or at least as soon as the underlying
   // connection allows them to be.) It is recommended to call these from within
   // a function added via `enqueueOperation()` or they may inadvertently be
   // performed during another enqueued operation, which may be a transaction
   // that is rolled back.
   // ---------------------------------------------------------------------------
   /**
    * Set state for a provider.
    *
    * Providers have the ability to register persistent state with the backend.
    * Persistent state doesn't expire. The format of the data is completely up
    * to the provider beyond the requirement that values be UTF-8 strings.
    *
    * This returns a promise that will be resolved when the underlying database
    * operation has completed.
    *
    * @param provider
    *        (string) Name of the provider.
    * @param key
    *        (string) Key under which to store this state.
    * @param value
    *        (string) Value for this state.
    * @return Promise<>
    */
   setProviderState: function (provider, key, value) {
     if (typeof(key) != "string") {
       throw new Error("State key must be a string. Got: " + key);
     }
     if (typeof(value) != "string") {
       throw new Error("State value must be a string. Got: " + value);
     }
     let id = this.providerID(provider);
     if (!id) {
       throw new Error("Unknown provider: " + provider);
     }
     return this._connection.executeCached(SQL.setProviderState, {
       provider_id: id,
       name: key,
       value: value,
     });
   },
   /**
    * Obtain named state for a provider.
    *
    *
    * The returned promise will resolve to the state from the database or null
    * if the key is not stored.
    *
    * @param provider
    *        (string) The name of the provider whose state to obtain.
    * @param key
    *        (string) The state's key to retrieve.
    *
    * @return Promise<data>
    */
   getProviderState: function (provider, key) {
     let id = this.providerID(provider);
     if (!id) {
       throw new Error("Unknown provider: " + provider);
     }
     let conn = this._connection;
     return Task.spawn(function queryDB() {
       let rows = yield conn.executeCached(SQL.getProviderStateWithName, {
         provider_id: id,
         name: key,
       });
       if (!rows.length) {
         throw new Task.Result(null);
       }
       throw new Task.Result(rows[0].getResultByIndex(0));
     });
   },
   /**
    * Increment a daily counter from a numeric field id.
    *
    * @param id
    *        (integer) Primary key of field to increment.
    * @param date
    *        (Date) When the increment occurred. This is typically "now" but can
    *        be explicitly defined for events that occurred in the past.
    * @param by
    *        (integer) How much to increment the value by. Defaults to 1.
    */
   incrementDailyCounterFromFieldID: function (id, date=new Date(), by=1) {
     this._ensureFieldType(id, this.FIELD_DAILY_COUNTER);
     let params = {
       field_id: id,
       days: dateToDays(date),
       by: by,
     };
     return this._connection.executeCached(SQL.incrementDailyCounterFromFieldID,
                                           params);
   },
   /**
    * Obtain all counts for a specific daily counter.
    *
    * @param id
    *        (integer) The ID of the field being retrieved.
    */
   getDailyCounterCountsFromFieldID: function (id) {
     this._ensureFieldType(id, this.FIELD_DAILY_COUNTER);
     let self = this;
     return Task.spawn(function fetchCounterDays() {
       let rows = yield self._connection.executeCached(SQL.getDailyCounterCountsFromFieldID,
                                                       {field_id: id});
       let result = new DailyValues();
       for (let row of rows) {
         let days = row.getResultByIndex(0);
         let counter = row.getResultByIndex(1);
         let date = daysToDate(days);
         result.setDay(date, counter);
       }
       throw new Task.Result(result);
     });
   },
   /**
    * Get the value of a daily counter for a given day.
    *
    * @param field
    *        (integer) Field ID to retrieve.
    * @param date
    *        (Date) Date for day from which to obtain data.
    */
   getDailyCounterCountFromFieldID: function (field, date) {
     this._ensureFieldType(field, this.FIELD_DAILY_COUNTER);
     let params = {
       field_id: field,
       days: dateToDays(date),
     };
     let self = this;
     return Task.spawn(function fetchCounter() {
       let rows = yield self._connection.executeCached(SQL.getDailyCounterCountFromFieldID,
                                                       params);
       if (!rows.length) {
         throw new Task.Result(null);
       }
       throw new Task.Result(rows[0].getResultByIndex(0));
     });
   },
   /**
    * Define the value for a "last numeric" field.
    *
    * The previous value (if any) will be replaced by the value passed, even if
    * the date of the incoming value is older than what's recorded in the
    * database.
    *
    * @param fieldID
    *        (Number) Integer primary key of field to update.
    * @param value
    *        (Number) Value to record.
    * @param date
    *        (Date) When this value was produced.
    */
   setLastNumericFromFieldID: function (fieldID, value, date=new Date()) {
     this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC);
     if (typeof(value) != "number") {
       throw new Error("Value is not a number: " + value);
     }
     let params = {
       field_id: fieldID,
       days: dateToDays(date),
       value: value,
     };
     return this._connection.executeCached(SQL.setLastNumeric, params);
   },
   /**
    * Define the value of a "last text" field.
    *
    * See `setLastNumericFromFieldID` for behavior.
    */
   setLastTextFromFieldID: function (fieldID, value, date=new Date()) {
     this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT);
     if (typeof(value) != "string") {
       throw new Error("Value is not a string: " + value);
     }
     let params = {
       field_id: fieldID,
       days: dateToDays(date),
       value: value,
     };
     return this._connection.executeCached(SQL.setLastText, params);
   },
   /**
    * Obtain the value of a "last numeric" field.
    *
    * This returns a promise that will be resolved with an Array of [date, value]
    * if a value is known or null if no last value is present.
    *
    * @param fieldID
    *        (Number) Integer primary key of field to retrieve.
    */
   getLastNumericFromFieldID: function (fieldID) {
     this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC);
     let self = this;
     return Task.spawn(function fetchLastField() {
       let rows = yield self._connection.executeCached(SQL.getLastNumericFromFieldID,
                                                       {field_id: fieldID});
       if (!rows.length) {
         throw new Task.Result(null);
       }
       let row = rows[0];
       let days = row.getResultByIndex(0);
       let value = row.getResultByIndex(1);
       throw new Task.Result([daysToDate(days), value]);
     });
   },
   /**
    * Obtain the value of a "last text" field.
    *
    * See `getLastNumericFromFieldID` for behavior.
    */
   getLastTextFromFieldID: function (fieldID) {
     this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT);
     let self = this;
     return Task.spawn(function fetchLastField() {
       let rows = yield self._connection.executeCached(SQL.getLastTextFromFieldID,
                                                       {field_id: fieldID});
       if (!rows.length) {
         throw new Task.Result(null);
       }
       let row = rows[0];
       let days = row.getResultByIndex(0);
       let value = row.getResultByIndex(1);
       throw new Task.Result([daysToDate(days), value]);
     });
   },
   /**
    * Delete the value (if any) in a "last numeric" field.
    */
   deleteLastNumericFromFieldID: function (fieldID) {
     this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC);
     return this._connection.executeCached(SQL.deleteLastNumericFromFieldID,
                                           {field_id: fieldID});
   },
   /**
    * Delete the value (if any) in a "last text" field.
    */
   deleteLastTextFromFieldID: function (fieldID) {
     this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT);
     return this._connection.executeCached(SQL.deleteLastTextFromFieldID,
                                           {field_id: fieldID});
   },
   /**
    * Record a value for a "daily last numeric" field.
    *
    * The field can hold 1 value per calendar day. If the field already has a
    * value for the day specified (defaults to now), that value will be
    * replaced, even if the date specified is older (within the day) than the
    * previously recorded value.
    *
    * @param fieldID
    *        (Number) Integer primary key of field.
    * @param value
    *        (Number) Value to record.
    * @param date
    *        (Date) When the value was produced. Defaults to now.
    */
   setDailyLastNumericFromFieldID: function (fieldID, value, date=new Date()) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_NUMERIC);
     let params = {
       field_id: fieldID,
       days: dateToDays(date),
       value: value,
     };
     return this._connection.executeCached(SQL.setDailyLastNumeric, params);
   },
   /**
    * Record a value for a "daily last text" field.
    *
    * See `setDailyLastNumericFromFieldID` for behavior.
    */
   setDailyLastTextFromFieldID: function (fieldID, value, date=new Date()) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_TEXT);
     let params = {
       field_id: fieldID,
       days: dateToDays(date),
       value: value,
     };
     return this._connection.executeCached(SQL.setDailyLastText, params);
   },
   /**
    * Obtain value(s) from a "daily last numeric" field.
    *
    * This returns a promise that resolves to a DailyValues instance. If `date`
    * is specified, that instance will have at most 1 entry. If there is no
    * `date` constraint, then all stored values will be retrieved.
    *
    * @param fieldID
    *        (Number) Integer primary key of field to retrieve.
    * @param date optional
    *        (Date) If specified, only return data for this day.
    *
    * @return Promise<DailyValues>
    */
   getDailyLastNumericFromFieldID: function (fieldID, date=null) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_NUMERIC);
     let params = {field_id: fieldID};
     let name = "getDailyLastNumericFromFieldID";
     if (date) {
       params.days = dateToDays(date);
       name = "getDailyLastNumericFromFieldIDAndDay";
     }
     return this._getDailyLastFromFieldID(name, params);
   },
   /**
    * Obtain value(s) from a "daily last text" field.
    *
    * See `getDailyLastNumericFromFieldID` for behavior.
    */
   getDailyLastTextFromFieldID: function (fieldID, date=null) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_TEXT);
     let params = {field_id: fieldID};
     let name = "getDailyLastTextFromFieldID";
     if (date) {
       params.days = dateToDays(date);
       name = "getDailyLastTextFromFieldIDAndDay";
     }
     return this._getDailyLastFromFieldID(name, params);
   },
   _getDailyLastFromFieldID: function (name, params) {
     let self = this;
     return Task.spawn(function fetchDailyLastForField() {
       let rows = yield self._connection.executeCached(SQL[name], params);
       let result = new DailyValues();
       for (let row of rows) {
         let d = daysToDate(row.getResultByIndex(0));
         let value = row.getResultByIndex(1);
         result.setDay(d, value);
       }
       throw new Task.Result(result);
     });
   },
   /**
    * Add a new value for a "daily discrete numeric" field.
    *
    * This appends a new value to the list of values for a specific field. All
    * values are retained. Duplicate values are allowed.
    *
    * @param fieldID
    *        (Number) Integer primary key of field.
    * @param value
    *        (Number) Value to record.
    * @param date optional
    *        (Date) When this value occurred. Values are bucketed by day.
    */
   addDailyDiscreteNumericFromFieldID: function (fieldID, value, date=new Date()) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_NUMERIC);
     if (typeof(value) != "number") {
       throw new Error("Number expected. Got: " + value);
     }
     let params = {
       field_id: fieldID,
       days: dateToDays(date),
       value: value,
     };
     return this._connection.executeCached(SQL.addDailyDiscreteNumeric, params);
   },
   /**
    * Add a new value for a "daily discrete text" field.
    *
    * See `addDailyDiscreteNumericFromFieldID` for behavior.
    */
   addDailyDiscreteTextFromFieldID: function (fieldID, value, date=new Date()) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_TEXT);
     if (typeof(value) != "string") {
       throw new Error("String expected. Got: " + value);
     }
     let params = {
       field_id: fieldID,
       days: dateToDays(date),
       value: value,
     };
     return this._connection.executeCached(SQL.addDailyDiscreteText, params);
   },
   /**
    * Obtain values for a "daily discrete numeric" field.
    *
    * This returns a promise that resolves to a `DailyValues` instance. If
    * `date` is specified, there will be at most 1 key in that instance. If
    * not, all data from the database will be retrieved.
    *
    * Values in that instance will be arrays of the raw values.
    *
    * @param fieldID
    *        (Number) Integer primary key of field to retrieve.
    * @param date optional
    *        (Date) Day to obtain data for. Date can be any time in the day.
    */
   getDailyDiscreteNumericFromFieldID: function (fieldID, date=null) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_NUMERIC);
     let params = {field_id: fieldID};
     let name = "getDailyDiscreteNumericFromFieldID";
     if (date) {
       params.days = dateToDays(date);
       name = "getDailyDiscreteNumericFromFieldIDAndDay";
     }
     return this._getDailyDiscreteFromFieldID(name, params);
   },
   /**
    * Obtain values for a "daily discrete text" field.
    *
    * See `getDailyDiscreteNumericFromFieldID` for behavior.
    */
   getDailyDiscreteTextFromFieldID: function (fieldID, date=null) {
     this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_TEXT);
     let params = {field_id: fieldID};
     let name = "getDailyDiscreteTextFromFieldID";
     if (date) {
       params.days = dateToDays(date);
       name = "getDailyDiscreteTextFromFieldIDAndDay";
     }
     return this._getDailyDiscreteFromFieldID(name, params);
   },
   _getDailyDiscreteFromFieldID: function (name, params) {
     let self = this;
     return Task.spawn(function fetchDailyDiscreteValuesForField() {
       let rows = yield self._connection.executeCached(SQL[name], params);
       let result = new DailyValues();
       for (let row of rows) {
         let d = daysToDate(row.getResultByIndex(0));
         let value = row.getResultByIndex(1);
         result.appendValue(d, value);
       }
       throw new Task.Result(result);
     });
   },
   /**
    * Obtain the counts of daily counters in a measurement.
    *
    * This returns a promise that resolves to a Map of field name strings to
    * DailyValues that hold per-day counts.
    *
    * @param id
    *        (Number) Integer primary key of measurement.
    *
    * @return Promise<Map>
    */
   getMeasurementDailyCountersFromMeasurementID: function (id) {
     let self = this;
     return Task.spawn(function fetchDailyCounters() {
       let rows = yield self._connection.execute(SQL.getMeasurementDailyCounters,
                                                 {measurement_id: id});
       let result = new Map();
       for (let row of rows) {
         let field = row.getResultByName("field_name");
         let date = daysToDate(row.getResultByName("day"));
         let value = row.getResultByName("value");
         if (!result.has(field)) {
           result.set(field, new DailyValues());
         }
         result.get(field).setDay(date, value);
       }
       throw new Task.Result(result);
     });
   },
   /**
    * Obtain the values of "last" fields from a measurement.
    *
    * This returns a promise that resolves to a Map of field name to an array
    * of [date, value].
    *
    * @param id
    *        (Number) Integer primary key of measurement whose data to retrieve.
    *
    * @return Promise<Map>
    */
   getMeasurementLastValuesFromMeasurementID: function (id) {
     let self = this;
     return Task.spawn(function fetchMeasurementLastValues() {
       let rows = yield self._connection.execute(SQL.getMeasurementLastValues,
                                                 {measurement_id: id});
       let result = new Map();
       for (let row of rows) {
         let date = daysToDate(row.getResultByIndex(1));
         let value = row.getResultByIndex(2);
         result.set(row.getResultByIndex(0), [date, value]);
       }
       throw new Task.Result(result);
     });
   },
   /**
    * Obtain the values of "last daily" fields from a measurement.
    *
    * This returns a promise that resolves to a Map of field name to DailyValues
    * instances. Each DailyValues instance has days for which a daily last value
    * is defined. The values in each DailyValues are the raw last value for that
    * day.
    *
    * @param id
    *        (Number) Integer primary key of measurement whose data to retrieve.
    *
    * @return Promise<Map>
    */
   getMeasurementDailyLastValuesFromMeasurementID: function (id) {
     let self = this;
     return Task.spawn(function fetchMeasurementDailyLastValues() {
       let rows = yield self._connection.execute(SQL.getMeasurementDailyLastValues,
                                                 {measurement_id: id});
       let result = new Map();
       for (let row of rows) {
         let field = row.getResultByName("field_name");
         let date = daysToDate(row.getResultByName("day"));
         let value = row.getResultByName("value");
         if (!result.has(field)) {
           result.set(field, new DailyValues());
         }
         result.get(field).setDay(date, value);
       }
       throw new Task.Result(result);
     });
   },
   /**
    * Obtain the values of "daily discrete" fields from a measurement.
    *
    * This obtains all discrete values for all "daily discrete" fields in a
    * measurement.
    *
    * This returns a promise that resolves to a Map. The Map's keys are field
    * string names. Values are `DailyValues` instances. The values inside
    * the `DailyValues` are arrays of the raw discrete values.
    *
    * @param id
    *        (Number) Integer primary key of measurement.
    *
    * @return Promise<Map>
    */
   getMeasurementDailyDiscreteValuesFromMeasurementID: function (id) {
     let deferred = Promise.defer();
     let result = new Map();
     this._connection.execute(SQL.getMeasurementDailyDiscreteValues,
                              {measurement_id: id}, function onRow(row) {
       let field = row.getResultByName("field_name");
       let date = daysToDate(row.getResultByName("day"));
       let value = row.getResultByName("value");
       if (!result.has(field)) {
         result.set(field, new DailyValues());
       }
       result.get(field).appendValue(date, value);
     }).then(function onComplete() {
       deferred.resolve(result);
     }, function onError(error) {
       deferred.reject(error);
     });
     return deferred.promise;
   },
 });
 // Alias built-in field types to public API.
 for (let property of MetricsStorageSqliteBackend.prototype._BUILTIN_TYPES) {
   this.MetricsStorageBackend[property] = MetricsStorageSqliteBackend.prototype[property];
 }

The Tor Browser / annotate

services/metrics/storage.jsm@b8a032363ba2 (annotated)

services/metrics/storage.jsm