1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/services/metrics/storage.jsm Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,2179 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +"use strict";
1.9 +
1.10 +#ifndef MERGED_COMPARTMENT
1.11 +
1.12 +this.EXPORTED_SYMBOLS = [
1.13 + "DailyValues",
1.14 + "MetricsStorageBackend",
1.15 + "dateToDays",
1.16 + "daysToDate",
1.17 +];
1.18 +
1.19 +const {utils: Cu} = Components;
1.20 +
1.21 +const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
1.22 +
1.23 +#endif
1.24 +
1.25 +Cu.import("resource://gre/modules/Promise.jsm");
1.26 +Cu.import("resource://gre/modules/Sqlite.jsm");
1.27 +Cu.import("resource://gre/modules/Task.jsm");
1.28 +Cu.import("resource://gre/modules/Log.jsm");
1.29 +Cu.import("resource://services-common/utils.js");
1.30 +
1.31 +
1.32 +// These do not account for leap seconds. Meh.
1.33 +function dateToDays(date) {
1.34 + return Math.floor(date.getTime() / MILLISECONDS_PER_DAY);
1.35 +}
1.36 +
1.37 +function daysToDate(days) {
1.38 + return new Date(days * MILLISECONDS_PER_DAY);
1.39 +}
1.40 +
1.41 +/**
1.42 + * Represents a collection of per-day values.
1.43 + *
1.44 + * This is a proxy around a Map which can transparently round Date instances to
1.45 + * their appropriate key.
1.46 + *
1.47 + * This emulates Map by providing .size and iterator support. Note that keys
1.48 + * from the iterator are Date instances corresponding to midnight of the start
1.49 + * of the day. get(), has(), and set() are modeled as getDay(), hasDay(), and
1.50 + * setDay(), respectively.
1.51 + *
1.52 + * All days are defined in terms of UTC (as opposed to local time).
1.53 + */
1.54 +this.DailyValues = function () {
1.55 + this._days = new Map();
1.56 +};
1.57 +
1.58 +DailyValues.prototype = Object.freeze({
1.59 + __iterator__: function () {
1.60 + for (let [k, v] of this._days) {
1.61 + yield [daysToDate(k), v];
1.62 + }
1.63 + },
1.64 +
1.65 + get size() {
1.66 + return this._days.size;
1.67 + },
1.68 +
1.69 + hasDay: function (date) {
1.70 + return this._days.has(dateToDays(date));
1.71 + },
1.72 +
1.73 + getDay: function (date) {
1.74 + return this._days.get(dateToDays(date));
1.75 + },
1.76 +
1.77 + setDay: function (date, value) {
1.78 + this._days.set(dateToDays(date), value);
1.79 + },
1.80 +
1.81 + appendValue: function (date, value) {
1.82 + let key = dateToDays(date);
1.83 +
1.84 + if (this._days.has(key)) {
1.85 + return this._days.get(key).push(value);
1.86 + }
1.87 +
1.88 + this._days.set(key, [value]);
1.89 + },
1.90 +});
1.91 +
1.92 +
1.93 +/**
1.94 + * DATABASE INFO
1.95 + * =============
1.96 + *
1.97 + * We use a SQLite database as the backend for persistent storage of metrics
1.98 + * data.
1.99 + *
1.100 + * Every piece of recorded data is associated with a measurement. A measurement
1.101 + * is an entity with a name and version. Each measurement is associated with a
1.102 + * named provider.
1.103 + *
1.104 + * When the metrics system is initialized, we ask providers (the entities that
1.105 + * emit data) to configure the database for storage of their data. They tell
1.106 + * storage what their requirements are. For example, they'll register
1.107 + * named daily counters associated with specific measurements.
1.108 + *
1.109 + * Recorded data is stored differently depending on the requirements for
1.110 + * storing it. We have facilities for storing the following classes of data:
1.111 + *
1.112 + * 1) Counts of event/field occurrences aggregated by day.
1.113 + * 2) Discrete values of fields aggregated by day.
1.114 + * 3) Discrete values of fields aggregated by day max 1 per day (last write
1.115 + * wins).
1.116 + * 4) Discrete values of fields max 1 (last write wins).
1.117 + *
1.118 + * Most data is aggregated per day mainly for privacy reasons. This does throw
1.119 + * away potentially useful data. But, it's not currently used, so there is no
1.120 + * need to keep the granular information.
1.121 + *
1.122 + * Database Schema
1.123 + * ---------------
1.124 + *
1.125 + * This database contains the following tables:
1.126 + *
1.127 + * providers -- Maps provider string name to an internal ID.
1.128 + * provider_state -- Holds opaque persisted state for providers.
1.129 + * measurements -- Holds the set of known measurements (name, version,
1.130 + * provider tuples).
1.131 + * types -- The data types that can be stored in measurements/fields.
1.132 + * fields -- Describes entities that occur within measurements.
1.133 + * daily_counters -- Holds daily-aggregated counts of events. Each row is
1.134 + * associated with a field and a day.
1.135 + * daily_discrete_numeric -- Holds numeric values for fields grouped by day.
1.136 + * Each row contains a discrete value associated with a field that occurred
1.137 + * on a specific day. There can be multiple rows per field per day.
1.138 + * daily_discrete_text -- Holds text values for fields grouped by day. Each
1.139 + * row contains a discrete value associated with a field that occurred on a
1.140 + * specific day.
1.141 + * daily_last_numeric -- Holds numeric values where the last encountered
1.142 + * value for a given day is retained.
1.143 + * daily_last_text -- Like daily_last_numeric except for text values.
1.144 + * last_numeric -- Holds the most recent value for a numeric field.
1.145 + * last_text -- Like last_numeric except for text fields.
1.146 + *
1.147 + * Notes
1.148 + * -----
1.149 + *
1.150 + * It is tempting to use SQLite's julianday() function to store days that
1.151 + * things happened. However, a Julian Day begins at *noon* in 4714 B.C. This
1.152 + * results in weird half day offsets from UNIX time. So, we instead store
1.153 + * number of days since UNIX epoch, not Julian.
1.154 + */
1.155 +
1.156 +/**
1.157 + * All of our SQL statements are stored in a central mapping so they can easily
1.158 + * be audited for security, perf, etc.
1.159 + */
1.160 +const SQL = {
1.161 + // Create the providers table.
1.162 + createProvidersTable: "\
1.163 +CREATE TABLE providers (\
1.164 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.165 + name TEXT, \
1.166 + UNIQUE (name) \
1.167 +)",
1.168 +
1.169 + createProviderStateTable: "\
1.170 +CREATE TABLE provider_state (\
1.171 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.172 + provider_id INTEGER, \
1.173 + name TEXT, \
1.174 + VALUE TEXT, \
1.175 + UNIQUE (provider_id, name), \
1.176 + FOREIGN KEY (provider_id) REFERENCES providers(id) ON DELETE CASCADE\
1.177 +)",
1.178 +
1.179 + createProviderStateProviderIndex: "\
1.180 +CREATE INDEX i_provider_state_provider_id ON provider_state (provider_id)",
1.181 +
1.182 + createMeasurementsTable: "\
1.183 +CREATE TABLE measurements (\
1.184 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.185 + provider_id INTEGER, \
1.186 + name TEXT, \
1.187 + version INTEGER, \
1.188 + UNIQUE (provider_id, name, version), \
1.189 + FOREIGN KEY (provider_id) REFERENCES providers(id) ON DELETE CASCADE\
1.190 +)",
1.191 +
1.192 + createMeasurementsProviderIndex: "\
1.193 +CREATE INDEX i_measurements_provider_id ON measurements (provider_id)",
1.194 +
1.195 + createMeasurementsView: "\
1.196 +CREATE VIEW v_measurements AS \
1.197 + SELECT \
1.198 + providers.id AS provider_id, \
1.199 + providers.name AS provider_name, \
1.200 + measurements.id AS measurement_id, \
1.201 + measurements.name AS measurement_name, \
1.202 + measurements.version AS measurement_version \
1.203 + FROM providers, measurements \
1.204 + WHERE \
1.205 + measurements.provider_id = providers.id",
1.206 +
1.207 + createTypesTable: "\
1.208 +CREATE TABLE types (\
1.209 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.210 + name TEXT, \
1.211 + UNIQUE (name)\
1.212 +)",
1.213 +
1.214 + createFieldsTable: "\
1.215 +CREATE TABLE fields (\
1.216 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.217 + measurement_id INTEGER, \
1.218 + name TEXT, \
1.219 + value_type INTEGER , \
1.220 + UNIQUE (measurement_id, name), \
1.221 + FOREIGN KEY (measurement_id) REFERENCES measurements(id) ON DELETE CASCADE \
1.222 + FOREIGN KEY (value_type) REFERENCES types(id) ON DELETE CASCADE \
1.223 +)",
1.224 +
1.225 + createFieldsMeasurementIndex: "\
1.226 +CREATE INDEX i_fields_measurement_id ON fields (measurement_id)",
1.227 +
1.228 + createFieldsView: "\
1.229 +CREATE VIEW v_fields AS \
1.230 + SELECT \
1.231 + providers.id AS provider_id, \
1.232 + providers.name AS provider_name, \
1.233 + measurements.id AS measurement_id, \
1.234 + measurements.name AS measurement_name, \
1.235 + measurements.version AS measurement_version, \
1.236 + fields.id AS field_id, \
1.237 + fields.name AS field_name, \
1.238 + types.id AS type_id, \
1.239 + types.name AS type_name \
1.240 + FROM providers, measurements, fields, types \
1.241 + WHERE \
1.242 + fields.measurement_id = measurements.id \
1.243 + AND measurements.provider_id = providers.id \
1.244 + AND fields.value_type = types.id",
1.245 +
1.246 + createDailyCountersTable: "\
1.247 +CREATE TABLE daily_counters (\
1.248 + field_id INTEGER, \
1.249 + day INTEGER, \
1.250 + value INTEGER, \
1.251 + UNIQUE(field_id, day), \
1.252 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.253 +)",
1.254 +
1.255 + createDailyCountersFieldIndex: "\
1.256 +CREATE INDEX i_daily_counters_field_id ON daily_counters (field_id)",
1.257 +
1.258 + createDailyCountersDayIndex: "\
1.259 +CREATE INDEX i_daily_counters_day ON daily_counters (day)",
1.260 +
1.261 + createDailyCountersView: "\
1.262 +CREATE VIEW v_daily_counters AS SELECT \
1.263 + providers.id AS provider_id, \
1.264 + providers.name AS provider_name, \
1.265 + measurements.id AS measurement_id, \
1.266 + measurements.name AS measurement_name, \
1.267 + measurements.version AS measurement_version, \
1.268 + fields.id AS field_id, \
1.269 + fields.name AS field_name, \
1.270 + daily_counters.day AS day, \
1.271 + daily_counters.value AS value \
1.272 +FROM providers, measurements, fields, daily_counters \
1.273 +WHERE \
1.274 + daily_counters.field_id = fields.id \
1.275 + AND fields.measurement_id = measurements.id \
1.276 + AND measurements.provider_id = providers.id",
1.277 +
1.278 + createDailyDiscreteNumericsTable: "\
1.279 +CREATE TABLE daily_discrete_numeric (\
1.280 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.281 + field_id INTEGER, \
1.282 + day INTEGER, \
1.283 + value INTEGER, \
1.284 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.285 +)",
1.286 +
1.287 + createDailyDiscreteNumericsFieldIndex: "\
1.288 +CREATE INDEX i_daily_discrete_numeric_field_id \
1.289 +ON daily_discrete_numeric (field_id)",
1.290 +
1.291 + createDailyDiscreteNumericsDayIndex: "\
1.292 +CREATE INDEX i_daily_discrete_numeric_day \
1.293 +ON daily_discrete_numeric (day)",
1.294 +
1.295 + createDailyDiscreteTextTable: "\
1.296 +CREATE TABLE daily_discrete_text (\
1.297 + id INTEGER PRIMARY KEY AUTOINCREMENT, \
1.298 + field_id INTEGER, \
1.299 + day INTEGER, \
1.300 + value TEXT, \
1.301 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.302 +)",
1.303 +
1.304 + createDailyDiscreteTextFieldIndex: "\
1.305 +CREATE INDEX i_daily_discrete_text_field_id \
1.306 +ON daily_discrete_text (field_id)",
1.307 +
1.308 + createDailyDiscreteTextDayIndex: "\
1.309 +CREATE INDEX i_daily_discrete_text_day \
1.310 +ON daily_discrete_text (day)",
1.311 +
1.312 + createDailyDiscreteView: "\
1.313 +CREATE VIEW v_daily_discrete AS \
1.314 + SELECT \
1.315 + providers.id AS provider_id, \
1.316 + providers.name AS provider_name, \
1.317 + measurements.id AS measurement_id, \
1.318 + measurements.name AS measurement_name, \
1.319 + measurements.version AS measurement_version, \
1.320 + fields.id AS field_id, \
1.321 + fields.name AS field_name, \
1.322 + daily_discrete_numeric.id AS value_id, \
1.323 + daily_discrete_numeric.day AS day, \
1.324 + daily_discrete_numeric.value AS value, \
1.325 + \"numeric\" AS value_type \
1.326 + FROM providers, measurements, fields, daily_discrete_numeric \
1.327 + WHERE \
1.328 + daily_discrete_numeric.field_id = fields.id \
1.329 + AND fields.measurement_id = measurements.id \
1.330 + AND measurements.provider_id = providers.id \
1.331 + UNION ALL \
1.332 + SELECT \
1.333 + providers.id AS provider_id, \
1.334 + providers.name AS provider_name, \
1.335 + measurements.id AS measurement_id, \
1.336 + measurements.name AS measurement_name, \
1.337 + measurements.version AS measurement_version, \
1.338 + fields.id AS field_id, \
1.339 + fields.name AS field_name, \
1.340 + daily_discrete_text.id AS value_id, \
1.341 + daily_discrete_text.day AS day, \
1.342 + daily_discrete_text.value AS value, \
1.343 + \"text\" AS value_type \
1.344 + FROM providers, measurements, fields, daily_discrete_text \
1.345 + WHERE \
1.346 + daily_discrete_text.field_id = fields.id \
1.347 + AND fields.measurement_id = measurements.id \
1.348 + AND measurements.provider_id = providers.id \
1.349 + ORDER BY day ASC, value_id ASC",
1.350 +
1.351 + createLastNumericTable: "\
1.352 +CREATE TABLE last_numeric (\
1.353 + field_id INTEGER PRIMARY KEY, \
1.354 + day INTEGER, \
1.355 + value NUMERIC, \
1.356 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.357 +)",
1.358 +
1.359 + createLastTextTable: "\
1.360 +CREATE TABLE last_text (\
1.361 + field_id INTEGER PRIMARY KEY, \
1.362 + day INTEGER, \
1.363 + value TEXT, \
1.364 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.365 +)",
1.366 +
1.367 + createLastView: "\
1.368 +CREATE VIEW v_last AS \
1.369 + SELECT \
1.370 + providers.id AS provider_id, \
1.371 + providers.name AS provider_name, \
1.372 + measurements.id AS measurement_id, \
1.373 + measurements.name AS measurement_name, \
1.374 + measurements.version AS measurement_version, \
1.375 + fields.id AS field_id, \
1.376 + fields.name AS field_name, \
1.377 + last_numeric.day AS day, \
1.378 + last_numeric.value AS value, \
1.379 + \"numeric\" AS value_type \
1.380 + FROM providers, measurements, fields, last_numeric \
1.381 + WHERE \
1.382 + last_numeric.field_id = fields.id \
1.383 + AND fields.measurement_id = measurements.id \
1.384 + AND measurements.provider_id = providers.id \
1.385 + UNION ALL \
1.386 + SELECT \
1.387 + providers.id AS provider_id, \
1.388 + providers.name AS provider_name, \
1.389 + measurements.id AS measurement_id, \
1.390 + measurements.name AS measurement_name, \
1.391 + measurements.version AS measurement_version, \
1.392 + fields.id AS field_id, \
1.393 + fields.name AS field_name, \
1.394 + last_text.day AS day, \
1.395 + last_text.value AS value, \
1.396 + \"text\" AS value_type \
1.397 + FROM providers, measurements, fields, last_text \
1.398 + WHERE \
1.399 + last_text.field_id = fields.id \
1.400 + AND fields.measurement_id = measurements.id \
1.401 + AND measurements.provider_id = providers.id",
1.402 +
1.403 + createDailyLastNumericTable: "\
1.404 +CREATE TABLE daily_last_numeric (\
1.405 + field_id INTEGER, \
1.406 + day INTEGER, \
1.407 + value NUMERIC, \
1.408 + UNIQUE (field_id, day) \
1.409 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.410 +)",
1.411 +
1.412 + createDailyLastNumericFieldIndex: "\
1.413 +CREATE INDEX i_daily_last_numeric_field_id ON daily_last_numeric (field_id)",
1.414 +
1.415 + createDailyLastNumericDayIndex: "\
1.416 +CREATE INDEX i_daily_last_numeric_day ON daily_last_numeric (day)",
1.417 +
1.418 + createDailyLastTextTable: "\
1.419 +CREATE TABLE daily_last_text (\
1.420 + field_id INTEGER, \
1.421 + day INTEGER, \
1.422 + value TEXT, \
1.423 + UNIQUE (field_id, day) \
1.424 + FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\
1.425 +)",
1.426 +
1.427 + createDailyLastTextFieldIndex: "\
1.428 +CREATE INDEX i_daily_last_text_field_id ON daily_last_text (field_id)",
1.429 +
1.430 + createDailyLastTextDayIndex: "\
1.431 +CREATE INDEX i_daily_last_text_day ON daily_last_text (day)",
1.432 +
1.433 + createDailyLastView: "\
1.434 +CREATE VIEW v_daily_last AS \
1.435 + SELECT \
1.436 + providers.id AS provider_id, \
1.437 + providers.name AS provider_name, \
1.438 + measurements.id AS measurement_id, \
1.439 + measurements.name AS measurement_name, \
1.440 + measurements.version AS measurement_version, \
1.441 + fields.id AS field_id, \
1.442 + fields.name AS field_name, \
1.443 + daily_last_numeric.day AS day, \
1.444 + daily_last_numeric.value AS value, \
1.445 + \"numeric\" as value_type \
1.446 + FROM providers, measurements, fields, daily_last_numeric \
1.447 + WHERE \
1.448 + daily_last_numeric.field_id = fields.id \
1.449 + AND fields.measurement_id = measurements.id \
1.450 + AND measurements.provider_id = providers.id \
1.451 + UNION ALL \
1.452 + SELECT \
1.453 + providers.id AS provider_id, \
1.454 + providers.name AS provider_name, \
1.455 + measurements.id AS measurement_id, \
1.456 + measurements.name AS measurement_name, \
1.457 + measurements.version AS measurement_version, \
1.458 + fields.id AS field_id, \
1.459 + fields.name AS field_name, \
1.460 + daily_last_text.day AS day, \
1.461 + daily_last_text.value AS value, \
1.462 + \"text\" as value_type \
1.463 + FROM providers, measurements, fields, daily_last_text \
1.464 + WHERE \
1.465 + daily_last_text.field_id = fields.id \
1.466 + AND fields.measurement_id = measurements.id \
1.467 + AND measurements.provider_id = providers.id",
1.468 +
1.469 + // Mutation.
1.470 +
1.471 + addProvider: "INSERT INTO providers (name) VALUES (:provider)",
1.472 +
1.473 + setProviderState: "\
1.474 +INSERT OR REPLACE INTO provider_state \
1.475 + (provider_id, name, value) \
1.476 + VALUES (:provider_id, :name, :value)",
1.477 +
1.478 + addMeasurement: "\
1.479 +INSERT INTO measurements (provider_id, name, version) \
1.480 + VALUES (:provider_id, :measurement, :version)",
1.481 +
1.482 + addType: "INSERT INTO types (name) VALUES (:name)",
1.483 +
1.484 + addField: "\
1.485 +INSERT INTO fields (measurement_id, name, value_type) \
1.486 + VALUES (:measurement_id, :field, :value_type)",
1.487 +
1.488 + incrementDailyCounterFromFieldID: "\
1.489 +INSERT OR REPLACE INTO daily_counters VALUES (\
1.490 + :field_id, \
1.491 + :days, \
1.492 + COALESCE(\
1.493 + (SELECT value FROM daily_counters WHERE \
1.494 + field_id = :field_id AND day = :days \
1.495 + ), \
1.496 + 0\
1.497 + ) + :by)",
1.498 +
1.499 + deleteLastNumericFromFieldID: "\
1.500 +DELETE FROM last_numeric WHERE field_id = :field_id",
1.501 +
1.502 + deleteLastTextFromFieldID: "\
1.503 +DELETE FROM last_text WHERE field_id = :field_id",
1.504 +
1.505 + setLastNumeric: "\
1.506 +INSERT OR REPLACE INTO last_numeric VALUES (:field_id, :days, :value)",
1.507 +
1.508 + setLastText: "\
1.509 +INSERT OR REPLACE INTO last_text VALUES (:field_id, :days, :value)",
1.510 +
1.511 + setDailyLastNumeric: "\
1.512 +INSERT OR REPLACE INTO daily_last_numeric VALUES (:field_id, :days, :value)",
1.513 +
1.514 + setDailyLastText: "\
1.515 +INSERT OR REPLACE INTO daily_last_text VALUES (:field_id, :days, :value)",
1.516 +
1.517 + addDailyDiscreteNumeric: "\
1.518 +INSERT INTO daily_discrete_numeric \
1.519 +(field_id, day, value) VALUES (:field_id, :days, :value)",
1.520 +
1.521 + addDailyDiscreteText: "\
1.522 +INSERT INTO daily_discrete_text \
1.523 +(field_id, day, value) VALUES (:field_id, :days, :value)",
1.524 +
1.525 + pruneOldDailyCounters: "DELETE FROM daily_counters WHERE day < :days",
1.526 + pruneOldDailyDiscreteNumeric: "DELETE FROM daily_discrete_numeric WHERE day < :days",
1.527 + pruneOldDailyDiscreteText: "DELETE FROM daily_discrete_text WHERE day < :days",
1.528 + pruneOldDailyLastNumeric: "DELETE FROM daily_last_numeric WHERE day < :days",
1.529 + pruneOldDailyLastText: "DELETE FROM daily_last_text WHERE day < :days",
1.530 + pruneOldLastNumeric: "DELETE FROM last_numeric WHERE day < :days",
1.531 + pruneOldLastText: "DELETE FROM last_text WHERE day < :days",
1.532 +
1.533 + // Retrieval.
1.534 +
1.535 + getProviderID: "SELECT id FROM providers WHERE name = :provider",
1.536 +
1.537 + getProviders: "SELECT id, name FROM providers",
1.538 +
1.539 + getProviderStateWithName: "\
1.540 +SELECT value FROM provider_state \
1.541 + WHERE provider_id = :provider_id \
1.542 + AND name = :name",
1.543 +
1.544 + getMeasurements: "SELECT * FROM v_measurements",
1.545 +
1.546 + getMeasurementID: "\
1.547 +SELECT id FROM measurements \
1.548 + WHERE provider_id = :provider_id \
1.549 + AND name = :measurement \
1.550 + AND version = :version",
1.551 +
1.552 + getFieldID: "\
1.553 +SELECT id FROM fields \
1.554 + WHERE measurement_id = :measurement_id \
1.555 + AND name = :field \
1.556 + AND value_type = :value_type \
1.557 +",
1.558 +
1.559 + getTypes: "SELECT * FROM types",
1.560 +
1.561 + getTypeID: "SELECT id FROM types WHERE name = :name",
1.562 +
1.563 + getDailyCounterCountsFromFieldID: "\
1.564 +SELECT day, value FROM daily_counters \
1.565 + WHERE field_id = :field_id \
1.566 + ORDER BY day ASC",
1.567 +
1.568 + getDailyCounterCountFromFieldID: "\
1.569 +SELECT value FROM daily_counters \
1.570 + WHERE field_id = :field_id \
1.571 + AND day = :days",
1.572 +
1.573 + getMeasurementDailyCounters: "\
1.574 +SELECT field_name, day, value FROM v_daily_counters \
1.575 +WHERE measurement_id = :measurement_id",
1.576 +
1.577 + getFieldInfo: "SELECT * FROM v_fields",
1.578 +
1.579 + getLastNumericFromFieldID: "\
1.580 +SELECT day, value FROM last_numeric WHERE field_id = :field_id",
1.581 +
1.582 + getLastTextFromFieldID: "\
1.583 +SELECT day, value FROM last_text WHERE field_id = :field_id",
1.584 +
1.585 + getMeasurementLastValues: "\
1.586 +SELECT field_name, day, value FROM v_last \
1.587 +WHERE measurement_id = :measurement_id",
1.588 +
1.589 + getDailyDiscreteNumericFromFieldID: "\
1.590 +SELECT day, value FROM daily_discrete_numeric \
1.591 + WHERE field_id = :field_id \
1.592 + ORDER BY day ASC, id ASC",
1.593 +
1.594 + getDailyDiscreteNumericFromFieldIDAndDay: "\
1.595 +SELECT day, value FROM daily_discrete_numeric \
1.596 + WHERE field_id = :field_id AND day = :days \
1.597 + ORDER BY id ASC",
1.598 +
1.599 + getDailyDiscreteTextFromFieldID: "\
1.600 +SELECT day, value FROM daily_discrete_text \
1.601 + WHERE field_id = :field_id \
1.602 + ORDER BY day ASC, id ASC",
1.603 +
1.604 + getDailyDiscreteTextFromFieldIDAndDay: "\
1.605 +SELECT day, value FROM daily_discrete_text \
1.606 + WHERE field_id = :field_id AND day = :days \
1.607 + ORDER BY id ASC",
1.608 +
1.609 + getMeasurementDailyDiscreteValues: "\
1.610 +SELECT field_name, day, value_id, value FROM v_daily_discrete \
1.611 +WHERE measurement_id = :measurement_id \
1.612 +ORDER BY day ASC, value_id ASC",
1.613 +
1.614 + getDailyLastNumericFromFieldID: "\
1.615 +SELECT day, value FROM daily_last_numeric \
1.616 + WHERE field_id = :field_id \
1.617 + ORDER BY day ASC",
1.618 +
1.619 + getDailyLastNumericFromFieldIDAndDay: "\
1.620 +SELECT day, value FROM daily_last_numeric \
1.621 + WHERE field_id = :field_id AND day = :days",
1.622 +
1.623 + getDailyLastTextFromFieldID: "\
1.624 +SELECT day, value FROM daily_last_text \
1.625 + WHERE field_id = :field_id \
1.626 + ORDER BY day ASC",
1.627 +
1.628 + getDailyLastTextFromFieldIDAndDay: "\
1.629 +SELECT day, value FROM daily_last_text \
1.630 + WHERE field_id = :field_id AND day = :days",
1.631 +
1.632 + getMeasurementDailyLastValues: "\
1.633 +SELECT field_name, day, value FROM v_daily_last \
1.634 +WHERE measurement_id = :measurement_id",
1.635 +};
1.636 +
1.637 +
1.638 +function dailyKeyFromDate(date) {
1.639 + let year = String(date.getUTCFullYear());
1.640 + let month = String(date.getUTCMonth() + 1);
1.641 + let day = String(date.getUTCDate());
1.642 +
1.643 + if (month.length < 2) {
1.644 + month = "0" + month;
1.645 + }
1.646 +
1.647 + if (day.length < 2) {
1.648 + day = "0" + day;
1.649 + }
1.650 +
1.651 + return year + "-" + month + "-" + day;
1.652 +}
1.653 +
1.654 +
1.655 +/**
1.656 + * Create a new backend instance bound to a SQLite database at the given path.
1.657 + *
1.658 + * This returns a promise that will resolve to a `MetricsStorageSqliteBackend`
1.659 + * instance. The resolved instance will be initialized and ready for use.
1.660 + *
1.661 + * Very few consumers have a need to call this. Instead, a higher-level entity
1.662 + * likely calls this and sets up the database connection for a service or
1.663 + * singleton.
1.664 + */
1.665 +this.MetricsStorageBackend = function (path) {
1.666 + return Task.spawn(function initTask() {
1.667 + let connection = yield Sqlite.openConnection({
1.668 + path: path,
1.669 +
1.670 + // There should only be one connection per database, so we disable this
1.671 + // for perf reasons.
1.672 + sharedMemoryCache: false,
1.673 + });
1.674 +
1.675 + // If we fail initializing the storage object, we need to close the
1.676 + // database connection or else Storage will assert on shutdown.
1.677 + let storage;
1.678 + try {
1.679 + storage = new MetricsStorageSqliteBackend(connection);
1.680 + yield storage._init();
1.681 + } catch (ex) {
1.682 + yield connection.close();
1.683 + throw ex;
1.684 + }
1.685 +
1.686 + throw new Task.Result(storage);
1.687 + });
1.688 +};
1.689 +
1.690 +
1.691 +/**
1.692 + * Manages storage of metrics data in a SQLite database.
1.693 + *
1.694 + * This is the main type used for interfacing with the database.
1.695 + *
1.696 + * Instances of this should be obtained by calling MetricsStorageConnection().
1.697 + *
1.698 + * The current implementation will not work if the database is mutated by
1.699 + * multiple connections because of the way we cache primary keys.
1.700 + *
1.701 + * FUTURE enforce 1 read/write connection per database limit.
1.702 + */
1.703 +function MetricsStorageSqliteBackend(connection) {
1.704 + this._log = Log.repository.getLogger("Services.Metrics.MetricsStorage");
1.705 +
1.706 + this._connection = connection;
1.707 + this._enabledWALCheckpointPages = null;
1.708 +
1.709 + // Integer IDs to string name.
1.710 + this._typesByID = new Map();
1.711 +
1.712 + // String name to integer IDs.
1.713 + this._typesByName = new Map();
1.714 +
1.715 + // Maps provider names to integer IDs.
1.716 + this._providerIDs = new Map();
1.717 +
1.718 + // Maps :-delimited strings of [provider name, name, version] to integer IDs.
1.719 + this._measurementsByInfo = new Map();
1.720 +
1.721 + // Integer IDs to Arrays of [provider name, name, version].
1.722 + this._measurementsByID = new Map();
1.723 +
1.724 + // Integer IDs to Arrays of [measurement id, field name, value name]
1.725 + this._fieldsByID = new Map();
1.726 +
1.727 + // Maps :-delimited strings of [measurement id, field name] to integer ID.
1.728 + this._fieldsByInfo = new Map();
1.729 +
1.730 + // Maps measurement ID to sets of field IDs.
1.731 + this._fieldsByMeasurement = new Map();
1.732 +
1.733 + this._queuedOperations = [];
1.734 + this._queuedInProgress = false;
1.735 +}
1.736 +
1.737 +MetricsStorageSqliteBackend.prototype = Object.freeze({
1.738 + // Max size (in kibibytes) the WAL log is allowed to grow to before it is
1.739 + // checkpointed.
1.740 + //
1.741 + // This was first deployed in bug 848136. We want a value large enough
1.742 + // that we aren't checkpointing all the time. However, we want it
1.743 + // small enough so we don't have to read so much when we open the
1.744 + // database.
1.745 + MAX_WAL_SIZE_KB: 512,
1.746 +
1.747 + FIELD_DAILY_COUNTER: "daily-counter",
1.748 + FIELD_DAILY_DISCRETE_NUMERIC: "daily-discrete-numeric",
1.749 + FIELD_DAILY_DISCRETE_TEXT: "daily-discrete-text",
1.750 + FIELD_DAILY_LAST_NUMERIC: "daily-last-numeric",
1.751 + FIELD_DAILY_LAST_TEXT: "daily-last-text",
1.752 + FIELD_LAST_NUMERIC: "last-numeric",
1.753 + FIELD_LAST_TEXT: "last-text",
1.754 +
1.755 + _BUILTIN_TYPES: [
1.756 + "FIELD_DAILY_COUNTER",
1.757 + "FIELD_DAILY_DISCRETE_NUMERIC",
1.758 + "FIELD_DAILY_DISCRETE_TEXT",
1.759 + "FIELD_DAILY_LAST_NUMERIC",
1.760 + "FIELD_DAILY_LAST_TEXT",
1.761 + "FIELD_LAST_NUMERIC",
1.762 + "FIELD_LAST_TEXT",
1.763 + ],
1.764 +
1.765 + // Statements that are used to create the initial DB schema.
1.766 + _SCHEMA_STATEMENTS: [
1.767 + "createProvidersTable",
1.768 + "createProviderStateTable",
1.769 + "createProviderStateProviderIndex",
1.770 + "createMeasurementsTable",
1.771 + "createMeasurementsProviderIndex",
1.772 + "createMeasurementsView",
1.773 + "createTypesTable",
1.774 + "createFieldsTable",
1.775 + "createFieldsMeasurementIndex",
1.776 + "createFieldsView",
1.777 + "createDailyCountersTable",
1.778 + "createDailyCountersFieldIndex",
1.779 + "createDailyCountersDayIndex",
1.780 + "createDailyCountersView",
1.781 + "createDailyDiscreteNumericsTable",
1.782 + "createDailyDiscreteNumericsFieldIndex",
1.783 + "createDailyDiscreteNumericsDayIndex",
1.784 + "createDailyDiscreteTextTable",
1.785 + "createDailyDiscreteTextFieldIndex",
1.786 + "createDailyDiscreteTextDayIndex",
1.787 + "createDailyDiscreteView",
1.788 + "createDailyLastNumericTable",
1.789 + "createDailyLastNumericFieldIndex",
1.790 + "createDailyLastNumericDayIndex",
1.791 + "createDailyLastTextTable",
1.792 + "createDailyLastTextFieldIndex",
1.793 + "createDailyLastTextDayIndex",
1.794 + "createDailyLastView",
1.795 + "createLastNumericTable",
1.796 + "createLastTextTable",
1.797 + "createLastView",
1.798 + ],
1.799 +
1.800 + // Statements that are used to prune old data.
1.801 + _PRUNE_STATEMENTS: [
1.802 + "pruneOldDailyCounters",
1.803 + "pruneOldDailyDiscreteNumeric",
1.804 + "pruneOldDailyDiscreteText",
1.805 + "pruneOldDailyLastNumeric",
1.806 + "pruneOldDailyLastText",
1.807 + "pruneOldLastNumeric",
1.808 + "pruneOldLastText",
1.809 + ],
1.810 +
1.811 + /**
1.812 + * Close the database connection.
1.813 + *
1.814 + * This should be called on all instances or the SQLite layer may complain
1.815 + * loudly. After this has been called, the connection cannot be used.
1.816 + *
1.817 + * @return Promise<>
1.818 + */
1.819 + close: function () {
1.820 + return Task.spawn(function doClose() {
1.821 + // There is some light magic involved here. First, we enqueue an
1.822 + // operation to ensure that all pending operations have the opportunity
1.823 + // to execute. We additionally execute a SQL operation. Due to the FIFO
1.824 + // execution order of issued statements, this will cause us to wait on
1.825 + // any outstanding statements before closing.
1.826 + try {
1.827 + yield this.enqueueOperation(function dummyOperation() {
1.828 + return this._connection.execute("SELECT 1");
1.829 + }.bind(this));
1.830 + } catch (ex) {}
1.831 +
1.832 + try {
1.833 + yield this._connection.close();
1.834 + } finally {
1.835 + this._connection = null;
1.836 + }
1.837 + }.bind(this));
1.838 + },
1.839 +
1.840 + /**
1.841 + * Whether a provider is known to exist.
1.842 + *
1.843 + * @param provider
1.844 + * (string) Name of the provider.
1.845 + */
1.846 + hasProvider: function (provider) {
1.847 + return this._providerIDs.has(provider);
1.848 + },
1.849 +
1.850 + /**
1.851 + * Whether a measurement is known to exist.
1.852 + *
1.853 + * @param provider
1.854 + * (string) Name of the provider.
1.855 + * @param name
1.856 + * (string) Name of the measurement.
1.857 + * @param version
1.858 + * (Number) Integer measurement version.
1.859 + */
1.860 + hasMeasurement: function (provider, name, version) {
1.861 + return this._measurementsByInfo.has([provider, name, version].join(":"));
1.862 + },
1.863 +
1.864 + /**
1.865 + * Whether a named field exists in a measurement.
1.866 + *
1.867 + * @param measurementID
1.868 + * (Number) The integer primary key of the measurement.
1.869 + * @param field
1.870 + * (string) The name of the field to look for.
1.871 + */
1.872 + hasFieldFromMeasurement: function (measurementID, field) {
1.873 + return this._fieldsByInfo.has([measurementID, field].join(":"));
1.874 + },
1.875 +
1.876 + /**
1.877 + * Whether a field is known.
1.878 + *
1.879 + * @param provider
1.880 + * (string) Name of the provider having the field.
1.881 + * @param measurement
1.882 + * (string) Name of the measurement in the provider having the field.
1.883 + * @param field
1.884 + * (string) Name of the field in the measurement.
1.885 + */
1.886 + hasField: function (provider, measurement, version, field) {
1.887 + let key = [provider, measurement, version].join(":");
1.888 + let measurementID = this._measurementsByInfo.get(key);
1.889 + if (!measurementID) {
1.890 + return false;
1.891 + }
1.892 +
1.893 + return this.hasFieldFromMeasurement(measurementID, field);
1.894 + },
1.895 +
1.896 + /**
1.897 + * Look up the integer primary key of a provider.
1.898 + *
1.899 + * @param provider
1.900 + * (string) Name of the provider.
1.901 + */
1.902 + providerID: function (provider) {
1.903 + return this._providerIDs.get(provider);
1.904 + },
1.905 +
1.906 + /**
1.907 + * Look up the integer primary key of a measurement.
1.908 + *
1.909 + * @param provider
1.910 + * (string) Name of the provider.
1.911 + * @param measurement
1.912 + * (string) Name of the measurement.
1.913 + * @param version
1.914 + * (Number) Integer version of the measurement.
1.915 + */
1.916 + measurementID: function (provider, measurement, version) {
1.917 + return this._measurementsByInfo.get([provider, measurement, version].join(":"));
1.918 + },
1.919 +
1.920 + fieldIDFromMeasurement: function (measurementID, field) {
1.921 + return this._fieldsByInfo.get([measurementID, field].join(":"));
1.922 + },
1.923 +
1.924 + fieldID: function (provider, measurement, version, field) {
1.925 + let measurementID = this.measurementID(provider, measurement, version);
1.926 + if (!measurementID) {
1.927 + return null;
1.928 + }
1.929 +
1.930 + return this.fieldIDFromMeasurement(measurementID, field);
1.931 + },
1.932 +
1.933 + measurementHasAnyDailyCounterFields: function (measurementID) {
1.934 + return this.measurementHasAnyFieldsOfTypes(measurementID,
1.935 + [this.FIELD_DAILY_COUNTER]);
1.936 + },
1.937 +
1.938 + measurementHasAnyLastFields: function (measurementID) {
1.939 + return this.measurementHasAnyFieldsOfTypes(measurementID,
1.940 + [this.FIELD_LAST_NUMERIC,
1.941 + this.FIELD_LAST_TEXT]);
1.942 + },
1.943 +
1.944 + measurementHasAnyDailyLastFields: function (measurementID) {
1.945 + return this.measurementHasAnyFieldsOfTypes(measurementID,
1.946 + [this.FIELD_DAILY_LAST_NUMERIC,
1.947 + this.FIELD_DAILY_LAST_TEXT]);
1.948 + },
1.949 +
1.950 + measurementHasAnyDailyDiscreteFields: function (measurementID) {
1.951 + return this.measurementHasAnyFieldsOfTypes(measurementID,
1.952 + [this.FIELD_DAILY_DISCRETE_NUMERIC,
1.953 + this.FIELD_DAILY_DISCRETE_TEXT]);
1.954 + },
1.955 +
1.956 + measurementHasAnyFieldsOfTypes: function (measurementID, types) {
1.957 + if (!this._fieldsByMeasurement.has(measurementID)) {
1.958 + return false;
1.959 + }
1.960 +
1.961 + let fieldIDs = this._fieldsByMeasurement.get(measurementID);
1.962 + for (let fieldID of fieldIDs) {
1.963 + let fieldType = this._fieldsByID.get(fieldID)[2];
1.964 + if (types.indexOf(fieldType) != -1) {
1.965 + return true;
1.966 + }
1.967 + }
1.968 +
1.969 + return false;
1.970 + },
1.971 +
1.972 + /**
1.973 + * Register a measurement with the backend.
1.974 + *
1.975 + * Measurements must be registered before storage can be allocated to them.
1.976 + *
1.977 + * A measurement consists of a string name and integer version attached
1.978 + * to a named provider.
1.979 + *
1.980 + * This returns a promise that resolves to the storage ID for this
1.981 + * measurement.
1.982 + *
1.983 + * If the measurement is not known to exist, it is registered with storage.
1.984 + * If the measurement has already been registered, this is effectively a
1.985 + * no-op (that still returns a promise resolving to the storage ID).
1.986 + *
1.987 + * @param provider
1.988 + * (string) Name of the provider this measurement belongs to.
1.989 + * @param name
1.990 + * (string) Name of this measurement.
1.991 + * @param version
1.992 + * (Number) Integer version of this measurement.
1.993 + */
1.994 + registerMeasurement: function (provider, name, version) {
1.995 + if (this.hasMeasurement(provider, name, version)) {
1.996 + return CommonUtils.laterTickResolvingPromise(
1.997 + this.measurementID(provider, name, version));
1.998 + }
1.999 +
1.1000 + // Registrations might not be safe to perform in parallel with provider
1.1001 + // operations. So, we queue them.
1.1002 + let self = this;
1.1003 + return this.enqueueOperation(function createMeasurementOperation() {
1.1004 + return Task.spawn(function createMeasurement() {
1.1005 + let providerID = self._providerIDs.get(provider);
1.1006 +
1.1007 + if (!providerID) {
1.1008 + yield self._connection.executeCached(SQL.addProvider, {provider: provider});
1.1009 + let rows = yield self._connection.executeCached(SQL.getProviderID,
1.1010 + {provider: provider});
1.1011 +
1.1012 + providerID = rows[0].getResultByIndex(0);
1.1013 +
1.1014 + self._providerIDs.set(provider, providerID);
1.1015 + }
1.1016 +
1.1017 + let params = {
1.1018 + provider_id: providerID,
1.1019 + measurement: name,
1.1020 + version: version,
1.1021 + };
1.1022 +
1.1023 + yield self._connection.executeCached(SQL.addMeasurement, params);
1.1024 + let rows = yield self._connection.executeCached(SQL.getMeasurementID, params);
1.1025 +
1.1026 + let measurementID = rows[0].getResultByIndex(0);
1.1027 +
1.1028 + self._measurementsByInfo.set([provider, name, version].join(":"), measurementID);
1.1029 + self._measurementsByID.set(measurementID, [provider, name, version]);
1.1030 + self._fieldsByMeasurement.set(measurementID, new Set());
1.1031 +
1.1032 + throw new Task.Result(measurementID);
1.1033 + });
1.1034 + });
1.1035 + },
1.1036 +
1.1037 + /**
1.1038 + * Register a field with the backend.
1.1039 + *
1.1040 + * Fields are what recorded pieces of data are primarily associated with.
1.1041 + *
1.1042 + * Fields are associated with measurements. Measurements must be registered
1.1043 + * via `registerMeasurement` before fields can be registered. This is
1.1044 + * enforced by this function requiring the database primary key of the
1.1045 + * measurement as an argument.
1.1046 + *
1.1047 + * @param measurementID
1.1048 + * (Number) Integer primary key of measurement this field belongs to.
1.1049 + * @param field
1.1050 + * (string) Name of this field.
1.1051 + * @param valueType
1.1052 + * (string) Type name of this field. Must be a registered type. Is
1.1053 + * likely one of the FIELD_ constants on this type.
1.1054 + *
1.1055 + * @return Promise<integer>
1.1056 + */
1.1057 + registerField: function (measurementID, field, valueType) {
1.1058 + if (!valueType) {
1.1059 + throw new Error("Value type must be defined.");
1.1060 + }
1.1061 +
1.1062 + if (!this._measurementsByID.has(measurementID)) {
1.1063 + throw new Error("Measurement not known: " + measurementID);
1.1064 + }
1.1065 +
1.1066 + if (!this._typesByName.has(valueType)) {
1.1067 + throw new Error("Unknown value type: " + valueType);
1.1068 + }
1.1069 +
1.1070 + let typeID = this._typesByName.get(valueType);
1.1071 +
1.1072 + if (!typeID) {
1.1073 + throw new Error("Undefined type: " + valueType);
1.1074 + }
1.1075 +
1.1076 + if (this.hasFieldFromMeasurement(measurementID, field)) {
1.1077 + let id = this.fieldIDFromMeasurement(measurementID, field);
1.1078 + let existingType = this._fieldsByID.get(id)[2];
1.1079 +
1.1080 + if (valueType != existingType) {
1.1081 + throw new Error("Field already defined with different type: " + existingType);
1.1082 + }
1.1083 +
1.1084 + return CommonUtils.laterTickResolvingPromise(
1.1085 + this.fieldIDFromMeasurement(measurementID, field));
1.1086 + }
1.1087 +
1.1088 + let self = this;
1.1089 + return Task.spawn(function createField() {
1.1090 + let params = {
1.1091 + measurement_id: measurementID,
1.1092 + field: field,
1.1093 + value_type: typeID,
1.1094 + };
1.1095 +
1.1096 + yield self._connection.executeCached(SQL.addField, params);
1.1097 +
1.1098 + let rows = yield self._connection.executeCached(SQL.getFieldID, params);
1.1099 +
1.1100 + let fieldID = rows[0].getResultByIndex(0);
1.1101 +
1.1102 + self._fieldsByID.set(fieldID, [measurementID, field, valueType]);
1.1103 + self._fieldsByInfo.set([measurementID, field].join(":"), fieldID);
1.1104 + self._fieldsByMeasurement.get(measurementID).add(fieldID);
1.1105 +
1.1106 + throw new Task.Result(fieldID);
1.1107 + });
1.1108 + },
1.1109 +
1.1110 + /**
1.1111 + * Initializes this instance with the database.
1.1112 + *
1.1113 + * This performs 2 major roles:
1.1114 + *
1.1115 + * 1) Set up database schema (creates tables).
1.1116 + * 2) Synchronize database with local instance.
1.1117 + */
1.1118 + _init: function() {
1.1119 + let self = this;
1.1120 + return Task.spawn(function initTask() {
1.1121 + // 0. Database file and connection configuration.
1.1122 +
1.1123 + // This should never fail. But, we assume the default of 1024 in case it
1.1124 + // does.
1.1125 + let rows = yield self._connection.execute("PRAGMA page_size");
1.1126 + let pageSize = 1024;
1.1127 + if (rows.length) {
1.1128 + pageSize = rows[0].getResultByIndex(0);
1.1129 + }
1.1130 +
1.1131 + self._log.debug("Page size is " + pageSize);
1.1132 +
1.1133 + // Ensure temp tables are stored in memory, not on disk.
1.1134 + yield self._connection.execute("PRAGMA temp_store=MEMORY");
1.1135 +
1.1136 + let journalMode;
1.1137 + rows = yield self._connection.execute("PRAGMA journal_mode=WAL");
1.1138 + if (rows.length) {
1.1139 + journalMode = rows[0].getResultByIndex(0);
1.1140 + }
1.1141 +
1.1142 + self._log.info("Journal mode is " + journalMode);
1.1143 +
1.1144 + if (journalMode == "wal") {
1.1145 + self._enabledWALCheckpointPages =
1.1146 + Math.ceil(self.MAX_WAL_SIZE_KB * 1024 / pageSize);
1.1147 +
1.1148 + self._log.info("WAL auto checkpoint pages: " +
1.1149 + self._enabledWALCheckpointPages);
1.1150 +
1.1151 + // We disable auto checkpoint during initialization to make it
1.1152 + // quicker.
1.1153 + yield self.setAutoCheckpoint(0);
1.1154 + } else {
1.1155 + if (journalMode != "truncate") {
1.1156 + // Fall back to truncate (which is faster than delete).
1.1157 + yield self._connection.execute("PRAGMA journal_mode=TRUNCATE");
1.1158 + }
1.1159 +
1.1160 + // And always use full synchronous mode to reduce possibility for data
1.1161 + // loss.
1.1162 + yield self._connection.execute("PRAGMA synchronous=FULL");
1.1163 + }
1.1164 +
1.1165 + let doCheckpoint = false;
1.1166 +
1.1167 + // 1. Create the schema.
1.1168 + yield self._connection.executeTransaction(function ensureSchema(conn) {
1.1169 + let schema = yield conn.getSchemaVersion();
1.1170 +
1.1171 + if (schema == 0) {
1.1172 + self._log.info("Creating database schema.");
1.1173 +
1.1174 + for (let k of self._SCHEMA_STATEMENTS) {
1.1175 + yield self._connection.execute(SQL[k]);
1.1176 + }
1.1177 +
1.1178 + yield self._connection.setSchemaVersion(1);
1.1179 + doCheckpoint = true;
1.1180 + } else if (schema != 1) {
1.1181 + throw new Error("Unknown database schema: " + schema);
1.1182 + } else {
1.1183 + self._log.debug("Database schema up to date.");
1.1184 + }
1.1185 + });
1.1186 +
1.1187 + // 2. Retrieve existing types.
1.1188 + yield self._connection.execute(SQL.getTypes, null, function onRow(row) {
1.1189 + let id = row.getResultByName("id");
1.1190 + let name = row.getResultByName("name");
1.1191 +
1.1192 + self._typesByID.set(id, name);
1.1193 + self._typesByName.set(name, id);
1.1194 + });
1.1195 +
1.1196 + // 3. Populate built-in types with database.
1.1197 + let missingTypes = [];
1.1198 + for (let type of self._BUILTIN_TYPES) {
1.1199 + type = self[type];
1.1200 + if (self._typesByName.has(type)) {
1.1201 + continue;
1.1202 + }
1.1203 +
1.1204 + missingTypes.push(type);
1.1205 + }
1.1206 +
1.1207 + // Don't perform DB transaction unless there is work to do.
1.1208 + if (missingTypes.length) {
1.1209 + yield self._connection.executeTransaction(function populateBuiltinTypes() {
1.1210 + for (let type of missingTypes) {
1.1211 + let params = {name: type};
1.1212 + yield self._connection.executeCached(SQL.addType, params);
1.1213 + let rows = yield self._connection.executeCached(SQL.getTypeID, params);
1.1214 + let id = rows[0].getResultByIndex(0);
1.1215 +
1.1216 + self._typesByID.set(id, type);
1.1217 + self._typesByName.set(type, id);
1.1218 + }
1.1219 + });
1.1220 +
1.1221 + doCheckpoint = true;
1.1222 + }
1.1223 +
1.1224 + // 4. Obtain measurement info.
1.1225 + yield self._connection.execute(SQL.getMeasurements, null, function onRow(row) {
1.1226 + let providerID = row.getResultByName("provider_id");
1.1227 + let providerName = row.getResultByName("provider_name");
1.1228 + let measurementID = row.getResultByName("measurement_id");
1.1229 + let measurementName = row.getResultByName("measurement_name");
1.1230 + let measurementVersion = row.getResultByName("measurement_version");
1.1231 +
1.1232 + self._providerIDs.set(providerName, providerID);
1.1233 +
1.1234 + let info = [providerName, measurementName, measurementVersion].join(":");
1.1235 +
1.1236 + self._measurementsByInfo.set(info, measurementID);
1.1237 + self._measurementsByID.set(measurementID, info);
1.1238 + self._fieldsByMeasurement.set(measurementID, new Set());
1.1239 + });
1.1240 +
1.1241 + // 5. Obtain field info.
1.1242 + yield self._connection.execute(SQL.getFieldInfo, null, function onRow(row) {
1.1243 + let measurementID = row.getResultByName("measurement_id");
1.1244 + let fieldID = row.getResultByName("field_id");
1.1245 + let fieldName = row.getResultByName("field_name");
1.1246 + let typeName = row.getResultByName("type_name");
1.1247 +
1.1248 + self._fieldsByID.set(fieldID, [measurementID, fieldName, typeName]);
1.1249 + self._fieldsByInfo.set([measurementID, fieldName].join(":"), fieldID);
1.1250 + self._fieldsByMeasurement.get(measurementID).add(fieldID);
1.1251 + });
1.1252 +
1.1253 + // Perform a checkpoint after initialization (if needed) and
1.1254 + // enable auto checkpoint during regular operation.
1.1255 + if (doCheckpoint) {
1.1256 + yield self.checkpoint();
1.1257 + }
1.1258 +
1.1259 + yield self.setAutoCheckpoint(1);
1.1260 + });
1.1261 + },
1.1262 +
1.1263 + /**
1.1264 + * Prune all data from earlier than the specified date.
1.1265 + *
1.1266 + * Data stored on days before the specified Date will be permanently
1.1267 + * deleted.
1.1268 + *
1.1269 + * This returns a promise that will be resolved when data has been deleted.
1.1270 + *
1.1271 + * @param date
1.1272 + * (Date) Old data threshold.
1.1273 + * @return Promise<>
1.1274 + */
1.1275 + pruneDataBefore: function (date) {
1.1276 + let statements = this._PRUNE_STATEMENTS;
1.1277 +
1.1278 + let self = this;
1.1279 + return this.enqueueOperation(function doPrune() {
1.1280 + return self._connection.executeTransaction(function prune(conn) {
1.1281 + let days = dateToDays(date);
1.1282 +
1.1283 + let params = {days: days};
1.1284 + for (let name of statements) {
1.1285 + yield conn.execute(SQL[name], params);
1.1286 + }
1.1287 + });
1.1288 + });
1.1289 + },
1.1290 +
1.1291 + /**
1.1292 + * Reduce memory usage as much as possible.
1.1293 + *
1.1294 + * This returns a promise that will be resolved on completion.
1.1295 + *
1.1296 + * @return Promise<>
1.1297 + */
1.1298 + compact: function () {
1.1299 + let self = this;
1.1300 + return this.enqueueOperation(function doCompact() {
1.1301 + self._connection.discardCachedStatements();
1.1302 + return self._connection.shrinkMemory();
1.1303 + });
1.1304 + },
1.1305 +
1.1306 + /**
1.1307 + * Checkpoint writes requiring flush to disk.
1.1308 + *
1.1309 + * This is called to persist queued and non-flushed writes to disk.
1.1310 + * It will force an fsync, so it is expensive and should be used
1.1311 + * sparingly.
1.1312 + */
1.1313 + checkpoint: function () {
1.1314 + if (!this._enabledWALCheckpointPages) {
1.1315 + return CommonUtils.laterTickResolvingPromise();
1.1316 + }
1.1317 +
1.1318 + return this.enqueueOperation(function checkpoint() {
1.1319 + this._log.info("Performing manual WAL checkpoint.");
1.1320 + return this._connection.execute("PRAGMA wal_checkpoint");
1.1321 + }.bind(this));
1.1322 + },
1.1323 +
1.1324 + setAutoCheckpoint: function (on) {
1.1325 + // If we aren't in WAL mode, wal_autocheckpoint won't do anything so
1.1326 + // we no-op.
1.1327 + if (!this._enabledWALCheckpointPages) {
1.1328 + return CommonUtils.laterTickResolvingPromise();
1.1329 + }
1.1330 +
1.1331 + let val = on ? this._enabledWALCheckpointPages : 0;
1.1332 +
1.1333 + return this.enqueueOperation(function setWALCheckpoint() {
1.1334 + this._log.info("Setting WAL auto checkpoint to " + val);
1.1335 + return this._connection.execute("PRAGMA wal_autocheckpoint=" + val);
1.1336 + }.bind(this));
1.1337 + },
1.1338 +
1.1339 + /**
1.1340 + * Ensure a field ID matches a specified type.
1.1341 + *
1.1342 + * This is called internally as part of adding values to ensure that
1.1343 + * the type of a field matches the operation being performed.
1.1344 + */
1.1345 + _ensureFieldType: function (id, type) {
1.1346 + let info = this._fieldsByID.get(id);
1.1347 +
1.1348 + if (!info || !Array.isArray(info)) {
1.1349 + throw new Error("Unknown field ID: " + id);
1.1350 + }
1.1351 +
1.1352 + if (type != info[2]) {
1.1353 + throw new Error("Field type does not match the expected for this " +
1.1354 + "operation. Actual: " + info[2] + "; Expected: " +
1.1355 + type);
1.1356 + }
1.1357 + },
1.1358 +
1.1359 + /**
1.1360 + * Enqueue a storage operation to be performed when the database is ready.
1.1361 + *
1.1362 + * The primary use case of this function is to prevent potentially
1.1363 + * conflicting storage operations from being performed in parallel. By
1.1364 + * calling this function, passed storage operations will be serially
1.1365 + * executed, avoiding potential order of operation issues.
1.1366 + *
1.1367 + * The passed argument is a function that will perform storage operations.
1.1368 + * The function should return a promise that will be resolved when all
1.1369 + * storage operations have been completed.
1.1370 + *
1.1371 + * The passed function may be executed immediately. If there are already
1.1372 + * queued operations, it will be appended to the queue and executed after all
1.1373 + * before it have finished.
1.1374 + *
1.1375 + * This function returns a promise that will be resolved or rejected with
1.1376 + * the same value that the function's promise was resolved or rejected with.
1.1377 + *
1.1378 + * @param func
1.1379 + * (function) Function performing storage interactions.
1.1380 + * @return Promise<>
1.1381 + */
1.1382 + enqueueOperation: function (func) {
1.1383 + if (typeof(func) != "function") {
1.1384 + throw new Error("enqueueOperation expects a function. Got: " + typeof(func));
1.1385 + }
1.1386 +
1.1387 + this._log.trace("Enqueueing operation.");
1.1388 + let deferred = Promise.defer();
1.1389 +
1.1390 + this._queuedOperations.push([func, deferred]);
1.1391 +
1.1392 + if (this._queuedOperations.length == 1) {
1.1393 + this._popAndPerformQueuedOperation();
1.1394 + }
1.1395 +
1.1396 + return deferred.promise;
1.1397 + },
1.1398 +
1.1399 + /**
1.1400 + * Enqueue a function to be performed as a transaction.
1.1401 + *
1.1402 + * The passed function should be a generator suitable for calling with
1.1403 + * `executeTransaction` from the SQLite connection.
1.1404 + */
1.1405 + enqueueTransaction: function (func, type) {
1.1406 + return this.enqueueOperation(
1.1407 + this._connection.executeTransaction.bind(this._connection, func, type)
1.1408 + );
1.1409 + },
1.1410 +
1.1411 + _popAndPerformQueuedOperation: function () {
1.1412 + if (!this._queuedOperations.length || this._queuedInProgress) {
1.1413 + return;
1.1414 + }
1.1415 +
1.1416 + this._log.trace("Performing queued operation.");
1.1417 + let [func, deferred] = this._queuedOperations.shift();
1.1418 + let promise;
1.1419 +
1.1420 + try {
1.1421 + this._queuedInProgress = true;
1.1422 + promise = func();
1.1423 + } catch (ex) {
1.1424 + this._log.warn("Queued operation threw during execution: " +
1.1425 + CommonUtils.exceptionStr(ex));
1.1426 + this._queuedInProgress = false;
1.1427 + deferred.reject(ex);
1.1428 + this._popAndPerformQueuedOperation();
1.1429 + return;
1.1430 + }
1.1431 +
1.1432 + if (!promise || typeof(promise.then) != "function") {
1.1433 + let msg = "Queued operation did not return a promise: " + func;
1.1434 + this._log.warn(msg);
1.1435 +
1.1436 + this._queuedInProgress = false;
1.1437 + deferred.reject(new Error(msg));
1.1438 + this._popAndPerformQueuedOperation();
1.1439 + return;
1.1440 + }
1.1441 +
1.1442 + promise.then(
1.1443 + function onSuccess(result) {
1.1444 + this._log.trace("Queued operation completed.");
1.1445 + this._queuedInProgress = false;
1.1446 + deferred.resolve(result);
1.1447 + this._popAndPerformQueuedOperation();
1.1448 + }.bind(this),
1.1449 + function onError(error) {
1.1450 + this._log.warn("Failure when performing queued operation: " +
1.1451 + CommonUtils.exceptionStr(error));
1.1452 + this._queuedInProgress = false;
1.1453 + deferred.reject(error);
1.1454 + this._popAndPerformQueuedOperation();
1.1455 + }.bind(this)
1.1456 + );
1.1457 + },
1.1458 +
1.1459 + /**
1.1460 + * Obtain all values associated with a measurement.
1.1461 + *
1.1462 + * This returns a promise that resolves to an object. The keys of the object
1.1463 + * are:
1.1464 + *
1.1465 + * days -- DailyValues where the values are Maps of field name to data
1.1466 + * structures. The data structures could be simple (string or number) or
1.1467 + * Arrays if the field type allows multiple values per day.
1.1468 + *
1.1469 + * singular -- Map of field names to values. This holds all fields that
1.1470 + * don't have a temporal component.
1.1471 + *
1.1472 + * @param id
1.1473 + * (Number) Primary key of measurement whose values to retrieve.
1.1474 + */
1.1475 + getMeasurementValues: function (id) {
1.1476 + let deferred = Promise.defer();
1.1477 + let days = new DailyValues();
1.1478 + let singular = new Map();
1.1479 +
1.1480 + let self = this;
1.1481 + this.enqueueOperation(function enqueuedGetMeasurementValues() {
1.1482 + return Task.spawn(function fetchMeasurementValues() {
1.1483 + function handleResult(data) {
1.1484 + for (let [field, values] of data) {
1.1485 + for (let [day, value] of Iterator(values)) {
1.1486 + if (!days.hasDay(day)) {
1.1487 + days.setDay(day, new Map());
1.1488 + }
1.1489 +
1.1490 + days.getDay(day).set(field, value);
1.1491 + }
1.1492 + }
1.1493 + }
1.1494 +
1.1495 + if (self.measurementHasAnyDailyCounterFields(id)) {
1.1496 + let counters = yield self.getMeasurementDailyCountersFromMeasurementID(id);
1.1497 + handleResult(counters);
1.1498 + }
1.1499 +
1.1500 + if (self.measurementHasAnyDailyLastFields(id)) {
1.1501 + let dailyLast = yield self.getMeasurementDailyLastValuesFromMeasurementID(id);
1.1502 + handleResult(dailyLast);
1.1503 + }
1.1504 +
1.1505 + if (self.measurementHasAnyDailyDiscreteFields(id)) {
1.1506 + let dailyDiscrete = yield self.getMeasurementDailyDiscreteValuesFromMeasurementID(id);
1.1507 + handleResult(dailyDiscrete);
1.1508 + }
1.1509 +
1.1510 + if (self.measurementHasAnyLastFields(id)) {
1.1511 + let last = yield self.getMeasurementLastValuesFromMeasurementID(id);
1.1512 +
1.1513 + for (let [field, value] of last) {
1.1514 + singular.set(field, value);
1.1515 + }
1.1516 + }
1.1517 +
1.1518 + });
1.1519 + }).then(function onSuccess() {
1.1520 + deferred.resolve({singular: singular, days: days});
1.1521 + }, function onError(error) {
1.1522 + deferred.reject(error);
1.1523 + });
1.1524 +
1.1525 + return deferred.promise;
1.1526 + },
1.1527 +
1.1528 + //---------------------------------------------------------------------------
1.1529 + // Low-level storage operations
1.1530 + //
1.1531 + // These will be performed immediately (or at least as soon as the underlying
1.1532 + // connection allows them to be.) It is recommended to call these from within
1.1533 + // a function added via `enqueueOperation()` or they may inadvertently be
1.1534 + // performed during another enqueued operation, which may be a transaction
1.1535 + // that is rolled back.
1.1536 + // ---------------------------------------------------------------------------
1.1537 +
1.1538 + /**
1.1539 + * Set state for a provider.
1.1540 + *
1.1541 + * Providers have the ability to register persistent state with the backend.
1.1542 + * Persistent state doesn't expire. The format of the data is completely up
1.1543 + * to the provider beyond the requirement that values be UTF-8 strings.
1.1544 + *
1.1545 + * This returns a promise that will be resolved when the underlying database
1.1546 + * operation has completed.
1.1547 + *
1.1548 + * @param provider
1.1549 + * (string) Name of the provider.
1.1550 + * @param key
1.1551 + * (string) Key under which to store this state.
1.1552 + * @param value
1.1553 + * (string) Value for this state.
1.1554 + * @return Promise<>
1.1555 + */
1.1556 + setProviderState: function (provider, key, value) {
1.1557 + if (typeof(key) != "string") {
1.1558 + throw new Error("State key must be a string. Got: " + key);
1.1559 + }
1.1560 +
1.1561 + if (typeof(value) != "string") {
1.1562 + throw new Error("State value must be a string. Got: " + value);
1.1563 + }
1.1564 +
1.1565 + let id = this.providerID(provider);
1.1566 + if (!id) {
1.1567 + throw new Error("Unknown provider: " + provider);
1.1568 + }
1.1569 +
1.1570 + return this._connection.executeCached(SQL.setProviderState, {
1.1571 + provider_id: id,
1.1572 + name: key,
1.1573 + value: value,
1.1574 + });
1.1575 + },
1.1576 +
1.1577 + /**
1.1578 + * Obtain named state for a provider.
1.1579 + *
1.1580 + *
1.1581 + * The returned promise will resolve to the state from the database or null
1.1582 + * if the key is not stored.
1.1583 + *
1.1584 + * @param provider
1.1585 + * (string) The name of the provider whose state to obtain.
1.1586 + * @param key
1.1587 + * (string) The state's key to retrieve.
1.1588 + *
1.1589 + * @return Promise<data>
1.1590 + */
1.1591 + getProviderState: function (provider, key) {
1.1592 + let id = this.providerID(provider);
1.1593 + if (!id) {
1.1594 + throw new Error("Unknown provider: " + provider);
1.1595 + }
1.1596 +
1.1597 + let conn = this._connection;
1.1598 + return Task.spawn(function queryDB() {
1.1599 + let rows = yield conn.executeCached(SQL.getProviderStateWithName, {
1.1600 + provider_id: id,
1.1601 + name: key,
1.1602 + });
1.1603 +
1.1604 + if (!rows.length) {
1.1605 + throw new Task.Result(null);
1.1606 + }
1.1607 +
1.1608 + throw new Task.Result(rows[0].getResultByIndex(0));
1.1609 + });
1.1610 + },
1.1611 +
1.1612 + /**
1.1613 + * Increment a daily counter from a numeric field id.
1.1614 + *
1.1615 + * @param id
1.1616 + * (integer) Primary key of field to increment.
1.1617 + * @param date
1.1618 + * (Date) When the increment occurred. This is typically "now" but can
1.1619 + * be explicitly defined for events that occurred in the past.
1.1620 + * @param by
1.1621 + * (integer) How much to increment the value by. Defaults to 1.
1.1622 + */
1.1623 + incrementDailyCounterFromFieldID: function (id, date=new Date(), by=1) {
1.1624 + this._ensureFieldType(id, this.FIELD_DAILY_COUNTER);
1.1625 +
1.1626 + let params = {
1.1627 + field_id: id,
1.1628 + days: dateToDays(date),
1.1629 + by: by,
1.1630 + };
1.1631 +
1.1632 + return this._connection.executeCached(SQL.incrementDailyCounterFromFieldID,
1.1633 + params);
1.1634 + },
1.1635 +
1.1636 + /**
1.1637 + * Obtain all counts for a specific daily counter.
1.1638 + *
1.1639 + * @param id
1.1640 + * (integer) The ID of the field being retrieved.
1.1641 + */
1.1642 + getDailyCounterCountsFromFieldID: function (id) {
1.1643 + this._ensureFieldType(id, this.FIELD_DAILY_COUNTER);
1.1644 +
1.1645 + let self = this;
1.1646 + return Task.spawn(function fetchCounterDays() {
1.1647 + let rows = yield self._connection.executeCached(SQL.getDailyCounterCountsFromFieldID,
1.1648 + {field_id: id});
1.1649 +
1.1650 + let result = new DailyValues();
1.1651 + for (let row of rows) {
1.1652 + let days = row.getResultByIndex(0);
1.1653 + let counter = row.getResultByIndex(1);
1.1654 +
1.1655 + let date = daysToDate(days);
1.1656 + result.setDay(date, counter);
1.1657 + }
1.1658 +
1.1659 + throw new Task.Result(result);
1.1660 + });
1.1661 + },
1.1662 +
1.1663 + /**
1.1664 + * Get the value of a daily counter for a given day.
1.1665 + *
1.1666 + * @param field
1.1667 + * (integer) Field ID to retrieve.
1.1668 + * @param date
1.1669 + * (Date) Date for day from which to obtain data.
1.1670 + */
1.1671 + getDailyCounterCountFromFieldID: function (field, date) {
1.1672 + this._ensureFieldType(field, this.FIELD_DAILY_COUNTER);
1.1673 +
1.1674 + let params = {
1.1675 + field_id: field,
1.1676 + days: dateToDays(date),
1.1677 + };
1.1678 +
1.1679 + let self = this;
1.1680 + return Task.spawn(function fetchCounter() {
1.1681 + let rows = yield self._connection.executeCached(SQL.getDailyCounterCountFromFieldID,
1.1682 + params);
1.1683 + if (!rows.length) {
1.1684 + throw new Task.Result(null);
1.1685 + }
1.1686 +
1.1687 + throw new Task.Result(rows[0].getResultByIndex(0));
1.1688 + });
1.1689 + },
1.1690 +
1.1691 + /**
1.1692 + * Define the value for a "last numeric" field.
1.1693 + *
1.1694 + * The previous value (if any) will be replaced by the value passed, even if
1.1695 + * the date of the incoming value is older than what's recorded in the
1.1696 + * database.
1.1697 + *
1.1698 + * @param fieldID
1.1699 + * (Number) Integer primary key of field to update.
1.1700 + * @param value
1.1701 + * (Number) Value to record.
1.1702 + * @param date
1.1703 + * (Date) When this value was produced.
1.1704 + */
1.1705 + setLastNumericFromFieldID: function (fieldID, value, date=new Date()) {
1.1706 + this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC);
1.1707 +
1.1708 + if (typeof(value) != "number") {
1.1709 + throw new Error("Value is not a number: " + value);
1.1710 + }
1.1711 +
1.1712 + let params = {
1.1713 + field_id: fieldID,
1.1714 + days: dateToDays(date),
1.1715 + value: value,
1.1716 + };
1.1717 +
1.1718 + return this._connection.executeCached(SQL.setLastNumeric, params);
1.1719 + },
1.1720 +
1.1721 + /**
1.1722 + * Define the value of a "last text" field.
1.1723 + *
1.1724 + * See `setLastNumericFromFieldID` for behavior.
1.1725 + */
1.1726 + setLastTextFromFieldID: function (fieldID, value, date=new Date()) {
1.1727 + this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT);
1.1728 +
1.1729 + if (typeof(value) != "string") {
1.1730 + throw new Error("Value is not a string: " + value);
1.1731 + }
1.1732 +
1.1733 + let params = {
1.1734 + field_id: fieldID,
1.1735 + days: dateToDays(date),
1.1736 + value: value,
1.1737 + };
1.1738 +
1.1739 + return this._connection.executeCached(SQL.setLastText, params);
1.1740 + },
1.1741 +
1.1742 + /**
1.1743 + * Obtain the value of a "last numeric" field.
1.1744 + *
1.1745 + * This returns a promise that will be resolved with an Array of [date, value]
1.1746 + * if a value is known or null if no last value is present.
1.1747 + *
1.1748 + * @param fieldID
1.1749 + * (Number) Integer primary key of field to retrieve.
1.1750 + */
1.1751 + getLastNumericFromFieldID: function (fieldID) {
1.1752 + this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC);
1.1753 +
1.1754 + let self = this;
1.1755 + return Task.spawn(function fetchLastField() {
1.1756 + let rows = yield self._connection.executeCached(SQL.getLastNumericFromFieldID,
1.1757 + {field_id: fieldID});
1.1758 +
1.1759 + if (!rows.length) {
1.1760 + throw new Task.Result(null);
1.1761 + }
1.1762 +
1.1763 + let row = rows[0];
1.1764 + let days = row.getResultByIndex(0);
1.1765 + let value = row.getResultByIndex(1);
1.1766 +
1.1767 + throw new Task.Result([daysToDate(days), value]);
1.1768 + });
1.1769 + },
1.1770 +
1.1771 + /**
1.1772 + * Obtain the value of a "last text" field.
1.1773 + *
1.1774 + * See `getLastNumericFromFieldID` for behavior.
1.1775 + */
1.1776 + getLastTextFromFieldID: function (fieldID) {
1.1777 + this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT);
1.1778 +
1.1779 + let self = this;
1.1780 + return Task.spawn(function fetchLastField() {
1.1781 + let rows = yield self._connection.executeCached(SQL.getLastTextFromFieldID,
1.1782 + {field_id: fieldID});
1.1783 +
1.1784 + if (!rows.length) {
1.1785 + throw new Task.Result(null);
1.1786 + }
1.1787 +
1.1788 + let row = rows[0];
1.1789 + let days = row.getResultByIndex(0);
1.1790 + let value = row.getResultByIndex(1);
1.1791 +
1.1792 + throw new Task.Result([daysToDate(days), value]);
1.1793 + });
1.1794 + },
1.1795 +
1.1796 + /**
1.1797 + * Delete the value (if any) in a "last numeric" field.
1.1798 + */
1.1799 + deleteLastNumericFromFieldID: function (fieldID) {
1.1800 + this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC);
1.1801 +
1.1802 + return this._connection.executeCached(SQL.deleteLastNumericFromFieldID,
1.1803 + {field_id: fieldID});
1.1804 + },
1.1805 +
1.1806 + /**
1.1807 + * Delete the value (if any) in a "last text" field.
1.1808 + */
1.1809 + deleteLastTextFromFieldID: function (fieldID) {
1.1810 + this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT);
1.1811 +
1.1812 + return this._connection.executeCached(SQL.deleteLastTextFromFieldID,
1.1813 + {field_id: fieldID});
1.1814 + },
1.1815 +
1.1816 + /**
1.1817 + * Record a value for a "daily last numeric" field.
1.1818 + *
1.1819 + * The field can hold 1 value per calendar day. If the field already has a
1.1820 + * value for the day specified (defaults to now), that value will be
1.1821 + * replaced, even if the date specified is older (within the day) than the
1.1822 + * previously recorded value.
1.1823 + *
1.1824 + * @param fieldID
1.1825 + * (Number) Integer primary key of field.
1.1826 + * @param value
1.1827 + * (Number) Value to record.
1.1828 + * @param date
1.1829 + * (Date) When the value was produced. Defaults to now.
1.1830 + */
1.1831 + setDailyLastNumericFromFieldID: function (fieldID, value, date=new Date()) {
1.1832 + this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_NUMERIC);
1.1833 +
1.1834 + let params = {
1.1835 + field_id: fieldID,
1.1836 + days: dateToDays(date),
1.1837 + value: value,
1.1838 + };
1.1839 +
1.1840 + return this._connection.executeCached(SQL.setDailyLastNumeric, params);
1.1841 + },
1.1842 +
1.1843 + /**
1.1844 + * Record a value for a "daily last text" field.
1.1845 + *
1.1846 + * See `setDailyLastNumericFromFieldID` for behavior.
1.1847 + */
1.1848 + setDailyLastTextFromFieldID: function (fieldID, value, date=new Date()) {
1.1849 + this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_TEXT);
1.1850 +
1.1851 + let params = {
1.1852 + field_id: fieldID,
1.1853 + days: dateToDays(date),
1.1854 + value: value,
1.1855 + };
1.1856 +
1.1857 + return this._connection.executeCached(SQL.setDailyLastText, params);
1.1858 + },
1.1859 +
1.1860 + /**
1.1861 + * Obtain value(s) from a "daily last numeric" field.
1.1862 + *
1.1863 + * This returns a promise that resolves to a DailyValues instance. If `date`
1.1864 + * is specified, that instance will have at most 1 entry. If there is no
1.1865 + * `date` constraint, then all stored values will be retrieved.
1.1866 + *
1.1867 + * @param fieldID
1.1868 + * (Number) Integer primary key of field to retrieve.
1.1869 + * @param date optional
1.1870 + * (Date) If specified, only return data for this day.
1.1871 + *
1.1872 + * @return Promise<DailyValues>
1.1873 + */
1.1874 + getDailyLastNumericFromFieldID: function (fieldID, date=null) {
1.1875 + this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_NUMERIC);
1.1876 +
1.1877 + let params = {field_id: fieldID};
1.1878 + let name = "getDailyLastNumericFromFieldID";
1.1879 +
1.1880 + if (date) {
1.1881 + params.days = dateToDays(date);
1.1882 + name = "getDailyLastNumericFromFieldIDAndDay";
1.1883 + }
1.1884 +
1.1885 + return this._getDailyLastFromFieldID(name, params);
1.1886 + },
1.1887 +
1.1888 + /**
1.1889 + * Obtain value(s) from a "daily last text" field.
1.1890 + *
1.1891 + * See `getDailyLastNumericFromFieldID` for behavior.
1.1892 + */
1.1893 + getDailyLastTextFromFieldID: function (fieldID, date=null) {
1.1894 + this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_TEXT);
1.1895 +
1.1896 + let params = {field_id: fieldID};
1.1897 + let name = "getDailyLastTextFromFieldID";
1.1898 +
1.1899 + if (date) {
1.1900 + params.days = dateToDays(date);
1.1901 + name = "getDailyLastTextFromFieldIDAndDay";
1.1902 + }
1.1903 +
1.1904 + return this._getDailyLastFromFieldID(name, params);
1.1905 + },
1.1906 +
1.1907 + _getDailyLastFromFieldID: function (name, params) {
1.1908 + let self = this;
1.1909 + return Task.spawn(function fetchDailyLastForField() {
1.1910 + let rows = yield self._connection.executeCached(SQL[name], params);
1.1911 +
1.1912 + let result = new DailyValues();
1.1913 + for (let row of rows) {
1.1914 + let d = daysToDate(row.getResultByIndex(0));
1.1915 + let value = row.getResultByIndex(1);
1.1916 +
1.1917 + result.setDay(d, value);
1.1918 + }
1.1919 +
1.1920 + throw new Task.Result(result);
1.1921 + });
1.1922 + },
1.1923 +
1.1924 + /**
1.1925 + * Add a new value for a "daily discrete numeric" field.
1.1926 + *
1.1927 + * This appends a new value to the list of values for a specific field. All
1.1928 + * values are retained. Duplicate values are allowed.
1.1929 + *
1.1930 + * @param fieldID
1.1931 + * (Number) Integer primary key of field.
1.1932 + * @param value
1.1933 + * (Number) Value to record.
1.1934 + * @param date optional
1.1935 + * (Date) When this value occurred. Values are bucketed by day.
1.1936 + */
1.1937 + addDailyDiscreteNumericFromFieldID: function (fieldID, value, date=new Date()) {
1.1938 + this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_NUMERIC);
1.1939 +
1.1940 + if (typeof(value) != "number") {
1.1941 + throw new Error("Number expected. Got: " + value);
1.1942 + }
1.1943 +
1.1944 + let params = {
1.1945 + field_id: fieldID,
1.1946 + days: dateToDays(date),
1.1947 + value: value,
1.1948 + };
1.1949 +
1.1950 + return this._connection.executeCached(SQL.addDailyDiscreteNumeric, params);
1.1951 + },
1.1952 +
1.1953 + /**
1.1954 + * Add a new value for a "daily discrete text" field.
1.1955 + *
1.1956 + * See `addDailyDiscreteNumericFromFieldID` for behavior.
1.1957 + */
1.1958 + addDailyDiscreteTextFromFieldID: function (fieldID, value, date=new Date()) {
1.1959 + this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_TEXT);
1.1960 +
1.1961 + if (typeof(value) != "string") {
1.1962 + throw new Error("String expected. Got: " + value);
1.1963 + }
1.1964 +
1.1965 + let params = {
1.1966 + field_id: fieldID,
1.1967 + days: dateToDays(date),
1.1968 + value: value,
1.1969 + };
1.1970 +
1.1971 + return this._connection.executeCached(SQL.addDailyDiscreteText, params);
1.1972 + },
1.1973 +
1.1974 + /**
1.1975 + * Obtain values for a "daily discrete numeric" field.
1.1976 + *
1.1977 + * This returns a promise that resolves to a `DailyValues` instance. If
1.1978 + * `date` is specified, there will be at most 1 key in that instance. If
1.1979 + * not, all data from the database will be retrieved.
1.1980 + *
1.1981 + * Values in that instance will be arrays of the raw values.
1.1982 + *
1.1983 + * @param fieldID
1.1984 + * (Number) Integer primary key of field to retrieve.
1.1985 + * @param date optional
1.1986 + * (Date) Day to obtain data for. Date can be any time in the day.
1.1987 + */
1.1988 + getDailyDiscreteNumericFromFieldID: function (fieldID, date=null) {
1.1989 + this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_NUMERIC);
1.1990 +
1.1991 + let params = {field_id: fieldID};
1.1992 +
1.1993 + let name = "getDailyDiscreteNumericFromFieldID";
1.1994 +
1.1995 + if (date) {
1.1996 + params.days = dateToDays(date);
1.1997 + name = "getDailyDiscreteNumericFromFieldIDAndDay";
1.1998 + }
1.1999 +
1.2000 + return this._getDailyDiscreteFromFieldID(name, params);
1.2001 + },
1.2002 +
1.2003 + /**
1.2004 + * Obtain values for a "daily discrete text" field.
1.2005 + *
1.2006 + * See `getDailyDiscreteNumericFromFieldID` for behavior.
1.2007 + */
1.2008 + getDailyDiscreteTextFromFieldID: function (fieldID, date=null) {
1.2009 + this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_TEXT);
1.2010 +
1.2011 + let params = {field_id: fieldID};
1.2012 +
1.2013 + let name = "getDailyDiscreteTextFromFieldID";
1.2014 +
1.2015 + if (date) {
1.2016 + params.days = dateToDays(date);
1.2017 + name = "getDailyDiscreteTextFromFieldIDAndDay";
1.2018 + }
1.2019 +
1.2020 + return this._getDailyDiscreteFromFieldID(name, params);
1.2021 + },
1.2022 +
1.2023 + _getDailyDiscreteFromFieldID: function (name, params) {
1.2024 + let self = this;
1.2025 + return Task.spawn(function fetchDailyDiscreteValuesForField() {
1.2026 + let rows = yield self._connection.executeCached(SQL[name], params);
1.2027 +
1.2028 + let result = new DailyValues();
1.2029 + for (let row of rows) {
1.2030 + let d = daysToDate(row.getResultByIndex(0));
1.2031 + let value = row.getResultByIndex(1);
1.2032 +
1.2033 + result.appendValue(d, value);
1.2034 + }
1.2035 +
1.2036 + throw new Task.Result(result);
1.2037 + });
1.2038 + },
1.2039 +
1.2040 + /**
1.2041 + * Obtain the counts of daily counters in a measurement.
1.2042 + *
1.2043 + * This returns a promise that resolves to a Map of field name strings to
1.2044 + * DailyValues that hold per-day counts.
1.2045 + *
1.2046 + * @param id
1.2047 + * (Number) Integer primary key of measurement.
1.2048 + *
1.2049 + * @return Promise<Map>
1.2050 + */
1.2051 + getMeasurementDailyCountersFromMeasurementID: function (id) {
1.2052 + let self = this;
1.2053 + return Task.spawn(function fetchDailyCounters() {
1.2054 + let rows = yield self._connection.execute(SQL.getMeasurementDailyCounters,
1.2055 + {measurement_id: id});
1.2056 +
1.2057 + let result = new Map();
1.2058 + for (let row of rows) {
1.2059 + let field = row.getResultByName("field_name");
1.2060 + let date = daysToDate(row.getResultByName("day"));
1.2061 + let value = row.getResultByName("value");
1.2062 +
1.2063 + if (!result.has(field)) {
1.2064 + result.set(field, new DailyValues());
1.2065 + }
1.2066 +
1.2067 + result.get(field).setDay(date, value);
1.2068 + }
1.2069 +
1.2070 + throw new Task.Result(result);
1.2071 + });
1.2072 + },
1.2073 +
1.2074 + /**
1.2075 + * Obtain the values of "last" fields from a measurement.
1.2076 + *
1.2077 + * This returns a promise that resolves to a Map of field name to an array
1.2078 + * of [date, value].
1.2079 + *
1.2080 + * @param id
1.2081 + * (Number) Integer primary key of measurement whose data to retrieve.
1.2082 + *
1.2083 + * @return Promise<Map>
1.2084 + */
1.2085 + getMeasurementLastValuesFromMeasurementID: function (id) {
1.2086 + let self = this;
1.2087 + return Task.spawn(function fetchMeasurementLastValues() {
1.2088 + let rows = yield self._connection.execute(SQL.getMeasurementLastValues,
1.2089 + {measurement_id: id});
1.2090 +
1.2091 + let result = new Map();
1.2092 + for (let row of rows) {
1.2093 + let date = daysToDate(row.getResultByIndex(1));
1.2094 + let value = row.getResultByIndex(2);
1.2095 + result.set(row.getResultByIndex(0), [date, value]);
1.2096 + }
1.2097 +
1.2098 + throw new Task.Result(result);
1.2099 + });
1.2100 + },
1.2101 +
1.2102 + /**
1.2103 + * Obtain the values of "last daily" fields from a measurement.
1.2104 + *
1.2105 + * This returns a promise that resolves to a Map of field name to DailyValues
1.2106 + * instances. Each DailyValues instance has days for which a daily last value
1.2107 + * is defined. The values in each DailyValues are the raw last value for that
1.2108 + * day.
1.2109 + *
1.2110 + * @param id
1.2111 + * (Number) Integer primary key of measurement whose data to retrieve.
1.2112 + *
1.2113 + * @return Promise<Map>
1.2114 + */
1.2115 + getMeasurementDailyLastValuesFromMeasurementID: function (id) {
1.2116 + let self = this;
1.2117 + return Task.spawn(function fetchMeasurementDailyLastValues() {
1.2118 + let rows = yield self._connection.execute(SQL.getMeasurementDailyLastValues,
1.2119 + {measurement_id: id});
1.2120 +
1.2121 + let result = new Map();
1.2122 + for (let row of rows) {
1.2123 + let field = row.getResultByName("field_name");
1.2124 + let date = daysToDate(row.getResultByName("day"));
1.2125 + let value = row.getResultByName("value");
1.2126 +
1.2127 + if (!result.has(field)) {
1.2128 + result.set(field, new DailyValues());
1.2129 + }
1.2130 +
1.2131 + result.get(field).setDay(date, value);
1.2132 + }
1.2133 +
1.2134 + throw new Task.Result(result);
1.2135 + });
1.2136 + },
1.2137 +
1.2138 + /**
1.2139 + * Obtain the values of "daily discrete" fields from a measurement.
1.2140 + *
1.2141 + * This obtains all discrete values for all "daily discrete" fields in a
1.2142 + * measurement.
1.2143 + *
1.2144 + * This returns a promise that resolves to a Map. The Map's keys are field
1.2145 + * string names. Values are `DailyValues` instances. The values inside
1.2146 + * the `DailyValues` are arrays of the raw discrete values.
1.2147 + *
1.2148 + * @param id
1.2149 + * (Number) Integer primary key of measurement.
1.2150 + *
1.2151 + * @return Promise<Map>
1.2152 + */
1.2153 + getMeasurementDailyDiscreteValuesFromMeasurementID: function (id) {
1.2154 + let deferred = Promise.defer();
1.2155 + let result = new Map();
1.2156 +
1.2157 + this._connection.execute(SQL.getMeasurementDailyDiscreteValues,
1.2158 + {measurement_id: id}, function onRow(row) {
1.2159 + let field = row.getResultByName("field_name");
1.2160 + let date = daysToDate(row.getResultByName("day"));
1.2161 + let value = row.getResultByName("value");
1.2162 +
1.2163 + if (!result.has(field)) {
1.2164 + result.set(field, new DailyValues());
1.2165 + }
1.2166 +
1.2167 + result.get(field).appendValue(date, value);
1.2168 + }).then(function onComplete() {
1.2169 + deferred.resolve(result);
1.2170 + }, function onError(error) {
1.2171 + deferred.reject(error);
1.2172 + });
1.2173 +
1.2174 + return deferred.promise;
1.2175 + },
1.2176 +});
1.2177 +
1.2178 +// Alias built-in field types to public API.
1.2179 +for (let property of MetricsStorageSqliteBackend.prototype._BUILTIN_TYPES) {
1.2180 + this.MetricsStorageBackend[property] = MetricsStorageSqliteBackend.prototype[property];
1.2181 +}
1.2182 +
