1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/toolkit/mozapps/extensions/internal/AddonRepository_SQLiteMigrator.jsm Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,518 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +"use strict";
1.9 +
1.10 +const Cc = Components.classes;
1.11 +const Ci = Components.interfaces;
1.12 +const Cu = Components.utils;
1.13 +
1.14 +Cu.import("resource://gre/modules/Services.jsm");
1.15 +Cu.import("resource://gre/modules/AddonManager.jsm");
1.16 +Cu.import("resource://gre/modules/FileUtils.jsm");
1.17 +
1.18 +const KEY_PROFILEDIR = "ProfD";
1.19 +const FILE_DATABASE = "addons.sqlite";
1.20 +const LAST_DB_SCHEMA = 4;
1.21 +
1.22 +// Add-on properties present in the columns of the database
1.23 +const PROP_SINGLE = ["id", "type", "name", "version", "creator", "description",
1.24 + "fullDescription", "developerComments", "eula",
1.25 + "homepageURL", "supportURL", "contributionURL",
1.26 + "contributionAmount", "averageRating", "reviewCount",
1.27 + "reviewURL", "totalDownloads", "weeklyDownloads",
1.28 + "dailyUsers", "sourceURI", "repositoryStatus", "size",
1.29 + "updateDate"];
1.30 +
1.31 +Cu.import("resource://gre/modules/Log.jsm");
1.32 +const LOGGER_ID = "addons.repository.sqlmigrator";
1.33 +
1.34 +// Create a new logger for use by the Addons Repository SQL Migrator
1.35 +// (Requires AddonManager.jsm)
1.36 +let logger = Log.repository.getLogger(LOGGER_ID);
1.37 +
1.38 +this.EXPORTED_SYMBOLS = ["AddonRepository_SQLiteMigrator"];
1.39 +
1.40 +
1.41 +this.AddonRepository_SQLiteMigrator = {
1.42 +
1.43 + /**
1.44 + * Migrates data from a previous SQLite version of the
1.45 + * database to the JSON version.
1.46 + *
1.47 + * @param structFunctions an object that contains functions
1.48 + * to create the various objects used
1.49 + * in the new JSON format
1.50 + * @param aCallback A callback to be called when migration
1.51 + * finishes, with the results in an array
1.52 + * @returns bool True if a migration will happen (DB was
1.53 + * found and succesfully opened)
1.54 + */
1.55 + migrate: function(aCallback) {
1.56 + if (!this._openConnection()) {
1.57 + this._closeConnection();
1.58 + aCallback([]);
1.59 + return false;
1.60 + }
1.61 +
1.62 + logger.debug("Importing addon repository from previous " + FILE_DATABASE + " storage.");
1.63 +
1.64 + this._retrieveStoredData((results) => {
1.65 + this._closeConnection();
1.66 + let resultArray = [addon for ([,addon] of Iterator(results))];
1.67 + logger.debug(resultArray.length + " addons imported.")
1.68 + aCallback(resultArray);
1.69 + });
1.70 +
1.71 + return true;
1.72 + },
1.73 +
1.74 + /**
1.75 + * Synchronously opens a new connection to the database file.
1.76 + *
1.77 + * @return bool Whether the DB was opened successfully.
1.78 + */
1.79 + _openConnection: function AD_openConnection() {
1.80 + delete this.connection;
1.81 +
1.82 + let dbfile = FileUtils.getFile(KEY_PROFILEDIR, [FILE_DATABASE], true);
1.83 + if (!dbfile.exists())
1.84 + return false;
1.85 +
1.86 + try {
1.87 + this.connection = Services.storage.openUnsharedDatabase(dbfile);
1.88 + } catch (e) {
1.89 + return false;
1.90 + }
1.91 +
1.92 + this.connection.executeSimpleSQL("PRAGMA locking_mode = EXCLUSIVE");
1.93 +
1.94 + // Any errors in here should rollback
1.95 + try {
1.96 + this.connection.beginTransaction();
1.97 +
1.98 + switch (this.connection.schemaVersion) {
1.99 + case 0:
1.100 + return false;
1.101 +
1.102 + case 1:
1.103 + logger.debug("Upgrading database schema to version 2");
1.104 + this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN width INTEGER");
1.105 + this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN height INTEGER");
1.106 + this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN thumbnailWidth INTEGER");
1.107 + this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN thumbnailHeight INTEGER");
1.108 + case 2:
1.109 + logger.debug("Upgrading database schema to version 3");
1.110 + this.connection.createTable("compatibility_override",
1.111 + "addon_internal_id INTEGER, " +
1.112 + "num INTEGER, " +
1.113 + "type TEXT, " +
1.114 + "minVersion TEXT, " +
1.115 + "maxVersion TEXT, " +
1.116 + "appID TEXT, " +
1.117 + "appMinVersion TEXT, " +
1.118 + "appMaxVersion TEXT, " +
1.119 + "PRIMARY KEY (addon_internal_id, num)");
1.120 + case 3:
1.121 + logger.debug("Upgrading database schema to version 4");
1.122 + this.connection.createTable("icon",
1.123 + "addon_internal_id INTEGER, " +
1.124 + "size INTEGER, " +
1.125 + "url TEXT, " +
1.126 + "PRIMARY KEY (addon_internal_id, size)");
1.127 + this._createIndices();
1.128 + this._createTriggers();
1.129 + this.connection.schemaVersion = LAST_DB_SCHEMA;
1.130 + case LAST_DB_SCHEMA:
1.131 + break;
1.132 + default:
1.133 + return false;
1.134 + }
1.135 + this.connection.commitTransaction();
1.136 + } catch (e) {
1.137 + logger.error("Failed to open " + FILE_DATABASE + ". Data import will not happen.", e);
1.138 + this.logSQLError(this.connection.lastError, this.connection.lastErrorString);
1.139 + this.connection.rollbackTransaction();
1.140 + return false;
1.141 + }
1.142 +
1.143 + return true;
1.144 + },
1.145 +
1.146 + _closeConnection: function() {
1.147 + for each (let stmt in this.asyncStatementsCache)
1.148 + stmt.finalize();
1.149 + this.asyncStatementsCache = {};
1.150 +
1.151 + if (this.connection)
1.152 + this.connection.asyncClose();
1.153 +
1.154 + delete this.connection;
1.155 + },
1.156 +
1.157 + /**
1.158 + * Asynchronously retrieve all add-ons from the database, and pass it
1.159 + * to the specified callback
1.160 + *
1.161 + * @param aCallback
1.162 + * The callback to pass the add-ons back to
1.163 + */
1.164 + _retrieveStoredData: function AD_retrieveStoredData(aCallback) {
1.165 + let self = this;
1.166 + let addons = {};
1.167 +
1.168 + // Retrieve all data from the addon table
1.169 + function getAllAddons() {
1.170 + self.getAsyncStatement("getAllAddons").executeAsync({
1.171 + handleResult: function getAllAddons_handleResult(aResults) {
1.172 + let row = null;
1.173 + while ((row = aResults.getNextRow())) {
1.174 + let internal_id = row.getResultByName("internal_id");
1.175 + addons[internal_id] = self._makeAddonFromAsyncRow(row);
1.176 + }
1.177 + },
1.178 +
1.179 + handleError: self.asyncErrorLogger,
1.180 +
1.181 + handleCompletion: function getAllAddons_handleCompletion(aReason) {
1.182 + if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
1.183 + logger.error("Error retrieving add-ons from database. Returning empty results");
1.184 + aCallback({});
1.185 + return;
1.186 + }
1.187 +
1.188 + getAllDevelopers();
1.189 + }
1.190 + });
1.191 + }
1.192 +
1.193 + // Retrieve all data from the developer table
1.194 + function getAllDevelopers() {
1.195 + self.getAsyncStatement("getAllDevelopers").executeAsync({
1.196 + handleResult: function getAllDevelopers_handleResult(aResults) {
1.197 + let row = null;
1.198 + while ((row = aResults.getNextRow())) {
1.199 + let addon_internal_id = row.getResultByName("addon_internal_id");
1.200 + if (!(addon_internal_id in addons)) {
1.201 + logger.warn("Found a developer not linked to an add-on in database");
1.202 + continue;
1.203 + }
1.204 +
1.205 + let addon = addons[addon_internal_id];
1.206 + if (!addon.developers)
1.207 + addon.developers = [];
1.208 +
1.209 + addon.developers.push(self._makeDeveloperFromAsyncRow(row));
1.210 + }
1.211 + },
1.212 +
1.213 + handleError: self.asyncErrorLogger,
1.214 +
1.215 + handleCompletion: function getAllDevelopers_handleCompletion(aReason) {
1.216 + if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
1.217 + logger.error("Error retrieving developers from database. Returning empty results");
1.218 + aCallback({});
1.219 + return;
1.220 + }
1.221 +
1.222 + getAllScreenshots();
1.223 + }
1.224 + });
1.225 + }
1.226 +
1.227 + // Retrieve all data from the screenshot table
1.228 + function getAllScreenshots() {
1.229 + self.getAsyncStatement("getAllScreenshots").executeAsync({
1.230 + handleResult: function getAllScreenshots_handleResult(aResults) {
1.231 + let row = null;
1.232 + while ((row = aResults.getNextRow())) {
1.233 + let addon_internal_id = row.getResultByName("addon_internal_id");
1.234 + if (!(addon_internal_id in addons)) {
1.235 + logger.warn("Found a screenshot not linked to an add-on in database");
1.236 + continue;
1.237 + }
1.238 +
1.239 + let addon = addons[addon_internal_id];
1.240 + if (!addon.screenshots)
1.241 + addon.screenshots = [];
1.242 + addon.screenshots.push(self._makeScreenshotFromAsyncRow(row));
1.243 + }
1.244 + },
1.245 +
1.246 + handleError: self.asyncErrorLogger,
1.247 +
1.248 + handleCompletion: function getAllScreenshots_handleCompletion(aReason) {
1.249 + if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
1.250 + logger.error("Error retrieving screenshots from database. Returning empty results");
1.251 + aCallback({});
1.252 + return;
1.253 + }
1.254 +
1.255 + getAllCompatOverrides();
1.256 + }
1.257 + });
1.258 + }
1.259 +
1.260 + function getAllCompatOverrides() {
1.261 + self.getAsyncStatement("getAllCompatOverrides").executeAsync({
1.262 + handleResult: function getAllCompatOverrides_handleResult(aResults) {
1.263 + let row = null;
1.264 + while ((row = aResults.getNextRow())) {
1.265 + let addon_internal_id = row.getResultByName("addon_internal_id");
1.266 + if (!(addon_internal_id in addons)) {
1.267 + logger.warn("Found a compatibility override not linked to an add-on in database");
1.268 + continue;
1.269 + }
1.270 +
1.271 + let addon = addons[addon_internal_id];
1.272 + if (!addon.compatibilityOverrides)
1.273 + addon.compatibilityOverrides = [];
1.274 + addon.compatibilityOverrides.push(self._makeCompatOverrideFromAsyncRow(row));
1.275 + }
1.276 + },
1.277 +
1.278 + handleError: self.asyncErrorLogger,
1.279 +
1.280 + handleCompletion: function getAllCompatOverrides_handleCompletion(aReason) {
1.281 + if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
1.282 + logger.error("Error retrieving compatibility overrides from database. Returning empty results");
1.283 + aCallback({});
1.284 + return;
1.285 + }
1.286 +
1.287 + getAllIcons();
1.288 + }
1.289 + });
1.290 + }
1.291 +
1.292 + function getAllIcons() {
1.293 + self.getAsyncStatement("getAllIcons").executeAsync({
1.294 + handleResult: function getAllIcons_handleResult(aResults) {
1.295 + let row = null;
1.296 + while ((row = aResults.getNextRow())) {
1.297 + let addon_internal_id = row.getResultByName("addon_internal_id");
1.298 + if (!(addon_internal_id in addons)) {
1.299 + logger.warn("Found an icon not linked to an add-on in database");
1.300 + continue;
1.301 + }
1.302 +
1.303 + let addon = addons[addon_internal_id];
1.304 + let { size, url } = self._makeIconFromAsyncRow(row);
1.305 + addon.icons[size] = url;
1.306 + if (size == 32)
1.307 + addon.iconURL = url;
1.308 + }
1.309 + },
1.310 +
1.311 + handleError: self.asyncErrorLogger,
1.312 +
1.313 + handleCompletion: function getAllIcons_handleCompletion(aReason) {
1.314 + if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
1.315 + logger.error("Error retrieving icons from database. Returning empty results");
1.316 + aCallback({});
1.317 + return;
1.318 + }
1.319 +
1.320 + let returnedAddons = {};
1.321 + for each (let addon in addons)
1.322 + returnedAddons[addon.id] = addon;
1.323 + aCallback(returnedAddons);
1.324 + }
1.325 + });
1.326 + }
1.327 +
1.328 + // Begin asynchronous process
1.329 + getAllAddons();
1.330 + },
1.331 +
1.332 + // A cache of statements that are used and need to be finalized on shutdown
1.333 + asyncStatementsCache: {},
1.334 +
1.335 + /**
1.336 + * Gets a cached async statement or creates a new statement if it doesn't
1.337 + * already exist.
1.338 + *
1.339 + * @param aKey
1.340 + * A unique key to reference the statement
1.341 + * @return a mozIStorageAsyncStatement for the SQL corresponding to the
1.342 + * unique key
1.343 + */
1.344 + getAsyncStatement: function AD_getAsyncStatement(aKey) {
1.345 + if (aKey in this.asyncStatementsCache)
1.346 + return this.asyncStatementsCache[aKey];
1.347 +
1.348 + let sql = this.queries[aKey];
1.349 + try {
1.350 + return this.asyncStatementsCache[aKey] = this.connection.createAsyncStatement(sql);
1.351 + } catch (e) {
1.352 + logger.error("Error creating statement " + aKey + " (" + sql + ")");
1.353 + throw Components.Exception("Error creating statement " + aKey + " (" + sql + "): " + e,
1.354 + e.result);
1.355 + }
1.356 + },
1.357 +
1.358 + // The queries used by the database
1.359 + queries: {
1.360 + getAllAddons: "SELECT internal_id, id, type, name, version, " +
1.361 + "creator, creatorURL, description, fullDescription, " +
1.362 + "developerComments, eula, homepageURL, supportURL, " +
1.363 + "contributionURL, contributionAmount, averageRating, " +
1.364 + "reviewCount, reviewURL, totalDownloads, weeklyDownloads, " +
1.365 + "dailyUsers, sourceURI, repositoryStatus, size, updateDate " +
1.366 + "FROM addon",
1.367 +
1.368 + getAllDevelopers: "SELECT addon_internal_id, name, url FROM developer " +
1.369 + "ORDER BY addon_internal_id, num",
1.370 +
1.371 + getAllScreenshots: "SELECT addon_internal_id, url, width, height, " +
1.372 + "thumbnailURL, thumbnailWidth, thumbnailHeight, caption " +
1.373 + "FROM screenshot ORDER BY addon_internal_id, num",
1.374 +
1.375 + getAllCompatOverrides: "SELECT addon_internal_id, type, minVersion, " +
1.376 + "maxVersion, appID, appMinVersion, appMaxVersion " +
1.377 + "FROM compatibility_override " +
1.378 + "ORDER BY addon_internal_id, num",
1.379 +
1.380 + getAllIcons: "SELECT addon_internal_id, size, url FROM icon " +
1.381 + "ORDER BY addon_internal_id, size",
1.382 + },
1.383 +
1.384 + /**
1.385 + * Make add-on structure from an asynchronous row.
1.386 + *
1.387 + * @param aRow
1.388 + * The asynchronous row to use
1.389 + * @return The created add-on
1.390 + */
1.391 + _makeAddonFromAsyncRow: function AD__makeAddonFromAsyncRow(aRow) {
1.392 + // This is intentionally not an AddonSearchResult object in order
1.393 + // to allow AddonDatabase._parseAddon to parse it, same as if it
1.394 + // was read from the JSON database.
1.395 +
1.396 + let addon = { icons: {} };
1.397 +
1.398 + for (let prop of PROP_SINGLE) {
1.399 + addon[prop] = aRow.getResultByName(prop)
1.400 + };
1.401 +
1.402 + return addon;
1.403 + },
1.404 +
1.405 + /**
1.406 + * Make a developer from an asynchronous row
1.407 + *
1.408 + * @param aRow
1.409 + * The asynchronous row to use
1.410 + * @return The created developer
1.411 + */
1.412 + _makeDeveloperFromAsyncRow: function AD__makeDeveloperFromAsyncRow(aRow) {
1.413 + let name = aRow.getResultByName("name");
1.414 + let url = aRow.getResultByName("url")
1.415 + return new AddonManagerPrivate.AddonAuthor(name, url);
1.416 + },
1.417 +
1.418 + /**
1.419 + * Make a screenshot from an asynchronous row
1.420 + *
1.421 + * @param aRow
1.422 + * The asynchronous row to use
1.423 + * @return The created screenshot
1.424 + */
1.425 + _makeScreenshotFromAsyncRow: function AD__makeScreenshotFromAsyncRow(aRow) {
1.426 + let url = aRow.getResultByName("url");
1.427 + let width = aRow.getResultByName("width");
1.428 + let height = aRow.getResultByName("height");
1.429 + let thumbnailURL = aRow.getResultByName("thumbnailURL");
1.430 + let thumbnailWidth = aRow.getResultByName("thumbnailWidth");
1.431 + let thumbnailHeight = aRow.getResultByName("thumbnailHeight");
1.432 + let caption = aRow.getResultByName("caption");
1.433 + return new AddonManagerPrivate.AddonScreenshot(url, width, height, thumbnailURL,
1.434 + thumbnailWidth, thumbnailHeight, caption);
1.435 + },
1.436 +
1.437 + /**
1.438 + * Make a CompatibilityOverride from an asynchronous row
1.439 + *
1.440 + * @param aRow
1.441 + * The asynchronous row to use
1.442 + * @return The created CompatibilityOverride
1.443 + */
1.444 + _makeCompatOverrideFromAsyncRow: function AD_makeCompatOverrideFromAsyncRow(aRow) {
1.445 + let type = aRow.getResultByName("type");
1.446 + let minVersion = aRow.getResultByName("minVersion");
1.447 + let maxVersion = aRow.getResultByName("maxVersion");
1.448 + let appID = aRow.getResultByName("appID");
1.449 + let appMinVersion = aRow.getResultByName("appMinVersion");
1.450 + let appMaxVersion = aRow.getResultByName("appMaxVersion");
1.451 + return new AddonManagerPrivate.AddonCompatibilityOverride(type,
1.452 + minVersion,
1.453 + maxVersion,
1.454 + appID,
1.455 + appMinVersion,
1.456 + appMaxVersion);
1.457 + },
1.458 +
1.459 + /**
1.460 + * Make an icon from an asynchronous row
1.461 + *
1.462 + * @param aRow
1.463 + * The asynchronous row to use
1.464 + * @return An object containing the size and URL of the icon
1.465 + */
1.466 + _makeIconFromAsyncRow: function AD_makeIconFromAsyncRow(aRow) {
1.467 + let size = aRow.getResultByName("size");
1.468 + let url = aRow.getResultByName("url");
1.469 + return { size: size, url: url };
1.470 + },
1.471 +
1.472 + /**
1.473 + * A helper function to log an SQL error.
1.474 + *
1.475 + * @param aError
1.476 + * The storage error code associated with the error
1.477 + * @param aErrorString
1.478 + * An error message
1.479 + */
1.480 + logSQLError: function AD_logSQLError(aError, aErrorString) {
1.481 + logger.error("SQL error " + aError + ": " + aErrorString);
1.482 + },
1.483 +
1.484 + /**
1.485 + * A helper function to log any errors that occur during async statements.
1.486 + *
1.487 + * @param aError
1.488 + * A mozIStorageError to log
1.489 + */
1.490 + asyncErrorLogger: function AD_asyncErrorLogger(aError) {
1.491 + logger.error("Async SQL error " + aError.result + ": " + aError.message);
1.492 + },
1.493 +
1.494 + /**
1.495 + * Synchronously creates the triggers in the database.
1.496 + */
1.497 + _createTriggers: function AD__createTriggers() {
1.498 + this.connection.executeSimpleSQL("DROP TRIGGER IF EXISTS delete_addon");
1.499 + this.connection.executeSimpleSQL("CREATE TRIGGER delete_addon AFTER DELETE " +
1.500 + "ON addon BEGIN " +
1.501 + "DELETE FROM developer WHERE addon_internal_id=old.internal_id; " +
1.502 + "DELETE FROM screenshot WHERE addon_internal_id=old.internal_id; " +
1.503 + "DELETE FROM compatibility_override WHERE addon_internal_id=old.internal_id; " +
1.504 + "DELETE FROM icon WHERE addon_internal_id=old.internal_id; " +
1.505 + "END");
1.506 + },
1.507 +
1.508 + /**
1.509 + * Synchronously creates the indices in the database.
1.510 + */
1.511 + _createIndices: function AD__createIndices() {
1.512 + this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS developer_idx " +
1.513 + "ON developer (addon_internal_id)");
1.514 + this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS screenshot_idx " +
1.515 + "ON screenshot (addon_internal_id)");
1.516 + this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS compatibility_override_idx " +
1.517 + "ON compatibility_override (addon_internal_id)");
1.518 + this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS icon_idx " +
1.519 + "ON icon (addon_internal_id)");
1.520 + }
1.521 +}
