michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: /**
michael@0:  * FormHistory
michael@0:  *
michael@0:  * Used to store values that have been entered into forms which may later
michael@0:  * be used to automatically fill in the values when the form is visited again.
michael@0:  *
michael@0:  * search(terms, queryData, callback)
michael@0:  *   Look up values that have been previously stored.
michael@0:  *     terms - array of terms to return data for
michael@0:  *     queryData - object that contains the query terms
michael@0:  *       The query object contains properties for each search criteria to match, where the value
michael@0:  *       of the property specifies the value that term must have. For example,
michael@0:  *       { term1: value1, term2: value2 }
michael@0:  *     callback - callback that is called when results are available or an error occurs.
michael@0:  *       The callback is passed a result array containing each found entry. Each element in
michael@0:  *       the array is an object containing a property for each search term specified by 'terms'.
michael@0:  * count(queryData, callback)
michael@0:  *   Find the number of stored entries that match the given criteria.
michael@0:  *     queryData - array of objects that indicate the query. See the search method for details.
michael@0:  *     callback - callback that is called when results are available or an error occurs.
michael@0:  *       The callback is passed the number of found entries.
michael@0:  * update(changes, callback)
michael@0:  *    Write data to form history storage.
michael@0:  *      changes - an array of changes to be made. If only one change is to be made, it
michael@0:  *                may be passed as an object rather than a one-element array.
michael@0:  *        Each change object is of the form:
michael@0:  *          { op: operation, term1: value1, term2: value2, ... }
michael@0:  *        Valid operations are:
michael@0:  *          add - add a new entry
michael@0:  *          update - update an existing entry
michael@0:  *          remove - remove an entry
michael@0:  *          bump - update the last accessed time on an entry
michael@0:  *        The terms specified allow matching of one or more specific entries. If no terms
michael@0:  *        are specified then all entries are matched. This means that { op: "remove" } is
michael@0:  *        used to remove all entries and clear the form history.
michael@0:  *      callback - callback that is called when results have been stored.
michael@0:  * getAutoCompeteResults(searchString, params, callback)
michael@0:  *   Retrieve an array of form history values suitable for display in an autocomplete list.
michael@0:  *   Returns an mozIStoragePendingStatement that can be used to cancel the operation if
michael@0:  *   needed.
michael@0:  *     searchString - the string to search for, typically the entered value of a textbox
michael@0:  *     params - zero or more filter arguments:
michael@0:  *       fieldname - form field name
michael@0:  *       agedWeight
michael@0:  *       bucketSize
michael@0:  *       expiryDate
michael@0:  *       maxTimeGroundings
michael@0:  *       timeGroupingSize
michael@0:  *       prefixWeight
michael@0:  *       boundaryWeight
michael@0:  *     callback - callback that is called with the array of results. Each result in the array
michael@0:  *                is an object with four arguments:
michael@0:  *                  text, textLowerCase, frecency, totalScore
michael@0:  * schemaVersion
michael@0:  *   This property holds the version of the database schema
michael@0:  *
michael@0:  * Terms:
michael@0:  *  guid - entry identifier. For 'add', a guid will be generated.
michael@0:  *  fieldname - form field name
michael@0:  *  value - form value
michael@0:  *  timesUsed - the number of times the entry has been accessed
michael@0:  *  firstUsed - the time the the entry was first created
michael@0:  *  lastUsed - the time the entry was last accessed
michael@0:  *  firstUsedStart - search for entries created after or at this time
michael@0:  *  firstUsedEnd - search for entries created before or at this time
michael@0:  *  lastUsedStart - search for entries last accessed after or at this time
michael@0:  *  lastUsedEnd - search for entries last accessed before or at this time
michael@0:  *  newGuid - a special case valid only for 'update' and allows the guid for
michael@0:  *            an existing record to be updated. The 'guid' term is the only
michael@0:  *            other term which can be used (ie, you can not also specify a
michael@0:  *            fieldname, value etc) and indicates the guid of the existing
michael@0:  *            record that should be updated.
michael@0:  *
michael@0:  * In all of the above methods, the callback argument should be an object with
michael@0:  * handleResult(result), handleFailure(error) and handleCompletion(reason) functions.
michael@0:  * For search and getAutoCompeteResults, result is an object containing the desired
michael@0:  * properties. For count, result is the integer count. For, update, handleResult is
michael@0:  * not called. For handleCompletion, reason is either 0 if successful or 1 if
michael@0:  * an error occurred.
michael@0:  */
michael@0: 
michael@0: this.EXPORTED_SYMBOLS = ["FormHistory"];
michael@0: 
michael@0: const Cc = Components.classes;
michael@0: const Ci = Components.interfaces;
michael@0: const Cr = Components.results;
michael@0: 
michael@0: Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
michael@0: Components.utils.import("resource://gre/modules/Services.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyServiceGetter(this, "uuidService",
michael@0:                                    "@mozilla.org/uuid-generator;1",
michael@0:                                    "nsIUUIDGenerator");
michael@0: 
michael@0: const DB_SCHEMA_VERSION = 4;
michael@0: const DAY_IN_MS  = 86400000; // 1 day in milliseconds
michael@0: const MAX_SEARCH_TOKENS = 10;
michael@0: const NOOP = function noop() {};
michael@0: 
michael@0: let supportsDeletedTable =
michael@0: #ifdef ANDROID
michael@0:   true;
michael@0: #else
michael@0:   false;
michael@0: #endif
michael@0: 
michael@0: let Prefs = {
michael@0:   initialized: false,
michael@0: 
michael@0:   get debug() { this.ensureInitialized(); return this._debug; },
michael@0:   get enabled() { this.ensureInitialized(); return this._enabled; },
michael@0:   get expireDays() { this.ensureInitialized(); return this._expireDays; },
michael@0: 
michael@0:   ensureInitialized: function() {
michael@0:     if (this.initialized)
michael@0:       return;
michael@0: 
michael@0:     this.initialized = true;
michael@0: 
michael@0:     this._debug = Services.prefs.getBoolPref("browser.formfill.debug");
michael@0:     this._enabled = Services.prefs.getBoolPref("browser.formfill.enable");
michael@0:     this._expireDays = Services.prefs.getIntPref("browser.formfill.expire_days");
michael@0:   }
michael@0: };
michael@0: 
michael@0: function log(aMessage) {
michael@0:   if (Prefs.debug) {
michael@0:     Services.console.logStringMessage("FormHistory: " + aMessage);
michael@0:   }
michael@0: }
michael@0: 
michael@0: function sendNotification(aType, aData) {
michael@0:   if (typeof aData == "string") {
michael@0:     let strWrapper = Cc["@mozilla.org/supports-string;1"].
michael@0:                      createInstance(Ci.nsISupportsString);
michael@0:     strWrapper.data = aData;
michael@0:     aData = strWrapper;
michael@0:   }
michael@0:   else if (typeof aData == "number") {
michael@0:     let intWrapper = Cc["@mozilla.org/supports-PRInt64;1"].
michael@0:                      createInstance(Ci.nsISupportsPRInt64);
michael@0:     intWrapper.data = aData;
michael@0:     aData = intWrapper;
michael@0:   }
michael@0:   else if (aData) {
michael@0:     throw Components.Exception("Invalid type " + (typeof aType) + " passed to sendNotification",
michael@0:                                Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:   }
michael@0: 
michael@0:   Services.obs.notifyObservers(aData, "satchel-storage-changed", aType);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Current database schema
michael@0:  */
michael@0: 
michael@0: const dbSchema = {
michael@0:   tables : {
michael@0:     moz_formhistory : {
michael@0:       "id"        : "INTEGER PRIMARY KEY",
michael@0:       "fieldname" : "TEXT NOT NULL",
michael@0:       "value"     : "TEXT NOT NULL",
michael@0:       "timesUsed" : "INTEGER",
michael@0:       "firstUsed" : "INTEGER",
michael@0:       "lastUsed"  : "INTEGER",
michael@0:       "guid"      : "TEXT",
michael@0:     },
michael@0:     moz_deleted_formhistory: {
michael@0:         "id"          : "INTEGER PRIMARY KEY",
michael@0:         "timeDeleted" : "INTEGER",
michael@0:         "guid"        : "TEXT"
michael@0:     }
michael@0:   },
michael@0:   indices : {
michael@0:     moz_formhistory_index : {
michael@0:       table   : "moz_formhistory",
michael@0:       columns : [ "fieldname" ]
michael@0:     },
michael@0:     moz_formhistory_lastused_index : {
michael@0:       table   : "moz_formhistory",
michael@0:       columns : [ "lastUsed" ]
michael@0:     },
michael@0:     moz_formhistory_guid_index : {
michael@0:       table   : "moz_formhistory",
michael@0:       columns : [ "guid" ]
michael@0:     },
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Validating and processing API querying data
michael@0:  */
michael@0: 
michael@0: const validFields = [
michael@0:   "fieldname",
michael@0:   "value",
michael@0:   "timesUsed",
michael@0:   "firstUsed",
michael@0:   "lastUsed",
michael@0:   "guid",
michael@0: ];
michael@0: 
michael@0: const searchFilters = [
michael@0:   "firstUsedStart",
michael@0:   "firstUsedEnd",
michael@0:   "lastUsedStart",
michael@0:   "lastUsedEnd",
michael@0: ];
michael@0: 
michael@0: function validateOpData(aData, aDataType) {
michael@0:   let thisValidFields = validFields;
michael@0:   // A special case to update the GUID - in this case there can be a 'newGuid'
michael@0:   // field and of the normally valid fields, only 'guid' is accepted.
michael@0:   if (aDataType == "Update" && "newGuid" in aData) {
michael@0:     thisValidFields = ["guid", "newGuid"];
michael@0:   }
michael@0:   for (let field in aData) {
michael@0:     if (field != "op" && thisValidFields.indexOf(field) == -1) {
michael@0:       throw Components.Exception(
michael@0:         aDataType + " query contains an unrecognized field: " + field,
michael@0:         Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:     }
michael@0:   }
michael@0:   return aData;
michael@0: }
michael@0: 
michael@0: function validateSearchData(aData, aDataType) {
michael@0:   for (let field in aData) {
michael@0:     if (field != "op" && validFields.indexOf(field) == -1 && searchFilters.indexOf(field) == -1) {
michael@0:       throw Components.Exception(
michael@0:         aDataType + " query contains an unrecognized field: " + field,
michael@0:         Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: function makeQueryPredicates(aQueryData, delimiter = ' AND ') {
michael@0:   return Object.keys(aQueryData).map(function(field) {
michael@0:     if (field == "firstUsedStart") {
michael@0:       return "firstUsed >= :" + field;
michael@0:     } else if (field == "firstUsedEnd") {
michael@0:       return "firstUsed <= :" + field;
michael@0:     } else if (field == "lastUsedStart") {
michael@0:       return "lastUsed >= :" + field;
michael@0:     } else if (field == "lastUsedEnd") {
michael@0:       return "lastUsed <= :" + field;
michael@0:     }
michael@0:     return field + " = :" + field;
michael@0:   }).join(delimiter);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Storage statement creation and parameter binding
michael@0:  */
michael@0: 
michael@0: function makeCountStatement(aSearchData) {
michael@0:   let query = "SELECT COUNT(*) AS numEntries FROM moz_formhistory";
michael@0:   let queryTerms = makeQueryPredicates(aSearchData);
michael@0:   if (queryTerms) {
michael@0:     query += " WHERE " + queryTerms;
michael@0:   }
michael@0:   return dbCreateAsyncStatement(query, aSearchData);
michael@0: }
michael@0: 
michael@0: function makeSearchStatement(aSearchData, aSelectTerms) {
michael@0:   let query = "SELECT " + aSelectTerms.join(", ") + " FROM moz_formhistory";
michael@0:   let queryTerms = makeQueryPredicates(aSearchData);
michael@0:   if (queryTerms) {
michael@0:     query += " WHERE " + queryTerms;
michael@0:   }
michael@0: 
michael@0:   return dbCreateAsyncStatement(query, aSearchData);
michael@0: }
michael@0: 
michael@0: function makeAddStatement(aNewData, aNow, aBindingArrays) {
michael@0:   let query = "INSERT INTO moz_formhistory (fieldname, value, timesUsed, firstUsed, lastUsed, guid) " +
michael@0:               "VALUES (:fieldname, :value, :timesUsed, :firstUsed, :lastUsed, :guid)";
michael@0: 
michael@0:   aNewData.timesUsed = aNewData.timesUsed || 1;
michael@0:   aNewData.firstUsed = aNewData.firstUsed || aNow;
michael@0:   aNewData.lastUsed = aNewData.lastUsed || aNow;
michael@0:   return dbCreateAsyncStatement(query, aNewData, aBindingArrays);
michael@0: }
michael@0: 
michael@0: function makeBumpStatement(aGuid, aNow, aBindingArrays) {
michael@0:   let query = "UPDATE moz_formhistory SET timesUsed = timesUsed + 1, lastUsed = :lastUsed WHERE guid = :guid";
michael@0:   let queryParams = {
michael@0:     lastUsed : aNow,
michael@0:     guid : aGuid,
michael@0:   };
michael@0: 
michael@0:   return dbCreateAsyncStatement(query, queryParams, aBindingArrays);
michael@0: }
michael@0: 
michael@0: function makeRemoveStatement(aSearchData, aBindingArrays) {
michael@0:   let query = "DELETE FROM moz_formhistory";
michael@0:   let queryTerms = makeQueryPredicates(aSearchData);
michael@0: 
michael@0:   if (queryTerms) {
michael@0:     log("removeEntries");
michael@0:     query += " WHERE " + queryTerms;
michael@0:   } else {
michael@0:     log("removeAllEntries");
michael@0:     // Not specifying any fields means we should remove all entries. We
michael@0:     // won't need to modify the query in this case.
michael@0:   }
michael@0: 
michael@0:   return dbCreateAsyncStatement(query, aSearchData, aBindingArrays);
michael@0: }
michael@0: 
michael@0: function makeUpdateStatement(aGuid, aNewData, aBindingArrays) {
michael@0:   let query = "UPDATE moz_formhistory SET ";
michael@0:   let queryTerms = makeQueryPredicates(aNewData, ', ');
michael@0: 
michael@0:   if (!queryTerms) {
michael@0:     throw Components.Exception("Update query must define fields to modify.",
michael@0:                                Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:   }
michael@0: 
michael@0:   query += queryTerms + " WHERE guid = :existing_guid";
michael@0:   aNewData["existing_guid"] = aGuid;
michael@0: 
michael@0:   return dbCreateAsyncStatement(query, aNewData, aBindingArrays);
michael@0: }
michael@0: 
michael@0: function makeMoveToDeletedStatement(aGuid, aNow, aData, aBindingArrays) {
michael@0:   if (supportsDeletedTable) {
michael@0:     let query = "INSERT INTO moz_deleted_formhistory (guid, timeDeleted)";
michael@0:     let queryTerms = makeQueryPredicates(aData);
michael@0: 
michael@0:     if (aGuid) {
michael@0:       query += " VALUES (:guid, :timeDeleted)";
michael@0:     } else {
michael@0:       // TODO: Add these items to the deleted items table once we've sorted
michael@0:       //       out the issues from bug 756701
michael@0:       if (!queryTerms)
michael@0:         return;
michael@0: 
michael@0:       query += " SELECT guid, :timeDeleted FROM moz_formhistory WHERE " + queryTerms;
michael@0:     }
michael@0: 
michael@0:     aData.timeDeleted = aNow;
michael@0: 
michael@0:     return dbCreateAsyncStatement(query, aData, aBindingArrays);
michael@0:   }
michael@0: 
michael@0:   return null;
michael@0: }
michael@0: 
michael@0: function generateGUID() {
michael@0:   // string like: "{f60d9eac-9421-4abc-8491-8e8322b063d4}"
michael@0:   let uuid = uuidService.generateUUID().toString();
michael@0:   let raw = ""; // A string with the low bytes set to random values
michael@0:   let bytes = 0;
michael@0:   for (let i = 1; bytes < 12 ; i+= 2) {
michael@0:     // Skip dashes
michael@0:     if (uuid[i] == "-")
michael@0:       i++;
michael@0:     let hexVal = parseInt(uuid[i] + uuid[i + 1], 16);
michael@0:     raw += String.fromCharCode(hexVal);
michael@0:     bytes++;
michael@0:   }
michael@0:   return btoa(raw);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Database creation and access
michael@0:  */
michael@0: 
michael@0: let _dbConnection = null;
michael@0: XPCOMUtils.defineLazyGetter(this, "dbConnection", function() {
michael@0:   let dbFile;
michael@0: 
michael@0:   try {
michael@0:     dbFile = Services.dirsvc.get("ProfD", Ci.nsIFile).clone();
michael@0:     dbFile.append("formhistory.sqlite");
michael@0:     log("Opening database at " + dbFile.path);
michael@0: 
michael@0:     _dbConnection = Services.storage.openUnsharedDatabase(dbFile);
michael@0:     dbInit();
michael@0:   } catch (e if e.result == Cr.NS_ERROR_FILE_CORRUPTED) {
michael@0:     dbCleanup(dbFile);
michael@0:     _dbConnection = Services.storage.openUnsharedDatabase(dbFile);
michael@0:     dbInit();
michael@0:   }
michael@0: 
michael@0:   return _dbConnection;
michael@0: });
michael@0: 
michael@0: 
michael@0: let dbStmts = new Map();
michael@0: 
michael@0: /*
michael@0:  * dbCreateAsyncStatement
michael@0:  *
michael@0:  * Creates a statement, wraps it, and then does parameter replacement
michael@0:  */
michael@0: function dbCreateAsyncStatement(aQuery, aParams, aBindingArrays) {
michael@0:   if (!aQuery)
michael@0:     return null;
michael@0: 
michael@0:   let stmt = dbStmts.get(aQuery);
michael@0:   if (!stmt) {
michael@0:     log("Creating new statement for query: " + aQuery);
michael@0:     stmt = dbConnection.createAsyncStatement(aQuery);
michael@0:     dbStmts.set(aQuery, stmt);
michael@0:   }
michael@0: 
michael@0:   if (aBindingArrays) {
michael@0:     let bindingArray = aBindingArrays.get(stmt);
michael@0:     if (!bindingArray) {
michael@0:       // first time using a particular statement in update
michael@0:       bindingArray = stmt.newBindingParamsArray();
michael@0:       aBindingArrays.set(stmt, bindingArray);
michael@0:     }
michael@0: 
michael@0:     if (aParams) {
michael@0:       let bindingParams = bindingArray.newBindingParams();
michael@0:       for (let field in aParams) {
michael@0:         bindingParams.bindByName(field, aParams[field]);
michael@0:       }
michael@0:       bindingArray.addParams(bindingParams);
michael@0:     }
michael@0:   } else {
michael@0:     if (aParams) {
michael@0:       for (let field in aParams) {
michael@0:         stmt.params[field] = aParams[field];
michael@0:       }
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return stmt;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * dbInit
michael@0:  *
michael@0:  * Attempts to initialize the database. This creates the file if it doesn't
michael@0:  * exist, performs any migrations, etc.
michael@0:  */
michael@0: function dbInit() {
michael@0:   log("Initializing Database");
michael@0: 
michael@0:   if (!_dbConnection.tableExists("moz_formhistory")) {
michael@0:     dbCreate();
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   // When FormHistory is released, we will no longer support the various schema versions prior to
michael@0:   // this release that nsIFormHistory2 once did.
michael@0:   let version = _dbConnection.schemaVersion;
michael@0:   if (version < 3) {
michael@0:     throw Components.Exception("DB version is unsupported.",
michael@0:                                Cr.NS_ERROR_FILE_CORRUPTED);
michael@0:   } else if (version != DB_SCHEMA_VERSION) {
michael@0:     dbMigrate(version);
michael@0:   }
michael@0: }
michael@0: 
michael@0: function dbCreate() {
michael@0:   log("Creating DB -- tables");
michael@0:   for (let name in dbSchema.tables) {
michael@0:     let table = dbSchema.tables[name];
michael@0:     let tSQL = [[col, table[col]].join(" ") for (col in table)].join(", ");
michael@0:     log("Creating table " + name + " with " + tSQL);
michael@0:     _dbConnection.createTable(name, tSQL);
michael@0:   }
michael@0: 
michael@0:   log("Creating DB -- indices");
michael@0:   for (let name in dbSchema.indices) {
michael@0:     let index = dbSchema.indices[name];
michael@0:     let statement = "CREATE INDEX IF NOT EXISTS " + name + " ON " + index.table +
michael@0:                     "(" + index.columns.join(", ") + ")";
michael@0:     _dbConnection.executeSimpleSQL(statement);
michael@0:   }
michael@0: 
michael@0:   _dbConnection.schemaVersion = DB_SCHEMA_VERSION;
michael@0: }
michael@0: 
michael@0: function dbMigrate(oldVersion) {
michael@0:   log("Attempting to migrate from version " + oldVersion);
michael@0: 
michael@0:   if (oldVersion > DB_SCHEMA_VERSION) {
michael@0:     log("Downgrading to version " + DB_SCHEMA_VERSION);
michael@0:     // User's DB is newer. Sanity check that our expected columns are
michael@0:     // present, and if so mark the lower version and merrily continue
michael@0:     // on. If the columns are borked, something is wrong so blow away
michael@0:     // the DB and start from scratch. [Future incompatible upgrades
michael@0:     // should switch to a different table or file.]
michael@0: 
michael@0:     if (!dbAreExpectedColumnsPresent()) {
michael@0:       throw Components.Exception("DB is missing expected columns",
michael@0:                                  Cr.NS_ERROR_FILE_CORRUPTED);
michael@0:     }
michael@0: 
michael@0:     // Change the stored version to the current version. If the user
michael@0:     // runs the newer code again, it will see the lower version number
michael@0:     // and re-upgrade (to fixup any entries the old code added).
michael@0:     _dbConnection.schemaVersion = DB_SCHEMA_VERSION;
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   // Note that migration is currently performed synchronously.
michael@0:   _dbConnection.beginTransaction();
michael@0: 
michael@0:   try {
michael@0:     for (let v = oldVersion + 1; v <= DB_SCHEMA_VERSION; v++) {
michael@0:       this.log("Upgrading to version " + v + "...");
michael@0:       Migrators["dbMigrateToVersion" + v]();
michael@0:     }
michael@0:   } catch (e) {
michael@0:     this.log("Migration failed: "  + e);
michael@0:     this.dbConnection.rollbackTransaction();
michael@0:     throw e;
michael@0:   }
michael@0: 
michael@0:   _dbConnection.schemaVersion = DB_SCHEMA_VERSION;
michael@0:   _dbConnection.commitTransaction();
michael@0: 
michael@0:   log("DB migration completed.");
michael@0: }
michael@0: 
michael@0: var Migrators = {
michael@0:   /*
michael@0:    * Updates the DB schema to v3 (bug 506402).
michael@0:    * Adds deleted form history table.
michael@0:    */
michael@0:   dbMigrateToVersion4: function dbMigrateToVersion4() {
michael@0:     if (!_dbConnection.tableExists("moz_deleted_formhistory")) {
michael@0:       let table = dbSchema.tables["moz_deleted_formhistory"];
michael@0:       let tSQL = [[col, table[col]].join(" ") for (col in table)].join(", ");
michael@0:       _dbConnection.createTable("moz_deleted_formhistory", tSQL);
michael@0:     }
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * dbAreExpectedColumnsPresent
michael@0:  *
michael@0:  * Sanity check to ensure that the columns this version of the code expects
michael@0:  * are present in the DB we're using.
michael@0:  */
michael@0: function dbAreExpectedColumnsPresent() {
michael@0:   for (let name in dbSchema.tables) {
michael@0:     let table = dbSchema.tables[name];
michael@0:     let query = "SELECT " +
michael@0:                 [col for (col in table)].join(", ") +
michael@0:                 " FROM " + name;
michael@0:     try {
michael@0:       let stmt = _dbConnection.createStatement(query);
michael@0:       // (no need to execute statement, if it compiled we're good)
michael@0:       stmt.finalize();
michael@0:     } catch (e) {
michael@0:       return false;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   log("verified that expected columns are present in DB.");
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * dbCleanup
michael@0:  *
michael@0:  * Called when database creation fails. Finalizes database statements,
michael@0:  * closes the database connection, deletes the database file.
michael@0:  */
michael@0: function dbCleanup(dbFile) {
michael@0:   log("Cleaning up DB file - close & remove & backup");
michael@0: 
michael@0:   // Create backup file
michael@0:   let backupFile = dbFile.leafName + ".corrupt";
michael@0:   Services.storage.backupDatabaseFile(dbFile, backupFile);
michael@0: 
michael@0:   dbClose(false);
michael@0:   dbFile.remove(false);
michael@0: }
michael@0: 
michael@0: function dbClose(aShutdown) {
michael@0:   log("dbClose(" + aShutdown + ")");
michael@0: 
michael@0:   if (aShutdown) {
michael@0:     sendNotification("formhistory-shutdown", null);
michael@0:   }
michael@0: 
michael@0:   // Connection may never have been created if say open failed but we still
michael@0:   // end up calling dbClose as part of the rest of dbCleanup.
michael@0:   if (!_dbConnection) {
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   log("dbClose finalize statements");
michael@0:   for (let stmt of dbStmts.values()) {
michael@0:     stmt.finalize();
michael@0:   }
michael@0: 
michael@0:   dbStmts = new Map();
michael@0: 
michael@0:   let closed = false;
michael@0:   _dbConnection.asyncClose(function () closed = true);
michael@0: 
michael@0:   if (!aShutdown) {
michael@0:     let thread = Services.tm.currentThread;
michael@0:     while (!closed) {
michael@0:       thread.processNextEvent(true);
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * updateFormHistoryWrite
michael@0:  *
michael@0:  * Constructs and executes database statements from a pre-processed list of
michael@0:  * inputted changes.
michael@0:  */
michael@0: function updateFormHistoryWrite(aChanges, aCallbacks) {
michael@0:   log("updateFormHistoryWrite  " + aChanges.length);
michael@0: 
michael@0:   // pass 'now' down so that every entry in the batch has the same timestamp
michael@0:   let now = Date.now() * 1000;
michael@0: 
michael@0:   // for each change, we either create and append a new storage statement to
michael@0:   // stmts or bind a new set of parameters to an existing storage statement.
michael@0:   // stmts and bindingArrays are updated when makeXXXStatement eventually
michael@0:   // calls dbCreateAsyncStatement.
michael@0:   let stmts = [];
michael@0:   let notifications = [];
michael@0:   let bindingArrays = new Map();
michael@0: 
michael@0:   for each (let change in aChanges) {
michael@0:     let operation = change.op;
michael@0:     delete change.op;
michael@0:     let stmt;
michael@0:     switch (operation) {
michael@0:       case "remove":
michael@0:         log("Remove from form history  " + change);
michael@0:         let delStmt = makeMoveToDeletedStatement(change.guid, now, change, bindingArrays);
michael@0:         if (delStmt && stmts.indexOf(delStmt) == -1)
michael@0:           stmts.push(delStmt);
michael@0:         if ("timeDeleted" in change)
michael@0:           delete change.timeDeleted;
michael@0:         stmt = makeRemoveStatement(change, bindingArrays);
michael@0:         notifications.push([ "formhistory-remove", change.guid ]);
michael@0:         break;
michael@0:       case "update":
michael@0:         log("Update form history " + change);
michael@0:         let guid = change.guid;
michael@0:         delete change.guid;
michael@0:         // a special case for updating the GUID - the new value can be
michael@0:         // specified in newGuid.
michael@0:         if (change.newGuid) {
michael@0:           change.guid = change.newGuid
michael@0:           delete change.newGuid;
michael@0:         }
michael@0:         stmt = makeUpdateStatement(guid, change, bindingArrays);
michael@0:         notifications.push([ "formhistory-update", guid ]);
michael@0:         break;
michael@0:       case "bump":
michael@0:         log("Bump form history " + change);
michael@0:         if (change.guid) {
michael@0:           stmt = makeBumpStatement(change.guid, now, bindingArrays);
michael@0:           notifications.push([ "formhistory-update", change.guid ]);
michael@0:         } else {
michael@0:           change.guid = generateGUID();
michael@0:           stmt = makeAddStatement(change, now, bindingArrays);
michael@0:           notifications.push([ "formhistory-add", change.guid ]);
michael@0:         }
michael@0:         break;
michael@0:       case "add":
michael@0:         log("Add to form history " + change);
michael@0:         change.guid = generateGUID();
michael@0:         stmt = makeAddStatement(change, now, bindingArrays);
michael@0:         notifications.push([ "formhistory-add", change.guid ]);
michael@0:         break;
michael@0:       default:
michael@0:         // We should've already guaranteed that change.op is one of the above
michael@0:         throw Components.Exception("Invalid operation " + operation,
michael@0:                                    Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:     }
michael@0: 
michael@0:     // As identical statements are reused, only add statements if they aren't already present.
michael@0:     if (stmt && stmts.indexOf(stmt) == -1) {
michael@0:       stmts.push(stmt);
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   for (let stmt of stmts) {
michael@0:     stmt.bindParameters(bindingArrays.get(stmt));
michael@0:   }
michael@0: 
michael@0:   let handlers = {
michael@0:     handleCompletion : function(aReason) {
michael@0:       if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
michael@0:         for (let [notification, param] of notifications) {
michael@0:           // We're either sending a GUID or nothing at all.
michael@0:           sendNotification(notification, param);
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       if (aCallbacks && aCallbacks.handleCompletion) {
michael@0:         aCallbacks.handleCompletion(aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED ? 0 : 1);
michael@0:       }
michael@0:     },
michael@0:     handleError : function(aError) {
michael@0:       if (aCallbacks && aCallbacks.handleError) {
michael@0:         aCallbacks.handleError(aError);
michael@0:       }
michael@0:     },
michael@0:     handleResult : NOOP
michael@0:   };
michael@0: 
michael@0:   dbConnection.executeAsync(stmts, stmts.length, handlers);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Functions that expire entries in form history and shrinks database
michael@0:  * afterwards as necessary initiated by expireOldEntries.
michael@0:  */
michael@0: 
michael@0: /**
michael@0:  * expireOldEntriesDeletion
michael@0:  *
michael@0:  * Removes entries from database.
michael@0:  */
michael@0: function expireOldEntriesDeletion(aExpireTime, aBeginningCount) {
michael@0:   log("expireOldEntriesDeletion(" + aExpireTime + "," + aBeginningCount + ")");
michael@0: 
michael@0:   FormHistory.update([
michael@0:     {
michael@0:       op: "remove",
michael@0:       lastUsedEnd : aExpireTime,
michael@0:     }], {
michael@0:       handleCompletion: function() {
michael@0:         expireOldEntriesVacuum(aExpireTime, aBeginningCount);
michael@0:       },
michael@0:       handleError: function(aError) {
michael@0:         log("expireOldEntriesDeletionFailure");
michael@0:       }
michael@0:   });
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * expireOldEntriesVacuum
michael@0:  *
michael@0:  * Counts number of entries removed and shrinks database as necessary.
michael@0:  */
michael@0: function expireOldEntriesVacuum(aExpireTime, aBeginningCount) {
michael@0:   FormHistory.count({}, {
michael@0:     handleResult: function(aEndingCount) {
michael@0:       if (aBeginningCount - aEndingCount > 500) {
michael@0:         log("expireOldEntriesVacuum");
michael@0: 
michael@0:         let stmt = dbCreateAsyncStatement("VACUUM");
michael@0:         stmt.executeAsync({
michael@0:           handleResult : NOOP,
michael@0:           handleError : function(aError) {
michael@0:             log("expireVacuumError");
michael@0:           },
michael@0:           handleCompletion : NOOP
michael@0:         });
michael@0:       }
michael@0: 
michael@0:       sendNotification("formhistory-expireoldentries", aExpireTime);
michael@0:     },
michael@0:     handleError: function(aError) {
michael@0:       log("expireEndCountFailure");
michael@0:     }
michael@0:   });
michael@0: }
michael@0: 
michael@0: this.FormHistory = {
michael@0:   get enabled() Prefs.enabled,
michael@0: 
michael@0:   search : function formHistorySearch(aSelectTerms, aSearchData, aCallbacks) {
michael@0:     // if no terms selected, select everything
michael@0:     aSelectTerms = (aSelectTerms) ?  aSelectTerms : validFields;
michael@0:     validateSearchData(aSearchData, "Search");
michael@0: 
michael@0:     let stmt = makeSearchStatement(aSearchData, aSelectTerms);
michael@0: 
michael@0:     let handlers = {
michael@0:       handleResult : function(aResultSet) {
michael@0:         let formHistoryFields = dbSchema.tables.moz_formhistory;
michael@0:         for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
michael@0:           let result = {};
michael@0:           for each (let field in aSelectTerms) {
michael@0:             result[field] = row.getResultByName(field);
michael@0:           }
michael@0: 
michael@0:           if (aCallbacks && aCallbacks.handleResult) {
michael@0:             aCallbacks.handleResult(result);
michael@0:           }
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleError : function(aError) {
michael@0:         if (aCallbacks && aCallbacks.handleError) {
michael@0:           aCallbacks.handleError(aError);
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleCompletion : function searchCompletionHandler(aReason) {
michael@0:         if (aCallbacks && aCallbacks.handleCompletion) {
michael@0:           aCallbacks.handleCompletion(aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED ? 0 : 1);
michael@0:         }
michael@0:       }
michael@0:     };
michael@0: 
michael@0:     stmt.executeAsync(handlers);
michael@0:   },
michael@0: 
michael@0:   count : function formHistoryCount(aSearchData, aCallbacks) {
michael@0:     validateSearchData(aSearchData, "Count");
michael@0:     let stmt = makeCountStatement(aSearchData);
michael@0:     let handlers = {
michael@0:       handleResult : function countResultHandler(aResultSet) {
michael@0:         let row = aResultSet.getNextRow();
michael@0:         let count = row.getResultByName("numEntries");
michael@0:         if (aCallbacks && aCallbacks.handleResult) {
michael@0:           aCallbacks.handleResult(count);
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleError : function(aError) {
michael@0:         if (aCallbacks && aCallbacks.handleError) {
michael@0:           aCallbacks.handleError(aError);
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleCompletion : function searchCompletionHandler(aReason) {
michael@0:         if (aCallbacks && aCallbacks.handleCompletion) {
michael@0:           aCallbacks.handleCompletion(aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED ? 0 : 1);
michael@0:         }
michael@0:       }
michael@0:     };
michael@0: 
michael@0:     stmt.executeAsync(handlers);
michael@0:   },
michael@0: 
michael@0:   update : function formHistoryUpdate(aChanges, aCallbacks) {
michael@0:     if (!Prefs.enabled) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     // Used to keep track of how many searches have been started. When that number
michael@0:     // are finished, updateFormHistoryWrite can be called.
michael@0:     let numSearches = 0;
michael@0:     let completedSearches = 0;
michael@0:     let searchFailed = false;
michael@0: 
michael@0:     function validIdentifier(change) {
michael@0:       // The identifier is only valid if one of either the guid or the (fieldname/value) are set
michael@0:       return Boolean(change.guid) != Boolean(change.fieldname && change.value);
michael@0:     }
michael@0: 
michael@0:     if (!("length" in aChanges))
michael@0:       aChanges = [aChanges];
michael@0: 
michael@0:     for each (let change in aChanges) {
michael@0:       switch (change.op) {
michael@0:         case "remove":
michael@0:           validateSearchData(change, "Remove");
michael@0:           continue;
michael@0:         case "update":
michael@0:           if (validIdentifier(change)) {
michael@0:             validateOpData(change, "Update");
michael@0:             if (change.guid) {
michael@0:               continue;
michael@0:             }
michael@0:           } else {
michael@0:             throw Components.Exception(
michael@0:               "update op='update' does not correctly reference a entry.",
michael@0:               Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:           }
michael@0:           break;
michael@0:         case "bump":
michael@0:           if (validIdentifier(change)) {
michael@0:             validateOpData(change, "Bump");
michael@0:             if (change.guid) {
michael@0:               continue;
michael@0:             }
michael@0:           } else {
michael@0:             throw Components.Exception(
michael@0:               "update op='bump' does not correctly reference a entry.",
michael@0:               Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:           }
michael@0:           break;
michael@0:         case "add":
michael@0:           if (change.guid) {
michael@0:             throw Components.Exception(
michael@0:               "op='add' cannot contain field 'guid'. Either use op='update' " +
michael@0:                 "explicitly or make 'guid' undefined.",
michael@0:               Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:           } else if (change.fieldname && change.value) {
michael@0:             validateOpData(change, "Add");
michael@0:           }
michael@0:           break;
michael@0:         default:
michael@0:           throw Components.Exception(
michael@0:             "update does not recognize op='" + change.op + "'",
michael@0:             Cr.NS_ERROR_ILLEGAL_VALUE);
michael@0:       }
michael@0: 
michael@0:       numSearches++;
michael@0:       let changeToUpdate = change;
michael@0:       FormHistory.search(
michael@0:         [ "guid" ],
michael@0:         {
michael@0:           fieldname : change.fieldname,
michael@0:           value : change.value
michael@0:         }, {
michael@0:           foundResult : false,
michael@0:           handleResult : function(aResult) {
michael@0:             if (this.foundResult) {
michael@0:               log("Database contains multiple entries with the same fieldname/value pair.");
michael@0:               if (aCallbacks && aCallbacks.handleError) {
michael@0:                 aCallbacks.handleError({
michael@0:                   message :
michael@0:                     "Database contains multiple entries with the same fieldname/value pair.",
michael@0:                   result : 19 // Constraint violation
michael@0:                 });
michael@0:               }
michael@0: 
michael@0:               searchFailed = true;
michael@0:               return;
michael@0:             }
michael@0: 
michael@0:             this.foundResult = true;
michael@0:             changeToUpdate.guid = aResult["guid"];
michael@0:           },
michael@0: 
michael@0:           handleError : function(aError) {
michael@0:             if (aCallbacks && aCallbacks.handleError) {
michael@0:               aCallbacks.handleError(aError);
michael@0:             }
michael@0:           },
michael@0: 
michael@0:           handleCompletion : function(aReason) {
michael@0:             completedSearches++;
michael@0:             if (completedSearches == numSearches) {
michael@0:               if (!aReason && !searchFailed) {
michael@0:                 updateFormHistoryWrite(aChanges, aCallbacks);
michael@0:               }
michael@0:               else if (aCallbacks && aCallbacks.handleCompletion) {
michael@0:                 aCallbacks.handleCompletion(1);
michael@0:               }
michael@0:             }
michael@0:           }
michael@0:         });
michael@0:     }
michael@0: 
michael@0:     if (numSearches == 0) {
michael@0:       // We don't have to wait for any statements to return.
michael@0:       updateFormHistoryWrite(aChanges, aCallbacks);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   getAutoCompleteResults: function getAutoCompleteResults(searchString, params, aCallbacks) {
michael@0:     // only do substring matching when the search string contains more than one character
michael@0:     let searchTokens;
michael@0:     let where = ""
michael@0:     let boundaryCalc = "";
michael@0:     if (searchString.length > 1) {
michael@0:         searchTokens = searchString.split(/\s+/);
michael@0: 
michael@0:         // build up the word boundary and prefix match bonus calculation
michael@0:         boundaryCalc = "MAX(1, :prefixWeight * (value LIKE :valuePrefix ESCAPE '/') + (";
michael@0:         // for each word, calculate word boundary weights for the SELECT clause and
michael@0:         // add word to the WHERE clause of the query
michael@0:         let tokenCalc = [];
michael@0:         let searchTokenCount = Math.min(searchTokens.length, MAX_SEARCH_TOKENS);
michael@0:         for (let i = 0; i < searchTokenCount; i++) {
michael@0:             tokenCalc.push("(value LIKE :tokenBegin" + i + " ESCAPE '/') + " +
michael@0:                             "(value LIKE :tokenBoundary" + i + " ESCAPE '/')");
michael@0:             where += "AND (value LIKE :tokenContains" + i + " ESCAPE '/') ";
michael@0:         }
michael@0:         // add more weight if we have a traditional prefix match and
michael@0:         // multiply boundary bonuses by boundary weight
michael@0:         boundaryCalc += tokenCalc.join(" + ") + ") * :boundaryWeight)";
michael@0:     } else if (searchString.length == 1) {
michael@0:         where = "AND (value LIKE :valuePrefix ESCAPE '/') ";
michael@0:         boundaryCalc = "1";
michael@0:         delete params.prefixWeight;
michael@0:         delete params.boundaryWeight;
michael@0:     } else {
michael@0:         where = "";
michael@0:         boundaryCalc = "1";
michael@0:         delete params.prefixWeight;
michael@0:         delete params.boundaryWeight;
michael@0:     }
michael@0: 
michael@0:     params.now = Date.now() * 1000; // convert from ms to microseconds
michael@0: 
michael@0:     /* Three factors in the frecency calculation for an entry (in order of use in calculation):
michael@0:      * 1) average number of times used - items used more are ranked higher
michael@0:      * 2) how recently it was last used - items used recently are ranked higher
michael@0:      * 3) additional weight for aged entries surviving expiry - these entries are relevant
michael@0:      *    since they have been used multiple times over a large time span so rank them higher
michael@0:      * The score is then divided by the bucket size and we round the result so that entries
michael@0:      * with a very similar frecency are bucketed together with an alphabetical sort. This is
michael@0:      * to reduce the amount of moving around by entries while typing.
michael@0:      */
michael@0: 
michael@0:     let query = "/* do not warn (bug 496471): can't use an index */ " +
michael@0:                 "SELECT value, " +
michael@0:                 "ROUND( " +
michael@0:                     "timesUsed / MAX(1.0, (lastUsed - firstUsed) / :timeGroupingSize) * " +
michael@0:                     "MAX(1.0, :maxTimeGroupings - (:now - lastUsed) / :timeGroupingSize) * "+
michael@0:                     "MAX(1.0, :agedWeight * (firstUsed < :expiryDate)) / " +
michael@0:                     ":bucketSize "+
michael@0:                 ", 3) AS frecency, " +
michael@0:                 boundaryCalc + " AS boundaryBonuses " +
michael@0:                 "FROM moz_formhistory " +
michael@0:                 "WHERE fieldname=:fieldname " + where +
michael@0:                 "ORDER BY ROUND(frecency * boundaryBonuses) DESC, UPPER(value) ASC";
michael@0: 
michael@0:     let stmt = dbCreateAsyncStatement(query, params);
michael@0: 
michael@0:     // Chicken and egg problem: Need the statement to escape the params we
michael@0:     // pass to the function that gives us the statement. So, fix it up now.
michael@0:     if (searchString.length >= 1)
michael@0:       stmt.params.valuePrefix = stmt.escapeStringForLIKE(searchString, "/") + "%";
michael@0:     if (searchString.length > 1) {
michael@0:       let searchTokenCount = Math.min(searchTokens.length, MAX_SEARCH_TOKENS);
michael@0:       for (let i = 0; i < searchTokenCount; i++) {
michael@0:         let escapedToken = stmt.escapeStringForLIKE(searchTokens[i], "/");
michael@0:         stmt.params["tokenBegin" + i] = escapedToken + "%";
michael@0:         stmt.params["tokenBoundary" + i] =  "% " + escapedToken + "%";
michael@0:         stmt.params["tokenContains" + i] = "%" + escapedToken + "%";
michael@0:       }
michael@0:     } else {
michael@0:       // no additional params need to be substituted into the query when the
michael@0:       // length is zero or one
michael@0:     }
michael@0: 
michael@0:     let pending = stmt.executeAsync({
michael@0:       handleResult : function (aResultSet) {
michael@0:         for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
michael@0:           let value = row.getResultByName("value");
michael@0:           let frecency = row.getResultByName("frecency");
michael@0:           let entry = {
michael@0:             text :          value,
michael@0:             textLowerCase : value.toLowerCase(),
michael@0:             frecency :      frecency,
michael@0:             totalScore :    Math.round(frecency * row.getResultByName("boundaryBonuses"))
michael@0:           };
michael@0:           if (aCallbacks && aCallbacks.handleResult) {
michael@0:             aCallbacks.handleResult(entry);
michael@0:           }
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleError : function (aError) {
michael@0:         if (aCallbacks && aCallbacks.handleError) {
michael@0:           aCallbacks.handleError(aError);
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleCompletion : function (aReason) {
michael@0:         if (aCallbacks && aCallbacks.handleCompletion) {
michael@0:           aCallbacks.handleCompletion(aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED ? 0 : 1);
michael@0:         }
michael@0:       }
michael@0:     });
michael@0:     return pending;
michael@0:   },
michael@0: 
michael@0:   get schemaVersion() {
michael@0:     return dbConnection.schemaVersion;
michael@0:   },
michael@0: 
michael@0:   // This is used only so that the test can verify deleted table support.
michael@0:   get _supportsDeletedTable() {
michael@0:     return supportsDeletedTable;
michael@0:   },
michael@0:   set _supportsDeletedTable(val) {
michael@0:     supportsDeletedTable = val;
michael@0:   },
michael@0: 
michael@0:   // The remaining methods are called by FormHistoryStartup.js
michael@0:   updatePrefs: function updatePrefs() {
michael@0:     Prefs.initialized = false;
michael@0:   },
michael@0: 
michael@0:   expireOldEntries: function expireOldEntries() {
michael@0:     log("expireOldEntries");
michael@0: 
michael@0:     // Determine how many days of history we're supposed to keep.
michael@0:     // Calculate expireTime in microseconds
michael@0:     let expireTime = (Date.now() - Prefs.expireDays * DAY_IN_MS) * 1000;
michael@0: 
michael@0:     sendNotification("formhistory-beforeexpireoldentries", expireTime);
michael@0: 
michael@0:     FormHistory.count({}, {
michael@0:       handleResult: function(aBeginningCount) {
michael@0:         expireOldEntriesDeletion(expireTime, aBeginningCount);
michael@0:       },
michael@0:       handleError: function(aError) {
michael@0:         log("expireStartCountFailure");
michael@0:       }
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   shutdown: function shutdown() { dbClose(true); }
michael@0: };
michael@0: 
michael@0: // Prevent add-ons from redefining this API
michael@0: Object.freeze(FormHistory);