diff -r 000000000000 -r 6474c204b198 toolkit/components/places/nsPlacesExpiration.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolkit/components/places/nsPlacesExpiration.js Wed Dec 31 06:09:35 2014 +0100 @@ -0,0 +1,996 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 sts=2 expandtab + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This component handles history and orphans expiration through asynchronous + * Storage statements. + * Expiration runs: + * - At idle, but just once, we stop any other kind of expiration during idle + * to preserve batteries in portable devices. + * - At shutdown, only if the database is dirty, we should still avoid to + * expire too heavily on shutdown. + * - On ClearHistory we run a full expiration for privacy reasons. + * - On a repeating timer we expire in small chunks. + * + * Expiration algorithm will adapt itself based on: + * - Memory size of the device. + * - Status of the database (clean or dirty). + */ + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); + +//////////////////////////////////////////////////////////////////////////////// +//// Constants + +// Last expiration step should run before the final sync. +const TOPIC_SHUTDOWN = "places-will-close-connection"; +const TOPIC_PREF_CHANGED = "nsPref:changed"; +const TOPIC_DEBUG_START_EXPIRATION = "places-debug-start-expiration"; +const TOPIC_EXPIRATION_FINISHED = "places-expiration-finished"; +const TOPIC_IDLE_BEGIN = "idle"; +const TOPIC_IDLE_END = "active"; +const TOPIC_IDLE_DAILY = "idle-daily"; + +// Branch for all expiration preferences. +const PREF_BRANCH = "places.history.expiration."; + +// Max number of unique URIs to retain in history. +// Notice this is a lazy limit. This means we will start to expire if we will +// go over it, but we won't ensure that we will stop exactly when we reach it, +// instead we will stop after the next expiration step that will bring us +// below it. +// If this preference does not exist or has a negative value, we will calculate +// a limit based on current hardware. +const PREF_MAX_URIS = "max_pages"; +const PREF_MAX_URIS_NOTSET = -1; // Use our internally calculated limit. + +// We save the current unique URIs limit to this pref, to make it available to +// other components without having to duplicate the full logic. +const PREF_READONLY_CALCULATED_MAX_URIS = "transient_current_max_pages"; + +// Seconds between each expiration step. +const PREF_INTERVAL_SECONDS = "interval_seconds"; +const PREF_INTERVAL_SECONDS_NOTSET = 3 * 60; + +// We calculate an optimal database size, based on hardware specs. +// This percentage of memory size is used to protect against calculating a too +// large database size on systems with small memory. +const DATABASE_TO_MEMORY_PERC = 4; +// This percentage of disk size is used to protect against calculating a too +// large database size on disks with tiny quota or available space. +const DATABASE_TO_DISK_PERC = 2; +// Maximum size of the optimal database. High-end hardware has plenty of +// memory and disk space, but performances don't grow linearly. +const DATABASE_MAX_SIZE = 167772160; // 160MiB +// If the physical memory size is bogus, fallback to this. +const MEMSIZE_FALLBACK_BYTES = 268435456; // 256 MiB +// If the disk available space is bogus, fallback to this. +const DISKSIZE_FALLBACK_BYTES = 268435456; // 256 MiB + +// Max number of entries to expire at each expiration step. +// This value is globally used for different kind of data we expire, can be +// tweaked based on data type. See below in getBoundStatement. +const EXPIRE_LIMIT_PER_STEP = 6; +// When we run a large expiration step, the above limit is multiplied by this. +const EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER = 10; + +// When history is clean or dirty enough we will adapt the expiration algorithm +// to be more lazy or more aggressive. +// This is done acting on the interval between expiration steps and the number +// of expirable items. +// 1. Clean history: +// We expire at (default interval * EXPIRE_AGGRESSIVITY_MULTIPLIER) the +// default number of entries. +// 2. Dirty history: +// We expire at the default interval, but a greater number of entries +// (default number of entries * EXPIRE_AGGRESSIVITY_MULTIPLIER). +const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3; + +// This is the average size in bytes of an URI entry in the database. +// Magic numbers are determined through analysis of the distribution of a ratio +// between number of unique URIs and database size among our users. +// Based on these values we evaluate how many unique URIs we can handle before +// starting expiring some. +const URIENTRY_AVG_SIZE = 1600; + +// Seconds of idle time before starting a larger expiration step. +// Notice during idle we stop the expiration timer since we don't want to hurt +// stand-by or mobile devices batteries. +const IDLE_TIMEOUT_SECONDS = 5 * 60; + +// If a clear history ran just before we shutdown, we will skip most of the +// expiration at shutdown. This is maximum number of seconds from last +// clearHistory to decide to skip expiration at shutdown. +const SHUTDOWN_WITH_RECENT_CLEARHISTORY_TIMEOUT_SECONDS = 10; + +// If the pages delta from the last ANALYZE is over this threashold, the tables +// should be analyzed again. +const ANALYZE_PAGES_THRESHOLD = 100; + +// If the number of pages over history limit is greater than this threshold, +// expiration will be more aggressive, to bring back history to a saner size. +const OVERLIMIT_PAGES_THRESHOLD = 1000; + +const USECS_PER_DAY = 86400000000; +const ANNOS_EXPIRE_POLICIES = [ + { bind: "expire_days", + type: Ci.nsIAnnotationService.EXPIRE_DAYS, + time: 7 * USECS_PER_DAY }, + { bind: "expire_weeks", + type: Ci.nsIAnnotationService.EXPIRE_WEEKS, + time: 30 * USECS_PER_DAY }, + { bind: "expire_months", + type: Ci.nsIAnnotationService.EXPIRE_MONTHS, + time: 180 * USECS_PER_DAY }, +]; + +// When we expire we can use these limits: +// - SMALL for usual partial expirations, will expire a small chunk. +// - LARGE for idle or shutdown expirations, will expire a large chunk. +// - UNLIMITED for clearHistory, will expire everything. +// - DEBUG will use a known limit, passed along with the debug notification. +const LIMIT = { + SMALL: 0, + LARGE: 1, + UNLIMITED: 2, + DEBUG: 3, +}; + +// Represents the status of history database. +const STATUS = { + CLEAN: 0, + DIRTY: 1, + UNKNOWN: 2, +}; + +// Represents actions on which a query will run. +const ACTION = { + TIMED: 1 << 0, // happens every this._interval + TIMED_OVERLIMIT: 1 << 1, // like TIMED but only when history is over limits + TIMED_ANALYZE: 1 << 2, // happens when ANALYZE statistics should be updated + CLEAR_HISTORY: 1 << 3, // happens when history is cleared + SHUTDOWN_DIRTY: 1 << 4, // happens at shutdown for DIRTY state + IDLE_DIRTY: 1 << 5, // happens on idle for DIRTY state + IDLE_DAILY: 1 << 6, // happens once a day on idle + DEBUG: 1 << 7, // happens on TOPIC_DEBUG_START_EXPIRATION +}; + +// The queries we use to expire. +const EXPIRATION_QUERIES = { + + // Finds visits to be expired when history is over the unique pages limit, + // otherwise will return nothing. + // This explicitly excludes any visits added in the last 7 days, to protect + // users with thousands of bookmarks from constantly losing history. + QUERY_FIND_VISITS_TO_EXPIRE: { + sql: "INSERT INTO expiration_notify " + + "(v_id, url, guid, visit_date, expected_results) " + + "SELECT v.id, h.url, h.guid, v.visit_date, :limit_visits " + + "FROM moz_historyvisits v " + + "JOIN moz_places h ON h.id = v.place_id " + + "WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris " + + "AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000 " + + "ORDER BY v.visit_date ASC " + + "LIMIT :limit_visits", + actions: ACTION.TIMED_OVERLIMIT | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Removes the previously found visits. + QUERY_EXPIRE_VISITS: { + sql: "DELETE FROM moz_historyvisits WHERE id IN ( " + + "SELECT v_id FROM expiration_notify WHERE v_id NOTNULL " + + ")", + actions: ACTION.TIMED_OVERLIMIT | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Finds orphan URIs in the database. + // Notice we won't notify single removed URIs on removeAllPages, so we don't + // run this query in such a case, but just delete URIs. + // This could run in the middle of adding a visit or bookmark to a new page. + // In such a case since it is async, could end up expiring the orphan page + // before it actually gets the new visit or bookmark. + // Thus, since new pages get frecency -1, we filter on that. + QUERY_FIND_URIS_TO_EXPIRE: { + sql: "INSERT INTO expiration_notify " + + "(p_id, url, guid, visit_date, expected_results) " + + "SELECT h.id, h.url, h.guid, h.last_visit_date, :limit_uris " + + "FROM moz_places h " + + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " + + "LEFT JOIN moz_bookmarks b ON h.id = b.fk " + + "WHERE h.last_visit_date IS NULL " + + "AND v.id IS NULL " + + "AND b.id IS NULL " + + "AND frecency <> -1 " + + "LIMIT :limit_uris", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + + // Expire found URIs from the database. + QUERY_EXPIRE_URIS: { + sql: "DELETE FROM moz_places WHERE id IN ( " + + "SELECT p_id FROM expiration_notify WHERE p_id NOTNULL " + + ")", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + + // Expire orphan URIs from the database. + QUERY_SILENT_EXPIRE_ORPHAN_URIS: { + sql: "DELETE FROM moz_places WHERE id IN ( " + + "SELECT h.id " + + "FROM moz_places h " + + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id " + + "LEFT JOIN moz_bookmarks b ON h.id = b.fk " + + "WHERE h.last_visit_date IS NULL " + + "AND v.id IS NULL " + + "AND b.id IS NULL " + + "LIMIT :limit_uris " + + ")", + actions: ACTION.CLEAR_HISTORY + }, + + // Expire orphan icons from the database. + QUERY_EXPIRE_FAVICONS: { + sql: "DELETE FROM moz_favicons WHERE id IN ( " + + "SELECT f.id FROM moz_favicons f " + + "LEFT JOIN moz_places h ON f.id = h.favicon_id " + + "WHERE h.favicon_id IS NULL " + + "LIMIT :limit_favicons " + + ")", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | + ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Expire orphan page annotations from the database. + QUERY_EXPIRE_ANNOS: { + sql: "DELETE FROM moz_annos WHERE id in ( " + + "SELECT a.id FROM moz_annos a " + + "LEFT JOIN moz_places h ON a.place_id = h.id " + + "WHERE h.id IS NULL " + + "LIMIT :limit_annos " + + ")", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | + ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Expire page annotations based on expiration policy. + QUERY_EXPIRE_ANNOS_WITH_POLICY: { + sql: "DELETE FROM moz_annos " + + "WHERE (expiration = :expire_days " + + "AND :expire_days_time > MAX(lastModified, dateAdded)) " + + "OR (expiration = :expire_weeks " + + "AND :expire_weeks_time > MAX(lastModified, dateAdded)) " + + "OR (expiration = :expire_months " + + "AND :expire_months_time > MAX(lastModified, dateAdded))", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | + ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Expire items annotations based on expiration policy. + QUERY_EXPIRE_ITEMS_ANNOS_WITH_POLICY: { + sql: "DELETE FROM moz_items_annos " + + "WHERE (expiration = :expire_days " + + "AND :expire_days_time > MAX(lastModified, dateAdded)) " + + "OR (expiration = :expire_weeks " + + "AND :expire_weeks_time > MAX(lastModified, dateAdded)) " + + "OR (expiration = :expire_months " + + "AND :expire_months_time > MAX(lastModified, dateAdded))", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | + ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Expire page annotations based on expiration policy. + QUERY_EXPIRE_ANNOS_WITH_HISTORY: { + sql: "DELETE FROM moz_annos " + + "WHERE expiration = :expire_with_history " + + "AND NOT EXISTS (SELECT id FROM moz_historyvisits " + + "WHERE place_id = moz_annos.place_id LIMIT 1)", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | + ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Expire item annos without a corresponding item id. + QUERY_EXPIRE_ITEMS_ANNOS: { + sql: "DELETE FROM moz_items_annos WHERE id IN ( " + + "SELECT a.id FROM moz_items_annos a " + + "LEFT JOIN moz_bookmarks b ON a.item_id = b.id " + + "WHERE b.id IS NULL " + + "LIMIT :limit_annos " + + ")", + actions: ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + + // Expire all annotation names without a corresponding annotation. + QUERY_EXPIRE_ANNO_ATTRIBUTES: { + sql: "DELETE FROM moz_anno_attributes WHERE id IN ( " + + "SELECT n.id FROM moz_anno_attributes n " + + "LEFT JOIN moz_annos a ON n.id = a.anno_attribute_id " + + "LEFT JOIN moz_items_annos t ON n.id = t.anno_attribute_id " + + "WHERE a.anno_attribute_id IS NULL " + + "AND t.anno_attribute_id IS NULL " + + "LIMIT :limit_annos" + + ")", + actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | + ACTION.IDLE_DAILY | ACTION.DEBUG + }, + + // Expire orphan inputhistory. + QUERY_EXPIRE_INPUTHISTORY: { + sql: "DELETE FROM moz_inputhistory WHERE place_id IN ( " + + "SELECT i.place_id FROM moz_inputhistory i " + + "LEFT JOIN moz_places h ON h.id = i.place_id " + + "WHERE h.id IS NULL " + + "LIMIT :limit_inputhistory " + + ")", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY | + ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | + ACTION.DEBUG + }, + + // Expire all session annotations. Should only be called at shutdown. + QUERY_EXPIRE_ANNOS_SESSION: { + sql: "DELETE FROM moz_annos WHERE expiration = :expire_session", + actions: ACTION.CLEAR_HISTORY | ACTION.DEBUG + }, + + // Expire all session item annotations. Should only be called at shutdown. + QUERY_EXPIRE_ITEMS_ANNOS_SESSION: { + sql: "DELETE FROM moz_items_annos WHERE expiration = :expire_session", + actions: ACTION.CLEAR_HISTORY | ACTION.DEBUG + }, + + // Select entries for notifications. + // If p_id is set whole_entry = 1, then we have expired the full page. + // Either p_id or v_id are always set. + QUERY_SELECT_NOTIFICATIONS: { + sql: "SELECT url, guid, MAX(visit_date) AS visit_date, " + + "MAX(IFNULL(MIN(p_id, 1), MIN(v_id, 0))) AS whole_entry, " + + "expected_results " + + "FROM expiration_notify " + + "GROUP BY url", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + + // Empty the notifications table. + QUERY_DELETE_NOTIFICATIONS: { + sql: "DELETE FROM expiration_notify", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY | + ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + + // The following queries are used to adjust the sqlite_stat1 table to help the + // query planner create better queries. These should always be run LAST, and + // are therefore at the end of the object. + // Since also nsNavHistory.cpp executes ANALYZE, the analyzed tables + // must be the same in both components. So ensure they are in sync. + + QUERY_ANALYZE_MOZ_PLACES: { + sql: "ANALYZE moz_places", + actions: ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE | + ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + QUERY_ANALYZE_MOZ_BOOKMARKS: { + sql: "ANALYZE moz_bookmarks", + actions: ACTION.TIMED_ANALYZE | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + QUERY_ANALYZE_MOZ_HISTORYVISITS: { + sql: "ANALYZE moz_historyvisits", + actions: ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE | + ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, + QUERY_ANALYZE_MOZ_INPUTHISTORY: { + sql: "ANALYZE moz_inputhistory", + actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE | + ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG + }, +}; + +//////////////////////////////////////////////////////////////////////////////// +//// nsPlacesExpiration definition + +function nsPlacesExpiration() +{ + ////////////////////////////////////////////////////////////////////////////// + //// Smart Getters + + XPCOMUtils.defineLazyGetter(this, "_db", function () { + let db = Cc["@mozilla.org/browser/nav-history-service;1"]. + getService(Ci.nsPIPlacesDatabase). + DBConnection; + + // Create the temporary notifications table. + let stmt = db.createAsyncStatement( + "CREATE TEMP TABLE expiration_notify ( " + + " id INTEGER PRIMARY KEY " + + ", v_id INTEGER " + + ", p_id INTEGER " + + ", url TEXT NOT NULL " + + ", guid TEXT NOT NULL " + + ", visit_date INTEGER " + + ", expected_results INTEGER NOT NULL " + + ") "); + stmt.executeAsync(); + stmt.finalize(); + + return db; + }); + + XPCOMUtils.defineLazyServiceGetter(this, "_hsn", + "@mozilla.org/browser/nav-history-service;1", + "nsPIPlacesHistoryListenersNotifier"); + XPCOMUtils.defineLazyServiceGetter(this, "_sys", + "@mozilla.org/system-info;1", + "nsIPropertyBag2"); + XPCOMUtils.defineLazyServiceGetter(this, "_idle", + "@mozilla.org/widget/idleservice;1", + "nsIIdleService"); + + this._prefBranch = Cc["@mozilla.org/preferences-service;1"]. + getService(Ci.nsIPrefService). + getBranch(PREF_BRANCH); + this._loadPrefs(); + + // Observe our preferences branch for changes. + this._prefBranch.addObserver("", this, false); + + // Register topic observers. + Services.obs.addObserver(this, TOPIC_SHUTDOWN, false); + Services.obs.addObserver(this, TOPIC_DEBUG_START_EXPIRATION, false); + Services.obs.addObserver(this, TOPIC_IDLE_DAILY, false); + + // Create our expiration timer. + this._newTimer(); +} + +nsPlacesExpiration.prototype = { + + ////////////////////////////////////////////////////////////////////////////// + //// nsIObserver + + observe: function PEX_observe(aSubject, aTopic, aData) + { + if (aTopic == TOPIC_SHUTDOWN) { + this._shuttingDown = true; + Services.obs.removeObserver(this, TOPIC_SHUTDOWN); + Services.obs.removeObserver(this, TOPIC_DEBUG_START_EXPIRATION); + Services.obs.removeObserver(this, TOPIC_IDLE_DAILY); + + this._prefBranch.removeObserver("", this); + + this.expireOnIdle = false; + + if (this._timer) { + this._timer.cancel(); + this._timer = null; + } + + // If we didn't ran a clearHistory recently and database is dirty, we + // want to expire some entries, to speed up the expiration process. + let hasRecentClearHistory = + Date.now() - this._lastClearHistoryTime < + SHUTDOWN_WITH_RECENT_CLEARHISTORY_TIMEOUT_SECONDS * 1000; + if (!hasRecentClearHistory && this.status == STATUS.DIRTY) { + this._expireWithActionAndLimit(ACTION.SHUTDOWN_DIRTY, LIMIT.LARGE); + } + + this._finalizeInternalStatements(); + } + else if (aTopic == TOPIC_PREF_CHANGED) { + this._loadPrefs(); + + if (aData == PREF_INTERVAL_SECONDS) { + // Renew the timer with the new interval value. + this._newTimer(); + } + } + else if (aTopic == TOPIC_DEBUG_START_EXPIRATION) { + // The passed-in limit is the maximum number of visits to expire when + // history is over capacity. Mind to correctly handle the NaN value. + let limit = parseInt(aData); + if (limit == -1) { + // Everything should be expired without any limit. If history is over + // capacity then all existing visits will be expired. + // Should only be used in tests, since may cause dataloss. + this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.UNLIMITED); + } + else if (limit > 0) { + // The number of expired visits is limited by this amount. It may be + // used for testing purposes, like checking that limited queries work. + this._debugLimit = limit; + this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.DEBUG); + } + else { + // Any other value is intended as a 0 limit, that means no visits + // will be expired. Even if this doesn't touch visits, it will remove + // any orphan pages, icons, annotations and similar from the database, + // so it may be used for cleanup purposes. + this._debugLimit = -1; + this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.DEBUG); + } + } + else if (aTopic == TOPIC_IDLE_BEGIN) { + // Stop the expiration timer. We don't want to keep up expiring on idle + // to preserve batteries on mobile devices and avoid killing stand-by. + if (this._timer) { + this._timer.cancel(); + this._timer = null; + } + if (this.expireOnIdle) + this._expireWithActionAndLimit(ACTION.IDLE_DIRTY, LIMIT.LARGE); + } + else if (aTopic == TOPIC_IDLE_END) { + // Restart the expiration timer. + if (!this._timer) + this._newTimer(); + } + else if (aTopic == TOPIC_IDLE_DAILY) { + this._expireWithActionAndLimit(ACTION.IDLE_DAILY, LIMIT.LARGE); + } + }, + + ////////////////////////////////////////////////////////////////////////////// + //// nsINavHistoryObserver + + _inBatchMode: false, + onBeginUpdateBatch: function PEX_onBeginUpdateBatch() + { + this._inBatchMode = true; + + // We do not want to expire while we are doing batch work. + if (this._timer) { + this._timer.cancel(); + this._timer = null; + } + }, + + onEndUpdateBatch: function PEX_onEndUpdateBatch() + { + this._inBatchMode = false; + + // Restore timer. + if (!this._timer) + this._newTimer(); + }, + + _lastClearHistoryTime: 0, + onClearHistory: function PEX_onClearHistory() { + this._lastClearHistoryTime = Date.now(); + // Expire orphans. History status is clean after a clear history. + this.status = STATUS.CLEAN; + this._expireWithActionAndLimit(ACTION.CLEAR_HISTORY, LIMIT.UNLIMITED); + }, + + onVisit: function() {}, + onTitleChanged: function() {}, + onDeleteURI: function() {}, + onPageChanged: function() {}, + onDeleteVisits: function() {}, + + ////////////////////////////////////////////////////////////////////////////// + //// nsITimerCallback + + notify: function PEX_timerCallback() + { + // Check if we are over history capacity, if so visits must be expired. + this._getPagesStats((function onPagesCount(aPagesCount, aStatsCount) { + let overLimitPages = aPagesCount - this._urisLimit; + this._overLimit = overLimitPages > 0; + + let action = this._overLimit ? ACTION.TIMED_OVERLIMIT : ACTION.TIMED; + // If the number of pages changed significantly from the last ANALYZE + // update SQLite statistics. + if (Math.abs(aPagesCount - aStatsCount) >= ANALYZE_PAGES_THRESHOLD) { + action = action | ACTION.TIMED_ANALYZE; + } + + // Adapt expiration aggressivity to the number of pages over the limit. + let limit = overLimitPages > OVERLIMIT_PAGES_THRESHOLD ? LIMIT.LARGE + : LIMIT.SMALL; + + this._expireWithActionAndLimit(action, limit); + }).bind(this)); + }, + + ////////////////////////////////////////////////////////////////////////////// + //// mozIStorageStatementCallback + + handleResult: function PEX_handleResult(aResultSet) + { + // We don't want to notify after shutdown. + if (this._shuttingDown) + return; + + let row; + while ((row = aResultSet.getNextRow())) { + if (!("_expectedResultsCount" in this)) + this._expectedResultsCount = row.getResultByName("expected_results"); + if (this._expectedResultsCount > 0) + this._expectedResultsCount--; + + let uri = Services.io.newURI(row.getResultByName("url"), null, null); + let guid = row.getResultByName("guid"); + let visitDate = row.getResultByName("visit_date"); + let wholeEntry = row.getResultByName("whole_entry"); + // Dispatch expiration notifications to history. + this._hsn.notifyOnPageExpired(uri, visitDate, wholeEntry, guid, + Ci.nsINavHistoryObserver.REASON_EXPIRED, 0); + } + }, + + handleError: function PEX_handleError(aError) + { + Cu.reportError("Async statement execution returned with '" + + aError.result + "', '" + aError.message + "'"); + }, + + // Number of expiration steps needed to reach a CLEAN status. + _telemetrySteps: 1, + handleCompletion: function PEX_handleCompletion(aReason) + { + if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) { + if ("_expectedResultsCount" in this) { + // Adapt the aggressivity of steps based on the status of history. + // A dirty history will return all the entries we are expecting bringing + // our countdown to zero, while a clean one will not. + let oldStatus = this.status; + this.status = this._expectedResultsCount == 0 ? STATUS.DIRTY + : STATUS.CLEAN; + + // Collect or send telemetry data. + if (this.status == STATUS.DIRTY) { + this._telemetrySteps++; + } + else { + // Avoid reporting the common cases where the database is clean, or + // a single step is needed. + if (oldStatus == STATUS.DIRTY) { + try { + Services.telemetry + .getHistogramById("PLACES_EXPIRATION_STEPS_TO_CLEAN2") + .add(this._telemetrySteps); + } catch (ex) { + Components.utils.reportError("Unable to report telemetry."); + } + } + this._telemetrySteps = 1; + } + + delete this._expectedResultsCount; + } + + // Dispatch a notification that expiration has finished. + Services.obs.notifyObservers(null, TOPIC_EXPIRATION_FINISHED, null); + } + }, + + ////////////////////////////////////////////////////////////////////////////// + //// nsPlacesExpiration + + _urisLimit: PREF_MAX_URIS_NOTSET, + _interval: PREF_INTERVAL_SECONDS_NOTSET, + _shuttingDown: false, + + _status: STATUS.UNKNOWN, + set status(aNewStatus) { + if (aNewStatus != this._status) { + // If status changes we should restart the timer. + this._status = aNewStatus; + this._newTimer(); + // If needed add/remove the cleanup step on idle. We want to expire on + // idle only if history is dirty, to preserve mobile devices batteries. + this.expireOnIdle = aNewStatus == STATUS.DIRTY; + } + return aNewStatus; + }, + get status() this._status, + + _isIdleObserver: false, + _expireOnIdle: false, + set expireOnIdle(aExpireOnIdle) { + // Observe idle regardless aExpireOnIdle, since we always want to stop + // timed expiration on idle, to preserve mobile battery life. + if (!this._isIdleObserver && !this._shuttingDown) { + this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS); + this._isIdleObserver = true; + } + else if (this._isIdleObserver && this._shuttingDown) { + this._idle.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS); + this._isIdleObserver = false; + } + + // If running a debug expiration we need full control of what happens + // but idle cleanup could activate in the middle, since tinderboxes are + // permanently idle. That would cause unexpected oranges, so disable it. + if (this._debugLimit !== undefined) + this._expireOnIdle = false; + else + this._expireOnIdle = aExpireOnIdle; + return this._expireOnIdle; + }, + get expireOnIdle() this._expireOnIdle, + + _loadPrefs: function PEX__loadPrefs() { + // Get the user's limit, if it was set. + try { + // We want to silently fail since getIntPref throws if it does not exist, + // and use a default to fallback to. + this._urisLimit = this._prefBranch.getIntPref(PREF_MAX_URIS); + } + catch(e) {} + + if (this._urisLimit < 0) { + // The preference did not exist or has a negative value. + // Calculate the number of unique places that may fit an optimal database + // size on this hardware. If there are more than these unique pages, + // some will be expired. + + let memSizeBytes = MEMSIZE_FALLBACK_BYTES; + try { + // Limit the size on systems with small memory. + memSizeBytes = this._sys.getProperty("memsize"); + } catch (ex) {} + if (memSizeBytes <= 0) { + memsize = MEMSIZE_FALLBACK_BYTES; + } + + let diskAvailableBytes = DISKSIZE_FALLBACK_BYTES; + try { + // Protect against a full disk or tiny quota. + let dbFile = this._db.databaseFile; + dbFile.QueryInterface(Ci.nsILocalFile); + diskAvailableBytes = dbFile.diskSpaceAvailable; + } catch (ex) {} + if (diskAvailableBytes <= 0) { + diskAvailableBytes = DISKSIZE_FALLBACK_BYTES; + } + + let optimalDatabaseSize = Math.min( + memSizeBytes * DATABASE_TO_MEMORY_PERC / 100, + diskAvailableBytes * DATABASE_TO_DISK_PERC / 100, + DATABASE_MAX_SIZE + ); + + this._urisLimit = Math.ceil(optimalDatabaseSize / URIENTRY_AVG_SIZE); + } + + // Expose the calculated limit to other components. + this._prefBranch.setIntPref(PREF_READONLY_CALCULATED_MAX_URIS, + this._urisLimit); + + // Get the expiration interval value. + try { + // We want to silently fail since getIntPref throws if it does not exist, + // and use a default to fallback to. + this._interval = this._prefBranch.getIntPref(PREF_INTERVAL_SECONDS); + } + catch (e) {} + if (this._interval <= 0) + this._interval = PREF_INTERVAL_SECONDS_NOTSET; + }, + + /** + * Evaluates the real number of pages in the database and the value currently + * used by the SQLite query planner. + * + * @param aCallback + * invoked on success, function (aPagesCount, aStatsCount). + */ + _getPagesStats: function PEX__getPagesStats(aCallback) { + if (!this._cachedStatements["LIMIT_COUNT"]) { + this._cachedStatements["LIMIT_COUNT"] = this._db.createAsyncStatement( + "SELECT (SELECT COUNT(*) FROM moz_places), " + + "(SELECT SUBSTR(stat,1,LENGTH(stat)-2) FROM sqlite_stat1 " + + "WHERE idx = 'moz_places_url_uniqueindex')" + ); + } + this._cachedStatements["LIMIT_COUNT"].executeAsync({ + _pagesCount: 0, + _statsCount: 0, + handleResult: function(aResults) { + let row = aResults.getNextRow(); + this._pagesCount = row.getResultByIndex(0); + this._statsCount = row.getResultByIndex(1); + }, + handleCompletion: function (aReason) { + if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) { + aCallback(this._pagesCount, this._statsCount); + } + }, + handleError: function(aError) { + Cu.reportError("Async statement execution returned with '" + + aError.result + "', '" + aError.message + "'"); + } + }); + }, + + /** + * Execute async statements to expire with the specified queries. + * + * @param aAction + * The ACTION we are expiring for. See the ACTION const for values. + * @param aLimit + * Whether to use small, large or no limits when expiring. See the + * LIMIT const for values. + */ + _expireWithActionAndLimit: + function PEX__expireWithActionAndLimit(aAction, aLimit) + { + // Skip expiration during batch mode. + if (this._inBatchMode) + return; + // Don't try to further expire after shutdown. + if (this._shuttingDown && aAction != ACTION.SHUTDOWN_DIRTY) { + return; + } + + let boundStatements = []; + for (let queryType in EXPIRATION_QUERIES) { + if (EXPIRATION_QUERIES[queryType].actions & aAction) + boundStatements.push(this._getBoundStatement(queryType, aLimit, aAction)); + } + + // Execute statements asynchronously in a transaction. + this._db.executeAsync(boundStatements, boundStatements.length, this); + }, + + /** + * Finalizes all of our mozIStorageStatements so we can properly close the + * database. + */ + _finalizeInternalStatements: function PEX__finalizeInternalStatements() + { + for each (let stmt in this._cachedStatements) { + stmt.finalize(); + } + }, + + /** + * Generate the statement used for expiration. + * + * @param aQueryType + * Type of the query to build statement for. + * @param aLimit + * Whether to use small, large or no limits when expiring. See the + * LIMIT const for values. + * @param aAction + * Current action causing the expiration. See the ACTION const. + */ + _cachedStatements: {}, + _getBoundStatement: function PEX__getBoundStatement(aQueryType, aLimit, aAction) + { + // Statements creation can be expensive, so we want to cache them. + let stmt = this._cachedStatements[aQueryType]; + if (stmt === undefined) { + stmt = this._cachedStatements[aQueryType] = + this._db.createAsyncStatement(EXPIRATION_QUERIES[aQueryType].sql); + } + + let baseLimit; + switch (aLimit) { + case LIMIT.UNLIMITED: + baseLimit = -1; + break; + case LIMIT.SMALL: + baseLimit = EXPIRE_LIMIT_PER_STEP; + break; + case LIMIT.LARGE: + baseLimit = EXPIRE_LIMIT_PER_STEP * EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER; + break; + case LIMIT.DEBUG: + baseLimit = this._debugLimit; + break; + } + if (this.status == STATUS.DIRTY && aAction != ACTION.DEBUG && + baseLimit > 0) { + baseLimit *= EXPIRE_AGGRESSIVITY_MULTIPLIER; + } + + // Bind the appropriate parameters. + let params = stmt.params; + switch (aQueryType) { + case "QUERY_FIND_VISITS_TO_EXPIRE": + params.max_uris = this._urisLimit; + // Avoid expiring all visits in case of an unlimited debug expiration, + // just remove orphans instead. + params.limit_visits = + aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit; + break; + case "QUERY_FIND_URIS_TO_EXPIRE": + params.limit_uris = baseLimit; + break; + case "QUERY_SILENT_EXPIRE_ORPHAN_URIS": + params.limit_uris = baseLimit; + break; + case "QUERY_EXPIRE_FAVICONS": + params.limit_favicons = baseLimit; + break; + case "QUERY_EXPIRE_ANNOS": + // Each page may have multiple annos. + params.limit_annos = baseLimit * EXPIRE_AGGRESSIVITY_MULTIPLIER; + break; + case "QUERY_EXPIRE_ANNOS_WITH_POLICY": + case "QUERY_EXPIRE_ITEMS_ANNOS_WITH_POLICY": + let microNow = Date.now() * 1000; + ANNOS_EXPIRE_POLICIES.forEach(function(policy) { + params[policy.bind] = policy.type; + params[policy.bind + "_time"] = microNow - policy.time; + }); + break; + case "QUERY_EXPIRE_ANNOS_WITH_HISTORY": + params.expire_with_history = Ci.nsIAnnotationService.EXPIRE_WITH_HISTORY; + break; + case "QUERY_EXPIRE_ITEMS_ANNOS": + params.limit_annos = baseLimit; + break; + case "QUERY_EXPIRE_ANNO_ATTRIBUTES": + params.limit_annos = baseLimit; + break; + case "QUERY_EXPIRE_INPUTHISTORY": + params.limit_inputhistory = baseLimit; + break; + case "QUERY_EXPIRE_ANNOS_SESSION": + case "QUERY_EXPIRE_ITEMS_ANNOS_SESSION": + params.expire_session = Ci.nsIAnnotationService.EXPIRE_SESSION; + break; + } + + return stmt; + }, + + /** + * Creates a new timer based on this._interval. + * + * @return a REPEATING_SLACK nsITimer that runs every this._interval. + */ + _newTimer: function PEX__newTimer() + { + if (this._timer) + this._timer.cancel(); + if (this._shuttingDown) + return; + let interval = this.status != STATUS.DIRTY ? + this._interval * EXPIRE_AGGRESSIVITY_MULTIPLIER : this._interval; + + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(this, interval * 1000, + Ci.nsITimer.TYPE_REPEATING_SLACK); + return this._timer = timer; + }, + + ////////////////////////////////////////////////////////////////////////////// + //// nsISupports + + classID: Components.ID("705a423f-2f69-42f3-b9fe-1517e0dee56f"), + + _xpcom_factory: XPCOMUtils.generateSingletonFactory(nsPlacesExpiration), + + QueryInterface: XPCOMUtils.generateQI([ + Ci.nsIObserver + , Ci.nsINavHistoryObserver + , Ci.nsITimerCallback + , Ci.mozIStorageStatementCallback + ]) +}; + +//////////////////////////////////////////////////////////////////////////////// +//// Module Registration + +let components = [nsPlacesExpiration]; +this.NSGetFactory = XPCOMUtils.generateNSGetFactory(components);