The Tor Browser: toolkit/components/places/nsPlacesExpiration.js@b8a032363ba2 (annotated)

toolkit/components/places/nsPlacesExpiration.js@b8a032363ba2 (annotated)

toolkit/components/places/nsPlacesExpiration.js

Thu, 22 Jan 2015 13:21:57 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 22 Jan 2015 13:21:57 +0100
branch: TOR_BUG_9701
changeset 15: b8a032363ba2
permissions: -rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
  * vim: sw=2 ts=2 sts=2 expandtab
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 /**
  * This component handles history and orphans expiration through asynchronous
  * Storage statements.
  * Expiration runs:
  * - At idle, but just once, we stop any other kind of expiration during idle
  *   to preserve batteries in portable devices.
  * - At shutdown, only if the database is dirty, we should still avoid to
  *   expire too heavily on shutdown.
  * - On ClearHistory we run a full expiration for privacy reasons.
  * - On a repeating timer we expire in small chunks.
  *
  * Expiration algorithm will adapt itself based on:
  * - Memory size of the device.
  * - Status of the database (clean or dirty).
  */
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cu = Components.utils;
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 ////////////////////////////////////////////////////////////////////////////////
 //// Constants
 // Last expiration step should run before the final sync.
 const TOPIC_SHUTDOWN = "places-will-close-connection";
 const TOPIC_PREF_CHANGED = "nsPref:changed";
 const TOPIC_DEBUG_START_EXPIRATION = "places-debug-start-expiration";
 const TOPIC_EXPIRATION_FINISHED = "places-expiration-finished";
 const TOPIC_IDLE_BEGIN = "idle";
 const TOPIC_IDLE_END = "active";
 const TOPIC_IDLE_DAILY = "idle-daily";
 // Branch for all expiration preferences.
 const PREF_BRANCH = "places.history.expiration.";
 // Max number of unique URIs to retain in history.
 // Notice this is a lazy limit.  This means we will start to expire if we will
 // go over it, but we won't ensure that we will stop exactly when we reach it,
 // instead we will stop after the next expiration step that will bring us
 // below it.
 // If this preference does not exist or has a negative value, we will calculate
 // a limit based on current hardware.
 const PREF_MAX_URIS = "max_pages";
 const PREF_MAX_URIS_NOTSET = -1; // Use our internally calculated limit.
 // We save the current unique URIs limit to this pref, to make it available to
 // other components without having to duplicate the full logic.
 const PREF_READONLY_CALCULATED_MAX_URIS = "transient_current_max_pages";
 // Seconds between each expiration step.
 const PREF_INTERVAL_SECONDS = "interval_seconds";
 const PREF_INTERVAL_SECONDS_NOTSET = 3 * 60;
 // We calculate an optimal database size, based on hardware specs.
 // This percentage of memory size is used to protect against calculating a too
 // large database size on systems with small memory.
 const DATABASE_TO_MEMORY_PERC = 4;
 // This percentage of disk size is used to protect against calculating a too
 // large database size on disks with tiny quota or available space.
 const DATABASE_TO_DISK_PERC = 2;
 // Maximum size of the optimal database.  High-end hardware has plenty of
 // memory and disk space, but performances don't grow linearly.
 const DATABASE_MAX_SIZE = 167772160; // 160MiB
 // If the physical memory size is bogus, fallback to this.
 const MEMSIZE_FALLBACK_BYTES = 268435456; // 256 MiB
 // If the disk available space is bogus, fallback to this.
 const DISKSIZE_FALLBACK_BYTES = 268435456; // 256 MiB
 // Max number of entries to expire at each expiration step.
 // This value is globally used for different kind of data we expire, can be
 // tweaked based on data type.  See below in getBoundStatement.
 const EXPIRE_LIMIT_PER_STEP = 6;
 // When we run a large expiration step, the above limit is multiplied by this.
 const EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER = 10;
 // When history is clean or dirty enough we will adapt the expiration algorithm
 // to be more lazy or more aggressive.
 // This is done acting on the interval between expiration steps and the number
 // of expirable items.
 // 1. Clean history:
 //   We expire at (default interval * EXPIRE_AGGRESSIVITY_MULTIPLIER) the
 //   default number of entries.
 // 2. Dirty history:
 //   We expire at the default interval, but a greater number of entries
 //   (default number of entries * EXPIRE_AGGRESSIVITY_MULTIPLIER).
 const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3;
 // This is the average size in bytes of an URI entry in the database.
 // Magic numbers are determined through analysis of the distribution of a ratio
 // between number of unique URIs and database size among our users.
 // Based on these values we evaluate how many unique URIs we can handle before
 // starting expiring some.
 const URIENTRY_AVG_SIZE = 1600;
 // Seconds of idle time before starting a larger expiration step.
 // Notice during idle we stop the expiration timer since we don't want to hurt
 // stand-by or mobile devices batteries.
 const IDLE_TIMEOUT_SECONDS = 5 * 60;
 // If a clear history ran just before we shutdown, we will skip most of the
 // expiration at shutdown.  This is maximum number of seconds from last
 // clearHistory to decide to skip expiration at shutdown.
 const SHUTDOWN_WITH_RECENT_CLEARHISTORY_TIMEOUT_SECONDS = 10;
 // If the pages delta from the last ANALYZE is over this threashold, the tables
 // should be analyzed again.
 const ANALYZE_PAGES_THRESHOLD = 100;
 // If the number of pages over history limit is greater than this threshold,
 // expiration will be more aggressive, to bring back history to a saner size.
 const OVERLIMIT_PAGES_THRESHOLD = 1000;
 const USECS_PER_DAY = 86400000000;
 const ANNOS_EXPIRE_POLICIES = [
   { bind: "expire_days",
     type: Ci.nsIAnnotationService.EXPIRE_DAYS,
     time: 7 * USECS_PER_DAY },
   { bind: "expire_weeks",
     type: Ci.nsIAnnotationService.EXPIRE_WEEKS,
     time: 30 * USECS_PER_DAY },
   { bind: "expire_months",
     type: Ci.nsIAnnotationService.EXPIRE_MONTHS,
     time: 180 * USECS_PER_DAY },
 ];
 // When we expire we can use these limits:
 // - SMALL for usual partial expirations, will expire a small chunk.
 // - LARGE for idle or shutdown expirations, will expire a large chunk.
 // - UNLIMITED for clearHistory, will expire everything.
 // - DEBUG will use a known limit, passed along with the debug notification.
 const LIMIT = {
   SMALL: 0,
   LARGE: 1,
   UNLIMITED: 2,
   DEBUG: 3,
 };
 // Represents the status of history database.
 const STATUS = {
   CLEAN: 0,
   DIRTY: 1,
   UNKNOWN: 2,
 };
 // Represents actions on which a query will run.
 const ACTION = {
   TIMED:           1 << 0, // happens every this._interval
   TIMED_OVERLIMIT: 1 << 1, // like TIMED but only when history is over limits
   TIMED_ANALYZE:   1 << 2, // happens when ANALYZE statistics should be updated
   CLEAR_HISTORY:   1 << 3, // happens when history is cleared
   SHUTDOWN_DIRTY:  1 << 4, // happens at shutdown for DIRTY state
   IDLE_DIRTY:      1 << 5, // happens on idle for DIRTY state
   IDLE_DAILY:      1 << 6, // happens once a day on idle
   DEBUG:           1 << 7, // happens on TOPIC_DEBUG_START_EXPIRATION
 };
 // The queries we use to expire.
 const EXPIRATION_QUERIES = {
   // Finds visits to be expired when history is over the unique pages limit,
   // otherwise will return nothing.
   // This explicitly excludes any visits added in the last 7 days, to protect
   // users with thousands of bookmarks from constantly losing history.
   QUERY_FIND_VISITS_TO_EXPIRE: {
     sql: "INSERT INTO expiration_notify "
        +   "(v_id, url, guid, visit_date, expected_results) "
        + "SELECT v.id, h.url, h.guid, v.visit_date, :limit_visits "
        + "FROM moz_historyvisits v "
        + "JOIN moz_places h ON h.id = v.place_id "
        + "WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris "
        + "AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000 "
        + "ORDER BY v.visit_date ASC "
        + "LIMIT :limit_visits",
     actions: ACTION.TIMED_OVERLIMIT | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Removes the previously found visits.
   QUERY_EXPIRE_VISITS: {
     sql: "DELETE FROM moz_historyvisits WHERE id IN ( "
        +   "SELECT v_id FROM expiration_notify WHERE v_id NOTNULL "
        + ")",
     actions: ACTION.TIMED_OVERLIMIT | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Finds orphan URIs in the database.
   // Notice we won't notify single removed URIs on removeAllPages, so we don't
   // run this query in such a case, but just delete URIs.
   // This could run in the middle of adding a visit or bookmark to a new page.
   // In such a case since it is async, could end up expiring the orphan page
   // before it actually gets the new visit or bookmark.
   // Thus, since new pages get frecency -1, we filter on that.
   QUERY_FIND_URIS_TO_EXPIRE: {
     sql: "INSERT INTO expiration_notify "
        +   "(p_id, url, guid, visit_date, expected_results) "
        + "SELECT h.id, h.url, h.guid, h.last_visit_date, :limit_uris "
        + "FROM moz_places h "
        + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id "
        + "LEFT JOIN moz_bookmarks b ON h.id = b.fk "
        + "WHERE h.last_visit_date IS NULL "
        +   "AND v.id IS NULL "
        +   "AND b.id IS NULL "
        +   "AND frecency <> -1 "
        + "LIMIT :limit_uris",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
              ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   // Expire found URIs from the database.
   QUERY_EXPIRE_URIS: {
     sql: "DELETE FROM moz_places WHERE id IN ( "
        +   "SELECT p_id FROM expiration_notify WHERE p_id NOTNULL "
        + ")",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
              ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   // Expire orphan URIs from the database.
   QUERY_SILENT_EXPIRE_ORPHAN_URIS: {
     sql: "DELETE FROM moz_places WHERE id IN ( "
        +   "SELECT h.id "
        +   "FROM moz_places h "
        +   "LEFT JOIN moz_historyvisits v ON h.id = v.place_id "
        +   "LEFT JOIN moz_bookmarks b ON h.id = b.fk "
        +   "WHERE h.last_visit_date IS NULL "
        +     "AND v.id IS NULL "
        +     "AND b.id IS NULL "
        +   "LIMIT :limit_uris "
        + ")",
     actions: ACTION.CLEAR_HISTORY
   },
   // Expire orphan icons from the database.
   QUERY_EXPIRE_FAVICONS: {
     sql: "DELETE FROM moz_favicons WHERE id IN ( "
        +   "SELECT f.id FROM moz_favicons f "
        +   "LEFT JOIN moz_places h ON f.id = h.favicon_id "
        +   "WHERE h.favicon_id IS NULL "
        +   "LIMIT :limit_favicons "
        + ")",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
              ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Expire orphan page annotations from the database.
   QUERY_EXPIRE_ANNOS: {
     sql: "DELETE FROM moz_annos WHERE id in ( "
        +   "SELECT a.id FROM moz_annos a "
        +   "LEFT JOIN moz_places h ON a.place_id = h.id "
        +   "WHERE h.id IS NULL "
        +   "LIMIT :limit_annos "
        + ")",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
              ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Expire page annotations based on expiration policy.
   QUERY_EXPIRE_ANNOS_WITH_POLICY: {
     sql: "DELETE FROM moz_annos "
        + "WHERE (expiration = :expire_days "
        +   "AND :expire_days_time > MAX(lastModified, dateAdded)) "
        +    "OR (expiration = :expire_weeks "
        +   "AND :expire_weeks_time > MAX(lastModified, dateAdded)) "
        +    "OR (expiration = :expire_months "
        +   "AND :expire_months_time > MAX(lastModified, dateAdded))",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
              ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Expire items annotations based on expiration policy.
   QUERY_EXPIRE_ITEMS_ANNOS_WITH_POLICY: {
     sql: "DELETE FROM moz_items_annos "
        + "WHERE (expiration = :expire_days "
        +   "AND :expire_days_time > MAX(lastModified, dateAdded)) "
        +    "OR (expiration = :expire_weeks "
        +   "AND :expire_weeks_time > MAX(lastModified, dateAdded)) "
        +    "OR (expiration = :expire_months "
        +   "AND :expire_months_time > MAX(lastModified, dateAdded))",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
              ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Expire page annotations based on expiration policy.
   QUERY_EXPIRE_ANNOS_WITH_HISTORY: {
     sql: "DELETE FROM moz_annos "
        + "WHERE expiration = :expire_with_history "
        +   "AND NOT EXISTS (SELECT id FROM moz_historyvisits "
        +                   "WHERE place_id = moz_annos.place_id LIMIT 1)",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
              ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Expire item annos without a corresponding item id.
   QUERY_EXPIRE_ITEMS_ANNOS: {
     sql: "DELETE FROM moz_items_annos WHERE id IN ( "
        +   "SELECT a.id FROM moz_items_annos a "
        +   "LEFT JOIN moz_bookmarks b ON a.item_id = b.id "
        +   "WHERE b.id IS NULL "
        +   "LIMIT :limit_annos "
        + ")",
     actions: ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   // Expire all annotation names without a corresponding annotation.
   QUERY_EXPIRE_ANNO_ATTRIBUTES: {
     sql: "DELETE FROM moz_anno_attributes WHERE id IN ( "
        +   "SELECT n.id FROM moz_anno_attributes n "
        +   "LEFT JOIN moz_annos a ON n.id = a.anno_attribute_id "
        +   "LEFT JOIN moz_items_annos t ON n.id = t.anno_attribute_id "
        +   "WHERE a.anno_attribute_id IS NULL "
        +     "AND t.anno_attribute_id IS NULL "
        +   "LIMIT :limit_annos"
        + ")",
     actions: ACTION.CLEAR_HISTORY | ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY |
              ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   // Expire orphan inputhistory.
   QUERY_EXPIRE_INPUTHISTORY: {
     sql: "DELETE FROM moz_inputhistory WHERE place_id IN ( "
        +   "SELECT i.place_id FROM moz_inputhistory i "
        +   "LEFT JOIN moz_places h ON h.id = i.place_id "
        +   "WHERE h.id IS NULL "
        +   "LIMIT :limit_inputhistory "
        + ")",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.CLEAR_HISTORY |
              ACTION.SHUTDOWN_DIRTY | ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY |
              ACTION.DEBUG
   },
   // Expire all session annotations.  Should only be called at shutdown.
   QUERY_EXPIRE_ANNOS_SESSION: {
     sql: "DELETE FROM moz_annos WHERE expiration = :expire_session",
     actions: ACTION.CLEAR_HISTORY | ACTION.DEBUG
   },
   // Expire all session item annotations.  Should only be called at shutdown.
   QUERY_EXPIRE_ITEMS_ANNOS_SESSION: {
     sql: "DELETE FROM moz_items_annos WHERE expiration = :expire_session",
     actions: ACTION.CLEAR_HISTORY | ACTION.DEBUG
   },
   // Select entries for notifications.
   // If p_id is set whole_entry = 1, then we have expired the full page.
   // Either p_id or v_id are always set.
   QUERY_SELECT_NOTIFICATIONS: {
     sql: "SELECT url, guid, MAX(visit_date) AS visit_date, "
        +        "MAX(IFNULL(MIN(p_id, 1), MIN(v_id, 0))) AS whole_entry, "
        +        "expected_results "
        + "FROM expiration_notify "
        + "GROUP BY url",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
              ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   // Empty the notifications table.
   QUERY_DELETE_NOTIFICATIONS: {
     sql: "DELETE FROM expiration_notify",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN_DIRTY |
              ACTION.IDLE_DIRTY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   // The following queries are used to adjust the sqlite_stat1 table to help the
   // query planner create better queries.  These should always be run LAST, and
   // are therefore at the end of the object.
   // Since also nsNavHistory.cpp executes ANALYZE, the analyzed tables
   // must be the same in both components.  So ensure they are in sync.
   QUERY_ANALYZE_MOZ_PLACES: {
     sql: "ANALYZE moz_places",
     actions: ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE |
              ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   QUERY_ANALYZE_MOZ_BOOKMARKS: {
     sql: "ANALYZE moz_bookmarks",
     actions: ACTION.TIMED_ANALYZE | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   QUERY_ANALYZE_MOZ_HISTORYVISITS: {
     sql: "ANALYZE moz_historyvisits",
     actions: ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE |
              ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
   QUERY_ANALYZE_MOZ_INPUTHISTORY: {
     sql: "ANALYZE moz_inputhistory",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.TIMED_ANALYZE |
              ACTION.CLEAR_HISTORY | ACTION.IDLE_DAILY | ACTION.DEBUG
   },
 };
 ////////////////////////////////////////////////////////////////////////////////
 //// nsPlacesExpiration definition
 function nsPlacesExpiration()
 {
   //////////////////////////////////////////////////////////////////////////////
   //// Smart Getters
   XPCOMUtils.defineLazyGetter(this, "_db", function () {
     let db = Cc["@mozilla.org/browser/nav-history-service;1"].
              getService(Ci.nsPIPlacesDatabase).
              DBConnection;
     // Create the temporary notifications table.
     let stmt = db.createAsyncStatement(
       "CREATE TEMP TABLE expiration_notify ( "
     + "  id INTEGER PRIMARY KEY "
     + ", v_id INTEGER "
     + ", p_id INTEGER "
     + ", url TEXT NOT NULL "
     + ", guid TEXT NOT NULL "
     + ", visit_date INTEGER "
     + ", expected_results INTEGER NOT NULL "
     + ") ");
     stmt.executeAsync();
     stmt.finalize();
     return db;
   });
   XPCOMUtils.defineLazyServiceGetter(this, "_hsn",
                                      "@mozilla.org/browser/nav-history-service;1",
                                      "nsPIPlacesHistoryListenersNotifier");
   XPCOMUtils.defineLazyServiceGetter(this, "_sys",
                                      "@mozilla.org/system-info;1",
                                      "nsIPropertyBag2");
   XPCOMUtils.defineLazyServiceGetter(this, "_idle",
                                      "@mozilla.org/widget/idleservice;1",
                                      "nsIIdleService");
   this._prefBranch = Cc["@mozilla.org/preferences-service;1"].
                      getService(Ci.nsIPrefService).
                      getBranch(PREF_BRANCH);
   this._loadPrefs();
   // Observe our preferences branch for changes.
   this._prefBranch.addObserver("", this, false);
   // Register topic observers.
   Services.obs.addObserver(this, TOPIC_SHUTDOWN, false);
   Services.obs.addObserver(this, TOPIC_DEBUG_START_EXPIRATION, false);
   Services.obs.addObserver(this, TOPIC_IDLE_DAILY, false);
   // Create our expiration timer.
   this._newTimer();
 }
 nsPlacesExpiration.prototype = {
   //////////////////////////////////////////////////////////////////////////////
   //// nsIObserver
   observe: function PEX_observe(aSubject, aTopic, aData)
   {
     if (aTopic == TOPIC_SHUTDOWN) {
       this._shuttingDown = true;
       Services.obs.removeObserver(this, TOPIC_SHUTDOWN);
       Services.obs.removeObserver(this, TOPIC_DEBUG_START_EXPIRATION);
       Services.obs.removeObserver(this, TOPIC_IDLE_DAILY);
       this._prefBranch.removeObserver("", this);
       this.expireOnIdle = false;
       if (this._timer) {
         this._timer.cancel();
         this._timer = null;
       }
       // If we didn't ran a clearHistory recently and database is dirty, we
       // want to expire some entries, to speed up the expiration process.
       let hasRecentClearHistory =
         Date.now() - this._lastClearHistoryTime <
           SHUTDOWN_WITH_RECENT_CLEARHISTORY_TIMEOUT_SECONDS * 1000;
       if (!hasRecentClearHistory && this.status == STATUS.DIRTY) {
         this._expireWithActionAndLimit(ACTION.SHUTDOWN_DIRTY, LIMIT.LARGE);
       }
       this._finalizeInternalStatements();
     }
     else if (aTopic == TOPIC_PREF_CHANGED) {
       this._loadPrefs();
       if (aData == PREF_INTERVAL_SECONDS) {
         // Renew the timer with the new interval value.
         this._newTimer();
       }
     }
     else if (aTopic == TOPIC_DEBUG_START_EXPIRATION) {
       // The passed-in limit is the maximum number of visits to expire when
       // history is over capacity.  Mind to correctly handle the NaN value.
       let limit = parseInt(aData);
       if (limit == -1) {
         // Everything should be expired without any limit.  If history is over
         // capacity then all existing visits will be expired.
         // Should only be used in tests, since may cause dataloss.
         this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.UNLIMITED);
       }
       else if (limit > 0) {
         // The number of expired visits is limited by this amount.  It may be
         // used for testing purposes, like checking that limited queries work.
         this._debugLimit = limit;
         this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.DEBUG);
       }
       else {
         // Any other value is intended as a 0 limit, that means no visits
         // will be expired.  Even if this doesn't touch visits, it will remove
         // any orphan pages, icons, annotations and similar from the database,
         // so it may be used for cleanup purposes.
         this._debugLimit = -1;
         this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.DEBUG);
       }
     }
     else if (aTopic == TOPIC_IDLE_BEGIN) {
       // Stop the expiration timer.  We don't want to keep up expiring on idle
       // to preserve batteries on mobile devices and avoid killing stand-by.
       if (this._timer) {
         this._timer.cancel();
         this._timer = null;
       }
       if (this.expireOnIdle)
         this._expireWithActionAndLimit(ACTION.IDLE_DIRTY, LIMIT.LARGE);
     }
     else if (aTopic == TOPIC_IDLE_END) {
       // Restart the expiration timer.
       if (!this._timer)
         this._newTimer();
     }
     else if (aTopic == TOPIC_IDLE_DAILY) {
       this._expireWithActionAndLimit(ACTION.IDLE_DAILY, LIMIT.LARGE);
     }
   },
   //////////////////////////////////////////////////////////////////////////////
   //// nsINavHistoryObserver
   _inBatchMode: false,
   onBeginUpdateBatch: function PEX_onBeginUpdateBatch()
   {
     this._inBatchMode = true;
     // We do not want to expire while we are doing batch work.
     if (this._timer) {
       this._timer.cancel();
       this._timer = null;
     }
   },
   onEndUpdateBatch: function PEX_onEndUpdateBatch()
   {
     this._inBatchMode = false;
     // Restore timer.
     if (!this._timer)
       this._newTimer();
   },
   _lastClearHistoryTime: 0,
   onClearHistory: function PEX_onClearHistory() {
     this._lastClearHistoryTime = Date.now();
     // Expire orphans.  History status is clean after a clear history.
     this.status = STATUS.CLEAN;
     this._expireWithActionAndLimit(ACTION.CLEAR_HISTORY, LIMIT.UNLIMITED);
   },
   onVisit: function() {},
   onTitleChanged: function() {},
   onDeleteURI: function() {},
   onPageChanged: function() {},
   onDeleteVisits: function() {},
   //////////////////////////////////////////////////////////////////////////////
   //// nsITimerCallback
   notify: function PEX_timerCallback()
   {
     // Check if we are over history capacity, if so visits must be expired.
     this._getPagesStats((function onPagesCount(aPagesCount, aStatsCount) {
       let overLimitPages = aPagesCount - this._urisLimit;
       this._overLimit = overLimitPages > 0;
       let action = this._overLimit ? ACTION.TIMED_OVERLIMIT : ACTION.TIMED;
       // If the number of pages changed significantly from the last ANALYZE
       // update SQLite statistics.
       if (Math.abs(aPagesCount - aStatsCount) >= ANALYZE_PAGES_THRESHOLD) {
         action = action | ACTION.TIMED_ANALYZE;
       }
       // Adapt expiration aggressivity to the number of pages over the limit.
       let limit = overLimitPages > OVERLIMIT_PAGES_THRESHOLD ? LIMIT.LARGE
                                                              : LIMIT.SMALL;
       this._expireWithActionAndLimit(action, limit);
     }).bind(this));
   },
   //////////////////////////////////////////////////////////////////////////////
   //// mozIStorageStatementCallback
   handleResult: function PEX_handleResult(aResultSet)
   {
     // We don't want to notify after shutdown.
     if (this._shuttingDown)
       return;
     let row;
     while ((row = aResultSet.getNextRow())) {
       if (!("_expectedResultsCount" in this))
         this._expectedResultsCount = row.getResultByName("expected_results");
       if (this._expectedResultsCount > 0)
         this._expectedResultsCount--;
       let uri = Services.io.newURI(row.getResultByName("url"), null, null);
       let guid = row.getResultByName("guid");
       let visitDate = row.getResultByName("visit_date");
       let wholeEntry = row.getResultByName("whole_entry");
       // Dispatch expiration notifications to history.
       this._hsn.notifyOnPageExpired(uri, visitDate, wholeEntry, guid,
                                     Ci.nsINavHistoryObserver.REASON_EXPIRED, 0);
     }
   },
   handleError: function PEX_handleError(aError)
   {
     Cu.reportError("Async statement execution returned with '" +
                    aError.result + "', '" + aError.message + "'");
   },
   // Number of expiration steps needed to reach a CLEAN status.
   _telemetrySteps: 1,
   handleCompletion: function PEX_handleCompletion(aReason)
   {
     if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
       if ("_expectedResultsCount" in this) {
         // Adapt the aggressivity of steps based on the status of history.
         // A dirty history will return all the entries we are expecting bringing
         // our countdown to zero, while a clean one will not.
         let oldStatus = this.status;
         this.status = this._expectedResultsCount == 0 ? STATUS.DIRTY
                                                       : STATUS.CLEAN;
         // Collect or send telemetry data.
         if (this.status == STATUS.DIRTY) {
           this._telemetrySteps++;
         }
         else {
           // Avoid reporting the common cases where the database is clean, or
           // a single step is needed.
           if (oldStatus == STATUS.DIRTY) {
             try {
               Services.telemetry
                       .getHistogramById("PLACES_EXPIRATION_STEPS_TO_CLEAN2")
                       .add(this._telemetrySteps);
             } catch (ex) {
               Components.utils.reportError("Unable to report telemetry.");
             }
           }
           this._telemetrySteps = 1;
         }
         delete this._expectedResultsCount;
       }
       // Dispatch a notification that expiration has finished.
       Services.obs.notifyObservers(null, TOPIC_EXPIRATION_FINISHED, null);
     }
   },
   //////////////////////////////////////////////////////////////////////////////
   //// nsPlacesExpiration
   _urisLimit: PREF_MAX_URIS_NOTSET,
   _interval: PREF_INTERVAL_SECONDS_NOTSET,
   _shuttingDown: false,
   _status: STATUS.UNKNOWN,
   set status(aNewStatus) {
     if (aNewStatus != this._status) {
       // If status changes we should restart the timer.
       this._status = aNewStatus;
       this._newTimer();
       // If needed add/remove the cleanup step on idle.  We want to expire on
       // idle only if history is dirty, to preserve mobile devices batteries.
       this.expireOnIdle = aNewStatus == STATUS.DIRTY;
     }
     return aNewStatus;
   },
   get status() this._status,
   _isIdleObserver: false,
   _expireOnIdle: false,
   set expireOnIdle(aExpireOnIdle) {
     // Observe idle regardless aExpireOnIdle, since we always want to stop
     // timed expiration on idle, to preserve mobile battery life.
     if (!this._isIdleObserver && !this._shuttingDown) {
       this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS);
       this._isIdleObserver = true;
     }
     else if (this._isIdleObserver && this._shuttingDown) {
       this._idle.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS);
       this._isIdleObserver = false;
     }
     // If running a debug expiration we need full control of what happens
     // but idle cleanup could activate in the middle, since tinderboxes are
     // permanently idle.  That would cause unexpected oranges, so disable it.
     if (this._debugLimit !== undefined)
       this._expireOnIdle = false;
     else
       this._expireOnIdle = aExpireOnIdle;
     return this._expireOnIdle;
   },
   get expireOnIdle() this._expireOnIdle,
   _loadPrefs: function PEX__loadPrefs() {
     // Get the user's limit, if it was set.
     try {
       // We want to silently fail since getIntPref throws if it does not exist,
       // and use a default to fallback to.
       this._urisLimit = this._prefBranch.getIntPref(PREF_MAX_URIS);
     }
     catch(e) {}
     if (this._urisLimit < 0) {
       // The preference did not exist or has a negative value.
       // Calculate the number of unique places that may fit an optimal database
       // size on this hardware.  If there are more than these unique pages,
       // some will be expired.
       let memSizeBytes = MEMSIZE_FALLBACK_BYTES;
       try {
         // Limit the size on systems with small memory.
          memSizeBytes = this._sys.getProperty("memsize");
       } catch (ex) {}
       if (memSizeBytes <= 0) {
         memsize = MEMSIZE_FALLBACK_BYTES;
       }
       let diskAvailableBytes = DISKSIZE_FALLBACK_BYTES;
       try {
         // Protect against a full disk or tiny quota.
         let dbFile = this._db.databaseFile;
         dbFile.QueryInterface(Ci.nsILocalFile);
         diskAvailableBytes = dbFile.diskSpaceAvailable;
       } catch (ex) {}
       if (diskAvailableBytes <= 0) {
         diskAvailableBytes = DISKSIZE_FALLBACK_BYTES;
       }
       let optimalDatabaseSize = Math.min(
         memSizeBytes * DATABASE_TO_MEMORY_PERC / 100,
         diskAvailableBytes * DATABASE_TO_DISK_PERC / 100,
         DATABASE_MAX_SIZE
       );
       this._urisLimit = Math.ceil(optimalDatabaseSize / URIENTRY_AVG_SIZE);
     }
     // Expose the calculated limit to other components.
     this._prefBranch.setIntPref(PREF_READONLY_CALCULATED_MAX_URIS,
                                 this._urisLimit);
     // Get the expiration interval value.
     try {
       // We want to silently fail since getIntPref throws if it does not exist,
       // and use a default to fallback to.
       this._interval = this._prefBranch.getIntPref(PREF_INTERVAL_SECONDS);
     }
     catch (e) {}
     if (this._interval <= 0)
       this._interval = PREF_INTERVAL_SECONDS_NOTSET;
   },
   /**
    * Evaluates the real number of pages in the database and the value currently
    * used by the SQLite query planner.
    *
    * @param aCallback
    *        invoked on success, function (aPagesCount, aStatsCount).
    */
   _getPagesStats: function PEX__getPagesStats(aCallback) {
     if (!this._cachedStatements["LIMIT_COUNT"]) {
       this._cachedStatements["LIMIT_COUNT"] = this._db.createAsyncStatement(
         "SELECT (SELECT COUNT(*) FROM moz_places), "
       +        "(SELECT SUBSTR(stat,1,LENGTH(stat)-2) FROM sqlite_stat1 "
       +         "WHERE idx = 'moz_places_url_uniqueindex')"
       );
     }
     this._cachedStatements["LIMIT_COUNT"].executeAsync({
       _pagesCount: 0,
       _statsCount: 0,
       handleResult: function(aResults) {
         let row = aResults.getNextRow();
         this._pagesCount = row.getResultByIndex(0);
         this._statsCount = row.getResultByIndex(1);
       },
       handleCompletion: function (aReason) {
         if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
           aCallback(this._pagesCount, this._statsCount);
         }
       },
       handleError: function(aError) {
         Cu.reportError("Async statement execution returned with '" +
                        aError.result + "', '" + aError.message + "'");
       }
     });
   },
   /**
    * Execute async statements to expire with the specified queries.
    *
    * @param aAction
    *        The ACTION we are expiring for.  See the ACTION const for values.
    * @param aLimit
    *        Whether to use small, large or no limits when expiring.  See the
    *        LIMIT const for values.
    */
   _expireWithActionAndLimit:
   function PEX__expireWithActionAndLimit(aAction, aLimit)
   {
     // Skip expiration during batch mode.
     if (this._inBatchMode)
       return;
     // Don't try to further expire after shutdown.
     if (this._shuttingDown && aAction != ACTION.SHUTDOWN_DIRTY) {
       return;
     }
     let boundStatements = [];
     for (let queryType in EXPIRATION_QUERIES) {
       if (EXPIRATION_QUERIES[queryType].actions & aAction)
         boundStatements.push(this._getBoundStatement(queryType, aLimit, aAction));
     }
     // Execute statements asynchronously in a transaction.
     this._db.executeAsync(boundStatements, boundStatements.length, this);
   },
   /**
    * Finalizes all of our mozIStorageStatements so we can properly close the
    * database.
    */
   _finalizeInternalStatements: function PEX__finalizeInternalStatements()
   {
     for each (let stmt in this._cachedStatements) {
       stmt.finalize();
     }
   },
   /**
    * Generate the statement used for expiration.
    *
    * @param aQueryType
    *        Type of the query to build statement for.
    * @param aLimit
    *        Whether to use small, large or no limits when expiring.  See the
    *        LIMIT const for values.
    * @param aAction
    *        Current action causing the expiration.  See the ACTION const.
    */
   _cachedStatements: {},
   _getBoundStatement: function PEX__getBoundStatement(aQueryType, aLimit, aAction)
   {
     // Statements creation can be expensive, so we want to cache them.
     let stmt = this._cachedStatements[aQueryType];
     if (stmt === undefined) {
       stmt = this._cachedStatements[aQueryType] =
         this._db.createAsyncStatement(EXPIRATION_QUERIES[aQueryType].sql);
     }
     let baseLimit;
     switch (aLimit) {
       case LIMIT.UNLIMITED:
         baseLimit = -1;
         break;
       case LIMIT.SMALL:
         baseLimit = EXPIRE_LIMIT_PER_STEP;
         break;
       case LIMIT.LARGE:
         baseLimit = EXPIRE_LIMIT_PER_STEP * EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER;
         break;
       case LIMIT.DEBUG:
         baseLimit = this._debugLimit;
         break;
     }
     if (this.status == STATUS.DIRTY && aAction != ACTION.DEBUG &&
         baseLimit > 0) {
       baseLimit *= EXPIRE_AGGRESSIVITY_MULTIPLIER;
     }
     // Bind the appropriate parameters.
     let params = stmt.params;
     switch (aQueryType) {
       case "QUERY_FIND_VISITS_TO_EXPIRE":
         params.max_uris = this._urisLimit;
         // Avoid expiring all visits in case of an unlimited debug expiration,
         // just remove orphans instead.
         params.limit_visits =
           aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit;
         break;
       case "QUERY_FIND_URIS_TO_EXPIRE":
         params.limit_uris = baseLimit;
         break;
       case "QUERY_SILENT_EXPIRE_ORPHAN_URIS":
         params.limit_uris = baseLimit;
         break;
       case "QUERY_EXPIRE_FAVICONS":
         params.limit_favicons = baseLimit;
         break;
       case "QUERY_EXPIRE_ANNOS":
         // Each page may have multiple annos.
         params.limit_annos = baseLimit * EXPIRE_AGGRESSIVITY_MULTIPLIER;
         break;
       case "QUERY_EXPIRE_ANNOS_WITH_POLICY":
       case "QUERY_EXPIRE_ITEMS_ANNOS_WITH_POLICY":
         let microNow = Date.now() * 1000;
         ANNOS_EXPIRE_POLICIES.forEach(function(policy) {
           params[policy.bind] = policy.type;
           params[policy.bind + "_time"] = microNow - policy.time;
         });
         break;
       case "QUERY_EXPIRE_ANNOS_WITH_HISTORY":
         params.expire_with_history = Ci.nsIAnnotationService.EXPIRE_WITH_HISTORY;
         break;
       case "QUERY_EXPIRE_ITEMS_ANNOS":
         params.limit_annos = baseLimit;
         break;
       case "QUERY_EXPIRE_ANNO_ATTRIBUTES":
         params.limit_annos = baseLimit;
         break;
       case "QUERY_EXPIRE_INPUTHISTORY":
         params.limit_inputhistory = baseLimit;
         break;
       case "QUERY_EXPIRE_ANNOS_SESSION":
       case "QUERY_EXPIRE_ITEMS_ANNOS_SESSION":
         params.expire_session = Ci.nsIAnnotationService.EXPIRE_SESSION;
         break;
     }
     return stmt;
   },
   /**
    * Creates a new timer based on this._interval.
    *
    * @return a REPEATING_SLACK nsITimer that runs every this._interval.
    */
   _newTimer: function PEX__newTimer()
   {
     if (this._timer)
       this._timer.cancel();
     if (this._shuttingDown)
       return;
     let interval = this.status != STATUS.DIRTY ?
       this._interval * EXPIRE_AGGRESSIVITY_MULTIPLIER : this._interval;
     let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
     timer.initWithCallback(this, interval * 1000,
                            Ci.nsITimer.TYPE_REPEATING_SLACK);
     return this._timer = timer;
   },
   //////////////////////////////////////////////////////////////////////////////
   //// nsISupports
   classID: Components.ID("705a423f-2f69-42f3-b9fe-1517e0dee56f"),
   _xpcom_factory: XPCOMUtils.generateSingletonFactory(nsPlacesExpiration),
   QueryInterface: XPCOMUtils.generateQI([
     Ci.nsIObserver
   , Ci.nsINavHistoryObserver
   , Ci.nsITimerCallback
   , Ci.mozIStorageStatementCallback
   ])
 };
 ////////////////////////////////////////////////////////////////////////////////
 //// Module Registration
 let components = [nsPlacesExpiration];
 this.NSGetFactory = XPCOMUtils.generateNSGetFactory(components);