michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: "use strict";
michael@0: 
michael@0: this.EXPORTED_SYMBOLS = ["NewTabUtils"];
michael@0: 
michael@0: const Ci = Components.interfaces;
michael@0: const Cc = Components.classes;
michael@0: const Cu = Components.utils;
michael@0: 
michael@0: Cu.import("resource://gre/modules/Services.jsm");
michael@0: Cu.import("resource://gre/modules/XPCOMUtils.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
michael@0:   "resource://gre/modules/PlacesUtils.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "PageThumbs",
michael@0:   "resource://gre/modules/PageThumbs.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "BinarySearch",
michael@0:   "resource://gre/modules/BinarySearch.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyGetter(this, "Timer", () => {
michael@0:   return Cu.import("resource://gre/modules/Timer.jsm", {});
michael@0: });
michael@0: 
michael@0: XPCOMUtils.defineLazyGetter(this, "gPrincipal", function () {
michael@0:   let uri = Services.io.newURI("about:newtab", null, null);
michael@0:   return Services.scriptSecurityManager.getNoAppCodebasePrincipal(uri);
michael@0: });
michael@0: 
michael@0: XPCOMUtils.defineLazyGetter(this, "gCryptoHash", function () {
michael@0:   return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
michael@0: });
michael@0: 
michael@0: XPCOMUtils.defineLazyGetter(this, "gUnicodeConverter", function () {
michael@0:   let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
michael@0:                     .createInstance(Ci.nsIScriptableUnicodeConverter);
michael@0:   converter.charset = 'utf8';
michael@0:   return converter;
michael@0: });
michael@0: 
michael@0: // The preference that tells whether this feature is enabled.
michael@0: const PREF_NEWTAB_ENABLED = "browser.newtabpage.enabled";
michael@0: 
michael@0: // The preference that tells the number of rows of the newtab grid.
michael@0: const PREF_NEWTAB_ROWS = "browser.newtabpage.rows";
michael@0: 
michael@0: // The preference that tells the number of columns of the newtab grid.
michael@0: const PREF_NEWTAB_COLUMNS = "browser.newtabpage.columns";
michael@0: 
michael@0: // The maximum number of results PlacesProvider retrieves from history.
michael@0: const HISTORY_RESULTS_LIMIT = 100;
michael@0: 
michael@0: // The maximum number of links Links.getLinks will return.
michael@0: const LINKS_GET_LINKS_LIMIT = 100;
michael@0: 
michael@0: // The gather telemetry topic.
michael@0: const TOPIC_GATHER_TELEMETRY = "gather-telemetry";
michael@0: 
michael@0: // The amount of time we wait while coalescing updates for hidden pages.
michael@0: const SCHEDULE_UPDATE_TIMEOUT_MS = 1000;
michael@0: 
michael@0: /**
michael@0:  * Calculate the MD5 hash for a string.
michael@0:  * @param aValue
michael@0:  *        The string to convert.
michael@0:  * @return The base64 representation of the MD5 hash.
michael@0:  */
michael@0: function toHash(aValue) {
michael@0:   let value = gUnicodeConverter.convertToByteArray(aValue);
michael@0:   gCryptoHash.init(gCryptoHash.MD5);
michael@0:   gCryptoHash.update(value, value.length);
michael@0:   return gCryptoHash.finish(true);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Singleton that provides storage functionality.
michael@0:  */
michael@0: XPCOMUtils.defineLazyGetter(this, "Storage", function() {
michael@0:   return new LinksStorage();
michael@0: });
michael@0: 
michael@0: function LinksStorage() {
michael@0:   // Handle migration of data across versions.
michael@0:   try {
michael@0:     if (this._storedVersion < this._version) {
michael@0:       // This is either an upgrade, or version information is missing.
michael@0:       if (this._storedVersion < 1) {
michael@0:         // Version 1 moved data from DOM Storage to prefs.  Since migrating from
michael@0:         // version 0 is no more supported, we just reportError a dataloss later.
michael@0:         throw new Error("Unsupported newTab storage version");
michael@0:       }
michael@0:       // Add further migration steps here.
michael@0:     }
michael@0:     else {
michael@0:       // This is a downgrade.  Since we cannot predict future, upgrades should
michael@0:       // be backwards compatible.  We will set the version to the old value
michael@0:       // regardless, so, on next upgrade, the migration steps will run again.
michael@0:       // For this reason, they should also be able to run multiple times, even
michael@0:       // on top of an already up-to-date storage.
michael@0:     }
michael@0:   } catch (ex) {
michael@0:     // Something went wrong in the update process, we can't recover from here,
michael@0:     // so just clear the storage and start from scratch (dataloss!).
michael@0:     Components.utils.reportError(
michael@0:       "Unable to migrate the newTab storage to the current version. "+
michael@0:       "Restarting from scratch.\n" + ex);
michael@0:     this.clear();
michael@0:   }
michael@0: 
michael@0:   // Set the version to the current one.
michael@0:   this._storedVersion = this._version;
michael@0: }
michael@0: 
michael@0: LinksStorage.prototype = {
michael@0:   get _version() 1,
michael@0: 
michael@0:   get _prefs() Object.freeze({
michael@0:     pinnedLinks: "browser.newtabpage.pinned",
michael@0:     blockedLinks: "browser.newtabpage.blocked",
michael@0:   }),
michael@0: 
michael@0:   get _storedVersion() {
michael@0:     if (this.__storedVersion === undefined) {
michael@0:       try {
michael@0:         this.__storedVersion =
michael@0:           Services.prefs.getIntPref("browser.newtabpage.storageVersion");
michael@0:       } catch (ex) {
michael@0:         // The storage version is unknown, so either:
michael@0:         // - it's a new profile
michael@0:         // - it's a profile where versioning information got lost
michael@0:         // In this case we still run through all of the valid migrations,
michael@0:         // starting from 1, as if it was a downgrade.  As previously stated the
michael@0:         // migrations should already support running on an updated store.
michael@0:         this.__storedVersion = 1;
michael@0:       }
michael@0:     }
michael@0:     return this.__storedVersion;
michael@0:   },
michael@0:   set _storedVersion(aValue) {
michael@0:     Services.prefs.setIntPref("browser.newtabpage.storageVersion", aValue);
michael@0:     this.__storedVersion = aValue;
michael@0:     return aValue;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Gets the value for a given key from the storage.
michael@0:    * @param aKey The storage key (a string).
michael@0:    * @param aDefault A default value if the key doesn't exist.
michael@0:    * @return The value for the given key.
michael@0:    */
michael@0:   get: function Storage_get(aKey, aDefault) {
michael@0:     let value;
michael@0:     try {
michael@0:       let prefValue = Services.prefs.getComplexValue(this._prefs[aKey],
michael@0:                                                      Ci.nsISupportsString).data;
michael@0:       value = JSON.parse(prefValue);
michael@0:     } catch (e) {}
michael@0:     return value || aDefault;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Sets the storage value for a given key.
michael@0:    * @param aKey The storage key (a string).
michael@0:    * @param aValue The value to set.
michael@0:    */
michael@0:   set: function Storage_set(aKey, aValue) {
michael@0:     // Page titles may contain unicode, thus use complex values.
michael@0:     let string = Cc["@mozilla.org/supports-string;1"]
michael@0:                    .createInstance(Ci.nsISupportsString);
michael@0:     string.data = JSON.stringify(aValue);
michael@0:     Services.prefs.setComplexValue(this._prefs[aKey], Ci.nsISupportsString,
michael@0:                                    string);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Removes the storage value for a given key.
michael@0:    * @param aKey The storage key (a string).
michael@0:    */
michael@0:   remove: function Storage_remove(aKey) {
michael@0:     Services.prefs.clearUserPref(this._prefs[aKey]);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Clears the storage and removes all values.
michael@0:    */
michael@0:   clear: function Storage_clear() {
michael@0:     for (let key in this._prefs) {
michael@0:       this.remove(key);
michael@0:     }
michael@0:   }
michael@0: };
michael@0: 
michael@0: 
michael@0: /**
michael@0:  * Singleton that serves as a registry for all open 'New Tab Page's.
michael@0:  */
michael@0: let AllPages = {
michael@0:   /**
michael@0:    * The array containing all active pages.
michael@0:    */
michael@0:   _pages: [],
michael@0: 
michael@0:   /**
michael@0:    * Cached value that tells whether the New Tab Page feature is enabled.
michael@0:    */
michael@0:   _enabled: null,
michael@0: 
michael@0:   /**
michael@0:    * Adds a page to the internal list of pages.
michael@0:    * @param aPage The page to register.
michael@0:    */
michael@0:   register: function AllPages_register(aPage) {
michael@0:     this._pages.push(aPage);
michael@0:     this._addObserver();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Removes a page from the internal list of pages.
michael@0:    * @param aPage The page to unregister.
michael@0:    */
michael@0:   unregister: function AllPages_unregister(aPage) {
michael@0:     let index = this._pages.indexOf(aPage);
michael@0:     if (index > -1)
michael@0:       this._pages.splice(index, 1);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Returns whether the 'New Tab Page' is enabled.
michael@0:    */
michael@0:   get enabled() {
michael@0:     if (this._enabled === null)
michael@0:       this._enabled = Services.prefs.getBoolPref(PREF_NEWTAB_ENABLED);
michael@0: 
michael@0:     return this._enabled;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Enables or disables the 'New Tab Page' feature.
michael@0:    */
michael@0:   set enabled(aEnabled) {
michael@0:     if (this.enabled != aEnabled)
michael@0:       Services.prefs.setBoolPref(PREF_NEWTAB_ENABLED, !!aEnabled);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Returns the number of registered New Tab Pages (i.e. the number of open
michael@0:    * about:newtab instances).
michael@0:    */
michael@0:   get length() {
michael@0:     return this._pages.length;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Updates all currently active pages but the given one.
michael@0:    * @param aExceptPage The page to exclude from updating.
michael@0:    * @param aHiddenPagesOnly If true, only pages hidden in the preloader are
michael@0:    *                         updated.
michael@0:    */
michael@0:   update: function AllPages_update(aExceptPage, aHiddenPagesOnly=false) {
michael@0:     this._pages.forEach(function (aPage) {
michael@0:       if (aExceptPage != aPage)
michael@0:         aPage.update(aHiddenPagesOnly);
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Many individual link changes may happen in a small amount of time over
michael@0:    * multiple turns of the event loop.  This method coalesces updates by waiting
michael@0:    * a small amount of time before updating hidden pages.
michael@0:    */
michael@0:   scheduleUpdateForHiddenPages: function AllPages_scheduleUpdateForHiddenPages() {
michael@0:     if (!this._scheduleUpdateTimeout) {
michael@0:       this._scheduleUpdateTimeout = Timer.setTimeout(() => {
michael@0:         delete this._scheduleUpdateTimeout;
michael@0:         this.update(null, true);
michael@0:       }, SCHEDULE_UPDATE_TIMEOUT_MS);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   get updateScheduledForHiddenPages() {
michael@0:     return !!this._scheduleUpdateTimeout;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Implements the nsIObserver interface to get notified when the preference
michael@0:    * value changes or when a new copy of a page thumbnail is available.
michael@0:    */
michael@0:   observe: function AllPages_observe(aSubject, aTopic, aData) {
michael@0:     if (aTopic == "nsPref:changed") {
michael@0:       // Clear the cached value.
michael@0:       this._enabled = null;
michael@0:     }
michael@0:     // and all notifications get forwarded to each page.
michael@0:     this._pages.forEach(function (aPage) {
michael@0:       aPage.observe(aSubject, aTopic, aData);
michael@0:     }, this);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Adds a preference and new thumbnail observer and turns itself into a
michael@0:    * no-op after the first invokation.
michael@0:    */
michael@0:   _addObserver: function AllPages_addObserver() {
michael@0:     Services.prefs.addObserver(PREF_NEWTAB_ENABLED, this, true);
michael@0:     Services.obs.addObserver(this, "page-thumbnail:create", true);
michael@0:     this._addObserver = function () {};
michael@0:   },
michael@0: 
michael@0:   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver,
michael@0:                                          Ci.nsISupportsWeakReference])
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton that keeps Grid preferences
michael@0:  */
michael@0: let GridPrefs = {
michael@0:   /**
michael@0:    * Cached value that tells the number of rows of newtab grid.
michael@0:    */
michael@0:   _gridRows: null,
michael@0:   get gridRows() {
michael@0:     if (!this._gridRows) {
michael@0:       this._gridRows = Math.max(1, Services.prefs.getIntPref(PREF_NEWTAB_ROWS));
michael@0:     }
michael@0: 
michael@0:     return this._gridRows;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Cached value that tells the number of columns of newtab grid.
michael@0:    */
michael@0:   _gridColumns: null,
michael@0:   get gridColumns() {
michael@0:     if (!this._gridColumns) {
michael@0:       this._gridColumns = Math.max(1, Services.prefs.getIntPref(PREF_NEWTAB_COLUMNS));
michael@0:     }
michael@0: 
michael@0:     return this._gridColumns;
michael@0:   },
michael@0: 
michael@0: 
michael@0:   /**
michael@0:    * Initializes object. Adds a preference observer
michael@0:    */
michael@0:   init: function GridPrefs_init() {
michael@0:     Services.prefs.addObserver(PREF_NEWTAB_ROWS, this, false);
michael@0:     Services.prefs.addObserver(PREF_NEWTAB_COLUMNS, this, false);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Implements the nsIObserver interface to get notified when the preference
michael@0:    * value changes.
michael@0:    */
michael@0:   observe: function GridPrefs_observe(aSubject, aTopic, aData) {
michael@0:     if (aData == PREF_NEWTAB_ROWS) {
michael@0:       this._gridRows = null;
michael@0:     } else {
michael@0:       this._gridColumns = null;
michael@0:     }
michael@0: 
michael@0:     AllPages.update();
michael@0:   }
michael@0: };
michael@0: 
michael@0: GridPrefs.init();
michael@0: 
michael@0: /**
michael@0:  * Singleton that keeps track of all pinned links and their positions in the
michael@0:  * grid.
michael@0:  */
michael@0: let PinnedLinks = {
michael@0:   /**
michael@0:    * The cached list of pinned links.
michael@0:    */
michael@0:   _links: null,
michael@0: 
michael@0:   /**
michael@0:    * The array of pinned links.
michael@0:    */
michael@0:   get links() {
michael@0:     if (!this._links)
michael@0:       this._links = Storage.get("pinnedLinks", []);
michael@0: 
michael@0:     return this._links;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Pins a link at the given position.
michael@0:    * @param aLink The link to pin.
michael@0:    * @param aIndex The grid index to pin the cell at.
michael@0:    */
michael@0:   pin: function PinnedLinks_pin(aLink, aIndex) {
michael@0:     // Clear the link's old position, if any.
michael@0:     this.unpin(aLink);
michael@0: 
michael@0:     this.links[aIndex] = aLink;
michael@0:     this.save();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Unpins a given link.
michael@0:    * @param aLink The link to unpin.
michael@0:    */
michael@0:   unpin: function PinnedLinks_unpin(aLink) {
michael@0:     let index = this._indexOfLink(aLink);
michael@0:     if (index == -1)
michael@0:       return;
michael@0:     let links = this.links;
michael@0:     links[index] = null;
michael@0:     // trim trailing nulls
michael@0:     let i=links.length-1;
michael@0:     while (i >= 0 && links[i] == null)
michael@0:       i--;
michael@0:     links.splice(i +1);
michael@0:     this.save();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Saves the current list of pinned links.
michael@0:    */
michael@0:   save: function PinnedLinks_save() {
michael@0:     Storage.set("pinnedLinks", this.links);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Checks whether a given link is pinned.
michael@0:    * @params aLink The link to check.
michael@0:    * @return whether The link is pinned.
michael@0:    */
michael@0:   isPinned: function PinnedLinks_isPinned(aLink) {
michael@0:     return this._indexOfLink(aLink) != -1;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Resets the links cache.
michael@0:    */
michael@0:   resetCache: function PinnedLinks_resetCache() {
michael@0:     this._links = null;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Finds the index of a given link in the list of pinned links.
michael@0:    * @param aLink The link to find an index for.
michael@0:    * @return The link's index.
michael@0:    */
michael@0:   _indexOfLink: function PinnedLinks_indexOfLink(aLink) {
michael@0:     for (let i = 0; i < this.links.length; i++) {
michael@0:       let link = this.links[i];
michael@0:       if (link && link.url == aLink.url)
michael@0:         return i;
michael@0:     }
michael@0: 
michael@0:     // The given link is unpinned.
michael@0:     return -1;
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton that keeps track of all blocked links in the grid.
michael@0:  */
michael@0: let BlockedLinks = {
michael@0:   /**
michael@0:    * The cached list of blocked links.
michael@0:    */
michael@0:   _links: null,
michael@0: 
michael@0:   /**
michael@0:    * The list of blocked links.
michael@0:    */
michael@0:   get links() {
michael@0:     if (!this._links)
michael@0:       this._links = Storage.get("blockedLinks", {});
michael@0: 
michael@0:     return this._links;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Blocks a given link.
michael@0:    * @param aLink The link to block.
michael@0:    */
michael@0:   block: function BlockedLinks_block(aLink) {
michael@0:     this.links[toHash(aLink.url)] = 1;
michael@0:     this.save();
michael@0: 
michael@0:     // Make sure we unpin blocked links.
michael@0:     PinnedLinks.unpin(aLink);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Unblocks a given link.
michael@0:    * @param aLink The link to unblock.
michael@0:    */
michael@0:   unblock: function BlockedLinks_unblock(aLink) {
michael@0:     if (this.isBlocked(aLink)) {
michael@0:       delete this.links[toHash(aLink.url)];
michael@0:       this.save();
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Saves the current list of blocked links.
michael@0:    */
michael@0:   save: function BlockedLinks_save() {
michael@0:     Storage.set("blockedLinks", this.links);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Returns whether a given link is blocked.
michael@0:    * @param aLink The link to check.
michael@0:    */
michael@0:   isBlocked: function BlockedLinks_isBlocked(aLink) {
michael@0:     return (toHash(aLink.url) in this.links);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Checks whether the list of blocked links is empty.
michael@0:    * @return Whether the list is empty.
michael@0:    */
michael@0:   isEmpty: function BlockedLinks_isEmpty() {
michael@0:     return Object.keys(this.links).length == 0;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Resets the links cache.
michael@0:    */
michael@0:   resetCache: function BlockedLinks_resetCache() {
michael@0:     this._links = null;
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton that serves as the default link provider for the grid. It queries
michael@0:  * the history to retrieve the most frequently visited sites.
michael@0:  */
michael@0: let PlacesProvider = {
michael@0:   /**
michael@0:    * Set this to change the maximum number of links the provider will provide.
michael@0:    */
michael@0:   maxNumLinks: HISTORY_RESULTS_LIMIT,
michael@0: 
michael@0:   /**
michael@0:    * Must be called before the provider is used.
michael@0:    */
michael@0:   init: function PlacesProvider_init() {
michael@0:     PlacesUtils.history.addObserver(this, true);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Gets the current set of links delivered by this provider.
michael@0:    * @param aCallback The function that the array of links is passed to.
michael@0:    */
michael@0:   getLinks: function PlacesProvider_getLinks(aCallback) {
michael@0:     let options = PlacesUtils.history.getNewQueryOptions();
michael@0:     options.maxResults = this.maxNumLinks;
michael@0: 
michael@0:     // Sort by frecency, descending.
michael@0:     options.sortingMode = Ci.nsINavHistoryQueryOptions.SORT_BY_FRECENCY_DESCENDING
michael@0: 
michael@0:     let links = [];
michael@0: 
michael@0:     let callback = {
michael@0:       handleResult: function (aResultSet) {
michael@0:         let row;
michael@0: 
michael@0:         while ((row = aResultSet.getNextRow())) {
michael@0:           let url = row.getResultByIndex(1);
michael@0:           if (LinkChecker.checkLoadURI(url)) {
michael@0:             let title = row.getResultByIndex(2);
michael@0:             let frecency = row.getResultByIndex(12);
michael@0:             let lastVisitDate = row.getResultByIndex(5);
michael@0:             links.push({
michael@0:               url: url,
michael@0:               title: title,
michael@0:               frecency: frecency,
michael@0:               lastVisitDate: lastVisitDate,
michael@0:               bgColor: "transparent",
michael@0:               type: "history",
michael@0:               imageURI: null,
michael@0:             });
michael@0:           }
michael@0:         }
michael@0:       },
michael@0: 
michael@0:       handleError: function (aError) {
michael@0:         // Should we somehow handle this error?
michael@0:         aCallback([]);
michael@0:       },
michael@0: 
michael@0:       handleCompletion: function (aReason) {
michael@0:         // The Places query breaks ties in frecency by place ID descending, but
michael@0:         // that's different from how Links.compareLinks breaks ties, because
michael@0:         // compareLinks doesn't have access to place IDs.  It's very important
michael@0:         // that the initial list of links is sorted in the same order imposed by
michael@0:         // compareLinks, because Links uses compareLinks to perform binary
michael@0:         // searches on the list.  So, ensure the list is so ordered.
michael@0:         let i = 1;
michael@0:         let outOfOrder = [];
michael@0:         while (i < links.length) {
michael@0:           if (Links.compareLinks(links[i - 1], links[i]) > 0)
michael@0:             outOfOrder.push(links.splice(i, 1)[0]);
michael@0:           else
michael@0:             i++;
michael@0:         }
michael@0:         for (let link of outOfOrder) {
michael@0:           i = BinarySearch.insertionIndexOf(links, link,
michael@0:                                             Links.compareLinks.bind(Links));
michael@0:           links.splice(i, 0, link);
michael@0:         }
michael@0: 
michael@0:         aCallback(links);
michael@0:       }
michael@0:     };
michael@0: 
michael@0:     // Execute the query.
michael@0:     let query = PlacesUtils.history.getNewQuery();
michael@0:     let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase);
michael@0:     db.asyncExecuteLegacyQueries([query], 1, options, callback);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Registers an object that will be notified when the provider's links change.
michael@0:    * @param aObserver An object with the following optional properties:
michael@0:    *        * onLinkChanged: A function that's called when a single link
michael@0:    *          changes.  It's passed the provider and the link object.  Only the
michael@0:    *          link's `url` property is guaranteed to be present.  If its `title`
michael@0:    *          property is present, then its title has changed, and the
michael@0:    *          property's value is the new title.  If any sort properties are
michael@0:    *          present, then its position within the provider's list of links may
michael@0:    *          have changed, and the properties' values are the new sort-related
michael@0:    *          values.  Note that this link may not necessarily have been present
michael@0:    *          in the lists returned from any previous calls to getLinks.
michael@0:    *        * onManyLinksChanged: A function that's called when many links
michael@0:    *          change at once.  It's passed the provider.  You should call
michael@0:    *          getLinks to get the provider's new list of links.
michael@0:    */
michael@0:   addObserver: function PlacesProvider_addObserver(aObserver) {
michael@0:     this._observers.push(aObserver);
michael@0:   },
michael@0: 
michael@0:   _observers: [],
michael@0: 
michael@0:   /**
michael@0:    * Called by the history service.
michael@0:    */
michael@0:   onFrecencyChanged: function PlacesProvider_onFrecencyChanged(aURI, aNewFrecency, aGUID, aHidden, aLastVisitDate) {
michael@0:     // The implementation of the query in getLinks excludes hidden and
michael@0:     // unvisited pages, so it's important to exclude them here, too.
michael@0:     if (!aHidden && aLastVisitDate) {
michael@0:       this._callObservers("onLinkChanged", {
michael@0:         url: aURI.spec,
michael@0:         frecency: aNewFrecency,
michael@0:         lastVisitDate: aLastVisitDate,
michael@0:       });
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Called by the history service.
michael@0:    */
michael@0:   onManyFrecenciesChanged: function PlacesProvider_onManyFrecenciesChanged() {
michael@0:     this._callObservers("onManyLinksChanged");
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Called by the history service.
michael@0:    */
michael@0:   onTitleChanged: function PlacesProvider_onTitleChanged(aURI, aNewTitle, aGUID) {
michael@0:     this._callObservers("onLinkChanged", {
michael@0:       url: aURI.spec,
michael@0:       title: aNewTitle
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   _callObservers: function PlacesProvider__callObservers(aMethodName, aArg) {
michael@0:     for (let obs of this._observers) {
michael@0:       if (obs[aMethodName]) {
michael@0:         try {
michael@0:           obs[aMethodName](this, aArg);
michael@0:         } catch (err) {
michael@0:           Cu.reportError(err);
michael@0:         }
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   QueryInterface: XPCOMUtils.generateQI([Ci.nsINavHistoryObserver,
michael@0:                                          Ci.nsISupportsWeakReference]),
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton that provides access to all links contained in the grid (including
michael@0:  * the ones that don't fit on the grid). A link is a plain object that looks
michael@0:  * like this:
michael@0:  *
michael@0:  * {
michael@0:  *   url: "http://www.mozilla.org/",
michael@0:  *   title: "Mozilla",
michael@0:  *   frecency: 1337,
michael@0:  *   lastVisitDate: 1394678824766431,
michael@0:  * }
michael@0:  */
michael@0: let Links = {
michael@0:   /**
michael@0:    * The maximum number of links returned by getLinks.
michael@0:    */
michael@0:   maxNumLinks: LINKS_GET_LINKS_LIMIT,
michael@0: 
michael@0:   /**
michael@0:    * The link providers.
michael@0:    */
michael@0:   _providers: new Set(),
michael@0: 
michael@0:   /**
michael@0:    * A mapping from each provider to an object { sortedLinks, linkMap }.
michael@0:    * sortedLinks is the cached, sorted array of links for the provider.  linkMap
michael@0:    * is a Map from link URLs to link objects.
michael@0:    */
michael@0:   _providerLinks: new Map(),
michael@0: 
michael@0:   /**
michael@0:    * The properties of link objects used to sort them.
michael@0:    */
michael@0:   _sortProperties: [
michael@0:     "frecency",
michael@0:     "lastVisitDate",
michael@0:     "url",
michael@0:   ],
michael@0: 
michael@0:   /**
michael@0:    * List of callbacks waiting for the cache to be populated.
michael@0:    */
michael@0:   _populateCallbacks: [],
michael@0: 
michael@0:   /**
michael@0:    * Adds a link provider.
michael@0:    * @param aProvider The link provider.
michael@0:    */
michael@0:   addProvider: function Links_addProvider(aProvider) {
michael@0:     this._providers.add(aProvider);
michael@0:     aProvider.addObserver(this);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Removes a link provider.
michael@0:    * @param aProvider The link provider.
michael@0:    */
michael@0:   removeProvider: function Links_removeProvider(aProvider) {
michael@0:     if (!this._providers.delete(aProvider))
michael@0:       throw new Error("Unknown provider");
michael@0:     this._providerLinks.delete(aProvider);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Populates the cache with fresh links from the providers.
michael@0:    * @param aCallback The callback to call when finished (optional).
michael@0:    * @param aForce When true, populates the cache even when it's already filled.
michael@0:    */
michael@0:   populateCache: function Links_populateCache(aCallback, aForce) {
michael@0:     let callbacks = this._populateCallbacks;
michael@0: 
michael@0:     // Enqueue the current callback.
michael@0:     callbacks.push(aCallback);
michael@0: 
michael@0:     // There was a callback waiting already, thus the cache has not yet been
michael@0:     // populated.
michael@0:     if (callbacks.length > 1)
michael@0:       return;
michael@0: 
michael@0:     function executeCallbacks() {
michael@0:       while (callbacks.length) {
michael@0:         let callback = callbacks.shift();
michael@0:         if (callback) {
michael@0:           try {
michael@0:             callback();
michael@0:           } catch (e) {
michael@0:             // We want to proceed even if a callback fails.
michael@0:           }
michael@0:         }
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     let numProvidersRemaining = this._providers.size;
michael@0:     for (let provider of this._providers) {
michael@0:       this._populateProviderCache(provider, () => {
michael@0:         if (--numProvidersRemaining == 0)
michael@0:           executeCallbacks();
michael@0:       }, aForce);
michael@0:     }
michael@0: 
michael@0:     this._addObserver();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Gets the current set of links contained in the grid.
michael@0:    * @return The links in the grid.
michael@0:    */
michael@0:   getLinks: function Links_getLinks() {
michael@0:     let pinnedLinks = Array.slice(PinnedLinks.links);
michael@0:     let links = this._getMergedProviderLinks();
michael@0: 
michael@0:     // Filter blocked and pinned links.
michael@0:     links = links.filter(function (link) {
michael@0:       return !BlockedLinks.isBlocked(link) && !PinnedLinks.isPinned(link);
michael@0:     });
michael@0: 
michael@0:     // Try to fill the gaps between pinned links.
michael@0:     for (let i = 0; i < pinnedLinks.length && links.length; i++)
michael@0:       if (!pinnedLinks[i])
michael@0:         pinnedLinks[i] = links.shift();
michael@0: 
michael@0:     // Append the remaining links if any.
michael@0:     if (links.length)
michael@0:       pinnedLinks = pinnedLinks.concat(links);
michael@0: 
michael@0:     return pinnedLinks;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Resets the links cache.
michael@0:    */
michael@0:   resetCache: function Links_resetCache() {
michael@0:     this._providerLinks.clear();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Compares two links.
michael@0:    * @param aLink1 The first link.
michael@0:    * @param aLink2 The second link.
michael@0:    * @return A negative number if aLink1 is ordered before aLink2, zero if
michael@0:    *         aLink1 and aLink2 have the same ordering, or a positive number if
michael@0:    *         aLink1 is ordered after aLink2.
michael@0:    */
michael@0:   compareLinks: function Links_compareLinks(aLink1, aLink2) {
michael@0:     for (let prop of this._sortProperties) {
michael@0:       if (!(prop in aLink1) || !(prop in aLink2))
michael@0:         throw new Error("Comparable link missing required property: " + prop);
michael@0:     }
michael@0:     return aLink2.frecency - aLink1.frecency ||
michael@0:            aLink2.lastVisitDate - aLink1.lastVisitDate ||
michael@0:            aLink1.url.localeCompare(aLink2.url);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Calls getLinks on the given provider and populates our cache for it.
michael@0:    * @param aProvider The provider whose cache will be populated.
michael@0:    * @param aCallback The callback to call when finished.
michael@0:    * @param aForce When true, populates the provider's cache even when it's
michael@0:    *               already filled.
michael@0:    */
michael@0:   _populateProviderCache: function Links_populateProviderCache(aProvider, aCallback, aForce) {
michael@0:     if (this._providerLinks.has(aProvider) && !aForce) {
michael@0:       aCallback();
michael@0:     } else {
michael@0:       aProvider.getLinks(links => {
michael@0:         // Filter out null and undefined links so we don't have to deal with
michael@0:         // them in getLinks when merging links from providers.
michael@0:         links = links.filter((link) => !!link);
michael@0:         this._providerLinks.set(aProvider, {
michael@0:           sortedLinks: links,
michael@0:           linkMap: links.reduce((map, link) => {
michael@0:             map.set(link.url, link);
michael@0:             return map;
michael@0:           }, new Map()),
michael@0:         });
michael@0:         aCallback();
michael@0:       });
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Merges the cached lists of links from all providers whose lists are cached.
michael@0:    * @return The merged list.
michael@0:    */
michael@0:   _getMergedProviderLinks: function Links__getMergedProviderLinks() {
michael@0:     // Build a list containing a copy of each provider's sortedLinks list.
michael@0:     let linkLists = [];
michael@0:     for (let links of this._providerLinks.values()) {
michael@0:       linkLists.push(links.sortedLinks.slice());
michael@0:     }
michael@0: 
michael@0:     function getNextLink() {
michael@0:       let minLinks = null;
michael@0:       for (let links of linkLists) {
michael@0:         if (links.length &&
michael@0:             (!minLinks || Links.compareLinks(links[0], minLinks[0]) < 0))
michael@0:           minLinks = links;
michael@0:       }
michael@0:       return minLinks ? minLinks.shift() : null;
michael@0:     }
michael@0: 
michael@0:     let finalLinks = [];
michael@0:     for (let nextLink = getNextLink();
michael@0:          nextLink && finalLinks.length < this.maxNumLinks;
michael@0:          nextLink = getNextLink()) {
michael@0:       finalLinks.push(nextLink);
michael@0:     }
michael@0: 
michael@0:     return finalLinks;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Called by a provider to notify us when a single link changes.
michael@0:    * @param aProvider The provider whose link changed.
michael@0:    * @param aLink The link that changed.  If the link is new, it must have all
michael@0:    *              of the _sortProperties.  Otherwise, it may have as few or as
michael@0:    *              many as is convenient.
michael@0:    */
michael@0:   onLinkChanged: function Links_onLinkChanged(aProvider, aLink) {
michael@0:     if (!("url" in aLink))
michael@0:       throw new Error("Changed links must have a url property");
michael@0: 
michael@0:     let links = this._providerLinks.get(aProvider);
michael@0:     if (!links)
michael@0:       // This is not an error, it just means that between the time the provider
michael@0:       // was added and the future time we call getLinks on it, it notified us of
michael@0:       // a change.
michael@0:       return;
michael@0: 
michael@0:     let { sortedLinks, linkMap } = links;
michael@0:     let existingLink = linkMap.get(aLink.url);
michael@0:     let insertionLink = null;
michael@0:     let updatePages = false;
michael@0: 
michael@0:     if (existingLink) {
michael@0:       // Update our copy's position in O(lg n) by first removing it from its
michael@0:       // list.  It's important to do this before modifying its properties.
michael@0:       if (this._sortProperties.some(prop => prop in aLink)) {
michael@0:         let idx = this._indexOf(sortedLinks, existingLink);
michael@0:         if (idx < 0) {
michael@0:           throw new Error("Link should be in _sortedLinks if in _linkMap");
michael@0:         }
michael@0:         sortedLinks.splice(idx, 1);
michael@0:         // Update our copy's properties.
michael@0:         for (let prop of this._sortProperties) {
michael@0:           if (prop in aLink) {
michael@0:             existingLink[prop] = aLink[prop];
michael@0:           }
michael@0:         }
michael@0:         // Finally, reinsert our copy below.
michael@0:         insertionLink = existingLink;
michael@0:       }
michael@0:       // Update our copy's title in O(1).
michael@0:       if ("title" in aLink && aLink.title != existingLink.title) {
michael@0:         existingLink.title = aLink.title;
michael@0:         updatePages = true;
michael@0:       }
michael@0:     }
michael@0:     else if (this._sortProperties.every(prop => prop in aLink)) {
michael@0:       // Before doing the O(lg n) insertion below, do an O(1) check for the
michael@0:       // common case where the new link is too low-ranked to be in the list.
michael@0:       if (sortedLinks.length && sortedLinks.length == aProvider.maxNumLinks) {
michael@0:         let lastLink = sortedLinks[sortedLinks.length - 1];
michael@0:         if (this.compareLinks(lastLink, aLink) < 0) {
michael@0:           return;
michael@0:         }
michael@0:       }
michael@0:       // Copy the link object so that changes later made to it by the caller
michael@0:       // don't affect our copy.
michael@0:       insertionLink = {};
michael@0:       for (let prop in aLink) {
michael@0:         insertionLink[prop] = aLink[prop];
michael@0:       }
michael@0:       linkMap.set(aLink.url, insertionLink);
michael@0:     }
michael@0: 
michael@0:     if (insertionLink) {
michael@0:       let idx = this._insertionIndexOf(sortedLinks, insertionLink);
michael@0:       sortedLinks.splice(idx, 0, insertionLink);
michael@0:       if (sortedLinks.length > aProvider.maxNumLinks) {
michael@0:         let lastLink = sortedLinks.pop();
michael@0:         linkMap.delete(lastLink.url);
michael@0:       }
michael@0:       updatePages = true;
michael@0:     }
michael@0: 
michael@0:     if (updatePages)
michael@0:       AllPages.scheduleUpdateForHiddenPages();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Called by a provider to notify us when many links change.
michael@0:    */
michael@0:   onManyLinksChanged: function Links_onManyLinksChanged(aProvider) {
michael@0:     this._populateProviderCache(aProvider, () => {
michael@0:       AllPages.scheduleUpdateForHiddenPages();
michael@0:     }, true);
michael@0:   },
michael@0: 
michael@0:   _indexOf: function Links__indexOf(aArray, aLink) {
michael@0:     return this._binsearch(aArray, aLink, "indexOf");
michael@0:   },
michael@0: 
michael@0:   _insertionIndexOf: function Links__insertionIndexOf(aArray, aLink) {
michael@0:     return this._binsearch(aArray, aLink, "insertionIndexOf");
michael@0:   },
michael@0: 
michael@0:   _binsearch: function Links__binsearch(aArray, aLink, aMethod) {
michael@0:     return BinarySearch[aMethod](aArray, aLink, this.compareLinks.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Implements the nsIObserver interface to get notified about browser history
michael@0:    * sanitization.
michael@0:    */
michael@0:   observe: function Links_observe(aSubject, aTopic, aData) {
michael@0:     // Make sure to update open about:newtab instances. If there are no opened
michael@0:     // pages we can just wait for the next new tab to populate the cache again.
michael@0:     if (AllPages.length && AllPages.enabled)
michael@0:       this.populateCache(function () { AllPages.update() }, true);
michael@0:     else
michael@0:       this.resetCache();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Adds a sanitization observer and turns itself into a no-op after the first
michael@0:    * invokation.
michael@0:    */
michael@0:   _addObserver: function Links_addObserver() {
michael@0:     Services.obs.addObserver(this, "browser:purge-session-history", true);
michael@0:     this._addObserver = function () {};
michael@0:   },
michael@0: 
michael@0:   QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver,
michael@0:                                          Ci.nsISupportsWeakReference])
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton used to collect telemetry data.
michael@0:  *
michael@0:  */
michael@0: let Telemetry = {
michael@0:   /**
michael@0:    * Initializes object.
michael@0:    */
michael@0:   init: function Telemetry_init() {
michael@0:     Services.obs.addObserver(this, TOPIC_GATHER_TELEMETRY, false);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Collects data.
michael@0:    */
michael@0:   _collect: function Telemetry_collect() {
michael@0:     let probes = [
michael@0:       { histogram: "NEWTAB_PAGE_ENABLED",
michael@0:         value: AllPages.enabled },
michael@0:       { histogram: "NEWTAB_PAGE_PINNED_SITES_COUNT",
michael@0:         value: PinnedLinks.links.length },
michael@0:       { histogram: "NEWTAB_PAGE_BLOCKED_SITES_COUNT",
michael@0:         value: Object.keys(BlockedLinks.links).length }
michael@0:     ];
michael@0: 
michael@0:     probes.forEach(function Telemetry_collect_forEach(aProbe) {
michael@0:       Services.telemetry.getHistogramById(aProbe.histogram)
michael@0:         .add(aProbe.value);
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Listens for gather telemetry topic.
michael@0:    */
michael@0:   observe: function Telemetry_observe(aSubject, aTopic, aData) {
michael@0:     this._collect();
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton that checks if a given link should be displayed on about:newtab
michael@0:  * or if we should rather not do it for security reasons. URIs that inherit
michael@0:  * their caller's principal will be filtered.
michael@0:  */
michael@0: let LinkChecker = {
michael@0:   _cache: {},
michael@0: 
michael@0:   get flags() {
michael@0:     return Ci.nsIScriptSecurityManager.DISALLOW_INHERIT_PRINCIPAL |
michael@0:            Ci.nsIScriptSecurityManager.DONT_REPORT_ERRORS;
michael@0:   },
michael@0: 
michael@0:   checkLoadURI: function LinkChecker_checkLoadURI(aURI) {
michael@0:     if (!(aURI in this._cache))
michael@0:       this._cache[aURI] = this._doCheckLoadURI(aURI);
michael@0: 
michael@0:     return this._cache[aURI];
michael@0:   },
michael@0: 
michael@0:   _doCheckLoadURI: function Links_doCheckLoadURI(aURI) {
michael@0:     try {
michael@0:       Services.scriptSecurityManager.
michael@0:         checkLoadURIStrWithPrincipal(gPrincipal, aURI, this.flags);
michael@0:       return true;
michael@0:     } catch (e) {
michael@0:       // We got a weird URI or one that would inherit the caller's principal.
michael@0:       return false;
michael@0:     }
michael@0:   }
michael@0: };
michael@0: 
michael@0: let ExpirationFilter = {
michael@0:   init: function ExpirationFilter_init() {
michael@0:     PageThumbs.addExpirationFilter(this);
michael@0:   },
michael@0: 
michael@0:   filterForThumbnailExpiration:
michael@0:   function ExpirationFilter_filterForThumbnailExpiration(aCallback) {
michael@0:     if (!AllPages.enabled) {
michael@0:       aCallback([]);
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     Links.populateCache(function () {
michael@0:       let urls = [];
michael@0: 
michael@0:       // Add all URLs to the list that we want to keep thumbnails for.
michael@0:       for (let link of Links.getLinks().slice(0, 25)) {
michael@0:         if (link && link.url)
michael@0:           urls.push(link.url);
michael@0:       }
michael@0: 
michael@0:       aCallback(urls);
michael@0:     });
michael@0:   }
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Singleton that provides the public API of this JSM.
michael@0:  */
michael@0: this.NewTabUtils = {
michael@0:   _initialized: false,
michael@0: 
michael@0:   init: function NewTabUtils_init() {
michael@0:     if (this.initWithoutProviders()) {
michael@0:       PlacesProvider.init();
michael@0:       Links.addProvider(PlacesProvider);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   initWithoutProviders: function NewTabUtils_initWithoutProviders() {
michael@0:     if (!this._initialized) {
michael@0:       this._initialized = true;
michael@0:       ExpirationFilter.init();
michael@0:       Telemetry.init();
michael@0:       return true;
michael@0:     }
michael@0:     return false;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Restores all sites that have been removed from the grid.
michael@0:    */
michael@0:   restore: function NewTabUtils_restore() {
michael@0:     Storage.clear();
michael@0:     Links.resetCache();
michael@0:     PinnedLinks.resetCache();
michael@0:     BlockedLinks.resetCache();
michael@0: 
michael@0:     Links.populateCache(function () {
michael@0:       AllPages.update();
michael@0:     }, true);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Undoes all sites that have been removed from the grid and keep the pinned
michael@0:    * tabs.
michael@0:    * @param aCallback the callback method.
michael@0:    */
michael@0:   undoAll: function NewTabUtils_undoAll(aCallback) {
michael@0:     Storage.remove("blockedLinks");
michael@0:     Links.resetCache();
michael@0:     BlockedLinks.resetCache();
michael@0:     Links.populateCache(aCallback, true);
michael@0:   },
michael@0: 
michael@0:   links: Links,
michael@0:   allPages: AllPages,
michael@0:   linkChecker: LinkChecker,
michael@0:   pinnedLinks: PinnedLinks,
michael@0:   blockedLinks: BlockedLinks,
michael@0:   gridPrefs: GridPrefs
michael@0: };