The Tor Browser: netwerk/base/src/Seer.cpp@b8a032363ba2

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

     1 /* vim: set ts=2 sts=2 et sw=2: */

     2 /* This Source Code Form is subject to the terms of the Mozilla Public

     3  * License, v. 2.0. If a copy of the MPL was not distributed with this

     4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

     6 #include <algorithm>

     8 #include "Seer.h"

    10 #include "nsAppDirectoryServiceDefs.h"

    11 #include "nsICancelable.h"

    12 #include "nsIChannel.h"

    13 #include "nsIDNSListener.h"

    14 #include "nsIDNSService.h"

    15 #include "nsIDocument.h"

    16 #include "nsIFile.h"

    17 #include "nsILoadContext.h"

    18 #include "nsILoadGroup.h"

    19 #include "nsINetworkSeerVerifier.h"

    20 #include "nsIObserverService.h"

    21 #include "nsIPrefBranch.h"

    22 #include "nsIPrefService.h"

    23 #include "nsISpeculativeConnect.h"

    24 #include "nsITimer.h"

    25 #include "nsIURI.h"

    26 #include "nsNetUtil.h"

    27 #include "nsServiceManagerUtils.h"

    28 #include "nsTArray.h"

    29 #include "nsThreadUtils.h"

    30 #ifdef MOZ_NUWA_PROCESS

    31 #include "ipc/Nuwa.h"

    32 #endif

    33 #include "prlog.h"

    35 #include "mozIStorageConnection.h"

    36 #include "mozIStorageService.h"

    37 #include "mozIStorageStatement.h"

    38 #include "mozStorageHelper.h"

    40 #include "mozilla/Preferences.h"

    41 #include "mozilla/storage.h"

    42 #include "mozilla/Telemetry.h"

    44 #if defined(ANDROID) && !defined(MOZ_WIDGET_GONK)

    45 #include "nsIPropertyBag2.h"

    46 static const int32_t ANDROID_23_VERSION = 10;

    47 #endif

    49 using namespace mozilla;

    51 namespace mozilla {

    52 namespace net {

    54 #define RETURN_IF_FAILED(_rv) \

    55   do { \

    56     if (NS_FAILED(_rv)) { \

    57       return; \

    58     } \

    59   } while (0)

    61 const char SEER_ENABLED_PREF[] = "network.seer.enabled";

    62 const char SEER_SSL_HOVER_PREF[] = "network.seer.enable-hover-on-ssl";

    64 const char SEER_PAGE_DELTA_DAY_PREF[] = "network.seer.page-degradation.day";

    65 const int SEER_PAGE_DELTA_DAY_DEFAULT = 0;

    66 const char SEER_PAGE_DELTA_WEEK_PREF[] = "network.seer.page-degradation.week";

    67 const int SEER_PAGE_DELTA_WEEK_DEFAULT = 5;

    68 const char SEER_PAGE_DELTA_MONTH_PREF[] = "network.seer.page-degradation.month";

    69 const int SEER_PAGE_DELTA_MONTH_DEFAULT = 10;

    70 const char SEER_PAGE_DELTA_YEAR_PREF[] = "network.seer.page-degradation.year";

    71 const int SEER_PAGE_DELTA_YEAR_DEFAULT = 25;

    72 const char SEER_PAGE_DELTA_MAX_PREF[] = "network.seer.page-degradation.max";

    73 const int SEER_PAGE_DELTA_MAX_DEFAULT = 50;

    74 const char SEER_SUB_DELTA_DAY_PREF[] =

    75   "network.seer.subresource-degradation.day";

    76 const int SEER_SUB_DELTA_DAY_DEFAULT = 1;

    77 const char SEER_SUB_DELTA_WEEK_PREF[] =

    78   "network.seer.subresource-degradation.week";

    79 const int SEER_SUB_DELTA_WEEK_DEFAULT = 10;

    80 const char SEER_SUB_DELTA_MONTH_PREF[] =

    81   "network.seer.subresource-degradation.month";

    82 const int SEER_SUB_DELTA_MONTH_DEFAULT = 25;

    83 const char SEER_SUB_DELTA_YEAR_PREF[] =

    84   "network.seer.subresource-degradation.year";

    85 const int SEER_SUB_DELTA_YEAR_DEFAULT = 50;

    86 const char SEER_SUB_DELTA_MAX_PREF[] =

    87   "network.seer.subresource-degradation.max";

    88 const int SEER_SUB_DELTA_MAX_DEFAULT = 100;

    90 const char SEER_PRECONNECT_MIN_PREF[] =

    91   "network.seer.preconnect-min-confidence";

    92 const int PRECONNECT_MIN_DEFAULT = 90;

    93 const char SEER_PRERESOLVE_MIN_PREF[] =

    94   "network.seer.preresolve-min-confidence";

    95 const int PRERESOLVE_MIN_DEFAULT = 60;

    96 const char SEER_REDIRECT_LIKELY_PREF[] =

    97   "network.seer.redirect-likely-confidence";

    98 const int REDIRECT_LIKELY_DEFAULT = 75;

   100 const char SEER_MAX_QUEUE_SIZE_PREF[] = "network.seer.max-queue-size";

   101 const uint32_t SEER_MAX_QUEUE_SIZE_DEFAULT = 50;

   103 const char SEER_MAX_DB_SIZE_PREF[] = "network.seer.max-db-size";

   104 const int32_t SEER_MAX_DB_SIZE_DEFAULT_BYTES = 150 * 1024 * 1024;

   105 const char SEER_PRESERVE_PERCENTAGE_PREF[] = "network.seer.preserve";

   106 const int32_t SEER_PRESERVE_PERCENTAGE_DEFAULT = 80;

   108 // All these time values are in usec

   109 const long long ONE_DAY = 86400LL * 1000000LL;

   110 const long long ONE_WEEK = 7LL * ONE_DAY;

   111 const long long ONE_MONTH = 30LL * ONE_DAY;

   112 const long long ONE_YEAR = 365LL * ONE_DAY;

   114 const long STARTUP_WINDOW = 5L * 60L * 1000000L; // 5min

   116 // Version for the database schema

   117 static const int32_t SEER_SCHEMA_VERSION = 1;

   119 struct SeerTelemetryAccumulators {

   120   Telemetry::AutoCounter<Telemetry::SEER_PREDICT_ATTEMPTS> mPredictAttempts;

   121   Telemetry::AutoCounter<Telemetry::SEER_LEARN_ATTEMPTS> mLearnAttempts;

   122   Telemetry::AutoCounter<Telemetry::SEER_PREDICT_FULL_QUEUE> mPredictFullQueue;

   123   Telemetry::AutoCounter<Telemetry::SEER_LEARN_FULL_QUEUE> mLearnFullQueue;

   124   Telemetry::AutoCounter<Telemetry::SEER_TOTAL_PREDICTIONS> mTotalPredictions;

   125   Telemetry::AutoCounter<Telemetry::SEER_TOTAL_PRECONNECTS> mTotalPreconnects;

   126   Telemetry::AutoCounter<Telemetry::SEER_TOTAL_PRERESOLVES> mTotalPreresolves;

   127   Telemetry::AutoCounter<Telemetry::SEER_PREDICTIONS_CALCULATED> mPredictionsCalculated;

   128   Telemetry::AutoCounter<Telemetry::SEER_LOAD_COUNT_IS_ZERO> mLoadCountZeroes;

   129   Telemetry::AutoCounter<Telemetry::SEER_LOAD_COUNT_OVERFLOWS> mLoadCountOverflows;

   130   Telemetry::AutoCounter<Telemetry::SEER_STARTUP_COUNT_IS_ZERO> mStartupCountZeroes;

   131   Telemetry::AutoCounter<Telemetry::SEER_STARTUP_COUNT_OVERFLOWS> mStartupCountOverflows;

   132 };

   134 // Listener for the speculative DNS requests we'll fire off, which just ignores

   135 // the result (since we're just trying to warm the cache). This also exists to

   136 // reduce round-trips to the main thread, by being something threadsafe the Seer

   137 // can use.

   139 class SeerDNSListener : public nsIDNSListener

   140 {

   141 public:

   142   NS_DECL_THREADSAFE_ISUPPORTS

   143   NS_DECL_NSIDNSLISTENER

   145   SeerDNSListener()

   146   { }

   148   virtual ~SeerDNSListener()

   149   { }

   150 };

   152 NS_IMPL_ISUPPORTS(SeerDNSListener, nsIDNSListener);

   154 NS_IMETHODIMP

   155 SeerDNSListener::OnLookupComplete(nsICancelable *request,

   156                                   nsIDNSRecord *rec,

   157                                   nsresult status)

   158 {

   159   return NS_OK;

   160 }

   162 // Are you ready for the fun part? Because here comes the fun part. The seer,

   163 // which will do awesome stuff as you browse to make your browsing experience

   164 // faster.

   166 static Seer *gSeer = nullptr;

   168 #if defined(PR_LOGGING)

   169 static PRLogModuleInfo *gSeerLog = nullptr;

   170 #define SEER_LOG(args) PR_LOG(gSeerLog, 4, args)

   171 #else

   172 #define SEER_LOG(args)

   173 #endif

   175 NS_IMPL_ISUPPORTS(Seer,

   176                   nsINetworkSeer,

   177                   nsIObserver,

   178                   nsISpeculativeConnectionOverrider,

   179                   nsIInterfaceRequestor)

   181 Seer::Seer()

   182   :mInitialized(false)

   183   ,mEnabled(true)

   184   ,mEnableHoverOnSSL(false)

   185   ,mPageDegradationDay(SEER_PAGE_DELTA_DAY_DEFAULT)

   186   ,mPageDegradationWeek(SEER_PAGE_DELTA_WEEK_DEFAULT)

   187   ,mPageDegradationMonth(SEER_PAGE_DELTA_MONTH_DEFAULT)

   188   ,mPageDegradationYear(SEER_PAGE_DELTA_YEAR_DEFAULT)

   189   ,mPageDegradationMax(SEER_PAGE_DELTA_MAX_DEFAULT)

   190   ,mSubresourceDegradationDay(SEER_SUB_DELTA_DAY_DEFAULT)

   191   ,mSubresourceDegradationWeek(SEER_SUB_DELTA_WEEK_DEFAULT)

   192   ,mSubresourceDegradationMonth(SEER_SUB_DELTA_MONTH_DEFAULT)

   193   ,mSubresourceDegradationYear(SEER_SUB_DELTA_YEAR_DEFAULT)

   194   ,mSubresourceDegradationMax(SEER_SUB_DELTA_MAX_DEFAULT)

   195   ,mPreconnectMinConfidence(PRECONNECT_MIN_DEFAULT)

   196   ,mPreresolveMinConfidence(PRERESOLVE_MIN_DEFAULT)

   197   ,mRedirectLikelyConfidence(REDIRECT_LIKELY_DEFAULT)

   198   ,mMaxQueueSize(SEER_MAX_QUEUE_SIZE_DEFAULT)

   199   ,mStatements(mDB)

   200   ,mLastStartupTime(0)

   201   ,mStartupCount(0)

   202   ,mQueueSize(0)

   203   ,mQueueSizeLock("Seer.mQueueSizeLock")

   204   ,mCleanupScheduled(false)

   205   ,mMaxDBSize(SEER_MAX_DB_SIZE_DEFAULT_BYTES)

   206   ,mPreservePercentage(SEER_PRESERVE_PERCENTAGE_DEFAULT)

   207   ,mLastCleanupTime(0)

   208 {

   209 #if defined(PR_LOGGING)

   210   gSeerLog = PR_NewLogModule("NetworkSeer");

   211 #endif

   213   MOZ_ASSERT(!gSeer, "multiple Seer instances!");

   214   gSeer = this;

   215 }

   217 Seer::~Seer()

   218 {

   219   if (mInitialized)

   220     Shutdown();

   222   RemoveObserver();

   224   gSeer = nullptr;

   225 }

   227 // Seer::nsIObserver

   229 nsresult

   230 Seer::InstallObserver()

   231 {

   232   MOZ_ASSERT(NS_IsMainThread(), "Installing observer off main thread");

   234   nsresult rv = NS_OK;

   235   nsCOMPtr<nsIObserverService> obs =

   236     mozilla::services::GetObserverService();

   237   if (!obs) {

   238     return NS_ERROR_NOT_AVAILABLE;

   239   }

   241   rv = obs->AddObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false);

   242   NS_ENSURE_SUCCESS(rv, rv);

   244   nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);

   245   if (!prefs) {

   246     return NS_ERROR_NOT_AVAILABLE;

   247   }

   249   Preferences::AddBoolVarCache(&mEnabled, SEER_ENABLED_PREF, true);

   250   Preferences::AddBoolVarCache(&mEnableHoverOnSSL, SEER_SSL_HOVER_PREF, false);

   251   Preferences::AddIntVarCache(&mPageDegradationDay, SEER_PAGE_DELTA_DAY_PREF,

   252                               SEER_PAGE_DELTA_DAY_DEFAULT);

   253   Preferences::AddIntVarCache(&mPageDegradationWeek, SEER_PAGE_DELTA_WEEK_PREF,

   254                               SEER_PAGE_DELTA_WEEK_DEFAULT);

   255   Preferences::AddIntVarCache(&mPageDegradationMonth,

   256                               SEER_PAGE_DELTA_MONTH_PREF,

   257                               SEER_PAGE_DELTA_MONTH_DEFAULT);

   258   Preferences::AddIntVarCache(&mPageDegradationYear, SEER_PAGE_DELTA_YEAR_PREF,

   259                               SEER_PAGE_DELTA_YEAR_DEFAULT);

   260   Preferences::AddIntVarCache(&mPageDegradationMax, SEER_PAGE_DELTA_MAX_PREF,

   261                               SEER_PAGE_DELTA_MAX_DEFAULT);

   263   Preferences::AddIntVarCache(&mSubresourceDegradationDay,

   264                               SEER_SUB_DELTA_DAY_PREF,

   265                               SEER_SUB_DELTA_DAY_DEFAULT);

   266   Preferences::AddIntVarCache(&mSubresourceDegradationWeek,

   267                               SEER_SUB_DELTA_WEEK_PREF,

   268                               SEER_SUB_DELTA_WEEK_DEFAULT);

   269   Preferences::AddIntVarCache(&mSubresourceDegradationMonth,

   270                               SEER_SUB_DELTA_MONTH_PREF,

   271                               SEER_SUB_DELTA_MONTH_DEFAULT);

   272   Preferences::AddIntVarCache(&mSubresourceDegradationYear,

   273                               SEER_SUB_DELTA_YEAR_PREF,

   274                               SEER_SUB_DELTA_YEAR_DEFAULT);

   275   Preferences::AddIntVarCache(&mSubresourceDegradationMax,

   276                               SEER_SUB_DELTA_MAX_PREF,

   277                               SEER_SUB_DELTA_MAX_DEFAULT);

   279   Preferences::AddIntVarCache(&mPreconnectMinConfidence,

   280                               SEER_PRECONNECT_MIN_PREF,

   281                               PRECONNECT_MIN_DEFAULT);

   282   Preferences::AddIntVarCache(&mPreresolveMinConfidence,

   283                               SEER_PRERESOLVE_MIN_PREF,

   284                               PRERESOLVE_MIN_DEFAULT);

   285   Preferences::AddIntVarCache(&mRedirectLikelyConfidence,

   286                               SEER_REDIRECT_LIKELY_PREF,

   287                               REDIRECT_LIKELY_DEFAULT);

   289   Preferences::AddIntVarCache(&mMaxQueueSize, SEER_MAX_QUEUE_SIZE_PREF,

   290                               SEER_MAX_QUEUE_SIZE_DEFAULT);

   292   Preferences::AddIntVarCache(&mMaxDBSize, SEER_MAX_DB_SIZE_PREF,

   293                               SEER_MAX_DB_SIZE_DEFAULT_BYTES);

   294   Preferences::AddIntVarCache(&mPreservePercentage,

   295                               SEER_PRESERVE_PERCENTAGE_PREF,

   296                               SEER_PRESERVE_PERCENTAGE_DEFAULT);

   298   return rv;

   299 }

   301 void

   302 Seer::RemoveObserver()

   303 {

   304   MOZ_ASSERT(NS_IsMainThread(), "Removing observer off main thread");

   306   nsCOMPtr<nsIObserverService> obs =

   307     mozilla::services::GetObserverService();

   308   if (obs) {

   309     obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID);

   310   }

   311 }

   313 static const uint32_t COMMIT_TIMER_DELTA_MS = 5 * 1000;

   315 class SeerCommitTimerInitEvent : public nsRunnable

   316 {

   317 public:

   318   NS_IMETHOD Run() MOZ_OVERRIDE

   319   {

   320     nsresult rv = NS_OK;

   322     if (!gSeer->mCommitTimer) {

   323       gSeer->mCommitTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv);

   324     } else {

   325       gSeer->mCommitTimer->Cancel();

   326     }

   327     if (NS_SUCCEEDED(rv)) {

   328       gSeer->mCommitTimer->Init(gSeer, COMMIT_TIMER_DELTA_MS,

   329                                 nsITimer::TYPE_ONE_SHOT);

   330     }

   332     return NS_OK;

   333   }

   334 };

   336 class SeerNewTransactionEvent : public nsRunnable

   337 {

   338   NS_IMETHODIMP Run() MOZ_OVERRIDE

   339   {

   340     gSeer->CommitTransaction();

   341     gSeer->BeginTransaction();

   342     gSeer->MaybeScheduleCleanup();

   343     nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();

   344     NS_DispatchToMainThread(event);

   345     return NS_OK;

   346   }

   347 };

   349 NS_IMETHODIMP

   350 Seer::Observe(nsISupports *subject, const char *topic,

   351               const char16_t *data_unicode)

   352 {

   353   nsresult rv = NS_OK;

   354   MOZ_ASSERT(NS_IsMainThread(), "Seer observing something off main thread!");

   356   if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) {

   357     Shutdown();

   358   } else if (!strcmp(NS_TIMER_CALLBACK_TOPIC, topic)) {

   359     if (mInitialized) { // Can't access io thread if we're not initialized!

   360       nsRefPtr<SeerNewTransactionEvent> event = new SeerNewTransactionEvent();

   361       mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);

   362     }

   363   }

   365   return rv;

   366 }

   368 // Seer::nsISpeculativeConnectionOverrider

   370 NS_IMETHODIMP

   371 Seer::GetIgnoreIdle(bool *ignoreIdle)

   372 {

   373   *ignoreIdle = true;

   374   return NS_OK;

   375 }

   377 NS_IMETHODIMP

   378 Seer::GetIgnorePossibleSpdyConnections(bool *ignorePossibleSpdyConnections)

   379 {

   380   *ignorePossibleSpdyConnections = true;

   381   return NS_OK;

   382 }

   384 NS_IMETHODIMP

   385 Seer::GetParallelSpeculativeConnectLimit(

   386     uint32_t *parallelSpeculativeConnectLimit)

   387 {

   388   *parallelSpeculativeConnectLimit = 6;

   389   return NS_OK;

   390 }

   392 // Seer::nsIInterfaceRequestor

   394 NS_IMETHODIMP

   395 Seer::GetInterface(const nsIID &iid, void **result)

   396 {

   397   return QueryInterface(iid, result);

   398 }

   400 #ifdef MOZ_NUWA_PROCESS

   401 class NuwaMarkSeerThreadRunner : public nsRunnable

   402 {

   403   NS_IMETHODIMP Run() MOZ_OVERRIDE

   404   {

   405     if (IsNuwaProcess()) {

   406       NS_ASSERTION(NuwaMarkCurrentThread != nullptr,

   407                    "NuwaMarkCurrentThread is undefined!");

   408       NuwaMarkCurrentThread(nullptr, nullptr);

   409     }

   410     return NS_OK;

   411   }

   412 };

   413 #endif

   415 // Seer::nsINetworkSeer

   417 nsresult

   418 Seer::Init()

   419 {

   420   if (!NS_IsMainThread()) {

   421     MOZ_ASSERT(false, "Seer::Init called off the main thread!");

   422     return NS_ERROR_UNEXPECTED;

   423   }

   425   nsresult rv = NS_OK;

   427 #if defined(ANDROID) && !defined(MOZ_WIDGET_GONK)

   428   // This is an ugly hack to disable the seer on android < 2.3, as it doesn't

   429   // play nicely with those android versions, at least on our infra. Causes

   430   // timeouts in reftests. See bug 881804 comment 86.

   431   nsCOMPtr<nsIPropertyBag2> infoService =

   432     do_GetService("@mozilla.org/system-info;1");

   433   if (infoService) {

   434     int32_t androidVersion = -1;

   435     rv = infoService->GetPropertyAsInt32(NS_LITERAL_STRING("version"),

   436                                          &androidVersion);

   437     if (NS_SUCCEEDED(rv) && (androidVersion < ANDROID_23_VERSION)) {

   438       return NS_ERROR_NOT_AVAILABLE;

   439     }

   440   }

   441 #endif

   443   mStartupTime = PR_Now();

   445   mAccumulators = new SeerTelemetryAccumulators();

   447   rv = InstallObserver();

   448   NS_ENSURE_SUCCESS(rv, rv);

   450   if (!mDNSListener) {

   451     mDNSListener = new SeerDNSListener();

   452   }

   454   rv = NS_NewNamedThread("Network Seer", getter_AddRefs(mIOThread));

   455   NS_ENSURE_SUCCESS(rv, rv);

   457 #ifdef MOZ_NUWA_PROCESS

   458   nsCOMPtr<nsIRunnable> runner = new NuwaMarkSeerThreadRunner();

   459   mIOThread->Dispatch(runner, NS_DISPATCH_NORMAL);

   460 #endif

   462   mSpeculativeService = do_GetService("@mozilla.org/network/io-service;1", &rv);

   463   NS_ENSURE_SUCCESS(rv, rv);

   465   mDnsService = do_GetService("@mozilla.org/network/dns-service;1", &rv);

   466   NS_ENSURE_SUCCESS(rv, rv);

   468   mStorageService = do_GetService("@mozilla.org/storage/service;1", &rv);

   469   NS_ENSURE_SUCCESS(rv, rv);

   471   rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,

   472                               getter_AddRefs(mDBFile));

   473   NS_ENSURE_SUCCESS(rv, rv);

   474   rv = mDBFile->AppendNative(NS_LITERAL_CSTRING("netpredictions.sqlite"));

   475   NS_ENSURE_SUCCESS(rv, rv);

   477   mInitialized = true;

   479   return rv;

   480 }

   482 void

   483 Seer::CheckForAndDeleteOldDBFile()

   484 {

   485   nsCOMPtr<nsIFile> oldDBFile;

   486   nsresult rv = mDBFile->GetParent(getter_AddRefs(oldDBFile));

   487   RETURN_IF_FAILED(rv);

   489   rv = oldDBFile->AppendNative(NS_LITERAL_CSTRING("seer.sqlite"));

   490   RETURN_IF_FAILED(rv);

   492   bool oldFileExists = false;

   493   rv = oldDBFile->Exists(&oldFileExists);

   494   if (NS_FAILED(rv) || !oldFileExists) {

   495     return;

   496   }

   498   oldDBFile->Remove(false);

   499 }

   501 // Make sure that our sqlite storage is all set up with all the tables we need

   502 // to do the work. It isn't the end of the world if this fails, since this is

   503 // all an optimization, anyway.

   505 nsresult

   506 Seer::EnsureInitStorage()

   507 {

   508   MOZ_ASSERT(!NS_IsMainThread(), "Initializing seer storage on main thread");

   510   if (mDB) {

   511     return NS_OK;

   512   }

   514   nsresult rv;

   516   CheckForAndDeleteOldDBFile();

   518   rv = mStorageService->OpenDatabase(mDBFile, getter_AddRefs(mDB));

   519   if (NS_FAILED(rv)) {

   520     // Retry once by trashing the file and trying to open again. If this fails,

   521     // we can just bail, and hope for better luck next time.

   522     rv = mDBFile->Remove(false);

   523     NS_ENSURE_SUCCESS(rv, rv);

   525     rv = mStorageService->OpenDatabase(mDBFile, getter_AddRefs(mDB));

   526     NS_ENSURE_SUCCESS(rv, rv);

   527   }

   529   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous = OFF;"));

   530   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA foreign_keys = ON;"));

   532   BeginTransaction();

   534   // A table to make sure we're working with the database layout we expect

   535   rv = mDB->ExecuteSimpleSQL(

   536       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_seer_version (\n"

   537                          "  version INTEGER NOT NULL\n"

   538                          ");\n"));

   539   NS_ENSURE_SUCCESS(rv, rv);

   541   nsCOMPtr<mozIStorageStatement> stmt;

   542   rv = mDB->CreateStatement(

   543       NS_LITERAL_CSTRING("SELECT version FROM moz_seer_version;\n"),

   544       getter_AddRefs(stmt));

   545   NS_ENSURE_SUCCESS(rv, rv);

   547   bool hasRows;

   548   rv = stmt->ExecuteStep(&hasRows);

   549   NS_ENSURE_SUCCESS(rv, rv);

   550   if (hasRows) {

   551     int32_t currentVersion;

   552     rv = stmt->GetInt32(0, &currentVersion);

   553     NS_ENSURE_SUCCESS(rv, rv);

   555     // This is what we do while we only have one schema version. Later, we'll

   556     // have to change this to actually upgrade things as appropriate.

   557     MOZ_ASSERT(currentVersion == SEER_SCHEMA_VERSION,

   558                "Invalid seer schema version!");

   559     if (currentVersion != SEER_SCHEMA_VERSION) {

   560       return NS_ERROR_UNEXPECTED;

   561     }

   562   } else {

   563     stmt = nullptr;

   564     rv = mDB->CreateStatement(

   565         NS_LITERAL_CSTRING("INSERT INTO moz_seer_version (version) VALUES "

   566                            "(:seer_version);"),

   567         getter_AddRefs(stmt));

   568     NS_ENSURE_SUCCESS(rv, rv);

   570     rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("seer_version"),

   571                                SEER_SCHEMA_VERSION);

   572     NS_ENSURE_SUCCESS(rv, rv);

   574     stmt->Execute();

   575   }

   577   stmt = nullptr;

   579   // This table keeps track of the hosts we've seen at the top level of a

   580   // pageload so we can map them to hosts used for subresources of a pageload.

   581   rv = mDB->ExecuteSimpleSQL(

   582       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_hosts (\n"

   583                          "  id INTEGER PRIMARY KEY AUTOINCREMENT,\n"

   584                          "  origin TEXT NOT NULL,\n"

   585                          "  loads INTEGER DEFAULT 0,\n"

   586                          "  last_load INTEGER DEFAULT 0\n"

   587                          ");\n"));

   588   NS_ENSURE_SUCCESS(rv, rv);

   590   rv = mDB->ExecuteSimpleSQL(

   591       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS host_id_origin_index "

   592                          "ON moz_hosts (id, origin);"));

   593   NS_ENSURE_SUCCESS(rv, rv);

   595   rv = mDB->ExecuteSimpleSQL(

   596       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS host_origin_index "

   597                          "ON moz_hosts (origin);"));

   598   NS_ENSURE_SUCCESS(rv, rv);

   600   rv = mDB->ExecuteSimpleSQL(

   601       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS host_load_index "

   602                          "ON moz_hosts (last_load);"));

   603   NS_ENSURE_SUCCESS(rv, rv);

   605   // And this is the table that keeps track of the hosts for subresources of a

   606   // pageload.

   607   rv = mDB->ExecuteSimpleSQL(

   608       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_subhosts (\n"

   609                          "  id INTEGER PRIMARY KEY AUTOINCREMENT,\n"

   610                          "  hid INTEGER NOT NULL,\n"

   611                          "  origin TEXT NOT NULL,\n"

   612                          "  hits INTEGER DEFAULT 0,\n"

   613                          "  last_hit INTEGER DEFAULT 0,\n"

   614                          "  FOREIGN KEY(hid) REFERENCES moz_hosts(id) ON DELETE CASCADE\n"

   615                          ");\n"));

   616   NS_ENSURE_SUCCESS(rv, rv);

   618   rv = mDB->ExecuteSimpleSQL(

   619       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subhost_hid_origin_index "

   620                          "ON moz_subhosts (hid, origin);"));

   621   NS_ENSURE_SUCCESS(rv, rv);

   623   rv = mDB->ExecuteSimpleSQL(

   624       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subhost_id_index "

   625                          "ON moz_subhosts (id);"));

   626   NS_ENSURE_SUCCESS(rv, rv);

   628   // Table to keep track of how many times we've started up, and when the last

   629   // time was.

   630   rv = mDB->ExecuteSimpleSQL(

   631       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_startups (\n"

   632                          "  startups INTEGER,\n"

   633                          "  last_startup INTEGER\n"

   634                          ");\n"));

   635   NS_ENSURE_SUCCESS(rv, rv);

   637   rv = mDB->CreateStatement(

   638       NS_LITERAL_CSTRING("SELECT startups, last_startup FROM moz_startups;\n"),

   639       getter_AddRefs(stmt));

   640   NS_ENSURE_SUCCESS(rv, rv);

   642   // We'll go ahead and keep track of our startup count here, since we can

   643   // (mostly) equate "the service was created and asked to do stuff" with

   644   // "the browser was started up".

   645   rv = stmt->ExecuteStep(&hasRows);

   646   NS_ENSURE_SUCCESS(rv, rv);

   647   if (hasRows) {

   648     // We've started up before. Update our startup statistics

   649     stmt->GetInt32(0, &mStartupCount);

   650     stmt->GetInt64(1, &mLastStartupTime);

   652     // This finalizes the statement

   653     stmt = nullptr;

   655     rv = mDB->CreateStatement(

   656         NS_LITERAL_CSTRING("UPDATE moz_startups SET startups = :startup_count, "

   657                            "last_startup = :startup_time;\n"),

   658         getter_AddRefs(stmt));

   659     NS_ENSURE_SUCCESS(rv, rv);

   661     int32_t newStartupCount = mStartupCount + 1;

   662     if (newStartupCount <= 0) {

   663       SEER_LOG(("Seer::EnsureInitStorage startup count overflow\n"));

   664       newStartupCount = mStartupCount;

   665       ++mAccumulators->mStartupCountOverflows;

   666     }

   668     rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("startup_count"),

   669                                mStartupCount + 1);

   670     NS_ENSURE_SUCCESS(rv, rv);

   672     rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),

   673                                mStartupTime);

   674     NS_ENSURE_SUCCESS(rv, rv);

   676     stmt->Execute();

   677   } else {

   678     // This is our first startup, so let's go ahead and mark it as such

   679     mStartupCount = 1;

   681     rv = mDB->CreateStatement(

   682         NS_LITERAL_CSTRING("INSERT INTO moz_startups (startups, last_startup) "

   683                            "VALUES (1, :startup_time);\n"),

   684         getter_AddRefs(stmt));

   685     NS_ENSURE_SUCCESS(rv, rv);

   687     rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),

   688                                mStartupTime);

   689     NS_ENSURE_SUCCESS(rv, rv);

   691     stmt->Execute();

   692   }

   694   // This finalizes the statement

   695   stmt = nullptr;

   697   // This table lists URIs loaded at startup, along with how many startups

   698   // they've been loaded during, and when the last time was.

   699   rv = mDB->ExecuteSimpleSQL(

   700       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_startup_pages (\n"

   701                          "  id INTEGER PRIMARY KEY AUTOINCREMENT,\n"

   702                          "  uri TEXT NOT NULL,\n"

   703                          "  hits INTEGER DEFAULT 0,\n"

   704                          "  last_hit INTEGER DEFAULT 0\n"

   705                          ");\n"));

   706   NS_ENSURE_SUCCESS(rv, rv);

   708   rv = mDB->ExecuteSimpleSQL(

   709       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS startup_page_uri_index "

   710                          "ON moz_startup_pages (uri);"));

   711   NS_ENSURE_SUCCESS(rv, rv);

   713   rv = mDB->ExecuteSimpleSQL(

   714       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS startup_page_hit_index "

   715                          "ON moz_startup_pages (last_hit);"));

   716   NS_ENSURE_SUCCESS(rv, rv);

   718   // This table is similar to moz_hosts above, but uses full URIs instead of

   719   // hosts so that we can get more specific predictions for URIs that people

   720   // visit often (such as their email or social network home pages).

   721   rv = mDB->ExecuteSimpleSQL(

   722       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_pages (\n"

   723                          "  id integer PRIMARY KEY AUTOINCREMENT,\n"

   724                          "  uri TEXT NOT NULL,\n"

   725                          "  loads INTEGER DEFAULT 0,\n"

   726                          "  last_load INTEGER DEFAULT 0\n"

   727                          ");\n"));

   728   NS_ENSURE_SUCCESS(rv, rv);

   730   rv = mDB->ExecuteSimpleSQL(

   731       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS page_id_uri_index "

   732                          "ON moz_pages (id, uri);"));

   733   NS_ENSURE_SUCCESS(rv, rv);

   735   rv = mDB->ExecuteSimpleSQL(

   736       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS page_uri_index "

   737                          "ON moz_pages (uri);"));

   738   NS_ENSURE_SUCCESS(rv, rv);

   740   // This table is similar to moz_subhosts above, but is instead related to

   741   // moz_pages for finer granularity.

   742   rv = mDB->ExecuteSimpleSQL(

   743       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_subresources (\n"

   744                          "  id integer PRIMARY KEY AUTOINCREMENT,\n"

   745                          "  pid INTEGER NOT NULL,\n"

   746                          "  uri TEXT NOT NULL,\n"

   747                          "  hits INTEGER DEFAULT 0,\n"

   748                          "  last_hit INTEGER DEFAULT 0,\n"

   749                          "  FOREIGN KEY(pid) REFERENCES moz_pages(id) ON DELETE CASCADE\n"

   750                          ");\n"));

   751   NS_ENSURE_SUCCESS(rv, rv);

   753   rv = mDB->ExecuteSimpleSQL(

   754       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subresource_pid_uri_index "

   755                          "ON moz_subresources (pid, uri);"));

   756   NS_ENSURE_SUCCESS(rv, rv);

   758   rv = mDB->ExecuteSimpleSQL(

   759       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subresource_id_index "

   760                          "ON moz_subresources (id);"));

   761   NS_ENSURE_SUCCESS(rv, rv);

   763   // This table keeps track of URIs and what they end up finally redirecting to

   764   // so we can handle redirects in a sane fashion, as well.

   765   rv = mDB->ExecuteSimpleSQL(

   766       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_redirects (\n"

   767                          "  id integer PRIMARY KEY AUTOINCREMENT,\n"

   768                          "  pid integer NOT NULL,\n"

   769                          "  uri TEXT NOT NULL,\n"

   770                          "  origin TEXT NOT NULL,\n"

   771                          "  hits INTEGER DEFAULT 0,\n"

   772                          "  last_hit INTEGER DEFAULT 0,\n"

   773                          "  FOREIGN KEY(pid) REFERENCES moz_pages(id) ON DELETE CASCADE\n"

   774                          ");\n"));

   775   NS_ENSURE_SUCCESS(rv, rv);

   777   rv = mDB->ExecuteSimpleSQL(

   778       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS redirect_pid_uri_index "

   779                          "ON moz_redirects (pid, uri);"));

   780   NS_ENSURE_SUCCESS(rv, rv);

   782   rv = mDB->ExecuteSimpleSQL(

   783       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS redirect_id_index "

   784                          "ON moz_redirects (id);"));

   785   NS_ENSURE_SUCCESS(rv, rv);

   787   CommitTransaction();

   788   BeginTransaction();

   790   nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();

   791   NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);

   793   return NS_OK;

   794 }

   796 class SeerThreadShutdownRunner : public nsRunnable

   797 {

   798 public:

   799   SeerThreadShutdownRunner(nsIThread *ioThread)

   800     :mIOThread(ioThread)

   801   { }

   803   NS_IMETHODIMP Run() MOZ_OVERRIDE

   804   {

   805     MOZ_ASSERT(NS_IsMainThread(), "Shut down seer io thread off main thread");

   806     mIOThread->Shutdown();

   807     return NS_OK;

   808   }

   810 private:

   811   nsCOMPtr<nsIThread> mIOThread;

   812 };

   814 class SeerDBShutdownRunner : public nsRunnable

   815 {

   816 public:

   817   SeerDBShutdownRunner(nsIThread *ioThread, nsINetworkSeer *seer)

   818     :mIOThread(ioThread)

   819   {

   820     mSeer = new nsMainThreadPtrHolder<nsINetworkSeer>(seer);

   821   }

   823   NS_IMETHODIMP Run() MOZ_OVERRIDE

   824   {

   825     MOZ_ASSERT(!NS_IsMainThread(), "Shutting down DB on main thread");

   827     // Ensure everything is written to disk before we shut down the db

   828     gSeer->CommitTransaction();

   830     gSeer->mStatements.FinalizeStatements();

   831     gSeer->mDB->Close();

   832     gSeer->mDB = nullptr;

   834     nsRefPtr<SeerThreadShutdownRunner> runner =

   835       new SeerThreadShutdownRunner(mIOThread);

   836     NS_DispatchToMainThread(runner);

   838     return NS_OK;

   839   }

   841 private:

   842   nsCOMPtr<nsIThread> mIOThread;

   844   // Death grip to keep seer alive while we cleanly close its DB connection

   845   nsMainThreadPtrHandle<nsINetworkSeer> mSeer;

   846 };

   848 void

   849 Seer::Shutdown()

   850 {

   851   if (!NS_IsMainThread()) {

   852     MOZ_ASSERT(false, "Seer::Shutdown called off the main thread!");

   853     return;

   854   }

   856   mInitialized = false;

   858   if (mCommitTimer) {

   859     mCommitTimer->Cancel();

   860   }

   862   if (mIOThread) {

   863     if (mDB) {

   864       nsRefPtr<SeerDBShutdownRunner> runner =

   865         new SeerDBShutdownRunner(mIOThread, this);

   866       mIOThread->Dispatch(runner, NS_DISPATCH_NORMAL);

   867     } else {

   868       nsRefPtr<SeerThreadShutdownRunner> runner =

   869         new SeerThreadShutdownRunner(mIOThread);

   870       NS_DispatchToMainThread(runner);

   871     }

   872   }

   873 }

   875 nsresult

   876 Seer::Create(nsISupports *aOuter, const nsIID& aIID,

   877              void **aResult)

   878 {

   879   nsresult rv;

   881   if (aOuter != nullptr) {

   882     return NS_ERROR_NO_AGGREGATION;

   883   }

   885   nsRefPtr<Seer> svc = new Seer();

   887   rv = svc->Init();

   888   if (NS_FAILED(rv)) {

   889     SEER_LOG(("Failed to initialize seer, seer will be a noop"));

   890   }

   892   // We treat init failure the same as the service being disabled, since this

   893   // is all an optimization anyway. No need to freak people out. That's why we

   894   // gladly continue on QI'ing here.

   895   rv = svc->QueryInterface(aIID, aResult);

   897   return rv;

   898 }

   900 // Get the full origin (scheme, host, port) out of a URI (maybe should be part

   901 // of nsIURI instead?)

   902 static void

   903 ExtractOrigin(nsIURI *uri, nsAutoCString &s)

   904 {

   905   s.Truncate();

   907   nsAutoCString scheme;

   908   nsresult rv = uri->GetScheme(scheme);

   909   RETURN_IF_FAILED(rv);

   911   nsAutoCString host;

   912   rv = uri->GetAsciiHost(host);

   913   RETURN_IF_FAILED(rv);

   915   int32_t port;

   916   rv = uri->GetPort(&port);

   917   RETURN_IF_FAILED(rv);

   919   s.Assign(scheme);

   920   s.AppendLiteral("://");

   921   s.Append(host);

   922   if (port != -1) {

   923     s.AppendLiteral(":");

   924     s.AppendInt(port);

   925   }

   926 }

   928 // An event to do the work for a prediction that needs to hit the sqlite

   929 // database. These events should be created on the main thread, and run on

   930 // the seer thread.

   931 class SeerPredictionEvent : public nsRunnable

   932 {

   933 public:

   934   SeerPredictionEvent(nsIURI *targetURI, nsIURI *sourceURI,

   935                       SeerPredictReason reason,

   936                       nsINetworkSeerVerifier *verifier)

   937     :mReason(reason)

   938   {

   939     MOZ_ASSERT(NS_IsMainThread(), "Creating prediction event off main thread");

   941     mEnqueueTime = TimeStamp::Now();

   943     if (verifier) {

   944       mVerifier = new nsMainThreadPtrHolder<nsINetworkSeerVerifier>(verifier);

   945     }

   946     if (targetURI) {

   947       targetURI->GetAsciiSpec(mTargetURI.spec);

   948       ExtractOrigin(targetURI, mTargetURI.origin);

   949     }

   950     if (sourceURI) {

   951       sourceURI->GetAsciiSpec(mSourceURI.spec);

   952       ExtractOrigin(sourceURI, mSourceURI.origin);

   953     }

   954   }

   956   NS_IMETHOD Run() MOZ_OVERRIDE

   957   {

   958     MOZ_ASSERT(!NS_IsMainThread(), "Running prediction event on main thread");

   960     Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_WAIT_TIME,

   961                                    mEnqueueTime);

   963     TimeStamp startTime = TimeStamp::Now();

   965     nsresult rv = NS_OK;

   967     switch (mReason) {

   968       case nsINetworkSeer::PREDICT_LOAD:

   969         gSeer->PredictForPageload(mTargetURI, mVerifier, 0, mEnqueueTime);

   970         break;

   971       case nsINetworkSeer::PREDICT_STARTUP:

   972         gSeer->PredictForStartup(mVerifier, mEnqueueTime);

   973         break;

   974       default:

   975         MOZ_ASSERT(false, "Got unexpected value for predict reason");

   976         rv = NS_ERROR_UNEXPECTED;

   977     }

   979     gSeer->FreeSpaceInQueue();

   981     Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_WORK_TIME,

   982                                    startTime);

   984     gSeer->MaybeScheduleCleanup();

   986     return rv;

   987   }

   989 private:

   990   Seer::UriInfo mTargetURI;

   991   Seer::UriInfo mSourceURI;

   992   SeerPredictReason mReason;

   993   SeerVerifierHandle mVerifier;

   994   TimeStamp mEnqueueTime;

   995 };

   997 // Predicting for a link is easy, and doesn't require the round-trip to the

   998 // seer thread and back to the main thread, since we don't have to hit the db

   999 // for that.

  1000 void

  1001 Seer::PredictForLink(nsIURI *targetURI, nsIURI *sourceURI,

  1002                      nsINetworkSeerVerifier *verifier)

  1003 {

  1004   MOZ_ASSERT(NS_IsMainThread(), "Predicting for link off main thread");

  1006   if (!mSpeculativeService) {

  1007     return;

  1008   }

  1010   if (!mEnableHoverOnSSL) {

  1011     bool isSSL = false;

  1012     sourceURI->SchemeIs("https", &isSSL);

  1013     if (isSSL) {

  1014       // We don't want to predict from an HTTPS page, to avoid info leakage

  1015       SEER_LOG(("Not predicting for link hover - on an SSL page"));

  1016       return;

  1017     }

  1018   }

  1020   mSpeculativeService->SpeculativeConnect(targetURI, nullptr);

  1021   if (verifier) {

  1022     verifier->OnPredictPreconnect(targetURI);

  1023   }

  1024 }

  1026 // This runnable runs on the main thread, and is responsible for actually

  1027 // firing off predictive actions (such as TCP/TLS preconnects and DNS lookups)

  1028 class SeerPredictionRunner : public nsRunnable

  1029 {

  1030 public:

  1031   SeerPredictionRunner(SeerVerifierHandle &verifier, TimeStamp predictStartTime)

  1032     :mVerifier(verifier)

  1033     ,mPredictStartTime(predictStartTime)

  1034   { }

  1036   void AddPreconnect(const nsACString &uri)

  1037   {

  1038     mPreconnects.AppendElement(uri);

  1039   }

  1041   void AddPreresolve(const nsACString &uri)

  1042   {

  1043     mPreresolves.AppendElement(uri);

  1044   }

  1046   bool HasWork() const

  1047   {

  1048     return !(mPreconnects.IsEmpty() && mPreresolves.IsEmpty());

  1049   }

  1051   NS_IMETHOD Run() MOZ_OVERRIDE

  1052   {

  1053     MOZ_ASSERT(NS_IsMainThread(), "Running prediction off main thread");

  1055     Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_TIME_TO_ACTION,

  1056                                    mPredictStartTime);

  1058     uint32_t len, i;

  1060     len = mPreconnects.Length();

  1061     for (i = 0; i < len; ++i) {

  1062       nsCOMPtr<nsIURI> uri;

  1063       nsresult rv = NS_NewURI(getter_AddRefs(uri), mPreconnects[i]);

  1064       if (NS_FAILED(rv)) {

  1065         continue;

  1066       }

  1068       ++gSeer->mAccumulators->mTotalPredictions;

  1069       ++gSeer->mAccumulators->mTotalPreconnects;

  1070       gSeer->mSpeculativeService->SpeculativeConnect(uri, gSeer);

  1071       if (mVerifier) {

  1072         mVerifier->OnPredictPreconnect(uri);

  1073       }

  1074     }

  1076     len = mPreresolves.Length();

  1077     nsCOMPtr<nsIThread> mainThread = do_GetMainThread();

  1078     for (i = 0; i < len; ++i) {

  1079       nsCOMPtr<nsIURI> uri;

  1080       nsresult rv = NS_NewURI(getter_AddRefs(uri), mPreresolves[i]);

  1081       if (NS_FAILED(rv)) {

  1082         continue;

  1083       }

  1085       ++gSeer->mAccumulators->mTotalPredictions;

  1086       ++gSeer->mAccumulators->mTotalPreresolves;

  1087       nsAutoCString hostname;

  1088       uri->GetAsciiHost(hostname);

  1089       nsCOMPtr<nsICancelable> tmpCancelable;

  1090       gSeer->mDnsService->AsyncResolve(hostname,

  1091                                        (nsIDNSService::RESOLVE_PRIORITY_MEDIUM |

  1092                                         nsIDNSService::RESOLVE_SPECULATE),

  1093                                        gSeer->mDNSListener, nullptr,

  1094                                        getter_AddRefs(tmpCancelable));

  1095       if (mVerifier) {

  1096         mVerifier->OnPredictDNS(uri);

  1097       }

  1098     }

  1100     mPreconnects.Clear();

  1101     mPreresolves.Clear();

  1103     return NS_OK;

  1104   }

  1106 private:

  1107   nsTArray<nsCString> mPreconnects;

  1108   nsTArray<nsCString> mPreresolves;

  1109   SeerVerifierHandle mVerifier;

  1110   TimeStamp mPredictStartTime;

  1111 };

  1113 // This calculates how much to degrade our confidence in our data based on

  1114 // the last time this top-level resource was loaded. This "global degradation"

  1115 // applies to *all* subresources we have associated with the top-level

  1116 // resource. This will be in addition to any reduction in confidence we have

  1117 // associated with a particular subresource.

  1118 int

  1119 Seer::CalculateGlobalDegradation(PRTime now, PRTime lastLoad)

  1120 {

  1121   int globalDegradation;

  1122   PRTime delta = now - lastLoad;

  1123   if (delta < ONE_DAY) {

  1124     globalDegradation = mPageDegradationDay;

  1125   } else if (delta < ONE_WEEK) {

  1126     globalDegradation = mPageDegradationWeek;

  1127   } else if (delta < ONE_MONTH) {

  1128     globalDegradation = mPageDegradationMonth;

  1129   } else if (delta < ONE_YEAR) {

  1130     globalDegradation = mPageDegradationYear;

  1131   } else {

  1132     globalDegradation = mPageDegradationMax;

  1133   }

  1135   Telemetry::Accumulate(Telemetry::SEER_GLOBAL_DEGRADATION, globalDegradation);

  1136   return globalDegradation;

  1137 }

  1139 // This calculates our overall confidence that a particular subresource will be

  1140 // loaded as part of a top-level load.

  1141 // @param baseConfidence - the basic confidence we have for this subresource,

  1142 //                         which is the percentage of time this top-level load

  1143 //                         loads the subresource in question

  1144 // @param lastHit - the timestamp of the last time we loaded this subresource as

  1145 //                  part of this top-level load

  1146 // @param lastPossible - the timestamp of the last time we performed this

  1147 //                       top-level load

  1148 // @param globalDegradation - the degradation for this top-level load as

  1149 //                            determined by CalculateGlobalDegradation

  1150 int

  1151 Seer::CalculateConfidence(int baseConfidence, PRTime lastHit,

  1152                           PRTime lastPossible, int globalDegradation)

  1153 {

  1154   ++mAccumulators->mPredictionsCalculated;

  1156   int maxConfidence = 100;

  1157   int confidenceDegradation = 0;

  1159   if (lastHit < lastPossible) {

  1160     // We didn't load this subresource the last time this top-level load was

  1161     // performed, so let's not bother preconnecting (at the very least).

  1162     maxConfidence = mPreconnectMinConfidence - 1;

  1164     // Now calculate how much we want to degrade our confidence based on how

  1165     // long it's been between the last time we did this top-level load and the

  1166     // last time this top-level load included this subresource.

  1167     PRTime delta = lastPossible - lastHit;

  1168     if (delta == 0) {

  1169       confidenceDegradation = 0;

  1170     } else if (delta < ONE_DAY) {

  1171       confidenceDegradation = mSubresourceDegradationDay;

  1172     } else if (delta < ONE_WEEK) {

  1173       confidenceDegradation = mSubresourceDegradationWeek;

  1174     } else if (delta < ONE_MONTH) {

  1175       confidenceDegradation = mSubresourceDegradationMonth;

  1176     } else if (delta < ONE_YEAR) {

  1177       confidenceDegradation = mSubresourceDegradationYear;

  1178     } else {

  1179       confidenceDegradation = mSubresourceDegradationMax;

  1180       maxConfidence = 0;

  1181     }

  1182   }

  1184   // Calculate our confidence and clamp it to between 0 and maxConfidence

  1185   // (<= 100)

  1186   int confidence = baseConfidence - confidenceDegradation - globalDegradation;

  1187   confidence = std::max(confidence, 0);

  1188   confidence = std::min(confidence, maxConfidence);

  1190   Telemetry::Accumulate(Telemetry::SEER_BASE_CONFIDENCE, baseConfidence);

  1191   Telemetry::Accumulate(Telemetry::SEER_SUBRESOURCE_DEGRADATION,

  1192                         confidenceDegradation);

  1193   Telemetry::Accumulate(Telemetry::SEER_CONFIDENCE, confidence);

  1194   return confidence;

  1195 }

  1197 // (Maybe) adds a predictive action to the prediction runner, based on our

  1198 // calculated confidence for the subresource in question.

  1199 void

  1200 Seer::SetupPrediction(int confidence, const nsACString &uri,

  1201                       SeerPredictionRunner *runner)

  1202 {

  1203     if (confidence >= mPreconnectMinConfidence) {

  1204       runner->AddPreconnect(uri);

  1205     } else if (confidence >= mPreresolveMinConfidence) {

  1206       runner->AddPreresolve(uri);

  1207     }

  1208 }

  1210 // This gets the data about the top-level load from our database, either from

  1211 // the pages table (which is specific to a particular URI), or from the hosts

  1212 // table (which is for a particular origin).

  1213 bool

  1214 Seer::LookupTopLevel(QueryType queryType, const nsACString &key,

  1215                      TopLevelInfo &info)

  1216 {

  1217   MOZ_ASSERT(!NS_IsMainThread(), "LookupTopLevel called on main thread.");

  1219   nsCOMPtr<mozIStorageStatement> stmt;

  1220   if (queryType == QUERY_PAGE) {

  1221     stmt = mStatements.GetCachedStatement(

  1222         NS_LITERAL_CSTRING("SELECT id, loads, last_load FROM moz_pages WHERE "

  1223                            "uri = :key;"));

  1224   } else {

  1225     stmt = mStatements.GetCachedStatement(

  1226         NS_LITERAL_CSTRING("SELECT id, loads, last_load FROM moz_hosts WHERE "

  1227                            "origin = :key;"));

  1228   }

  1229   NS_ENSURE_TRUE(stmt, false);

  1230   mozStorageStatementScoper scope(stmt);

  1232   nsresult rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);

  1233   NS_ENSURE_SUCCESS(rv, false);

  1235   bool hasRows;

  1236   rv = stmt->ExecuteStep(&hasRows);

  1237   NS_ENSURE_SUCCESS(rv, false);

  1239   if (!hasRows) {

  1240     return false;

  1241   }

  1243   rv = stmt->GetInt32(0, &info.id);

  1244   NS_ENSURE_SUCCESS(rv, false);

  1246   rv = stmt->GetInt32(1, &info.loadCount);

  1247   NS_ENSURE_SUCCESS(rv, false);

  1249   rv = stmt->GetInt64(2, &info.lastLoad);

  1250   NS_ENSURE_SUCCESS(rv, false);

  1252   return true;

  1253 }

  1255 // Insert data about either a top-level page or a top-level origin into

  1256 // the database.

  1257 void

  1258 Seer::AddTopLevel(QueryType queryType, const nsACString &key, PRTime now)

  1259 {

  1260   MOZ_ASSERT(!NS_IsMainThread(), "AddTopLevel called on main thread.");

  1262   nsCOMPtr<mozIStorageStatement> stmt;

  1263   if (queryType == QUERY_PAGE) {

  1264     stmt = mStatements.GetCachedStatement(

  1265         NS_LITERAL_CSTRING("INSERT INTO moz_pages (uri, loads, last_load) "

  1266                            "VALUES (:key, 1, :now);"));

  1267   } else {

  1268     stmt = mStatements.GetCachedStatement(

  1269         NS_LITERAL_CSTRING("INSERT INTO moz_hosts (origin, loads, last_load) "

  1270                            "VALUES (:key, 1, :now);"));

  1271   }

  1272   if (!stmt) {

  1273     return;

  1274   }

  1275   mozStorageStatementScoper scope(stmt);

  1277   // Loading a page implicitly makes the seer learn about the page,

  1278   // so since we don't have it already, let's add it.

  1279   nsresult rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);

  1280   RETURN_IF_FAILED(rv);

  1282   rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);

  1283   RETURN_IF_FAILED(rv);

  1285   rv = stmt->Execute();

  1286 }

  1288 // Update data about either a top-level page or a top-level origin in the

  1289 // database.

  1290 void

  1291 Seer::UpdateTopLevel(QueryType queryType, const TopLevelInfo &info, PRTime now)

  1292 {

  1293   MOZ_ASSERT(!NS_IsMainThread(), "UpdateTopLevel called on main thread.");

  1295   nsCOMPtr<mozIStorageStatement> stmt;

  1296   if (queryType == QUERY_PAGE) {

  1297     stmt = mStatements.GetCachedStatement(

  1298         NS_LITERAL_CSTRING("UPDATE moz_pages SET loads = :load_count, "

  1299                            "last_load = :now WHERE id = :id;"));

  1300   } else {

  1301     stmt = mStatements.GetCachedStatement(

  1302         NS_LITERAL_CSTRING("UPDATE moz_hosts SET loads = :load_count, "

  1303                            "last_load = :now WHERE id = :id;"));

  1304   }

  1305   if (!stmt) {

  1306     return;

  1307   }

  1308   mozStorageStatementScoper scope(stmt);

  1310   int32_t newLoadCount = info.loadCount + 1;

  1311   if (newLoadCount <= 0) {

  1312     SEER_LOG(("Seer::UpdateTopLevel type %d id %d load count overflow\n",

  1313               queryType, info.id));

  1314     newLoadCount = info.loadCount;

  1315     ++mAccumulators->mLoadCountOverflows;

  1316   }

  1318   // First, let's update the page in the database, since loading a page

  1319   // implicitly learns about the page.

  1320   nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("load_count"),

  1321                                       newLoadCount);

  1322   RETURN_IF_FAILED(rv);

  1324   rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);

  1325   RETURN_IF_FAILED(rv);

  1327   rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);

  1328   RETURN_IF_FAILED(rv);

  1330   rv = stmt->Execute();

  1331 }

  1333 // Tries to predict for a top-level load (either page-based or origin-based).

  1334 // Returns false if it failed to predict at all, true if it did some sort of

  1335 // prediction.

  1336 // @param queryType - whether to predict based on page or origin

  1337 // @param info - the db info about the top-level resource

  1338 bool

  1339 Seer::TryPredict(QueryType queryType, const TopLevelInfo &info, PRTime now,

  1340                  SeerVerifierHandle &verifier, TimeStamp &predictStartTime)

  1341 {

  1342   MOZ_ASSERT(!NS_IsMainThread(), "TryPredict called on main thread.");

  1344   if (!info.loadCount) {

  1345     SEER_LOG(("Seer::TryPredict info.loadCount is zero!\n"));

  1346     ++mAccumulators->mLoadCountZeroes;

  1347     return false;

  1348   }

  1350   int globalDegradation = CalculateGlobalDegradation(now, info.lastLoad);

  1352   // Now let's look up the subresources we know about for this page

  1353   nsCOMPtr<mozIStorageStatement> stmt;

  1354   if (queryType == QUERY_PAGE) {

  1355     stmt = mStatements.GetCachedStatement(

  1356         NS_LITERAL_CSTRING("SELECT uri, hits, last_hit FROM moz_subresources "

  1357                            "WHERE pid = :id;"));

  1358   } else {

  1359     stmt = mStatements.GetCachedStatement(

  1360         NS_LITERAL_CSTRING("SELECT origin, hits, last_hit FROM moz_subhosts "

  1361                            "WHERE hid = :id;"));

  1362   }

  1363   NS_ENSURE_TRUE(stmt, false);

  1364   mozStorageStatementScoper scope(stmt);

  1366   nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);

  1367   NS_ENSURE_SUCCESS(rv, false);

  1369   bool hasRows;

  1370   rv = stmt->ExecuteStep(&hasRows);

  1371   if (NS_FAILED(rv) || !hasRows) {

  1372     return false;

  1373   }

  1375   nsRefPtr<SeerPredictionRunner> runner =

  1376     new SeerPredictionRunner(verifier, predictStartTime);

  1378   while (hasRows) {

  1379     int32_t hitCount;

  1380     PRTime lastHit;

  1381     nsAutoCString subresource;

  1382     int baseConfidence, confidence;

  1384     // We use goto nextrow here instead of just failling, because we want

  1385     // to do some sort of prediction if at all possible. Of course, it's

  1386     // probably unlikely that subsequent rows will succeed if one fails, but

  1387     // it's worth a shot.

  1389     rv = stmt->GetUTF8String(0, subresource);

  1390     if NS_FAILED(rv) {

  1391       goto nextrow;

  1392     }

  1394     rv = stmt->GetInt32(1, &hitCount);

  1395     if (NS_FAILED(rv)) {

  1396       goto nextrow;

  1397     }

  1399     rv = stmt->GetInt64(2, &lastHit);

  1400     if (NS_FAILED(rv)) {

  1401       goto nextrow;

  1402     }

  1404     baseConfidence = (hitCount * 100) / info.loadCount;

  1405     confidence = CalculateConfidence(baseConfidence, lastHit, info.lastLoad,

  1406                                      globalDegradation);

  1407     SetupPrediction(confidence, subresource, runner);

  1409 nextrow:

  1410     rv = stmt->ExecuteStep(&hasRows);

  1411     NS_ENSURE_SUCCESS(rv, false);

  1412   }

  1414   bool predicted = false;

  1416   if (runner->HasWork()) {

  1417     NS_DispatchToMainThread(runner);

  1418     predicted = true;

  1419   }

  1421   return predicted;

  1422 }

  1424 // Find out if a top-level page is likely to redirect.

  1425 bool

  1426 Seer::WouldRedirect(const TopLevelInfo &info, PRTime now, UriInfo &newUri)

  1427 {

  1428   MOZ_ASSERT(!NS_IsMainThread(), "WouldRedirect called on main thread.");

  1430   if (!info.loadCount) {

  1431     SEER_LOG(("Seer::WouldRedirect info.loadCount is zero!\n"));

  1432     ++mAccumulators->mLoadCountZeroes;

  1433     return false;

  1434   }

  1436   nsCOMPtr<mozIStorageStatement> stmt = mStatements.GetCachedStatement(

  1437       NS_LITERAL_CSTRING("SELECT uri, origin, hits, last_hit "

  1438                          "FROM moz_redirects WHERE pid = :id;"));

  1439   NS_ENSURE_TRUE(stmt, false);

  1440   mozStorageStatementScoper scope(stmt);

  1442   nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);

  1443   NS_ENSURE_SUCCESS(rv, false);

  1445   bool hasRows;

  1446   rv = stmt->ExecuteStep(&hasRows);

  1447   if (NS_FAILED(rv) || !hasRows) {

  1448     return false;

  1449   }

  1451   rv = stmt->GetUTF8String(0, newUri.spec);

  1452   NS_ENSURE_SUCCESS(rv, false);

  1454   rv = stmt->GetUTF8String(1, newUri.origin);

  1455   NS_ENSURE_SUCCESS(rv, false);

  1457   int32_t hitCount;

  1458   rv = stmt->GetInt32(2, &hitCount);

  1459   NS_ENSURE_SUCCESS(rv, false);

  1461   PRTime lastHit;

  1462   rv = stmt->GetInt64(3, &lastHit);

  1463   NS_ENSURE_SUCCESS(rv, false);

  1465   int globalDegradation = CalculateGlobalDegradation(now, info.lastLoad);

  1466   int baseConfidence = (hitCount * 100) / info.loadCount;

  1467   int confidence = CalculateConfidence(baseConfidence, lastHit, info.lastLoad,

  1468                                        globalDegradation);

  1470   if (confidence > mRedirectLikelyConfidence) {

  1471     return true;

  1472   }

  1474   return false;

  1475 }

  1477 // This will add a page to our list of startup pages if it's being loaded

  1478 // before our startup window has expired.

  1479 void

  1480 Seer::MaybeLearnForStartup(const UriInfo &uri, const PRTime now)

  1481 {

  1482   MOZ_ASSERT(!NS_IsMainThread(), "MaybeLearnForStartup called on main thread.");

  1484   if ((now - mStartupTime) < STARTUP_WINDOW) {

  1485     LearnForStartup(uri);

  1486   }

  1487 }

  1489 const int MAX_PAGELOAD_DEPTH = 10;

  1491 // This is the driver for prediction based on a new pageload.

  1492 void

  1493 Seer::PredictForPageload(const UriInfo &uri, SeerVerifierHandle &verifier,

  1494                          int stackCount, TimeStamp &predictStartTime)

  1495 {

  1496   MOZ_ASSERT(!NS_IsMainThread(), "PredictForPageload called on main thread.");

  1498   if (stackCount > MAX_PAGELOAD_DEPTH) {

  1499     SEER_LOG(("Too deep into pageload prediction"));

  1500     return;

  1501   }

  1503   if (NS_FAILED(EnsureInitStorage())) {

  1504     return;

  1505   }

  1507   PRTime now = PR_Now();

  1509   MaybeLearnForStartup(uri, now);

  1511   TopLevelInfo pageInfo;

  1512   TopLevelInfo originInfo;

  1513   bool havePage = LookupTopLevel(QUERY_PAGE, uri.spec, pageInfo);

  1514   bool haveOrigin = LookupTopLevel(QUERY_ORIGIN, uri.origin, originInfo);

  1516   if (!havePage) {

  1517     AddTopLevel(QUERY_PAGE, uri.spec, now);

  1518   } else {

  1519     UpdateTopLevel(QUERY_PAGE, pageInfo, now);

  1520   }

  1522   if (!haveOrigin) {

  1523     AddTopLevel(QUERY_ORIGIN, uri.origin, now);

  1524   } else {

  1525     UpdateTopLevel(QUERY_ORIGIN, originInfo, now);

  1526   }

  1528   UriInfo newUri;

  1529   if (havePage && WouldRedirect(pageInfo, now, newUri)) {

  1530     nsRefPtr<SeerPredictionRunner> runner =

  1531       new SeerPredictionRunner(verifier, predictStartTime);

  1532     runner->AddPreconnect(newUri.spec);

  1533     NS_DispatchToMainThread(runner);

  1534     PredictForPageload(newUri, verifier, stackCount + 1, predictStartTime);

  1535     return;

  1536   }

  1538   bool predicted = false;

  1540   // We always try to be as specific as possible in our predictions, so try

  1541   // to predict based on the full URI before we fall back to the origin.

  1542   if (havePage) {

  1543     predicted = TryPredict(QUERY_PAGE, pageInfo, now, verifier,

  1544                            predictStartTime);

  1545   }

  1547   if (!predicted && haveOrigin) {

  1548     predicted = TryPredict(QUERY_ORIGIN, originInfo, now, verifier,

  1549                            predictStartTime);

  1550   }

  1552   if (!predicted) {

  1553     Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_TIME_TO_INACTION,

  1554                                    predictStartTime);

  1555   }

  1556 }

  1558 // This is the driver for predicting at browser startup time based on pages that

  1559 // have previously been loaded close to startup.

  1560 void

  1561 Seer::PredictForStartup(SeerVerifierHandle &verifier,

  1562                         TimeStamp &predictStartTime)

  1563 {

  1564   MOZ_ASSERT(!NS_IsMainThread(), "PredictForStartup called on main thread");

  1566   if (!mStartupCount) {

  1567     SEER_LOG(("Seer::PredictForStartup mStartupCount is zero!\n"));

  1568     ++mAccumulators->mStartupCountZeroes;

  1569     return;

  1570   }

  1572   if (NS_FAILED(EnsureInitStorage())) {

  1573     return;

  1574   }

  1576   nsCOMPtr<mozIStorageStatement> stmt = mStatements.GetCachedStatement(

  1577       NS_LITERAL_CSTRING("SELECT uri, hits, last_hit FROM moz_startup_pages;"));

  1578   if (!stmt) {

  1579     return;

  1580   }

  1581   mozStorageStatementScoper scope(stmt);

  1582   nsresult rv;

  1583   bool hasRows;

  1585   nsRefPtr<SeerPredictionRunner> runner =

  1586     new SeerPredictionRunner(verifier, predictStartTime);

  1588   rv = stmt->ExecuteStep(&hasRows);

  1589   RETURN_IF_FAILED(rv);

  1591   while (hasRows) {

  1592     nsAutoCString uri;

  1593     int32_t hitCount;

  1594     PRTime lastHit;

  1595     int baseConfidence, confidence;

  1597     // We use goto nextrow here instead of just failling, because we want

  1598     // to do some sort of prediction if at all possible. Of course, it's

  1599     // probably unlikely that subsequent rows will succeed if one fails, but

  1600     // it's worth a shot.

  1602     rv = stmt->GetUTF8String(0, uri);

  1603     if (NS_FAILED(rv)) {

  1604       goto nextrow;

  1605     }

  1607     rv = stmt->GetInt32(1, &hitCount);

  1608     if (NS_FAILED(rv)) {

  1609       goto nextrow;

  1610     }

  1612     rv = stmt->GetInt64(2, &lastHit);

  1613     if (NS_FAILED(rv)) {

  1614       goto nextrow;

  1615     }

  1617     baseConfidence = (hitCount * 100) / mStartupCount;

  1618     confidence = CalculateConfidence(baseConfidence, lastHit,

  1619                                      mLastStartupTime, 0);

  1620     SetupPrediction(confidence, uri, runner);

  1622 nextrow:

  1623     rv = stmt->ExecuteStep(&hasRows);

  1624     RETURN_IF_FAILED(rv);

  1625   }

  1627   if (runner->HasWork()) {

  1628     NS_DispatchToMainThread(runner);

  1629   } else {

  1630     Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_TIME_TO_INACTION,

  1631                                    predictStartTime);

  1632   }

  1633 }

  1635 // All URIs we get passed *must* be http or https if they're not null. This

  1636 // helps ensure that.

  1637 static bool

  1638 IsNullOrHttp(nsIURI *uri)

  1639 {

  1640   if (!uri) {

  1641     return true;

  1642   }

  1644   bool isHTTP = false;

  1645   uri->SchemeIs("http", &isHTTP);

  1646   if (!isHTTP) {

  1647     uri->SchemeIs("https", &isHTTP);

  1648   }

  1650   return isHTTP;

  1651 }

  1653 nsresult

  1654 Seer::ReserveSpaceInQueue()

  1655 {

  1656   MutexAutoLock lock(mQueueSizeLock);

  1658   if (mQueueSize >= mMaxQueueSize) {

  1659     SEER_LOG(("Not enqueuing event - queue too large"));

  1660     return NS_ERROR_NOT_AVAILABLE;

  1661   }

  1663   mQueueSize++;

  1664   return NS_OK;

  1665 }

  1667 void

  1668 Seer::FreeSpaceInQueue()

  1669 {

  1670   MutexAutoLock lock(mQueueSizeLock);

  1671   MOZ_ASSERT(mQueueSize > 0, "unexpected mQueueSize");

  1672   mQueueSize--;

  1673 }

  1675 // Called from the main thread to initiate predictive actions

  1676 NS_IMETHODIMP

  1677 Seer::Predict(nsIURI *targetURI, nsIURI *sourceURI, SeerPredictReason reason,

  1678               nsILoadContext *loadContext, nsINetworkSeerVerifier *verifier)

  1679 {

  1680   MOZ_ASSERT(NS_IsMainThread(),

  1681              "Seer interface methods must be called on the main thread");

  1683   if (!mInitialized) {

  1684     return NS_ERROR_NOT_AVAILABLE;

  1685   }

  1687   if (!mEnabled) {

  1688     return NS_ERROR_NOT_AVAILABLE;

  1689   }

  1691   if (loadContext && loadContext->UsePrivateBrowsing()) {

  1692     // Don't want to do anything in PB mode

  1693     return NS_OK;

  1694   }

  1696   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  1697     // Nothing we can do for non-HTTP[S] schemes

  1698     return NS_OK;

  1699   }

  1701   // Ensure we've been given the appropriate arguments for the kind of

  1702   // prediction we're being asked to do

  1703   switch (reason) {

  1704     case nsINetworkSeer::PREDICT_LINK:

  1705       if (!targetURI || !sourceURI) {

  1706         return NS_ERROR_INVALID_ARG;

  1707       }

  1708       // Link hover is a special case where we can predict without hitting the

  1709       // db, so let's go ahead and fire off that prediction here.

  1710       PredictForLink(targetURI, sourceURI, verifier);

  1711       return NS_OK;

  1712     case nsINetworkSeer::PREDICT_LOAD:

  1713       if (!targetURI || sourceURI) {

  1714         return NS_ERROR_INVALID_ARG;

  1715       }

  1716       break;

  1717     case nsINetworkSeer::PREDICT_STARTUP:

  1718       if (targetURI || sourceURI) {

  1719         return NS_ERROR_INVALID_ARG;

  1720       }

  1721       break;

  1722     default:

  1723       return NS_ERROR_INVALID_ARG;

  1724   }

  1726   ++mAccumulators->mPredictAttempts;

  1727   nsresult rv = ReserveSpaceInQueue();

  1728   if (NS_FAILED(rv)) {

  1729     ++mAccumulators->mPredictFullQueue;

  1730     return NS_ERROR_NOT_AVAILABLE;

  1731   }

  1733   nsRefPtr<SeerPredictionEvent> event = new SeerPredictionEvent(targetURI,

  1734                                                                 sourceURI,

  1735                                                                 reason,

  1736                                                                 verifier);

  1737   return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);

  1738 }

  1740 // A runnable for updating our information in the database. This must always

  1741 // be dispatched to the seer thread.

  1742 class SeerLearnEvent : public nsRunnable

  1743 {

  1744 public:

  1745   SeerLearnEvent(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason)

  1746     :mReason(reason)

  1747   {

  1748     MOZ_ASSERT(NS_IsMainThread(), "Creating learn event off main thread");

  1750     mEnqueueTime = TimeStamp::Now();

  1752     targetURI->GetAsciiSpec(mTargetURI.spec);

  1753     ExtractOrigin(targetURI, mTargetURI.origin);

  1754     if (sourceURI) {

  1755       sourceURI->GetAsciiSpec(mSourceURI.spec);

  1756       ExtractOrigin(sourceURI, mSourceURI.origin);

  1757     }

  1758   }

  1760   NS_IMETHOD Run() MOZ_OVERRIDE

  1761   {

  1762     MOZ_ASSERT(!NS_IsMainThread(), "Running learn off main thread");

  1764     nsresult rv = NS_OK;

  1766     Telemetry::AccumulateTimeDelta(Telemetry::SEER_LEARN_WAIT_TIME,

  1767                                    mEnqueueTime);

  1769     TimeStamp startTime = TimeStamp::Now();

  1771     switch (mReason) {

  1772     case nsINetworkSeer::LEARN_LOAD_TOPLEVEL:

  1773       gSeer->LearnForToplevel(mTargetURI);

  1774       break;

  1775     case nsINetworkSeer::LEARN_LOAD_REDIRECT:

  1776       gSeer->LearnForRedirect(mTargetURI, mSourceURI);

  1777       break;

  1778     case nsINetworkSeer::LEARN_LOAD_SUBRESOURCE:

  1779       gSeer->LearnForSubresource(mTargetURI, mSourceURI);

  1780       break;

  1781     case nsINetworkSeer::LEARN_STARTUP:

  1782       gSeer->LearnForStartup(mTargetURI);

  1783       break;

  1784     default:

  1785       MOZ_ASSERT(false, "Got unexpected value for learn reason");

  1786       rv = NS_ERROR_UNEXPECTED;

  1787     }

  1789     gSeer->FreeSpaceInQueue();

  1791     Telemetry::AccumulateTimeDelta(Telemetry::SEER_LEARN_WORK_TIME, startTime);

  1793     gSeer->MaybeScheduleCleanup();

  1795     return rv;

  1796   }

  1797 private:

  1798   Seer::UriInfo mTargetURI;

  1799   Seer::UriInfo mSourceURI;

  1800   SeerLearnReason mReason;

  1801   TimeStamp mEnqueueTime;

  1802 };

  1804 void

  1805 Seer::LearnForToplevel(const UriInfo &uri)

  1806 {

  1807   MOZ_ASSERT(!NS_IsMainThread(), "LearnForToplevel called on main thread.");

  1809   if (NS_FAILED(EnsureInitStorage())) {

  1810     return;

  1811   }

  1813   PRTime now = PR_Now();

  1815   MaybeLearnForStartup(uri, now);

  1817   TopLevelInfo pageInfo;

  1818   TopLevelInfo originInfo;

  1819   bool havePage = LookupTopLevel(QUERY_PAGE, uri.spec, pageInfo);

  1820   bool haveOrigin = LookupTopLevel(QUERY_ORIGIN, uri.origin, originInfo);

  1822   if (!havePage) {

  1823     AddTopLevel(QUERY_PAGE, uri.spec, now);

  1824   } else {

  1825     UpdateTopLevel(QUERY_PAGE, pageInfo, now);

  1826   }

  1828   if (!haveOrigin) {

  1829     AddTopLevel(QUERY_ORIGIN, uri.origin, now);

  1830   } else {

  1831     UpdateTopLevel(QUERY_ORIGIN, originInfo, now);

  1832   }

  1833 }

  1835 // Queries to look up information about a *specific* subresource associated

  1836 // with a *specific* top-level load.

  1837 bool

  1838 Seer::LookupSubresource(QueryType queryType, const int32_t parentId,

  1839                         const nsACString &key, SubresourceInfo &info)

  1840 {

  1841   MOZ_ASSERT(!NS_IsMainThread(), "LookupSubresource called on main thread.");

  1843   nsCOMPtr<mozIStorageStatement> stmt;

  1844   if (queryType == QUERY_PAGE) {

  1845     stmt = mStatements.GetCachedStatement(

  1846         NS_LITERAL_CSTRING("SELECT id, hits, last_hit FROM moz_subresources "

  1847                            "WHERE pid = :parent_id AND uri = :key;"));

  1848   } else {

  1849     stmt = mStatements.GetCachedStatement(

  1850         NS_LITERAL_CSTRING("SELECT id, hits, last_hit FROM moz_subhosts WHERE "

  1851                            "hid = :parent_id AND origin = :key;"));

  1852   }

  1853   NS_ENSURE_TRUE(stmt, false);

  1854   mozStorageStatementScoper scope(stmt);

  1856   nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("parent_id"),

  1857                                       parentId);

  1858   NS_ENSURE_SUCCESS(rv, false);

  1860   rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);

  1861   NS_ENSURE_SUCCESS(rv, false);

  1863   bool hasRows;

  1864   rv = stmt->ExecuteStep(&hasRows);

  1865   NS_ENSURE_SUCCESS(rv, false);

  1866   if (!hasRows) {

  1867     return false;

  1868   }

  1870   rv = stmt->GetInt32(0, &info.id);

  1871   NS_ENSURE_SUCCESS(rv, false);

  1873   rv = stmt->GetInt32(1, &info.hitCount);

  1874   NS_ENSURE_SUCCESS(rv, false);

  1876   rv = stmt->GetInt64(2, &info.lastHit);

  1877   NS_ENSURE_SUCCESS(rv, false);

  1879   return true;

  1880 }

  1882 // Add information about a new subresource associated with a top-level load.

  1883 void

  1884 Seer::AddSubresource(QueryType queryType, const int32_t parentId,

  1885                      const nsACString &key, const PRTime now)

  1886 {

  1887   MOZ_ASSERT(!NS_IsMainThread(), "AddSubresource called on main thread.");

  1889   nsCOMPtr<mozIStorageStatement> stmt;

  1890   if (queryType == QUERY_PAGE) {

  1891     stmt = mStatements.GetCachedStatement(

  1892         NS_LITERAL_CSTRING("INSERT INTO moz_subresources "

  1893                            "(pid, uri, hits, last_hit) VALUES "

  1894                            "(:parent_id, :key, 1, :now);"));

  1895   } else {

  1896     stmt = mStatements.GetCachedStatement(

  1897         NS_LITERAL_CSTRING("INSERT INTO moz_subhosts "

  1898                            "(hid, origin, hits, last_hit) VALUES "

  1899                            "(:parent_id, :key, 1, :now);"));

  1900   }

  1901   if (!stmt) {

  1902     return;

  1903   }

  1904   mozStorageStatementScoper scope(stmt);

  1906   nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("parent_id"),

  1907                                       parentId);

  1908   RETURN_IF_FAILED(rv);

  1910   rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);

  1911   RETURN_IF_FAILED(rv);

  1913   rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);

  1914   RETURN_IF_FAILED(rv);

  1916   rv = stmt->Execute();

  1917 }

  1919 // Update the information about a particular subresource associated with a

  1920 // top-level load

  1921 void

  1922 Seer::UpdateSubresource(QueryType queryType, const SubresourceInfo &info,

  1923                         const PRTime now)

  1924 {

  1925   MOZ_ASSERT(!NS_IsMainThread(), "UpdateSubresource called on main thread.");

  1927   nsCOMPtr<mozIStorageStatement> stmt;

  1928   if (queryType == QUERY_PAGE) {

  1929     stmt = mStatements.GetCachedStatement(

  1930         NS_LITERAL_CSTRING("UPDATE moz_subresources SET hits = :hit_count, "

  1931                            "last_hit = :now WHERE id = :id;"));

  1932   } else {

  1933     stmt = mStatements.GetCachedStatement(

  1934         NS_LITERAL_CSTRING("UPDATE moz_subhosts SET hits = :hit_count, "

  1935                            "last_hit = :now WHERE id = :id;"));

  1936   }

  1937   if (!stmt) {

  1938     return;

  1939   }

  1940   mozStorageStatementScoper scope(stmt);

  1942   nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("hit_count"),

  1943                                       info.hitCount + 1);

  1944   RETURN_IF_FAILED(rv);

  1946   rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);

  1947   RETURN_IF_FAILED(rv);

  1949   rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);

  1950   RETURN_IF_FAILED(rv);

  1952   rv = stmt->Execute();

  1953 }

  1955 // Called when a subresource has been hit from a top-level load. Uses the two

  1956 // helper functions above to update the database appropriately.

  1957 void

  1958 Seer::LearnForSubresource(const UriInfo &targetURI, const UriInfo &sourceURI)

  1959 {

  1960   MOZ_ASSERT(!NS_IsMainThread(), "LearnForSubresource called on main thread.");

  1962   if (NS_FAILED(EnsureInitStorage())) {

  1963     return;

  1964   }

  1966   TopLevelInfo pageInfo, originInfo;

  1967   bool havePage = LookupTopLevel(QUERY_PAGE, sourceURI.spec, pageInfo);

  1968   bool haveOrigin = LookupTopLevel(QUERY_ORIGIN, sourceURI.origin,

  1969                                    originInfo);

  1971   if (!havePage && !haveOrigin) {

  1972     // Nothing to do, since we know nothing about the top level resource

  1973     return;

  1974   }

  1976   SubresourceInfo resourceInfo;

  1977   bool haveResource = false;

  1978   if (havePage) {

  1979     haveResource = LookupSubresource(QUERY_PAGE, pageInfo.id, targetURI.spec,

  1980                                      resourceInfo);

  1981   }

  1983   SubresourceInfo hostInfo;

  1984   bool haveHost = false;

  1985   if (haveOrigin) {

  1986     haveHost = LookupSubresource(QUERY_ORIGIN, originInfo.id, targetURI.origin,

  1987                                  hostInfo);

  1988   }

  1990   PRTime now = PR_Now();

  1992   if (haveResource) {

  1993     UpdateSubresource(QUERY_PAGE, resourceInfo, now);

  1994   } else if (havePage) {

  1995     AddSubresource(QUERY_PAGE, pageInfo.id, targetURI.spec, now);

  1996   }

  1997   // Can't add a subresource to a page we don't have in our db.

  1999   if (haveHost) {

  2000     UpdateSubresource(QUERY_ORIGIN, hostInfo, now);

  2001   } else if (haveOrigin) {

  2002     AddSubresource(QUERY_ORIGIN, originInfo.id, targetURI.origin, now);

  2003   }

  2004   // Can't add a subhost to a host we don't have in our db

  2005 }

  2007 // This is called when a top-level loaded ended up redirecting to a different

  2008 // URI so we can keep track of that fact.

  2009 void

  2010 Seer::LearnForRedirect(const UriInfo &targetURI, const UriInfo &sourceURI)

  2011 {

  2012   MOZ_ASSERT(!NS_IsMainThread(), "LearnForRedirect called on main thread.");

  2014   if (NS_FAILED(EnsureInitStorage())) {

  2015     return;

  2016   }

  2018   PRTime now = PR_Now();

  2019   nsresult rv;

  2021   nsCOMPtr<mozIStorageStatement> getPage = mStatements.GetCachedStatement(

  2022       NS_LITERAL_CSTRING("SELECT id FROM moz_pages WHERE uri = :spec;"));

  2023   if (!getPage) {

  2024     return;

  2025   }

  2026   mozStorageStatementScoper scopedPage(getPage);

  2028   // look up source in moz_pages

  2029   rv = getPage->BindUTF8StringByName(NS_LITERAL_CSTRING("spec"),

  2030                                      sourceURI.spec);

  2031   RETURN_IF_FAILED(rv);

  2033   bool hasRows;

  2034   rv = getPage->ExecuteStep(&hasRows);

  2035   if (NS_FAILED(rv) || !hasRows) {

  2036     return;

  2037   }

  2039   int32_t pageId;

  2040   rv = getPage->GetInt32(0, &pageId);

  2041   RETURN_IF_FAILED(rv);

  2043   nsCOMPtr<mozIStorageStatement> getRedirect = mStatements.GetCachedStatement(

  2044       NS_LITERAL_CSTRING("SELECT id, hits FROM moz_redirects WHERE "

  2045                          "pid = :page_id AND uri = :spec;"));

  2046   if (!getRedirect) {

  2047     return;

  2048   }

  2049   mozStorageStatementScoper scopedRedirect(getRedirect);

  2051   rv = getRedirect->BindInt32ByName(NS_LITERAL_CSTRING("page_id"), pageId);

  2052   RETURN_IF_FAILED(rv);

  2054   rv = getRedirect->BindUTF8StringByName(NS_LITERAL_CSTRING("spec"),

  2055                                          targetURI.spec);

  2056   RETURN_IF_FAILED(rv);

  2058   rv = getRedirect->ExecuteStep(&hasRows);

  2059   RETURN_IF_FAILED(rv);

  2061   if (!hasRows) {

  2062     // This is the first time we've seen this top-level redirect to this URI

  2063     nsCOMPtr<mozIStorageStatement> addRedirect = mStatements.GetCachedStatement(

  2064         NS_LITERAL_CSTRING("INSERT INTO moz_redirects "

  2065                            "(pid, uri, origin, hits, last_hit) VALUES "

  2066                            "(:page_id, :spec, :origin, 1, :now);"));

  2067     if (!addRedirect) {

  2068       return;

  2069     }

  2070     mozStorageStatementScoper scopedAdd(addRedirect);

  2072     rv = addRedirect->BindInt32ByName(NS_LITERAL_CSTRING("page_id"), pageId);

  2073     RETURN_IF_FAILED(rv);

  2075     rv = addRedirect->BindUTF8StringByName(NS_LITERAL_CSTRING("spec"),

  2076                                            targetURI.spec);

  2077     RETURN_IF_FAILED(rv);

  2079     rv = addRedirect->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"),

  2080                                            targetURI.origin);

  2081     RETURN_IF_FAILED(rv);

  2083     rv = addRedirect->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);

  2084     RETURN_IF_FAILED(rv);

  2086     rv = addRedirect->Execute();

  2087   } else {

  2088     // We've seen this redirect before

  2089     int32_t redirId, hits;

  2090     rv = getRedirect->GetInt32(0, &redirId);

  2091     RETURN_IF_FAILED(rv);

  2093     rv = getRedirect->GetInt32(1, &hits);

  2094     RETURN_IF_FAILED(rv);

  2096     nsCOMPtr<mozIStorageStatement> updateRedirect =

  2097       mStatements.GetCachedStatement(

  2098           NS_LITERAL_CSTRING("UPDATE moz_redirects SET hits = :hits, "

  2099                              "last_hit = :now WHERE id = :redir;"));

  2100     if (!updateRedirect) {

  2101       return;

  2102     }

  2103     mozStorageStatementScoper scopedUpdate(updateRedirect);

  2105     rv = updateRedirect->BindInt32ByName(NS_LITERAL_CSTRING("hits"), hits + 1);

  2106     RETURN_IF_FAILED(rv);

  2108     rv = updateRedirect->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);

  2109     RETURN_IF_FAILED(rv);

  2111     rv = updateRedirect->BindInt32ByName(NS_LITERAL_CSTRING("redir"), redirId);

  2112     RETURN_IF_FAILED(rv);

  2114     updateRedirect->Execute();

  2115   }

  2116 }

  2118 // Add information about a top-level load to our list of startup pages

  2119 void

  2120 Seer::LearnForStartup(const UriInfo &uri)

  2121 {

  2122   MOZ_ASSERT(!NS_IsMainThread(), "LearnForStartup called on main thread.");

  2124   if (NS_FAILED(EnsureInitStorage())) {

  2125     return;

  2126   }

  2128   nsCOMPtr<mozIStorageStatement> getPage = mStatements.GetCachedStatement(

  2129       NS_LITERAL_CSTRING("SELECT id, hits FROM moz_startup_pages WHERE "

  2130                          "uri = :origin;"));

  2131   if (!getPage) {

  2132     return;

  2133   }

  2134   mozStorageStatementScoper scopedPage(getPage);

  2135   nsresult rv;

  2137   rv = getPage->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"), uri.origin);

  2138   RETURN_IF_FAILED(rv);

  2140   bool hasRows;

  2141   rv = getPage->ExecuteStep(&hasRows);

  2142   RETURN_IF_FAILED(rv);

  2144   if (hasRows) {

  2145     // We've loaded this page on startup before

  2146     int32_t pageId, hitCount;

  2148     rv = getPage->GetInt32(0, &pageId);

  2149     RETURN_IF_FAILED(rv);

  2151     rv = getPage->GetInt32(1, &hitCount);

  2152     RETURN_IF_FAILED(rv);

  2154     nsCOMPtr<mozIStorageStatement> updatePage = mStatements.GetCachedStatement(

  2155         NS_LITERAL_CSTRING("UPDATE moz_startup_pages SET hits = :hit_count, "

  2156                            "last_hit = :startup_time WHERE id = :page_id;"));

  2157     if (!updatePage) {

  2158       return;

  2159     }

  2160     mozStorageStatementScoper scopedUpdate(updatePage);

  2162     rv = updatePage->BindInt32ByName(NS_LITERAL_CSTRING("hit_count"),

  2163                                      hitCount + 1);

  2164     RETURN_IF_FAILED(rv);

  2166     rv = updatePage->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),

  2167                                      mStartupTime);

  2168     RETURN_IF_FAILED(rv);

  2170     rv = updatePage->BindInt32ByName(NS_LITERAL_CSTRING("page_id"), pageId);

  2171     RETURN_IF_FAILED(rv);

  2173     updatePage->Execute();

  2174   } else {

  2175     // New startup page

  2176     nsCOMPtr<mozIStorageStatement> addPage = mStatements.GetCachedStatement(

  2177         NS_LITERAL_CSTRING("INSERT INTO moz_startup_pages (uri, hits, "

  2178                            "last_hit) VALUES (:origin, 1, :startup_time);"));

  2179     if (!addPage) {

  2180       return;

  2181     }

  2182     mozStorageStatementScoper scopedAdd(addPage);

  2183     rv = addPage->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"),

  2184                                        uri.origin);

  2185     RETURN_IF_FAILED(rv);

  2187     rv = addPage->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),

  2188                                   mStartupTime);

  2189     RETURN_IF_FAILED(rv);

  2191     addPage->Execute();

  2192   }

  2193 }

  2195 // Called from the main thread to update the database

  2196 NS_IMETHODIMP

  2197 Seer::Learn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,

  2198             nsILoadContext *loadContext)

  2199 {

  2200   MOZ_ASSERT(NS_IsMainThread(),

  2201              "Seer interface methods must be called on the main thread");

  2203   if (!mInitialized) {

  2204     return NS_ERROR_NOT_AVAILABLE;

  2205   }

  2207   if (!mEnabled) {

  2208     return NS_ERROR_NOT_AVAILABLE;

  2209   }

  2211   if (loadContext && loadContext->UsePrivateBrowsing()) {

  2212     // Don't want to do anything in PB mode

  2213     return NS_OK;

  2214   }

  2216   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  2217     return NS_ERROR_INVALID_ARG;

  2218   }

  2220   switch (reason) {

  2221   case nsINetworkSeer::LEARN_LOAD_TOPLEVEL:

  2222   case nsINetworkSeer::LEARN_STARTUP:

  2223     if (!targetURI || sourceURI) {

  2224       return NS_ERROR_INVALID_ARG;

  2225     }

  2226     break;

  2227   case nsINetworkSeer::LEARN_LOAD_REDIRECT:

  2228   case nsINetworkSeer::LEARN_LOAD_SUBRESOURCE:

  2229     if (!targetURI || !sourceURI) {

  2230       return NS_ERROR_INVALID_ARG;

  2231     }

  2232     break;

  2233   default:

  2234     return NS_ERROR_INVALID_ARG;

  2235   }

  2237   ++mAccumulators->mLearnAttempts;

  2238   nsresult rv = ReserveSpaceInQueue();

  2239   if (NS_FAILED(rv)) {

  2240     ++mAccumulators->mLearnFullQueue;

  2241     return NS_ERROR_NOT_AVAILABLE;

  2242   }

  2244   nsRefPtr<SeerLearnEvent> event = new SeerLearnEvent(targetURI, sourceURI,

  2245                                                       reason);

  2246   return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);

  2247 }

  2249 // Runnable to clear out the database. Dispatched from the main thread to the

  2250 // seer thread

  2251 class SeerResetEvent : public nsRunnable

  2252 {

  2253 public:

  2254   SeerResetEvent()

  2255   { }

  2257   NS_IMETHOD Run() MOZ_OVERRIDE

  2258   {

  2259     MOZ_ASSERT(!NS_IsMainThread(), "Running reset on main thread");

  2261     gSeer->ResetInternal();

  2263     return NS_OK;

  2264   }

  2265 };

  2267 // Helper that actually does the database wipe.

  2268 void

  2269 Seer::ResetInternal()

  2270 {

  2271   MOZ_ASSERT(!NS_IsMainThread(), "Resetting db on main thread");

  2273   nsresult rv = EnsureInitStorage();

  2274   RETURN_IF_FAILED(rv);

  2276   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_redirects;"));

  2277   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages;"));

  2278   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startups;"));

  2280   // These cascade to moz_subresources and moz_subhosts, respectively.

  2281   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages;"));

  2282   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_hosts;"));

  2284   VacuumDatabase();

  2286   // Go ahead and ensure this is flushed to disk

  2287   CommitTransaction();

  2288   BeginTransaction();

  2289 }

  2291 // Called on the main thread to clear out all our knowledge. Tabula Rasa FTW!

  2292 NS_IMETHODIMP

  2293 Seer::Reset()

  2294 {

  2295   MOZ_ASSERT(NS_IsMainThread(),

  2296              "Seer interface methods must be called on the main thread");

  2298   if (!mInitialized) {

  2299     return NS_ERROR_NOT_AVAILABLE;

  2300   }

  2302   if (!mEnabled) {

  2303     return NS_OK;

  2304   }

  2306   nsRefPtr<SeerResetEvent> event = new SeerResetEvent();

  2307   return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);

  2308 }

  2310 class SeerCleanupEvent : public nsRunnable

  2311 {

  2312 public:

  2313   NS_IMETHOD Run() MOZ_OVERRIDE

  2314   {

  2315     gSeer->Cleanup();

  2316     gSeer->mCleanupScheduled = false;

  2317     return NS_OK;

  2318   }

  2319 };

  2321 // Returns the current size (in bytes) of the db file on disk

  2322 int64_t

  2323 Seer::GetDBFileSize()

  2324 {

  2325   MOZ_ASSERT(!NS_IsMainThread(), "GetDBFileSize called on main thread!");

  2327   nsresult rv = EnsureInitStorage();

  2328   if (NS_FAILED(rv)) {

  2329     SEER_LOG(("GetDBFileSize called without db available!"));

  2330     return 0;

  2331   }

  2333   CommitTransaction();

  2335   nsCOMPtr<mozIStorageStatement> countStmt = mStatements.GetCachedStatement(

  2336       NS_LITERAL_CSTRING("PRAGMA page_count;"));

  2337   if (!countStmt) {

  2338     return 0;

  2339   }

  2340   mozStorageStatementScoper scopedCount(countStmt);

  2341   bool hasRows;

  2342   rv = countStmt->ExecuteStep(&hasRows);

  2343   if (NS_FAILED(rv) || !hasRows) {

  2344     return 0;

  2345   }

  2346   int64_t pageCount;

  2347   rv = countStmt->GetInt64(0, &pageCount);

  2348   if (NS_FAILED(rv)) {

  2349     return 0;

  2350   }

  2352   nsCOMPtr<mozIStorageStatement> sizeStmt = mStatements.GetCachedStatement(

  2353       NS_LITERAL_CSTRING("PRAGMA page_size;"));

  2354   if (!sizeStmt) {

  2355     return 0;

  2356   }

  2357   mozStorageStatementScoper scopedSize(sizeStmt);

  2358   rv = sizeStmt->ExecuteStep(&hasRows);

  2359   if (NS_FAILED(rv) || !hasRows) {

  2360     return 0;

  2361   }

  2362   int64_t pageSize;

  2363   rv = sizeStmt->GetInt64(0, &pageSize);

  2364   if (NS_FAILED(rv)) {

  2365     return 0;

  2366   }

  2368   BeginTransaction();

  2370   return pageCount * pageSize;

  2371 }

  2373 // Returns the size (in bytes) that the db file will consume on disk AFTER we

  2374 // vacuum the db.

  2375 int64_t

  2376 Seer::GetDBFileSizeAfterVacuum()

  2377 {

  2378   MOZ_ASSERT(!NS_IsMainThread(), "GetDBFileSizeAfterVacuum called on main thread!");

  2380   CommitTransaction();

  2382   nsCOMPtr<mozIStorageStatement> countStmt = mStatements.GetCachedStatement(

  2383       NS_LITERAL_CSTRING("PRAGMA page_count;"));

  2384   if (!countStmt) {

  2385     return 0;

  2386   }

  2387   mozStorageStatementScoper scopedCount(countStmt);

  2388   bool hasRows;

  2389   nsresult rv = countStmt->ExecuteStep(&hasRows);

  2390   if (NS_FAILED(rv) || !hasRows) {

  2391     return 0;

  2392   }

  2393   int64_t pageCount;

  2394   rv = countStmt->GetInt64(0, &pageCount);

  2395   if (NS_FAILED(rv)) {

  2396     return 0;

  2397   }

  2399   nsCOMPtr<mozIStorageStatement> sizeStmt = mStatements.GetCachedStatement(

  2400       NS_LITERAL_CSTRING("PRAGMA page_size;"));

  2401   if (!sizeStmt) {

  2402     return 0;

  2403   }

  2404   mozStorageStatementScoper scopedSize(sizeStmt);

  2405   rv = sizeStmt->ExecuteStep(&hasRows);

  2406   if (NS_FAILED(rv) || !hasRows) {

  2407     return 0;

  2408   }

  2409   int64_t pageSize;

  2410   rv = sizeStmt->GetInt64(0, &pageSize);

  2411   if (NS_FAILED(rv)) {

  2412     return 0;

  2413   }

  2415   nsCOMPtr<mozIStorageStatement> freeStmt = mStatements.GetCachedStatement(

  2416       NS_LITERAL_CSTRING("PRAGMA freelist_count;"));

  2417   if (!freeStmt) {

  2418     return 0;

  2419   }

  2420   mozStorageStatementScoper scopedFree(freeStmt);

  2421   rv = freeStmt->ExecuteStep(&hasRows);

  2422   if (NS_FAILED(rv) || !hasRows) {

  2423     return 0;

  2424   }

  2425   int64_t freelistCount;

  2426   rv = freeStmt->GetInt64(0, &freelistCount);

  2427   if (NS_FAILED(rv)) {

  2428     return 0;

  2429   }

  2431   BeginTransaction();

  2433   return (pageCount - freelistCount) * pageSize;

  2434 }

  2436 void

  2437 Seer::MaybeScheduleCleanup()

  2438 {

  2439   MOZ_ASSERT(!NS_IsMainThread(), "MaybeScheduleCleanup called on main thread!");

  2441   // This is a little racy, but it's a nice little shutdown optimization if the

  2442   // race works out the right way.

  2443   if (!mInitialized) {

  2444     return;

  2445   }

  2447   if (mCleanupScheduled) {

  2448     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, false);

  2449     return;

  2450   }

  2452   int64_t dbFileSize = GetDBFileSize();

  2453   if (dbFileSize < mMaxDBSize) {

  2454     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, false);

  2455     return;

  2456   }

  2458   mCleanupScheduled = true;

  2460   nsRefPtr<SeerCleanupEvent> event = new SeerCleanupEvent();

  2461   nsresult rv = mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);

  2462   if (NS_FAILED(rv)) {

  2463     mCleanupScheduled = false;

  2464     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, false);

  2465   } else {

  2466     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, true);

  2467   }

  2468 }

  2470 #ifndef ANDROID

  2471 static const long long CLEANUP_CUTOFF = ONE_MONTH;

  2472 #else

  2473 static const long long CLEANUP_CUTOFF = ONE_WEEK;

  2474 #endif

  2476 void

  2477 Seer::CleanupOrigins(PRTime now)

  2478 {

  2479   PRTime cutoff = now - CLEANUP_CUTOFF;

  2481   nsCOMPtr<mozIStorageStatement> deleteOrigins = mStatements.GetCachedStatement(

  2482       NS_LITERAL_CSTRING("DELETE FROM moz_hosts WHERE last_load <= :cutoff"));

  2483   if (!deleteOrigins) {

  2484     return;

  2485   }

  2486   mozStorageStatementScoper scopedOrigins(deleteOrigins);

  2488   nsresult rv = deleteOrigins->BindInt32ByName(NS_LITERAL_CSTRING("cutoff"),

  2489                                                cutoff);

  2490   RETURN_IF_FAILED(rv);

  2492   deleteOrigins->Execute();

  2493 }

  2495 void

  2496 Seer::CleanupStartupPages(PRTime now)

  2497 {

  2498   PRTime cutoff = now - ONE_WEEK;

  2500   nsCOMPtr<mozIStorageStatement> deletePages = mStatements.GetCachedStatement(

  2501       NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages WHERE "

  2502                          "last_hit <= :cutoff"));

  2503   if (!deletePages) {

  2504     return;

  2505   }

  2506   mozStorageStatementScoper scopedPages(deletePages);

  2508   nsresult rv = deletePages->BindInt32ByName(NS_LITERAL_CSTRING("cutoff"),

  2509                                              cutoff);

  2510   RETURN_IF_FAILED(rv);

  2512   deletePages->Execute();

  2513 }

  2515 int32_t

  2516 Seer::GetSubresourceCount()

  2517 {

  2518   nsCOMPtr<mozIStorageStatement> count = mStatements.GetCachedStatement(

  2519       NS_LITERAL_CSTRING("SELECT COUNT(id) FROM moz_subresources"));

  2520   if (!count) {

  2521     return 0;

  2522   }

  2523   mozStorageStatementScoper scopedCount(count);

  2525   bool hasRows;

  2526   nsresult rv = count->ExecuteStep(&hasRows);

  2527   if (NS_FAILED(rv) || !hasRows) {

  2528     return 0;

  2529   }

  2531   int32_t subresourceCount = 0;

  2532   count->GetInt32(0, &subresourceCount);

  2534   return subresourceCount;

  2535 }

  2537 void

  2538 Seer::Cleanup()

  2539 {

  2540   MOZ_ASSERT(!NS_IsMainThread(), "Seer::Cleanup called on main thread!");

  2542   nsresult rv = EnsureInitStorage();

  2543   if (NS_FAILED(rv)) {

  2544     return;

  2545   }

  2547   int64_t dbFileSize = GetDBFileSize();

  2548   float preservePercentage = static_cast<float>(mPreservePercentage) / 100.0;

  2549   int64_t evictionCutoff = static_cast<int64_t>(mMaxDBSize) * preservePercentage;

  2550   if (dbFileSize < evictionCutoff) {

  2551     return;

  2552   }

  2554   CommitTransaction();

  2555   BeginTransaction();

  2557   PRTime now = PR_Now();

  2558   if (mLastCleanupTime) {

  2559     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_DELTA,

  2560                           (now - mLastCleanupTime) / 1000);

  2561   }

  2562   mLastCleanupTime = now;

  2564   CleanupOrigins(now);

  2565   CleanupStartupPages(now);

  2567   dbFileSize = GetDBFileSizeAfterVacuum();

  2568   if (dbFileSize < evictionCutoff) {

  2569     // We've deleted enough stuff, time to free up the disk space and be on

  2570     // our way.

  2571     VacuumDatabase();

  2572     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SUCCEEDED, true);

  2573     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_TIME,

  2574                           (PR_Now() - mLastCleanupTime) / 1000);

  2575     return;

  2576   }

  2578   bool canDelete = true;

  2579   while (canDelete && (dbFileSize >= evictionCutoff)) {

  2580     int32_t subresourceCount = GetSubresourceCount();

  2581     if (!subresourceCount) {

  2582       canDelete = false;

  2583       break;

  2584     }

  2586     // DB size scales pretty much linearly with the number of rows in

  2587     // moz_subresources, so we can guess how many rows we need to delete pretty

  2588     // accurately.

  2589     float percentNeeded = static_cast<float>(dbFileSize - evictionCutoff) /

  2590       static_cast<float>(dbFileSize);

  2592     int32_t subresourcesToDelete = static_cast<int32_t>(percentNeeded * subresourceCount);

  2593     if (!subresourcesToDelete) {

  2594       // We're getting pretty close to nothing here, anyway, so we may as well

  2595       // just trash it all. This delete cascades to moz_subresources, as well.

  2596       rv = mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages;"));

  2597       if (NS_FAILED(rv)) {

  2598         canDelete = false;

  2599         break;

  2600       }

  2601     } else {

  2602       nsCOMPtr<mozIStorageStatement> deleteStatement = mStatements.GetCachedStatement(

  2603           NS_LITERAL_CSTRING("DELETE FROM moz_subresources WHERE id IN "

  2604                             "(SELECT id FROM moz_subresources ORDER BY "

  2605                             "last_hit ASC LIMIT :limit);"));

  2606       if (!deleteStatement) {

  2607         canDelete = false;

  2608         break;

  2609       }

  2610       mozStorageStatementScoper scopedDelete(deleteStatement);

  2612       rv = deleteStatement->BindInt32ByName(NS_LITERAL_CSTRING("limit"),

  2613                                             subresourcesToDelete);

  2614       if (NS_FAILED(rv)) {

  2615         canDelete = false;

  2616         break;

  2617       }

  2619       rv = deleteStatement->Execute();

  2620       if (NS_FAILED(rv)) {

  2621         canDelete = false;

  2622         break;

  2623       }

  2625       // Now we clean up pages that no longer reference any subresources

  2626       rv = mDB->ExecuteSimpleSQL(

  2627           NS_LITERAL_CSTRING("DELETE FROM moz_pages WHERE id NOT IN "

  2628                              "(SELECT DISTINCT(pid) FROM moz_subresources);"));

  2629       if (NS_FAILED(rv)) {

  2630         canDelete = false;

  2631         break;

  2632       }

  2633     }

  2635     if (canDelete) {

  2636       dbFileSize = GetDBFileSizeAfterVacuum();

  2637     }

  2638   }

  2640   if (!canDelete || (dbFileSize >= evictionCutoff)) {

  2641     // Last-ditch effort to free up space

  2642     ResetInternal();

  2643     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SUCCEEDED, false);

  2644   } else {

  2645     // We do this to actually free up the space on disk

  2646     VacuumDatabase();

  2647     Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SUCCEEDED, true);

  2648   }

  2649   Telemetry::Accumulate(Telemetry::SEER_CLEANUP_TIME,

  2650                         (PR_Now() - mLastCleanupTime) / 1000);

  2651 }

  2653 void

  2654 Seer::VacuumDatabase()

  2655 {

  2656   MOZ_ASSERT(!NS_IsMainThread(), "VacuumDatabase called on main thread!");

  2658   CommitTransaction();

  2659   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM;"));

  2660   BeginTransaction();

  2661 }

  2663 #ifdef SEER_TESTS

  2664 class SeerPrepareForDnsTestEvent : public nsRunnable

  2665 {

  2666 public:

  2667   SeerPrepareForDnsTestEvent(int64_t timestamp, const char *uri)

  2668     :mTimestamp(timestamp)

  2669     ,mUri(uri)

  2670   { }

  2672   NS_IMETHOD Run() MOZ_OVERRIDE

  2673   {

  2674     MOZ_ASSERT(!NS_IsMainThread(), "Preparing for DNS Test on main thread!");

  2675     gSeer->PrepareForDnsTestInternal(mTimestamp, mUri);

  2676     return NS_OK;

  2677   }

  2679 private:

  2680   int64_t mTimestamp;

  2681   nsAutoCString mUri;

  2682 };

  2684 void

  2685 Seer::PrepareForDnsTestInternal(int64_t timestamp, const nsACString &uri)

  2686 {

  2687   nsCOMPtr<mozIStorageStatement> update = mStatements.GetCachedStatement(

  2688       NS_LITERAL_CSTRING("UPDATE moz_subresources SET last_hit = :timestamp, "

  2689                           "hits = 2 WHERE uri = :uri;"));

  2690   if (!update) {

  2691     return;

  2692   }

  2693   mozStorageStatementScoper scopedUpdate(update);

  2695   nsresult rv = update->BindInt64ByName(NS_LITERAL_CSTRING("timestamp"),

  2696                                         timestamp);

  2697   RETURN_IF_FAILED(rv);

  2699   rv = update->BindUTF8StringByName(NS_LITERAL_CSTRING("uri"), uri);

  2700   RETURN_IF_FAILED(rv);

  2702   update->Execute();

  2703 }

  2704 #endif

  2706 NS_IMETHODIMP

  2707 Seer::PrepareForDnsTest(int64_t timestamp, const char *uri)

  2708 {

  2709 #ifdef SEER_TESTS

  2710   MOZ_ASSERT(NS_IsMainThread(),

  2711              "Seer interface methods must be called on the main thread");

  2713   if (!mInitialized) {

  2714     return NS_ERROR_NOT_AVAILABLE;

  2715   }

  2717   nsRefPtr<SeerPrepareForDnsTestEvent> event =

  2718     new SeerPrepareForDnsTestEvent(timestamp, uri);

  2719   return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);

  2720 #else

  2721   return NS_ERROR_NOT_IMPLEMENTED;

  2722 #endif

  2723 }

  2725 // Helper functions to make using the seer easier from native code

  2727 static nsresult

  2728 EnsureGlobalSeer(nsINetworkSeer **aSeer)

  2729 {

  2730   nsresult rv;

  2731   nsCOMPtr<nsINetworkSeer> seer = do_GetService("@mozilla.org/network/seer;1",

  2732                                                 &rv);

  2733   NS_ENSURE_SUCCESS(rv, rv);

  2735   NS_IF_ADDREF(*aSeer = seer);

  2736   return NS_OK;

  2737 }

  2739 nsresult

  2740 SeerPredict(nsIURI *targetURI, nsIURI *sourceURI, SeerPredictReason reason,

  2741             nsILoadContext *loadContext, nsINetworkSeerVerifier *verifier)

  2742 {

  2743   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  2744     return NS_OK;

  2745   }

  2747   nsCOMPtr<nsINetworkSeer> seer;

  2748   nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));

  2749   NS_ENSURE_SUCCESS(rv, rv);

  2751   return seer->Predict(targetURI, sourceURI, reason, loadContext, verifier);

  2752 }

  2754 nsresult

  2755 SeerLearn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,

  2756           nsILoadContext *loadContext)

  2757 {

  2758   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  2759     return NS_OK;

  2760   }

  2762   nsCOMPtr<nsINetworkSeer> seer;

  2763   nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));

  2764   NS_ENSURE_SUCCESS(rv, rv);

  2766   return seer->Learn(targetURI, sourceURI, reason, loadContext);

  2767 }

  2769 nsresult

  2770 SeerLearn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,

  2771           nsILoadGroup *loadGroup)

  2772 {

  2773   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  2774     return NS_OK;

  2775   }

  2777   nsCOMPtr<nsINetworkSeer> seer;

  2778   nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));

  2779   NS_ENSURE_SUCCESS(rv, rv);

  2781   nsCOMPtr<nsILoadContext> loadContext;

  2783   if (loadGroup) {

  2784     nsCOMPtr<nsIInterfaceRequestor> callbacks;

  2785     loadGroup->GetNotificationCallbacks(getter_AddRefs(callbacks));

  2786     if (callbacks) {

  2787       loadContext = do_GetInterface(callbacks);

  2788     }

  2789   }

  2791   return seer->Learn(targetURI, sourceURI, reason, loadContext);

  2792 }

  2794 nsresult

  2795 SeerLearn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,

  2796           nsIDocument *document)

  2797 {

  2798   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  2799     return NS_OK;

  2800   }

  2802   nsCOMPtr<nsINetworkSeer> seer;

  2803   nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));

  2804   NS_ENSURE_SUCCESS(rv, rv);

  2806   nsCOMPtr<nsILoadContext> loadContext;

  2808   if (document) {

  2809     loadContext = document->GetLoadContext();

  2810   }

  2812   return seer->Learn(targetURI, sourceURI, reason, loadContext);

  2813 }

  2815 nsresult

  2816 SeerLearnRedirect(nsIURI *targetURI, nsIChannel *channel,

  2817                   nsILoadContext *loadContext)

  2818 {

  2819   nsCOMPtr<nsIURI> sourceURI;

  2820   nsresult rv = channel->GetOriginalURI(getter_AddRefs(sourceURI));

  2821   NS_ENSURE_SUCCESS(rv, rv);

  2823   bool sameUri;

  2824   rv = targetURI->Equals(sourceURI, &sameUri);

  2825   NS_ENSURE_SUCCESS(rv, rv);

  2827   if (sameUri) {

  2828     return NS_OK;

  2829   }

  2831   if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {

  2832     return NS_OK;

  2833   }

  2835   nsCOMPtr<nsINetworkSeer> seer;

  2836   rv = EnsureGlobalSeer(getter_AddRefs(seer));

  2837   NS_ENSURE_SUCCESS(rv, rv);

  2839   return seer->Learn(targetURI, sourceURI,

  2840                      nsINetworkSeer::LEARN_LOAD_REDIRECT, loadContext);

  2841 }

  2843 } // ::mozilla::net

  2844 } // ::mozilla

The Tor Browser / file revision

netwerk/base/src/Seer.cpp@b8a032363ba2

netwerk/base/src/Seer.cpp