1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/netwerk/base/src/Seer.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,2844 @@
1.4 +/* vim: set ts=2 sts=2 et sw=2: */
1.5 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.6 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.8 +
1.9 +#include <algorithm>
1.10 +
1.11 +#include "Seer.h"
1.12 +
1.13 +#include "nsAppDirectoryServiceDefs.h"
1.14 +#include "nsICancelable.h"
1.15 +#include "nsIChannel.h"
1.16 +#include "nsIDNSListener.h"
1.17 +#include "nsIDNSService.h"
1.18 +#include "nsIDocument.h"
1.19 +#include "nsIFile.h"
1.20 +#include "nsILoadContext.h"
1.21 +#include "nsILoadGroup.h"
1.22 +#include "nsINetworkSeerVerifier.h"
1.23 +#include "nsIObserverService.h"
1.24 +#include "nsIPrefBranch.h"
1.25 +#include "nsIPrefService.h"
1.26 +#include "nsISpeculativeConnect.h"
1.27 +#include "nsITimer.h"
1.28 +#include "nsIURI.h"
1.29 +#include "nsNetUtil.h"
1.30 +#include "nsServiceManagerUtils.h"
1.31 +#include "nsTArray.h"
1.32 +#include "nsThreadUtils.h"
1.33 +#ifdef MOZ_NUWA_PROCESS
1.34 +#include "ipc/Nuwa.h"
1.35 +#endif
1.36 +#include "prlog.h"
1.37 +
1.38 +#include "mozIStorageConnection.h"
1.39 +#include "mozIStorageService.h"
1.40 +#include "mozIStorageStatement.h"
1.41 +#include "mozStorageHelper.h"
1.42 +
1.43 +#include "mozilla/Preferences.h"
1.44 +#include "mozilla/storage.h"
1.45 +#include "mozilla/Telemetry.h"
1.46 +
1.47 +#if defined(ANDROID) && !defined(MOZ_WIDGET_GONK)
1.48 +#include "nsIPropertyBag2.h"
1.49 +static const int32_t ANDROID_23_VERSION = 10;
1.50 +#endif
1.51 +
1.52 +using namespace mozilla;
1.53 +
1.54 +namespace mozilla {
1.55 +namespace net {
1.56 +
1.57 +#define RETURN_IF_FAILED(_rv) \
1.58 + do { \
1.59 + if (NS_FAILED(_rv)) { \
1.60 + return; \
1.61 + } \
1.62 + } while (0)
1.63 +
1.64 +const char SEER_ENABLED_PREF[] = "network.seer.enabled";
1.65 +const char SEER_SSL_HOVER_PREF[] = "network.seer.enable-hover-on-ssl";
1.66 +
1.67 +const char SEER_PAGE_DELTA_DAY_PREF[] = "network.seer.page-degradation.day";
1.68 +const int SEER_PAGE_DELTA_DAY_DEFAULT = 0;
1.69 +const char SEER_PAGE_DELTA_WEEK_PREF[] = "network.seer.page-degradation.week";
1.70 +const int SEER_PAGE_DELTA_WEEK_DEFAULT = 5;
1.71 +const char SEER_PAGE_DELTA_MONTH_PREF[] = "network.seer.page-degradation.month";
1.72 +const int SEER_PAGE_DELTA_MONTH_DEFAULT = 10;
1.73 +const char SEER_PAGE_DELTA_YEAR_PREF[] = "network.seer.page-degradation.year";
1.74 +const int SEER_PAGE_DELTA_YEAR_DEFAULT = 25;
1.75 +const char SEER_PAGE_DELTA_MAX_PREF[] = "network.seer.page-degradation.max";
1.76 +const int SEER_PAGE_DELTA_MAX_DEFAULT = 50;
1.77 +const char SEER_SUB_DELTA_DAY_PREF[] =
1.78 + "network.seer.subresource-degradation.day";
1.79 +const int SEER_SUB_DELTA_DAY_DEFAULT = 1;
1.80 +const char SEER_SUB_DELTA_WEEK_PREF[] =
1.81 + "network.seer.subresource-degradation.week";
1.82 +const int SEER_SUB_DELTA_WEEK_DEFAULT = 10;
1.83 +const char SEER_SUB_DELTA_MONTH_PREF[] =
1.84 + "network.seer.subresource-degradation.month";
1.85 +const int SEER_SUB_DELTA_MONTH_DEFAULT = 25;
1.86 +const char SEER_SUB_DELTA_YEAR_PREF[] =
1.87 + "network.seer.subresource-degradation.year";
1.88 +const int SEER_SUB_DELTA_YEAR_DEFAULT = 50;
1.89 +const char SEER_SUB_DELTA_MAX_PREF[] =
1.90 + "network.seer.subresource-degradation.max";
1.91 +const int SEER_SUB_DELTA_MAX_DEFAULT = 100;
1.92 +
1.93 +const char SEER_PRECONNECT_MIN_PREF[] =
1.94 + "network.seer.preconnect-min-confidence";
1.95 +const int PRECONNECT_MIN_DEFAULT = 90;
1.96 +const char SEER_PRERESOLVE_MIN_PREF[] =
1.97 + "network.seer.preresolve-min-confidence";
1.98 +const int PRERESOLVE_MIN_DEFAULT = 60;
1.99 +const char SEER_REDIRECT_LIKELY_PREF[] =
1.100 + "network.seer.redirect-likely-confidence";
1.101 +const int REDIRECT_LIKELY_DEFAULT = 75;
1.102 +
1.103 +const char SEER_MAX_QUEUE_SIZE_PREF[] = "network.seer.max-queue-size";
1.104 +const uint32_t SEER_MAX_QUEUE_SIZE_DEFAULT = 50;
1.105 +
1.106 +const char SEER_MAX_DB_SIZE_PREF[] = "network.seer.max-db-size";
1.107 +const int32_t SEER_MAX_DB_SIZE_DEFAULT_BYTES = 150 * 1024 * 1024;
1.108 +const char SEER_PRESERVE_PERCENTAGE_PREF[] = "network.seer.preserve";
1.109 +const int32_t SEER_PRESERVE_PERCENTAGE_DEFAULT = 80;
1.110 +
1.111 +// All these time values are in usec
1.112 +const long long ONE_DAY = 86400LL * 1000000LL;
1.113 +const long long ONE_WEEK = 7LL * ONE_DAY;
1.114 +const long long ONE_MONTH = 30LL * ONE_DAY;
1.115 +const long long ONE_YEAR = 365LL * ONE_DAY;
1.116 +
1.117 +const long STARTUP_WINDOW = 5L * 60L * 1000000L; // 5min
1.118 +
1.119 +// Version for the database schema
1.120 +static const int32_t SEER_SCHEMA_VERSION = 1;
1.121 +
1.122 +struct SeerTelemetryAccumulators {
1.123 + Telemetry::AutoCounter<Telemetry::SEER_PREDICT_ATTEMPTS> mPredictAttempts;
1.124 + Telemetry::AutoCounter<Telemetry::SEER_LEARN_ATTEMPTS> mLearnAttempts;
1.125 + Telemetry::AutoCounter<Telemetry::SEER_PREDICT_FULL_QUEUE> mPredictFullQueue;
1.126 + Telemetry::AutoCounter<Telemetry::SEER_LEARN_FULL_QUEUE> mLearnFullQueue;
1.127 + Telemetry::AutoCounter<Telemetry::SEER_TOTAL_PREDICTIONS> mTotalPredictions;
1.128 + Telemetry::AutoCounter<Telemetry::SEER_TOTAL_PRECONNECTS> mTotalPreconnects;
1.129 + Telemetry::AutoCounter<Telemetry::SEER_TOTAL_PRERESOLVES> mTotalPreresolves;
1.130 + Telemetry::AutoCounter<Telemetry::SEER_PREDICTIONS_CALCULATED> mPredictionsCalculated;
1.131 + Telemetry::AutoCounter<Telemetry::SEER_LOAD_COUNT_IS_ZERO> mLoadCountZeroes;
1.132 + Telemetry::AutoCounter<Telemetry::SEER_LOAD_COUNT_OVERFLOWS> mLoadCountOverflows;
1.133 + Telemetry::AutoCounter<Telemetry::SEER_STARTUP_COUNT_IS_ZERO> mStartupCountZeroes;
1.134 + Telemetry::AutoCounter<Telemetry::SEER_STARTUP_COUNT_OVERFLOWS> mStartupCountOverflows;
1.135 +};
1.136 +
1.137 +// Listener for the speculative DNS requests we'll fire off, which just ignores
1.138 +// the result (since we're just trying to warm the cache). This also exists to
1.139 +// reduce round-trips to the main thread, by being something threadsafe the Seer
1.140 +// can use.
1.141 +
1.142 +class SeerDNSListener : public nsIDNSListener
1.143 +{
1.144 +public:
1.145 + NS_DECL_THREADSAFE_ISUPPORTS
1.146 + NS_DECL_NSIDNSLISTENER
1.147 +
1.148 + SeerDNSListener()
1.149 + { }
1.150 +
1.151 + virtual ~SeerDNSListener()
1.152 + { }
1.153 +};
1.154 +
1.155 +NS_IMPL_ISUPPORTS(SeerDNSListener, nsIDNSListener);
1.156 +
1.157 +NS_IMETHODIMP
1.158 +SeerDNSListener::OnLookupComplete(nsICancelable *request,
1.159 + nsIDNSRecord *rec,
1.160 + nsresult status)
1.161 +{
1.162 + return NS_OK;
1.163 +}
1.164 +
1.165 +// Are you ready for the fun part? Because here comes the fun part. The seer,
1.166 +// which will do awesome stuff as you browse to make your browsing experience
1.167 +// faster.
1.168 +
1.169 +static Seer *gSeer = nullptr;
1.170 +
1.171 +#if defined(PR_LOGGING)
1.172 +static PRLogModuleInfo *gSeerLog = nullptr;
1.173 +#define SEER_LOG(args) PR_LOG(gSeerLog, 4, args)
1.174 +#else
1.175 +#define SEER_LOG(args)
1.176 +#endif
1.177 +
1.178 +NS_IMPL_ISUPPORTS(Seer,
1.179 + nsINetworkSeer,
1.180 + nsIObserver,
1.181 + nsISpeculativeConnectionOverrider,
1.182 + nsIInterfaceRequestor)
1.183 +
1.184 +Seer::Seer()
1.185 + :mInitialized(false)
1.186 + ,mEnabled(true)
1.187 + ,mEnableHoverOnSSL(false)
1.188 + ,mPageDegradationDay(SEER_PAGE_DELTA_DAY_DEFAULT)
1.189 + ,mPageDegradationWeek(SEER_PAGE_DELTA_WEEK_DEFAULT)
1.190 + ,mPageDegradationMonth(SEER_PAGE_DELTA_MONTH_DEFAULT)
1.191 + ,mPageDegradationYear(SEER_PAGE_DELTA_YEAR_DEFAULT)
1.192 + ,mPageDegradationMax(SEER_PAGE_DELTA_MAX_DEFAULT)
1.193 + ,mSubresourceDegradationDay(SEER_SUB_DELTA_DAY_DEFAULT)
1.194 + ,mSubresourceDegradationWeek(SEER_SUB_DELTA_WEEK_DEFAULT)
1.195 + ,mSubresourceDegradationMonth(SEER_SUB_DELTA_MONTH_DEFAULT)
1.196 + ,mSubresourceDegradationYear(SEER_SUB_DELTA_YEAR_DEFAULT)
1.197 + ,mSubresourceDegradationMax(SEER_SUB_DELTA_MAX_DEFAULT)
1.198 + ,mPreconnectMinConfidence(PRECONNECT_MIN_DEFAULT)
1.199 + ,mPreresolveMinConfidence(PRERESOLVE_MIN_DEFAULT)
1.200 + ,mRedirectLikelyConfidence(REDIRECT_LIKELY_DEFAULT)
1.201 + ,mMaxQueueSize(SEER_MAX_QUEUE_SIZE_DEFAULT)
1.202 + ,mStatements(mDB)
1.203 + ,mLastStartupTime(0)
1.204 + ,mStartupCount(0)
1.205 + ,mQueueSize(0)
1.206 + ,mQueueSizeLock("Seer.mQueueSizeLock")
1.207 + ,mCleanupScheduled(false)
1.208 + ,mMaxDBSize(SEER_MAX_DB_SIZE_DEFAULT_BYTES)
1.209 + ,mPreservePercentage(SEER_PRESERVE_PERCENTAGE_DEFAULT)
1.210 + ,mLastCleanupTime(0)
1.211 +{
1.212 +#if defined(PR_LOGGING)
1.213 + gSeerLog = PR_NewLogModule("NetworkSeer");
1.214 +#endif
1.215 +
1.216 + MOZ_ASSERT(!gSeer, "multiple Seer instances!");
1.217 + gSeer = this;
1.218 +}
1.219 +
1.220 +Seer::~Seer()
1.221 +{
1.222 + if (mInitialized)
1.223 + Shutdown();
1.224 +
1.225 + RemoveObserver();
1.226 +
1.227 + gSeer = nullptr;
1.228 +}
1.229 +
1.230 +// Seer::nsIObserver
1.231 +
1.232 +nsresult
1.233 +Seer::InstallObserver()
1.234 +{
1.235 + MOZ_ASSERT(NS_IsMainThread(), "Installing observer off main thread");
1.236 +
1.237 + nsresult rv = NS_OK;
1.238 + nsCOMPtr<nsIObserverService> obs =
1.239 + mozilla::services::GetObserverService();
1.240 + if (!obs) {
1.241 + return NS_ERROR_NOT_AVAILABLE;
1.242 + }
1.243 +
1.244 + rv = obs->AddObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false);
1.245 + NS_ENSURE_SUCCESS(rv, rv);
1.246 +
1.247 + nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
1.248 + if (!prefs) {
1.249 + return NS_ERROR_NOT_AVAILABLE;
1.250 + }
1.251 +
1.252 + Preferences::AddBoolVarCache(&mEnabled, SEER_ENABLED_PREF, true);
1.253 + Preferences::AddBoolVarCache(&mEnableHoverOnSSL, SEER_SSL_HOVER_PREF, false);
1.254 + Preferences::AddIntVarCache(&mPageDegradationDay, SEER_PAGE_DELTA_DAY_PREF,
1.255 + SEER_PAGE_DELTA_DAY_DEFAULT);
1.256 + Preferences::AddIntVarCache(&mPageDegradationWeek, SEER_PAGE_DELTA_WEEK_PREF,
1.257 + SEER_PAGE_DELTA_WEEK_DEFAULT);
1.258 + Preferences::AddIntVarCache(&mPageDegradationMonth,
1.259 + SEER_PAGE_DELTA_MONTH_PREF,
1.260 + SEER_PAGE_DELTA_MONTH_DEFAULT);
1.261 + Preferences::AddIntVarCache(&mPageDegradationYear, SEER_PAGE_DELTA_YEAR_PREF,
1.262 + SEER_PAGE_DELTA_YEAR_DEFAULT);
1.263 + Preferences::AddIntVarCache(&mPageDegradationMax, SEER_PAGE_DELTA_MAX_PREF,
1.264 + SEER_PAGE_DELTA_MAX_DEFAULT);
1.265 +
1.266 + Preferences::AddIntVarCache(&mSubresourceDegradationDay,
1.267 + SEER_SUB_DELTA_DAY_PREF,
1.268 + SEER_SUB_DELTA_DAY_DEFAULT);
1.269 + Preferences::AddIntVarCache(&mSubresourceDegradationWeek,
1.270 + SEER_SUB_DELTA_WEEK_PREF,
1.271 + SEER_SUB_DELTA_WEEK_DEFAULT);
1.272 + Preferences::AddIntVarCache(&mSubresourceDegradationMonth,
1.273 + SEER_SUB_DELTA_MONTH_PREF,
1.274 + SEER_SUB_DELTA_MONTH_DEFAULT);
1.275 + Preferences::AddIntVarCache(&mSubresourceDegradationYear,
1.276 + SEER_SUB_DELTA_YEAR_PREF,
1.277 + SEER_SUB_DELTA_YEAR_DEFAULT);
1.278 + Preferences::AddIntVarCache(&mSubresourceDegradationMax,
1.279 + SEER_SUB_DELTA_MAX_PREF,
1.280 + SEER_SUB_DELTA_MAX_DEFAULT);
1.281 +
1.282 + Preferences::AddIntVarCache(&mPreconnectMinConfidence,
1.283 + SEER_PRECONNECT_MIN_PREF,
1.284 + PRECONNECT_MIN_DEFAULT);
1.285 + Preferences::AddIntVarCache(&mPreresolveMinConfidence,
1.286 + SEER_PRERESOLVE_MIN_PREF,
1.287 + PRERESOLVE_MIN_DEFAULT);
1.288 + Preferences::AddIntVarCache(&mRedirectLikelyConfidence,
1.289 + SEER_REDIRECT_LIKELY_PREF,
1.290 + REDIRECT_LIKELY_DEFAULT);
1.291 +
1.292 + Preferences::AddIntVarCache(&mMaxQueueSize, SEER_MAX_QUEUE_SIZE_PREF,
1.293 + SEER_MAX_QUEUE_SIZE_DEFAULT);
1.294 +
1.295 + Preferences::AddIntVarCache(&mMaxDBSize, SEER_MAX_DB_SIZE_PREF,
1.296 + SEER_MAX_DB_SIZE_DEFAULT_BYTES);
1.297 + Preferences::AddIntVarCache(&mPreservePercentage,
1.298 + SEER_PRESERVE_PERCENTAGE_PREF,
1.299 + SEER_PRESERVE_PERCENTAGE_DEFAULT);
1.300 +
1.301 + return rv;
1.302 +}
1.303 +
1.304 +void
1.305 +Seer::RemoveObserver()
1.306 +{
1.307 + MOZ_ASSERT(NS_IsMainThread(), "Removing observer off main thread");
1.308 +
1.309 + nsCOMPtr<nsIObserverService> obs =
1.310 + mozilla::services::GetObserverService();
1.311 + if (obs) {
1.312 + obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID);
1.313 + }
1.314 +}
1.315 +
1.316 +static const uint32_t COMMIT_TIMER_DELTA_MS = 5 * 1000;
1.317 +
1.318 +class SeerCommitTimerInitEvent : public nsRunnable
1.319 +{
1.320 +public:
1.321 + NS_IMETHOD Run() MOZ_OVERRIDE
1.322 + {
1.323 + nsresult rv = NS_OK;
1.324 +
1.325 + if (!gSeer->mCommitTimer) {
1.326 + gSeer->mCommitTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv);
1.327 + } else {
1.328 + gSeer->mCommitTimer->Cancel();
1.329 + }
1.330 + if (NS_SUCCEEDED(rv)) {
1.331 + gSeer->mCommitTimer->Init(gSeer, COMMIT_TIMER_DELTA_MS,
1.332 + nsITimer::TYPE_ONE_SHOT);
1.333 + }
1.334 +
1.335 + return NS_OK;
1.336 + }
1.337 +};
1.338 +
1.339 +class SeerNewTransactionEvent : public nsRunnable
1.340 +{
1.341 + NS_IMETHODIMP Run() MOZ_OVERRIDE
1.342 + {
1.343 + gSeer->CommitTransaction();
1.344 + gSeer->BeginTransaction();
1.345 + gSeer->MaybeScheduleCleanup();
1.346 + nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();
1.347 + NS_DispatchToMainThread(event);
1.348 + return NS_OK;
1.349 + }
1.350 +};
1.351 +
1.352 +NS_IMETHODIMP
1.353 +Seer::Observe(nsISupports *subject, const char *topic,
1.354 + const char16_t *data_unicode)
1.355 +{
1.356 + nsresult rv = NS_OK;
1.357 + MOZ_ASSERT(NS_IsMainThread(), "Seer observing something off main thread!");
1.358 +
1.359 + if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) {
1.360 + Shutdown();
1.361 + } else if (!strcmp(NS_TIMER_CALLBACK_TOPIC, topic)) {
1.362 + if (mInitialized) { // Can't access io thread if we're not initialized!
1.363 + nsRefPtr<SeerNewTransactionEvent> event = new SeerNewTransactionEvent();
1.364 + mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
1.365 + }
1.366 + }
1.367 +
1.368 + return rv;
1.369 +}
1.370 +
1.371 +// Seer::nsISpeculativeConnectionOverrider
1.372 +
1.373 +NS_IMETHODIMP
1.374 +Seer::GetIgnoreIdle(bool *ignoreIdle)
1.375 +{
1.376 + *ignoreIdle = true;
1.377 + return NS_OK;
1.378 +}
1.379 +
1.380 +NS_IMETHODIMP
1.381 +Seer::GetIgnorePossibleSpdyConnections(bool *ignorePossibleSpdyConnections)
1.382 +{
1.383 + *ignorePossibleSpdyConnections = true;
1.384 + return NS_OK;
1.385 +}
1.386 +
1.387 +NS_IMETHODIMP
1.388 +Seer::GetParallelSpeculativeConnectLimit(
1.389 + uint32_t *parallelSpeculativeConnectLimit)
1.390 +{
1.391 + *parallelSpeculativeConnectLimit = 6;
1.392 + return NS_OK;
1.393 +}
1.394 +
1.395 +// Seer::nsIInterfaceRequestor
1.396 +
1.397 +NS_IMETHODIMP
1.398 +Seer::GetInterface(const nsIID &iid, void **result)
1.399 +{
1.400 + return QueryInterface(iid, result);
1.401 +}
1.402 +
1.403 +#ifdef MOZ_NUWA_PROCESS
1.404 +class NuwaMarkSeerThreadRunner : public nsRunnable
1.405 +{
1.406 + NS_IMETHODIMP Run() MOZ_OVERRIDE
1.407 + {
1.408 + if (IsNuwaProcess()) {
1.409 + NS_ASSERTION(NuwaMarkCurrentThread != nullptr,
1.410 + "NuwaMarkCurrentThread is undefined!");
1.411 + NuwaMarkCurrentThread(nullptr, nullptr);
1.412 + }
1.413 + return NS_OK;
1.414 + }
1.415 +};
1.416 +#endif
1.417 +
1.418 +// Seer::nsINetworkSeer
1.419 +
1.420 +nsresult
1.421 +Seer::Init()
1.422 +{
1.423 + if (!NS_IsMainThread()) {
1.424 + MOZ_ASSERT(false, "Seer::Init called off the main thread!");
1.425 + return NS_ERROR_UNEXPECTED;
1.426 + }
1.427 +
1.428 + nsresult rv = NS_OK;
1.429 +
1.430 +#if defined(ANDROID) && !defined(MOZ_WIDGET_GONK)
1.431 + // This is an ugly hack to disable the seer on android < 2.3, as it doesn't
1.432 + // play nicely with those android versions, at least on our infra. Causes
1.433 + // timeouts in reftests. See bug 881804 comment 86.
1.434 + nsCOMPtr<nsIPropertyBag2> infoService =
1.435 + do_GetService("@mozilla.org/system-info;1");
1.436 + if (infoService) {
1.437 + int32_t androidVersion = -1;
1.438 + rv = infoService->GetPropertyAsInt32(NS_LITERAL_STRING("version"),
1.439 + &androidVersion);
1.440 + if (NS_SUCCEEDED(rv) && (androidVersion < ANDROID_23_VERSION)) {
1.441 + return NS_ERROR_NOT_AVAILABLE;
1.442 + }
1.443 + }
1.444 +#endif
1.445 +
1.446 + mStartupTime = PR_Now();
1.447 +
1.448 + mAccumulators = new SeerTelemetryAccumulators();
1.449 +
1.450 + rv = InstallObserver();
1.451 + NS_ENSURE_SUCCESS(rv, rv);
1.452 +
1.453 + if (!mDNSListener) {
1.454 + mDNSListener = new SeerDNSListener();
1.455 + }
1.456 +
1.457 + rv = NS_NewNamedThread("Network Seer", getter_AddRefs(mIOThread));
1.458 + NS_ENSURE_SUCCESS(rv, rv);
1.459 +
1.460 +#ifdef MOZ_NUWA_PROCESS
1.461 + nsCOMPtr<nsIRunnable> runner = new NuwaMarkSeerThreadRunner();
1.462 + mIOThread->Dispatch(runner, NS_DISPATCH_NORMAL);
1.463 +#endif
1.464 +
1.465 + mSpeculativeService = do_GetService("@mozilla.org/network/io-service;1", &rv);
1.466 + NS_ENSURE_SUCCESS(rv, rv);
1.467 +
1.468 + mDnsService = do_GetService("@mozilla.org/network/dns-service;1", &rv);
1.469 + NS_ENSURE_SUCCESS(rv, rv);
1.470 +
1.471 + mStorageService = do_GetService("@mozilla.org/storage/service;1", &rv);
1.472 + NS_ENSURE_SUCCESS(rv, rv);
1.473 +
1.474 + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
1.475 + getter_AddRefs(mDBFile));
1.476 + NS_ENSURE_SUCCESS(rv, rv);
1.477 + rv = mDBFile->AppendNative(NS_LITERAL_CSTRING("netpredictions.sqlite"));
1.478 + NS_ENSURE_SUCCESS(rv, rv);
1.479 +
1.480 + mInitialized = true;
1.481 +
1.482 + return rv;
1.483 +}
1.484 +
1.485 +void
1.486 +Seer::CheckForAndDeleteOldDBFile()
1.487 +{
1.488 + nsCOMPtr<nsIFile> oldDBFile;
1.489 + nsresult rv = mDBFile->GetParent(getter_AddRefs(oldDBFile));
1.490 + RETURN_IF_FAILED(rv);
1.491 +
1.492 + rv = oldDBFile->AppendNative(NS_LITERAL_CSTRING("seer.sqlite"));
1.493 + RETURN_IF_FAILED(rv);
1.494 +
1.495 + bool oldFileExists = false;
1.496 + rv = oldDBFile->Exists(&oldFileExists);
1.497 + if (NS_FAILED(rv) || !oldFileExists) {
1.498 + return;
1.499 + }
1.500 +
1.501 + oldDBFile->Remove(false);
1.502 +}
1.503 +
1.504 +// Make sure that our sqlite storage is all set up with all the tables we need
1.505 +// to do the work. It isn't the end of the world if this fails, since this is
1.506 +// all an optimization, anyway.
1.507 +
1.508 +nsresult
1.509 +Seer::EnsureInitStorage()
1.510 +{
1.511 + MOZ_ASSERT(!NS_IsMainThread(), "Initializing seer storage on main thread");
1.512 +
1.513 + if (mDB) {
1.514 + return NS_OK;
1.515 + }
1.516 +
1.517 + nsresult rv;
1.518 +
1.519 + CheckForAndDeleteOldDBFile();
1.520 +
1.521 + rv = mStorageService->OpenDatabase(mDBFile, getter_AddRefs(mDB));
1.522 + if (NS_FAILED(rv)) {
1.523 + // Retry once by trashing the file and trying to open again. If this fails,
1.524 + // we can just bail, and hope for better luck next time.
1.525 + rv = mDBFile->Remove(false);
1.526 + NS_ENSURE_SUCCESS(rv, rv);
1.527 +
1.528 + rv = mStorageService->OpenDatabase(mDBFile, getter_AddRefs(mDB));
1.529 + NS_ENSURE_SUCCESS(rv, rv);
1.530 + }
1.531 +
1.532 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous = OFF;"));
1.533 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA foreign_keys = ON;"));
1.534 +
1.535 + BeginTransaction();
1.536 +
1.537 + // A table to make sure we're working with the database layout we expect
1.538 + rv = mDB->ExecuteSimpleSQL(
1.539 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_seer_version (\n"
1.540 + " version INTEGER NOT NULL\n"
1.541 + ");\n"));
1.542 + NS_ENSURE_SUCCESS(rv, rv);
1.543 +
1.544 + nsCOMPtr<mozIStorageStatement> stmt;
1.545 + rv = mDB->CreateStatement(
1.546 + NS_LITERAL_CSTRING("SELECT version FROM moz_seer_version;\n"),
1.547 + getter_AddRefs(stmt));
1.548 + NS_ENSURE_SUCCESS(rv, rv);
1.549 +
1.550 + bool hasRows;
1.551 + rv = stmt->ExecuteStep(&hasRows);
1.552 + NS_ENSURE_SUCCESS(rv, rv);
1.553 + if (hasRows) {
1.554 + int32_t currentVersion;
1.555 + rv = stmt->GetInt32(0, ¤tVersion);
1.556 + NS_ENSURE_SUCCESS(rv, rv);
1.557 +
1.558 + // This is what we do while we only have one schema version. Later, we'll
1.559 + // have to change this to actually upgrade things as appropriate.
1.560 + MOZ_ASSERT(currentVersion == SEER_SCHEMA_VERSION,
1.561 + "Invalid seer schema version!");
1.562 + if (currentVersion != SEER_SCHEMA_VERSION) {
1.563 + return NS_ERROR_UNEXPECTED;
1.564 + }
1.565 + } else {
1.566 + stmt = nullptr;
1.567 + rv = mDB->CreateStatement(
1.568 + NS_LITERAL_CSTRING("INSERT INTO moz_seer_version (version) VALUES "
1.569 + "(:seer_version);"),
1.570 + getter_AddRefs(stmt));
1.571 + NS_ENSURE_SUCCESS(rv, rv);
1.572 +
1.573 + rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("seer_version"),
1.574 + SEER_SCHEMA_VERSION);
1.575 + NS_ENSURE_SUCCESS(rv, rv);
1.576 +
1.577 + stmt->Execute();
1.578 + }
1.579 +
1.580 + stmt = nullptr;
1.581 +
1.582 + // This table keeps track of the hosts we've seen at the top level of a
1.583 + // pageload so we can map them to hosts used for subresources of a pageload.
1.584 + rv = mDB->ExecuteSimpleSQL(
1.585 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_hosts (\n"
1.586 + " id INTEGER PRIMARY KEY AUTOINCREMENT,\n"
1.587 + " origin TEXT NOT NULL,\n"
1.588 + " loads INTEGER DEFAULT 0,\n"
1.589 + " last_load INTEGER DEFAULT 0\n"
1.590 + ");\n"));
1.591 + NS_ENSURE_SUCCESS(rv, rv);
1.592 +
1.593 + rv = mDB->ExecuteSimpleSQL(
1.594 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS host_id_origin_index "
1.595 + "ON moz_hosts (id, origin);"));
1.596 + NS_ENSURE_SUCCESS(rv, rv);
1.597 +
1.598 + rv = mDB->ExecuteSimpleSQL(
1.599 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS host_origin_index "
1.600 + "ON moz_hosts (origin);"));
1.601 + NS_ENSURE_SUCCESS(rv, rv);
1.602 +
1.603 + rv = mDB->ExecuteSimpleSQL(
1.604 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS host_load_index "
1.605 + "ON moz_hosts (last_load);"));
1.606 + NS_ENSURE_SUCCESS(rv, rv);
1.607 +
1.608 + // And this is the table that keeps track of the hosts for subresources of a
1.609 + // pageload.
1.610 + rv = mDB->ExecuteSimpleSQL(
1.611 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_subhosts (\n"
1.612 + " id INTEGER PRIMARY KEY AUTOINCREMENT,\n"
1.613 + " hid INTEGER NOT NULL,\n"
1.614 + " origin TEXT NOT NULL,\n"
1.615 + " hits INTEGER DEFAULT 0,\n"
1.616 + " last_hit INTEGER DEFAULT 0,\n"
1.617 + " FOREIGN KEY(hid) REFERENCES moz_hosts(id) ON DELETE CASCADE\n"
1.618 + ");\n"));
1.619 + NS_ENSURE_SUCCESS(rv, rv);
1.620 +
1.621 + rv = mDB->ExecuteSimpleSQL(
1.622 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subhost_hid_origin_index "
1.623 + "ON moz_subhosts (hid, origin);"));
1.624 + NS_ENSURE_SUCCESS(rv, rv);
1.625 +
1.626 + rv = mDB->ExecuteSimpleSQL(
1.627 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subhost_id_index "
1.628 + "ON moz_subhosts (id);"));
1.629 + NS_ENSURE_SUCCESS(rv, rv);
1.630 +
1.631 + // Table to keep track of how many times we've started up, and when the last
1.632 + // time was.
1.633 + rv = mDB->ExecuteSimpleSQL(
1.634 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_startups (\n"
1.635 + " startups INTEGER,\n"
1.636 + " last_startup INTEGER\n"
1.637 + ");\n"));
1.638 + NS_ENSURE_SUCCESS(rv, rv);
1.639 +
1.640 + rv = mDB->CreateStatement(
1.641 + NS_LITERAL_CSTRING("SELECT startups, last_startup FROM moz_startups;\n"),
1.642 + getter_AddRefs(stmt));
1.643 + NS_ENSURE_SUCCESS(rv, rv);
1.644 +
1.645 + // We'll go ahead and keep track of our startup count here, since we can
1.646 + // (mostly) equate "the service was created and asked to do stuff" with
1.647 + // "the browser was started up".
1.648 + rv = stmt->ExecuteStep(&hasRows);
1.649 + NS_ENSURE_SUCCESS(rv, rv);
1.650 + if (hasRows) {
1.651 + // We've started up before. Update our startup statistics
1.652 + stmt->GetInt32(0, &mStartupCount);
1.653 + stmt->GetInt64(1, &mLastStartupTime);
1.654 +
1.655 + // This finalizes the statement
1.656 + stmt = nullptr;
1.657 +
1.658 + rv = mDB->CreateStatement(
1.659 + NS_LITERAL_CSTRING("UPDATE moz_startups SET startups = :startup_count, "
1.660 + "last_startup = :startup_time;\n"),
1.661 + getter_AddRefs(stmt));
1.662 + NS_ENSURE_SUCCESS(rv, rv);
1.663 +
1.664 + int32_t newStartupCount = mStartupCount + 1;
1.665 + if (newStartupCount <= 0) {
1.666 + SEER_LOG(("Seer::EnsureInitStorage startup count overflow\n"));
1.667 + newStartupCount = mStartupCount;
1.668 + ++mAccumulators->mStartupCountOverflows;
1.669 + }
1.670 +
1.671 + rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("startup_count"),
1.672 + mStartupCount + 1);
1.673 + NS_ENSURE_SUCCESS(rv, rv);
1.674 +
1.675 + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),
1.676 + mStartupTime);
1.677 + NS_ENSURE_SUCCESS(rv, rv);
1.678 +
1.679 + stmt->Execute();
1.680 + } else {
1.681 + // This is our first startup, so let's go ahead and mark it as such
1.682 + mStartupCount = 1;
1.683 +
1.684 + rv = mDB->CreateStatement(
1.685 + NS_LITERAL_CSTRING("INSERT INTO moz_startups (startups, last_startup) "
1.686 + "VALUES (1, :startup_time);\n"),
1.687 + getter_AddRefs(stmt));
1.688 + NS_ENSURE_SUCCESS(rv, rv);
1.689 +
1.690 + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),
1.691 + mStartupTime);
1.692 + NS_ENSURE_SUCCESS(rv, rv);
1.693 +
1.694 + stmt->Execute();
1.695 + }
1.696 +
1.697 + // This finalizes the statement
1.698 + stmt = nullptr;
1.699 +
1.700 + // This table lists URIs loaded at startup, along with how many startups
1.701 + // they've been loaded during, and when the last time was.
1.702 + rv = mDB->ExecuteSimpleSQL(
1.703 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_startup_pages (\n"
1.704 + " id INTEGER PRIMARY KEY AUTOINCREMENT,\n"
1.705 + " uri TEXT NOT NULL,\n"
1.706 + " hits INTEGER DEFAULT 0,\n"
1.707 + " last_hit INTEGER DEFAULT 0\n"
1.708 + ");\n"));
1.709 + NS_ENSURE_SUCCESS(rv, rv);
1.710 +
1.711 + rv = mDB->ExecuteSimpleSQL(
1.712 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS startup_page_uri_index "
1.713 + "ON moz_startup_pages (uri);"));
1.714 + NS_ENSURE_SUCCESS(rv, rv);
1.715 +
1.716 + rv = mDB->ExecuteSimpleSQL(
1.717 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS startup_page_hit_index "
1.718 + "ON moz_startup_pages (last_hit);"));
1.719 + NS_ENSURE_SUCCESS(rv, rv);
1.720 +
1.721 + // This table is similar to moz_hosts above, but uses full URIs instead of
1.722 + // hosts so that we can get more specific predictions for URIs that people
1.723 + // visit often (such as their email or social network home pages).
1.724 + rv = mDB->ExecuteSimpleSQL(
1.725 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_pages (\n"
1.726 + " id integer PRIMARY KEY AUTOINCREMENT,\n"
1.727 + " uri TEXT NOT NULL,\n"
1.728 + " loads INTEGER DEFAULT 0,\n"
1.729 + " last_load INTEGER DEFAULT 0\n"
1.730 + ");\n"));
1.731 + NS_ENSURE_SUCCESS(rv, rv);
1.732 +
1.733 + rv = mDB->ExecuteSimpleSQL(
1.734 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS page_id_uri_index "
1.735 + "ON moz_pages (id, uri);"));
1.736 + NS_ENSURE_SUCCESS(rv, rv);
1.737 +
1.738 + rv = mDB->ExecuteSimpleSQL(
1.739 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS page_uri_index "
1.740 + "ON moz_pages (uri);"));
1.741 + NS_ENSURE_SUCCESS(rv, rv);
1.742 +
1.743 + // This table is similar to moz_subhosts above, but is instead related to
1.744 + // moz_pages for finer granularity.
1.745 + rv = mDB->ExecuteSimpleSQL(
1.746 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_subresources (\n"
1.747 + " id integer PRIMARY KEY AUTOINCREMENT,\n"
1.748 + " pid INTEGER NOT NULL,\n"
1.749 + " uri TEXT NOT NULL,\n"
1.750 + " hits INTEGER DEFAULT 0,\n"
1.751 + " last_hit INTEGER DEFAULT 0,\n"
1.752 + " FOREIGN KEY(pid) REFERENCES moz_pages(id) ON DELETE CASCADE\n"
1.753 + ");\n"));
1.754 + NS_ENSURE_SUCCESS(rv, rv);
1.755 +
1.756 + rv = mDB->ExecuteSimpleSQL(
1.757 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subresource_pid_uri_index "
1.758 + "ON moz_subresources (pid, uri);"));
1.759 + NS_ENSURE_SUCCESS(rv, rv);
1.760 +
1.761 + rv = mDB->ExecuteSimpleSQL(
1.762 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS subresource_id_index "
1.763 + "ON moz_subresources (id);"));
1.764 + NS_ENSURE_SUCCESS(rv, rv);
1.765 +
1.766 + // This table keeps track of URIs and what they end up finally redirecting to
1.767 + // so we can handle redirects in a sane fashion, as well.
1.768 + rv = mDB->ExecuteSimpleSQL(
1.769 + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_redirects (\n"
1.770 + " id integer PRIMARY KEY AUTOINCREMENT,\n"
1.771 + " pid integer NOT NULL,\n"
1.772 + " uri TEXT NOT NULL,\n"
1.773 + " origin TEXT NOT NULL,\n"
1.774 + " hits INTEGER DEFAULT 0,\n"
1.775 + " last_hit INTEGER DEFAULT 0,\n"
1.776 + " FOREIGN KEY(pid) REFERENCES moz_pages(id) ON DELETE CASCADE\n"
1.777 + ");\n"));
1.778 + NS_ENSURE_SUCCESS(rv, rv);
1.779 +
1.780 + rv = mDB->ExecuteSimpleSQL(
1.781 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS redirect_pid_uri_index "
1.782 + "ON moz_redirects (pid, uri);"));
1.783 + NS_ENSURE_SUCCESS(rv, rv);
1.784 +
1.785 + rv = mDB->ExecuteSimpleSQL(
1.786 + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS redirect_id_index "
1.787 + "ON moz_redirects (id);"));
1.788 + NS_ENSURE_SUCCESS(rv, rv);
1.789 +
1.790 + CommitTransaction();
1.791 + BeginTransaction();
1.792 +
1.793 + nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();
1.794 + NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
1.795 +
1.796 + return NS_OK;
1.797 +}
1.798 +
1.799 +class SeerThreadShutdownRunner : public nsRunnable
1.800 +{
1.801 +public:
1.802 + SeerThreadShutdownRunner(nsIThread *ioThread)
1.803 + :mIOThread(ioThread)
1.804 + { }
1.805 +
1.806 + NS_IMETHODIMP Run() MOZ_OVERRIDE
1.807 + {
1.808 + MOZ_ASSERT(NS_IsMainThread(), "Shut down seer io thread off main thread");
1.809 + mIOThread->Shutdown();
1.810 + return NS_OK;
1.811 + }
1.812 +
1.813 +private:
1.814 + nsCOMPtr<nsIThread> mIOThread;
1.815 +};
1.816 +
1.817 +class SeerDBShutdownRunner : public nsRunnable
1.818 +{
1.819 +public:
1.820 + SeerDBShutdownRunner(nsIThread *ioThread, nsINetworkSeer *seer)
1.821 + :mIOThread(ioThread)
1.822 + {
1.823 + mSeer = new nsMainThreadPtrHolder<nsINetworkSeer>(seer);
1.824 + }
1.825 +
1.826 + NS_IMETHODIMP Run() MOZ_OVERRIDE
1.827 + {
1.828 + MOZ_ASSERT(!NS_IsMainThread(), "Shutting down DB on main thread");
1.829 +
1.830 + // Ensure everything is written to disk before we shut down the db
1.831 + gSeer->CommitTransaction();
1.832 +
1.833 + gSeer->mStatements.FinalizeStatements();
1.834 + gSeer->mDB->Close();
1.835 + gSeer->mDB = nullptr;
1.836 +
1.837 + nsRefPtr<SeerThreadShutdownRunner> runner =
1.838 + new SeerThreadShutdownRunner(mIOThread);
1.839 + NS_DispatchToMainThread(runner);
1.840 +
1.841 + return NS_OK;
1.842 + }
1.843 +
1.844 +private:
1.845 + nsCOMPtr<nsIThread> mIOThread;
1.846 +
1.847 + // Death grip to keep seer alive while we cleanly close its DB connection
1.848 + nsMainThreadPtrHandle<nsINetworkSeer> mSeer;
1.849 +};
1.850 +
1.851 +void
1.852 +Seer::Shutdown()
1.853 +{
1.854 + if (!NS_IsMainThread()) {
1.855 + MOZ_ASSERT(false, "Seer::Shutdown called off the main thread!");
1.856 + return;
1.857 + }
1.858 +
1.859 + mInitialized = false;
1.860 +
1.861 + if (mCommitTimer) {
1.862 + mCommitTimer->Cancel();
1.863 + }
1.864 +
1.865 + if (mIOThread) {
1.866 + if (mDB) {
1.867 + nsRefPtr<SeerDBShutdownRunner> runner =
1.868 + new SeerDBShutdownRunner(mIOThread, this);
1.869 + mIOThread->Dispatch(runner, NS_DISPATCH_NORMAL);
1.870 + } else {
1.871 + nsRefPtr<SeerThreadShutdownRunner> runner =
1.872 + new SeerThreadShutdownRunner(mIOThread);
1.873 + NS_DispatchToMainThread(runner);
1.874 + }
1.875 + }
1.876 +}
1.877 +
1.878 +nsresult
1.879 +Seer::Create(nsISupports *aOuter, const nsIID& aIID,
1.880 + void **aResult)
1.881 +{
1.882 + nsresult rv;
1.883 +
1.884 + if (aOuter != nullptr) {
1.885 + return NS_ERROR_NO_AGGREGATION;
1.886 + }
1.887 +
1.888 + nsRefPtr<Seer> svc = new Seer();
1.889 +
1.890 + rv = svc->Init();
1.891 + if (NS_FAILED(rv)) {
1.892 + SEER_LOG(("Failed to initialize seer, seer will be a noop"));
1.893 + }
1.894 +
1.895 + // We treat init failure the same as the service being disabled, since this
1.896 + // is all an optimization anyway. No need to freak people out. That's why we
1.897 + // gladly continue on QI'ing here.
1.898 + rv = svc->QueryInterface(aIID, aResult);
1.899 +
1.900 + return rv;
1.901 +}
1.902 +
1.903 +// Get the full origin (scheme, host, port) out of a URI (maybe should be part
1.904 +// of nsIURI instead?)
1.905 +static void
1.906 +ExtractOrigin(nsIURI *uri, nsAutoCString &s)
1.907 +{
1.908 + s.Truncate();
1.909 +
1.910 + nsAutoCString scheme;
1.911 + nsresult rv = uri->GetScheme(scheme);
1.912 + RETURN_IF_FAILED(rv);
1.913 +
1.914 + nsAutoCString host;
1.915 + rv = uri->GetAsciiHost(host);
1.916 + RETURN_IF_FAILED(rv);
1.917 +
1.918 + int32_t port;
1.919 + rv = uri->GetPort(&port);
1.920 + RETURN_IF_FAILED(rv);
1.921 +
1.922 + s.Assign(scheme);
1.923 + s.AppendLiteral("://");
1.924 + s.Append(host);
1.925 + if (port != -1) {
1.926 + s.AppendLiteral(":");
1.927 + s.AppendInt(port);
1.928 + }
1.929 +}
1.930 +
1.931 +// An event to do the work for a prediction that needs to hit the sqlite
1.932 +// database. These events should be created on the main thread, and run on
1.933 +// the seer thread.
1.934 +class SeerPredictionEvent : public nsRunnable
1.935 +{
1.936 +public:
1.937 + SeerPredictionEvent(nsIURI *targetURI, nsIURI *sourceURI,
1.938 + SeerPredictReason reason,
1.939 + nsINetworkSeerVerifier *verifier)
1.940 + :mReason(reason)
1.941 + {
1.942 + MOZ_ASSERT(NS_IsMainThread(), "Creating prediction event off main thread");
1.943 +
1.944 + mEnqueueTime = TimeStamp::Now();
1.945 +
1.946 + if (verifier) {
1.947 + mVerifier = new nsMainThreadPtrHolder<nsINetworkSeerVerifier>(verifier);
1.948 + }
1.949 + if (targetURI) {
1.950 + targetURI->GetAsciiSpec(mTargetURI.spec);
1.951 + ExtractOrigin(targetURI, mTargetURI.origin);
1.952 + }
1.953 + if (sourceURI) {
1.954 + sourceURI->GetAsciiSpec(mSourceURI.spec);
1.955 + ExtractOrigin(sourceURI, mSourceURI.origin);
1.956 + }
1.957 + }
1.958 +
1.959 + NS_IMETHOD Run() MOZ_OVERRIDE
1.960 + {
1.961 + MOZ_ASSERT(!NS_IsMainThread(), "Running prediction event on main thread");
1.962 +
1.963 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_WAIT_TIME,
1.964 + mEnqueueTime);
1.965 +
1.966 + TimeStamp startTime = TimeStamp::Now();
1.967 +
1.968 + nsresult rv = NS_OK;
1.969 +
1.970 + switch (mReason) {
1.971 + case nsINetworkSeer::PREDICT_LOAD:
1.972 + gSeer->PredictForPageload(mTargetURI, mVerifier, 0, mEnqueueTime);
1.973 + break;
1.974 + case nsINetworkSeer::PREDICT_STARTUP:
1.975 + gSeer->PredictForStartup(mVerifier, mEnqueueTime);
1.976 + break;
1.977 + default:
1.978 + MOZ_ASSERT(false, "Got unexpected value for predict reason");
1.979 + rv = NS_ERROR_UNEXPECTED;
1.980 + }
1.981 +
1.982 + gSeer->FreeSpaceInQueue();
1.983 +
1.984 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_WORK_TIME,
1.985 + startTime);
1.986 +
1.987 + gSeer->MaybeScheduleCleanup();
1.988 +
1.989 + return rv;
1.990 + }
1.991 +
1.992 +private:
1.993 + Seer::UriInfo mTargetURI;
1.994 + Seer::UriInfo mSourceURI;
1.995 + SeerPredictReason mReason;
1.996 + SeerVerifierHandle mVerifier;
1.997 + TimeStamp mEnqueueTime;
1.998 +};
1.999 +
1.1000 +// Predicting for a link is easy, and doesn't require the round-trip to the
1.1001 +// seer thread and back to the main thread, since we don't have to hit the db
1.1002 +// for that.
1.1003 +void
1.1004 +Seer::PredictForLink(nsIURI *targetURI, nsIURI *sourceURI,
1.1005 + nsINetworkSeerVerifier *verifier)
1.1006 +{
1.1007 + MOZ_ASSERT(NS_IsMainThread(), "Predicting for link off main thread");
1.1008 +
1.1009 + if (!mSpeculativeService) {
1.1010 + return;
1.1011 + }
1.1012 +
1.1013 + if (!mEnableHoverOnSSL) {
1.1014 + bool isSSL = false;
1.1015 + sourceURI->SchemeIs("https", &isSSL);
1.1016 + if (isSSL) {
1.1017 + // We don't want to predict from an HTTPS page, to avoid info leakage
1.1018 + SEER_LOG(("Not predicting for link hover - on an SSL page"));
1.1019 + return;
1.1020 + }
1.1021 + }
1.1022 +
1.1023 + mSpeculativeService->SpeculativeConnect(targetURI, nullptr);
1.1024 + if (verifier) {
1.1025 + verifier->OnPredictPreconnect(targetURI);
1.1026 + }
1.1027 +}
1.1028 +
1.1029 +// This runnable runs on the main thread, and is responsible for actually
1.1030 +// firing off predictive actions (such as TCP/TLS preconnects and DNS lookups)
1.1031 +class SeerPredictionRunner : public nsRunnable
1.1032 +{
1.1033 +public:
1.1034 + SeerPredictionRunner(SeerVerifierHandle &verifier, TimeStamp predictStartTime)
1.1035 + :mVerifier(verifier)
1.1036 + ,mPredictStartTime(predictStartTime)
1.1037 + { }
1.1038 +
1.1039 + void AddPreconnect(const nsACString &uri)
1.1040 + {
1.1041 + mPreconnects.AppendElement(uri);
1.1042 + }
1.1043 +
1.1044 + void AddPreresolve(const nsACString &uri)
1.1045 + {
1.1046 + mPreresolves.AppendElement(uri);
1.1047 + }
1.1048 +
1.1049 + bool HasWork() const
1.1050 + {
1.1051 + return !(mPreconnects.IsEmpty() && mPreresolves.IsEmpty());
1.1052 + }
1.1053 +
1.1054 + NS_IMETHOD Run() MOZ_OVERRIDE
1.1055 + {
1.1056 + MOZ_ASSERT(NS_IsMainThread(), "Running prediction off main thread");
1.1057 +
1.1058 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_TIME_TO_ACTION,
1.1059 + mPredictStartTime);
1.1060 +
1.1061 + uint32_t len, i;
1.1062 +
1.1063 + len = mPreconnects.Length();
1.1064 + for (i = 0; i < len; ++i) {
1.1065 + nsCOMPtr<nsIURI> uri;
1.1066 + nsresult rv = NS_NewURI(getter_AddRefs(uri), mPreconnects[i]);
1.1067 + if (NS_FAILED(rv)) {
1.1068 + continue;
1.1069 + }
1.1070 +
1.1071 + ++gSeer->mAccumulators->mTotalPredictions;
1.1072 + ++gSeer->mAccumulators->mTotalPreconnects;
1.1073 + gSeer->mSpeculativeService->SpeculativeConnect(uri, gSeer);
1.1074 + if (mVerifier) {
1.1075 + mVerifier->OnPredictPreconnect(uri);
1.1076 + }
1.1077 + }
1.1078 +
1.1079 + len = mPreresolves.Length();
1.1080 + nsCOMPtr<nsIThread> mainThread = do_GetMainThread();
1.1081 + for (i = 0; i < len; ++i) {
1.1082 + nsCOMPtr<nsIURI> uri;
1.1083 + nsresult rv = NS_NewURI(getter_AddRefs(uri), mPreresolves[i]);
1.1084 + if (NS_FAILED(rv)) {
1.1085 + continue;
1.1086 + }
1.1087 +
1.1088 + ++gSeer->mAccumulators->mTotalPredictions;
1.1089 + ++gSeer->mAccumulators->mTotalPreresolves;
1.1090 + nsAutoCString hostname;
1.1091 + uri->GetAsciiHost(hostname);
1.1092 + nsCOMPtr<nsICancelable> tmpCancelable;
1.1093 + gSeer->mDnsService->AsyncResolve(hostname,
1.1094 + (nsIDNSService::RESOLVE_PRIORITY_MEDIUM |
1.1095 + nsIDNSService::RESOLVE_SPECULATE),
1.1096 + gSeer->mDNSListener, nullptr,
1.1097 + getter_AddRefs(tmpCancelable));
1.1098 + if (mVerifier) {
1.1099 + mVerifier->OnPredictDNS(uri);
1.1100 + }
1.1101 + }
1.1102 +
1.1103 + mPreconnects.Clear();
1.1104 + mPreresolves.Clear();
1.1105 +
1.1106 + return NS_OK;
1.1107 + }
1.1108 +
1.1109 +private:
1.1110 + nsTArray<nsCString> mPreconnects;
1.1111 + nsTArray<nsCString> mPreresolves;
1.1112 + SeerVerifierHandle mVerifier;
1.1113 + TimeStamp mPredictStartTime;
1.1114 +};
1.1115 +
1.1116 +// This calculates how much to degrade our confidence in our data based on
1.1117 +// the last time this top-level resource was loaded. This "global degradation"
1.1118 +// applies to *all* subresources we have associated with the top-level
1.1119 +// resource. This will be in addition to any reduction in confidence we have
1.1120 +// associated with a particular subresource.
1.1121 +int
1.1122 +Seer::CalculateGlobalDegradation(PRTime now, PRTime lastLoad)
1.1123 +{
1.1124 + int globalDegradation;
1.1125 + PRTime delta = now - lastLoad;
1.1126 + if (delta < ONE_DAY) {
1.1127 + globalDegradation = mPageDegradationDay;
1.1128 + } else if (delta < ONE_WEEK) {
1.1129 + globalDegradation = mPageDegradationWeek;
1.1130 + } else if (delta < ONE_MONTH) {
1.1131 + globalDegradation = mPageDegradationMonth;
1.1132 + } else if (delta < ONE_YEAR) {
1.1133 + globalDegradation = mPageDegradationYear;
1.1134 + } else {
1.1135 + globalDegradation = mPageDegradationMax;
1.1136 + }
1.1137 +
1.1138 + Telemetry::Accumulate(Telemetry::SEER_GLOBAL_DEGRADATION, globalDegradation);
1.1139 + return globalDegradation;
1.1140 +}
1.1141 +
1.1142 +// This calculates our overall confidence that a particular subresource will be
1.1143 +// loaded as part of a top-level load.
1.1144 +// @param baseConfidence - the basic confidence we have for this subresource,
1.1145 +// which is the percentage of time this top-level load
1.1146 +// loads the subresource in question
1.1147 +// @param lastHit - the timestamp of the last time we loaded this subresource as
1.1148 +// part of this top-level load
1.1149 +// @param lastPossible - the timestamp of the last time we performed this
1.1150 +// top-level load
1.1151 +// @param globalDegradation - the degradation for this top-level load as
1.1152 +// determined by CalculateGlobalDegradation
1.1153 +int
1.1154 +Seer::CalculateConfidence(int baseConfidence, PRTime lastHit,
1.1155 + PRTime lastPossible, int globalDegradation)
1.1156 +{
1.1157 + ++mAccumulators->mPredictionsCalculated;
1.1158 +
1.1159 + int maxConfidence = 100;
1.1160 + int confidenceDegradation = 0;
1.1161 +
1.1162 + if (lastHit < lastPossible) {
1.1163 + // We didn't load this subresource the last time this top-level load was
1.1164 + // performed, so let's not bother preconnecting (at the very least).
1.1165 + maxConfidence = mPreconnectMinConfidence - 1;
1.1166 +
1.1167 + // Now calculate how much we want to degrade our confidence based on how
1.1168 + // long it's been between the last time we did this top-level load and the
1.1169 + // last time this top-level load included this subresource.
1.1170 + PRTime delta = lastPossible - lastHit;
1.1171 + if (delta == 0) {
1.1172 + confidenceDegradation = 0;
1.1173 + } else if (delta < ONE_DAY) {
1.1174 + confidenceDegradation = mSubresourceDegradationDay;
1.1175 + } else if (delta < ONE_WEEK) {
1.1176 + confidenceDegradation = mSubresourceDegradationWeek;
1.1177 + } else if (delta < ONE_MONTH) {
1.1178 + confidenceDegradation = mSubresourceDegradationMonth;
1.1179 + } else if (delta < ONE_YEAR) {
1.1180 + confidenceDegradation = mSubresourceDegradationYear;
1.1181 + } else {
1.1182 + confidenceDegradation = mSubresourceDegradationMax;
1.1183 + maxConfidence = 0;
1.1184 + }
1.1185 + }
1.1186 +
1.1187 + // Calculate our confidence and clamp it to between 0 and maxConfidence
1.1188 + // (<= 100)
1.1189 + int confidence = baseConfidence - confidenceDegradation - globalDegradation;
1.1190 + confidence = std::max(confidence, 0);
1.1191 + confidence = std::min(confidence, maxConfidence);
1.1192 +
1.1193 + Telemetry::Accumulate(Telemetry::SEER_BASE_CONFIDENCE, baseConfidence);
1.1194 + Telemetry::Accumulate(Telemetry::SEER_SUBRESOURCE_DEGRADATION,
1.1195 + confidenceDegradation);
1.1196 + Telemetry::Accumulate(Telemetry::SEER_CONFIDENCE, confidence);
1.1197 + return confidence;
1.1198 +}
1.1199 +
1.1200 +// (Maybe) adds a predictive action to the prediction runner, based on our
1.1201 +// calculated confidence for the subresource in question.
1.1202 +void
1.1203 +Seer::SetupPrediction(int confidence, const nsACString &uri,
1.1204 + SeerPredictionRunner *runner)
1.1205 +{
1.1206 + if (confidence >= mPreconnectMinConfidence) {
1.1207 + runner->AddPreconnect(uri);
1.1208 + } else if (confidence >= mPreresolveMinConfidence) {
1.1209 + runner->AddPreresolve(uri);
1.1210 + }
1.1211 +}
1.1212 +
1.1213 +// This gets the data about the top-level load from our database, either from
1.1214 +// the pages table (which is specific to a particular URI), or from the hosts
1.1215 +// table (which is for a particular origin).
1.1216 +bool
1.1217 +Seer::LookupTopLevel(QueryType queryType, const nsACString &key,
1.1218 + TopLevelInfo &info)
1.1219 +{
1.1220 + MOZ_ASSERT(!NS_IsMainThread(), "LookupTopLevel called on main thread.");
1.1221 +
1.1222 + nsCOMPtr<mozIStorageStatement> stmt;
1.1223 + if (queryType == QUERY_PAGE) {
1.1224 + stmt = mStatements.GetCachedStatement(
1.1225 + NS_LITERAL_CSTRING("SELECT id, loads, last_load FROM moz_pages WHERE "
1.1226 + "uri = :key;"));
1.1227 + } else {
1.1228 + stmt = mStatements.GetCachedStatement(
1.1229 + NS_LITERAL_CSTRING("SELECT id, loads, last_load FROM moz_hosts WHERE "
1.1230 + "origin = :key;"));
1.1231 + }
1.1232 + NS_ENSURE_TRUE(stmt, false);
1.1233 + mozStorageStatementScoper scope(stmt);
1.1234 +
1.1235 + nsresult rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);
1.1236 + NS_ENSURE_SUCCESS(rv, false);
1.1237 +
1.1238 + bool hasRows;
1.1239 + rv = stmt->ExecuteStep(&hasRows);
1.1240 + NS_ENSURE_SUCCESS(rv, false);
1.1241 +
1.1242 + if (!hasRows) {
1.1243 + return false;
1.1244 + }
1.1245 +
1.1246 + rv = stmt->GetInt32(0, &info.id);
1.1247 + NS_ENSURE_SUCCESS(rv, false);
1.1248 +
1.1249 + rv = stmt->GetInt32(1, &info.loadCount);
1.1250 + NS_ENSURE_SUCCESS(rv, false);
1.1251 +
1.1252 + rv = stmt->GetInt64(2, &info.lastLoad);
1.1253 + NS_ENSURE_SUCCESS(rv, false);
1.1254 +
1.1255 + return true;
1.1256 +}
1.1257 +
1.1258 +// Insert data about either a top-level page or a top-level origin into
1.1259 +// the database.
1.1260 +void
1.1261 +Seer::AddTopLevel(QueryType queryType, const nsACString &key, PRTime now)
1.1262 +{
1.1263 + MOZ_ASSERT(!NS_IsMainThread(), "AddTopLevel called on main thread.");
1.1264 +
1.1265 + nsCOMPtr<mozIStorageStatement> stmt;
1.1266 + if (queryType == QUERY_PAGE) {
1.1267 + stmt = mStatements.GetCachedStatement(
1.1268 + NS_LITERAL_CSTRING("INSERT INTO moz_pages (uri, loads, last_load) "
1.1269 + "VALUES (:key, 1, :now);"));
1.1270 + } else {
1.1271 + stmt = mStatements.GetCachedStatement(
1.1272 + NS_LITERAL_CSTRING("INSERT INTO moz_hosts (origin, loads, last_load) "
1.1273 + "VALUES (:key, 1, :now);"));
1.1274 + }
1.1275 + if (!stmt) {
1.1276 + return;
1.1277 + }
1.1278 + mozStorageStatementScoper scope(stmt);
1.1279 +
1.1280 + // Loading a page implicitly makes the seer learn about the page,
1.1281 + // so since we don't have it already, let's add it.
1.1282 + nsresult rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);
1.1283 + RETURN_IF_FAILED(rv);
1.1284 +
1.1285 + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);
1.1286 + RETURN_IF_FAILED(rv);
1.1287 +
1.1288 + rv = stmt->Execute();
1.1289 +}
1.1290 +
1.1291 +// Update data about either a top-level page or a top-level origin in the
1.1292 +// database.
1.1293 +void
1.1294 +Seer::UpdateTopLevel(QueryType queryType, const TopLevelInfo &info, PRTime now)
1.1295 +{
1.1296 + MOZ_ASSERT(!NS_IsMainThread(), "UpdateTopLevel called on main thread.");
1.1297 +
1.1298 + nsCOMPtr<mozIStorageStatement> stmt;
1.1299 + if (queryType == QUERY_PAGE) {
1.1300 + stmt = mStatements.GetCachedStatement(
1.1301 + NS_LITERAL_CSTRING("UPDATE moz_pages SET loads = :load_count, "
1.1302 + "last_load = :now WHERE id = :id;"));
1.1303 + } else {
1.1304 + stmt = mStatements.GetCachedStatement(
1.1305 + NS_LITERAL_CSTRING("UPDATE moz_hosts SET loads = :load_count, "
1.1306 + "last_load = :now WHERE id = :id;"));
1.1307 + }
1.1308 + if (!stmt) {
1.1309 + return;
1.1310 + }
1.1311 + mozStorageStatementScoper scope(stmt);
1.1312 +
1.1313 + int32_t newLoadCount = info.loadCount + 1;
1.1314 + if (newLoadCount <= 0) {
1.1315 + SEER_LOG(("Seer::UpdateTopLevel type %d id %d load count overflow\n",
1.1316 + queryType, info.id));
1.1317 + newLoadCount = info.loadCount;
1.1318 + ++mAccumulators->mLoadCountOverflows;
1.1319 + }
1.1320 +
1.1321 + // First, let's update the page in the database, since loading a page
1.1322 + // implicitly learns about the page.
1.1323 + nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("load_count"),
1.1324 + newLoadCount);
1.1325 + RETURN_IF_FAILED(rv);
1.1326 +
1.1327 + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);
1.1328 + RETURN_IF_FAILED(rv);
1.1329 +
1.1330 + rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);
1.1331 + RETURN_IF_FAILED(rv);
1.1332 +
1.1333 + rv = stmt->Execute();
1.1334 +}
1.1335 +
1.1336 +// Tries to predict for a top-level load (either page-based or origin-based).
1.1337 +// Returns false if it failed to predict at all, true if it did some sort of
1.1338 +// prediction.
1.1339 +// @param queryType - whether to predict based on page or origin
1.1340 +// @param info - the db info about the top-level resource
1.1341 +bool
1.1342 +Seer::TryPredict(QueryType queryType, const TopLevelInfo &info, PRTime now,
1.1343 + SeerVerifierHandle &verifier, TimeStamp &predictStartTime)
1.1344 +{
1.1345 + MOZ_ASSERT(!NS_IsMainThread(), "TryPredict called on main thread.");
1.1346 +
1.1347 + if (!info.loadCount) {
1.1348 + SEER_LOG(("Seer::TryPredict info.loadCount is zero!\n"));
1.1349 + ++mAccumulators->mLoadCountZeroes;
1.1350 + return false;
1.1351 + }
1.1352 +
1.1353 + int globalDegradation = CalculateGlobalDegradation(now, info.lastLoad);
1.1354 +
1.1355 + // Now let's look up the subresources we know about for this page
1.1356 + nsCOMPtr<mozIStorageStatement> stmt;
1.1357 + if (queryType == QUERY_PAGE) {
1.1358 + stmt = mStatements.GetCachedStatement(
1.1359 + NS_LITERAL_CSTRING("SELECT uri, hits, last_hit FROM moz_subresources "
1.1360 + "WHERE pid = :id;"));
1.1361 + } else {
1.1362 + stmt = mStatements.GetCachedStatement(
1.1363 + NS_LITERAL_CSTRING("SELECT origin, hits, last_hit FROM moz_subhosts "
1.1364 + "WHERE hid = :id;"));
1.1365 + }
1.1366 + NS_ENSURE_TRUE(stmt, false);
1.1367 + mozStorageStatementScoper scope(stmt);
1.1368 +
1.1369 + nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);
1.1370 + NS_ENSURE_SUCCESS(rv, false);
1.1371 +
1.1372 + bool hasRows;
1.1373 + rv = stmt->ExecuteStep(&hasRows);
1.1374 + if (NS_FAILED(rv) || !hasRows) {
1.1375 + return false;
1.1376 + }
1.1377 +
1.1378 + nsRefPtr<SeerPredictionRunner> runner =
1.1379 + new SeerPredictionRunner(verifier, predictStartTime);
1.1380 +
1.1381 + while (hasRows) {
1.1382 + int32_t hitCount;
1.1383 + PRTime lastHit;
1.1384 + nsAutoCString subresource;
1.1385 + int baseConfidence, confidence;
1.1386 +
1.1387 + // We use goto nextrow here instead of just failling, because we want
1.1388 + // to do some sort of prediction if at all possible. Of course, it's
1.1389 + // probably unlikely that subsequent rows will succeed if one fails, but
1.1390 + // it's worth a shot.
1.1391 +
1.1392 + rv = stmt->GetUTF8String(0, subresource);
1.1393 + if NS_FAILED(rv) {
1.1394 + goto nextrow;
1.1395 + }
1.1396 +
1.1397 + rv = stmt->GetInt32(1, &hitCount);
1.1398 + if (NS_FAILED(rv)) {
1.1399 + goto nextrow;
1.1400 + }
1.1401 +
1.1402 + rv = stmt->GetInt64(2, &lastHit);
1.1403 + if (NS_FAILED(rv)) {
1.1404 + goto nextrow;
1.1405 + }
1.1406 +
1.1407 + baseConfidence = (hitCount * 100) / info.loadCount;
1.1408 + confidence = CalculateConfidence(baseConfidence, lastHit, info.lastLoad,
1.1409 + globalDegradation);
1.1410 + SetupPrediction(confidence, subresource, runner);
1.1411 +
1.1412 +nextrow:
1.1413 + rv = stmt->ExecuteStep(&hasRows);
1.1414 + NS_ENSURE_SUCCESS(rv, false);
1.1415 + }
1.1416 +
1.1417 + bool predicted = false;
1.1418 +
1.1419 + if (runner->HasWork()) {
1.1420 + NS_DispatchToMainThread(runner);
1.1421 + predicted = true;
1.1422 + }
1.1423 +
1.1424 + return predicted;
1.1425 +}
1.1426 +
1.1427 +// Find out if a top-level page is likely to redirect.
1.1428 +bool
1.1429 +Seer::WouldRedirect(const TopLevelInfo &info, PRTime now, UriInfo &newUri)
1.1430 +{
1.1431 + MOZ_ASSERT(!NS_IsMainThread(), "WouldRedirect called on main thread.");
1.1432 +
1.1433 + if (!info.loadCount) {
1.1434 + SEER_LOG(("Seer::WouldRedirect info.loadCount is zero!\n"));
1.1435 + ++mAccumulators->mLoadCountZeroes;
1.1436 + return false;
1.1437 + }
1.1438 +
1.1439 + nsCOMPtr<mozIStorageStatement> stmt = mStatements.GetCachedStatement(
1.1440 + NS_LITERAL_CSTRING("SELECT uri, origin, hits, last_hit "
1.1441 + "FROM moz_redirects WHERE pid = :id;"));
1.1442 + NS_ENSURE_TRUE(stmt, false);
1.1443 + mozStorageStatementScoper scope(stmt);
1.1444 +
1.1445 + nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);
1.1446 + NS_ENSURE_SUCCESS(rv, false);
1.1447 +
1.1448 + bool hasRows;
1.1449 + rv = stmt->ExecuteStep(&hasRows);
1.1450 + if (NS_FAILED(rv) || !hasRows) {
1.1451 + return false;
1.1452 + }
1.1453 +
1.1454 + rv = stmt->GetUTF8String(0, newUri.spec);
1.1455 + NS_ENSURE_SUCCESS(rv, false);
1.1456 +
1.1457 + rv = stmt->GetUTF8String(1, newUri.origin);
1.1458 + NS_ENSURE_SUCCESS(rv, false);
1.1459 +
1.1460 + int32_t hitCount;
1.1461 + rv = stmt->GetInt32(2, &hitCount);
1.1462 + NS_ENSURE_SUCCESS(rv, false);
1.1463 +
1.1464 + PRTime lastHit;
1.1465 + rv = stmt->GetInt64(3, &lastHit);
1.1466 + NS_ENSURE_SUCCESS(rv, false);
1.1467 +
1.1468 + int globalDegradation = CalculateGlobalDegradation(now, info.lastLoad);
1.1469 + int baseConfidence = (hitCount * 100) / info.loadCount;
1.1470 + int confidence = CalculateConfidence(baseConfidence, lastHit, info.lastLoad,
1.1471 + globalDegradation);
1.1472 +
1.1473 + if (confidence > mRedirectLikelyConfidence) {
1.1474 + return true;
1.1475 + }
1.1476 +
1.1477 + return false;
1.1478 +}
1.1479 +
1.1480 +// This will add a page to our list of startup pages if it's being loaded
1.1481 +// before our startup window has expired.
1.1482 +void
1.1483 +Seer::MaybeLearnForStartup(const UriInfo &uri, const PRTime now)
1.1484 +{
1.1485 + MOZ_ASSERT(!NS_IsMainThread(), "MaybeLearnForStartup called on main thread.");
1.1486 +
1.1487 + if ((now - mStartupTime) < STARTUP_WINDOW) {
1.1488 + LearnForStartup(uri);
1.1489 + }
1.1490 +}
1.1491 +
1.1492 +const int MAX_PAGELOAD_DEPTH = 10;
1.1493 +
1.1494 +// This is the driver for prediction based on a new pageload.
1.1495 +void
1.1496 +Seer::PredictForPageload(const UriInfo &uri, SeerVerifierHandle &verifier,
1.1497 + int stackCount, TimeStamp &predictStartTime)
1.1498 +{
1.1499 + MOZ_ASSERT(!NS_IsMainThread(), "PredictForPageload called on main thread.");
1.1500 +
1.1501 + if (stackCount > MAX_PAGELOAD_DEPTH) {
1.1502 + SEER_LOG(("Too deep into pageload prediction"));
1.1503 + return;
1.1504 + }
1.1505 +
1.1506 + if (NS_FAILED(EnsureInitStorage())) {
1.1507 + return;
1.1508 + }
1.1509 +
1.1510 + PRTime now = PR_Now();
1.1511 +
1.1512 + MaybeLearnForStartup(uri, now);
1.1513 +
1.1514 + TopLevelInfo pageInfo;
1.1515 + TopLevelInfo originInfo;
1.1516 + bool havePage = LookupTopLevel(QUERY_PAGE, uri.spec, pageInfo);
1.1517 + bool haveOrigin = LookupTopLevel(QUERY_ORIGIN, uri.origin, originInfo);
1.1518 +
1.1519 + if (!havePage) {
1.1520 + AddTopLevel(QUERY_PAGE, uri.spec, now);
1.1521 + } else {
1.1522 + UpdateTopLevel(QUERY_PAGE, pageInfo, now);
1.1523 + }
1.1524 +
1.1525 + if (!haveOrigin) {
1.1526 + AddTopLevel(QUERY_ORIGIN, uri.origin, now);
1.1527 + } else {
1.1528 + UpdateTopLevel(QUERY_ORIGIN, originInfo, now);
1.1529 + }
1.1530 +
1.1531 + UriInfo newUri;
1.1532 + if (havePage && WouldRedirect(pageInfo, now, newUri)) {
1.1533 + nsRefPtr<SeerPredictionRunner> runner =
1.1534 + new SeerPredictionRunner(verifier, predictStartTime);
1.1535 + runner->AddPreconnect(newUri.spec);
1.1536 + NS_DispatchToMainThread(runner);
1.1537 + PredictForPageload(newUri, verifier, stackCount + 1, predictStartTime);
1.1538 + return;
1.1539 + }
1.1540 +
1.1541 + bool predicted = false;
1.1542 +
1.1543 + // We always try to be as specific as possible in our predictions, so try
1.1544 + // to predict based on the full URI before we fall back to the origin.
1.1545 + if (havePage) {
1.1546 + predicted = TryPredict(QUERY_PAGE, pageInfo, now, verifier,
1.1547 + predictStartTime);
1.1548 + }
1.1549 +
1.1550 + if (!predicted && haveOrigin) {
1.1551 + predicted = TryPredict(QUERY_ORIGIN, originInfo, now, verifier,
1.1552 + predictStartTime);
1.1553 + }
1.1554 +
1.1555 + if (!predicted) {
1.1556 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_TIME_TO_INACTION,
1.1557 + predictStartTime);
1.1558 + }
1.1559 +}
1.1560 +
1.1561 +// This is the driver for predicting at browser startup time based on pages that
1.1562 +// have previously been loaded close to startup.
1.1563 +void
1.1564 +Seer::PredictForStartup(SeerVerifierHandle &verifier,
1.1565 + TimeStamp &predictStartTime)
1.1566 +{
1.1567 + MOZ_ASSERT(!NS_IsMainThread(), "PredictForStartup called on main thread");
1.1568 +
1.1569 + if (!mStartupCount) {
1.1570 + SEER_LOG(("Seer::PredictForStartup mStartupCount is zero!\n"));
1.1571 + ++mAccumulators->mStartupCountZeroes;
1.1572 + return;
1.1573 + }
1.1574 +
1.1575 + if (NS_FAILED(EnsureInitStorage())) {
1.1576 + return;
1.1577 + }
1.1578 +
1.1579 + nsCOMPtr<mozIStorageStatement> stmt = mStatements.GetCachedStatement(
1.1580 + NS_LITERAL_CSTRING("SELECT uri, hits, last_hit FROM moz_startup_pages;"));
1.1581 + if (!stmt) {
1.1582 + return;
1.1583 + }
1.1584 + mozStorageStatementScoper scope(stmt);
1.1585 + nsresult rv;
1.1586 + bool hasRows;
1.1587 +
1.1588 + nsRefPtr<SeerPredictionRunner> runner =
1.1589 + new SeerPredictionRunner(verifier, predictStartTime);
1.1590 +
1.1591 + rv = stmt->ExecuteStep(&hasRows);
1.1592 + RETURN_IF_FAILED(rv);
1.1593 +
1.1594 + while (hasRows) {
1.1595 + nsAutoCString uri;
1.1596 + int32_t hitCount;
1.1597 + PRTime lastHit;
1.1598 + int baseConfidence, confidence;
1.1599 +
1.1600 + // We use goto nextrow here instead of just failling, because we want
1.1601 + // to do some sort of prediction if at all possible. Of course, it's
1.1602 + // probably unlikely that subsequent rows will succeed if one fails, but
1.1603 + // it's worth a shot.
1.1604 +
1.1605 + rv = stmt->GetUTF8String(0, uri);
1.1606 + if (NS_FAILED(rv)) {
1.1607 + goto nextrow;
1.1608 + }
1.1609 +
1.1610 + rv = stmt->GetInt32(1, &hitCount);
1.1611 + if (NS_FAILED(rv)) {
1.1612 + goto nextrow;
1.1613 + }
1.1614 +
1.1615 + rv = stmt->GetInt64(2, &lastHit);
1.1616 + if (NS_FAILED(rv)) {
1.1617 + goto nextrow;
1.1618 + }
1.1619 +
1.1620 + baseConfidence = (hitCount * 100) / mStartupCount;
1.1621 + confidence = CalculateConfidence(baseConfidence, lastHit,
1.1622 + mLastStartupTime, 0);
1.1623 + SetupPrediction(confidence, uri, runner);
1.1624 +
1.1625 +nextrow:
1.1626 + rv = stmt->ExecuteStep(&hasRows);
1.1627 + RETURN_IF_FAILED(rv);
1.1628 + }
1.1629 +
1.1630 + if (runner->HasWork()) {
1.1631 + NS_DispatchToMainThread(runner);
1.1632 + } else {
1.1633 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_PREDICT_TIME_TO_INACTION,
1.1634 + predictStartTime);
1.1635 + }
1.1636 +}
1.1637 +
1.1638 +// All URIs we get passed *must* be http or https if they're not null. This
1.1639 +// helps ensure that.
1.1640 +static bool
1.1641 +IsNullOrHttp(nsIURI *uri)
1.1642 +{
1.1643 + if (!uri) {
1.1644 + return true;
1.1645 + }
1.1646 +
1.1647 + bool isHTTP = false;
1.1648 + uri->SchemeIs("http", &isHTTP);
1.1649 + if (!isHTTP) {
1.1650 + uri->SchemeIs("https", &isHTTP);
1.1651 + }
1.1652 +
1.1653 + return isHTTP;
1.1654 +}
1.1655 +
1.1656 +nsresult
1.1657 +Seer::ReserveSpaceInQueue()
1.1658 +{
1.1659 + MutexAutoLock lock(mQueueSizeLock);
1.1660 +
1.1661 + if (mQueueSize >= mMaxQueueSize) {
1.1662 + SEER_LOG(("Not enqueuing event - queue too large"));
1.1663 + return NS_ERROR_NOT_AVAILABLE;
1.1664 + }
1.1665 +
1.1666 + mQueueSize++;
1.1667 + return NS_OK;
1.1668 +}
1.1669 +
1.1670 +void
1.1671 +Seer::FreeSpaceInQueue()
1.1672 +{
1.1673 + MutexAutoLock lock(mQueueSizeLock);
1.1674 + MOZ_ASSERT(mQueueSize > 0, "unexpected mQueueSize");
1.1675 + mQueueSize--;
1.1676 +}
1.1677 +
1.1678 +// Called from the main thread to initiate predictive actions
1.1679 +NS_IMETHODIMP
1.1680 +Seer::Predict(nsIURI *targetURI, nsIURI *sourceURI, SeerPredictReason reason,
1.1681 + nsILoadContext *loadContext, nsINetworkSeerVerifier *verifier)
1.1682 +{
1.1683 + MOZ_ASSERT(NS_IsMainThread(),
1.1684 + "Seer interface methods must be called on the main thread");
1.1685 +
1.1686 + if (!mInitialized) {
1.1687 + return NS_ERROR_NOT_AVAILABLE;
1.1688 + }
1.1689 +
1.1690 + if (!mEnabled) {
1.1691 + return NS_ERROR_NOT_AVAILABLE;
1.1692 + }
1.1693 +
1.1694 + if (loadContext && loadContext->UsePrivateBrowsing()) {
1.1695 + // Don't want to do anything in PB mode
1.1696 + return NS_OK;
1.1697 + }
1.1698 +
1.1699 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.1700 + // Nothing we can do for non-HTTP[S] schemes
1.1701 + return NS_OK;
1.1702 + }
1.1703 +
1.1704 + // Ensure we've been given the appropriate arguments for the kind of
1.1705 + // prediction we're being asked to do
1.1706 + switch (reason) {
1.1707 + case nsINetworkSeer::PREDICT_LINK:
1.1708 + if (!targetURI || !sourceURI) {
1.1709 + return NS_ERROR_INVALID_ARG;
1.1710 + }
1.1711 + // Link hover is a special case where we can predict without hitting the
1.1712 + // db, so let's go ahead and fire off that prediction here.
1.1713 + PredictForLink(targetURI, sourceURI, verifier);
1.1714 + return NS_OK;
1.1715 + case nsINetworkSeer::PREDICT_LOAD:
1.1716 + if (!targetURI || sourceURI) {
1.1717 + return NS_ERROR_INVALID_ARG;
1.1718 + }
1.1719 + break;
1.1720 + case nsINetworkSeer::PREDICT_STARTUP:
1.1721 + if (targetURI || sourceURI) {
1.1722 + return NS_ERROR_INVALID_ARG;
1.1723 + }
1.1724 + break;
1.1725 + default:
1.1726 + return NS_ERROR_INVALID_ARG;
1.1727 + }
1.1728 +
1.1729 + ++mAccumulators->mPredictAttempts;
1.1730 + nsresult rv = ReserveSpaceInQueue();
1.1731 + if (NS_FAILED(rv)) {
1.1732 + ++mAccumulators->mPredictFullQueue;
1.1733 + return NS_ERROR_NOT_AVAILABLE;
1.1734 + }
1.1735 +
1.1736 + nsRefPtr<SeerPredictionEvent> event = new SeerPredictionEvent(targetURI,
1.1737 + sourceURI,
1.1738 + reason,
1.1739 + verifier);
1.1740 + return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
1.1741 +}
1.1742 +
1.1743 +// A runnable for updating our information in the database. This must always
1.1744 +// be dispatched to the seer thread.
1.1745 +class SeerLearnEvent : public nsRunnable
1.1746 +{
1.1747 +public:
1.1748 + SeerLearnEvent(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason)
1.1749 + :mReason(reason)
1.1750 + {
1.1751 + MOZ_ASSERT(NS_IsMainThread(), "Creating learn event off main thread");
1.1752 +
1.1753 + mEnqueueTime = TimeStamp::Now();
1.1754 +
1.1755 + targetURI->GetAsciiSpec(mTargetURI.spec);
1.1756 + ExtractOrigin(targetURI, mTargetURI.origin);
1.1757 + if (sourceURI) {
1.1758 + sourceURI->GetAsciiSpec(mSourceURI.spec);
1.1759 + ExtractOrigin(sourceURI, mSourceURI.origin);
1.1760 + }
1.1761 + }
1.1762 +
1.1763 + NS_IMETHOD Run() MOZ_OVERRIDE
1.1764 + {
1.1765 + MOZ_ASSERT(!NS_IsMainThread(), "Running learn off main thread");
1.1766 +
1.1767 + nsresult rv = NS_OK;
1.1768 +
1.1769 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_LEARN_WAIT_TIME,
1.1770 + mEnqueueTime);
1.1771 +
1.1772 + TimeStamp startTime = TimeStamp::Now();
1.1773 +
1.1774 + switch (mReason) {
1.1775 + case nsINetworkSeer::LEARN_LOAD_TOPLEVEL:
1.1776 + gSeer->LearnForToplevel(mTargetURI);
1.1777 + break;
1.1778 + case nsINetworkSeer::LEARN_LOAD_REDIRECT:
1.1779 + gSeer->LearnForRedirect(mTargetURI, mSourceURI);
1.1780 + break;
1.1781 + case nsINetworkSeer::LEARN_LOAD_SUBRESOURCE:
1.1782 + gSeer->LearnForSubresource(mTargetURI, mSourceURI);
1.1783 + break;
1.1784 + case nsINetworkSeer::LEARN_STARTUP:
1.1785 + gSeer->LearnForStartup(mTargetURI);
1.1786 + break;
1.1787 + default:
1.1788 + MOZ_ASSERT(false, "Got unexpected value for learn reason");
1.1789 + rv = NS_ERROR_UNEXPECTED;
1.1790 + }
1.1791 +
1.1792 + gSeer->FreeSpaceInQueue();
1.1793 +
1.1794 + Telemetry::AccumulateTimeDelta(Telemetry::SEER_LEARN_WORK_TIME, startTime);
1.1795 +
1.1796 + gSeer->MaybeScheduleCleanup();
1.1797 +
1.1798 + return rv;
1.1799 + }
1.1800 +private:
1.1801 + Seer::UriInfo mTargetURI;
1.1802 + Seer::UriInfo mSourceURI;
1.1803 + SeerLearnReason mReason;
1.1804 + TimeStamp mEnqueueTime;
1.1805 +};
1.1806 +
1.1807 +void
1.1808 +Seer::LearnForToplevel(const UriInfo &uri)
1.1809 +{
1.1810 + MOZ_ASSERT(!NS_IsMainThread(), "LearnForToplevel called on main thread.");
1.1811 +
1.1812 + if (NS_FAILED(EnsureInitStorage())) {
1.1813 + return;
1.1814 + }
1.1815 +
1.1816 + PRTime now = PR_Now();
1.1817 +
1.1818 + MaybeLearnForStartup(uri, now);
1.1819 +
1.1820 + TopLevelInfo pageInfo;
1.1821 + TopLevelInfo originInfo;
1.1822 + bool havePage = LookupTopLevel(QUERY_PAGE, uri.spec, pageInfo);
1.1823 + bool haveOrigin = LookupTopLevel(QUERY_ORIGIN, uri.origin, originInfo);
1.1824 +
1.1825 + if (!havePage) {
1.1826 + AddTopLevel(QUERY_PAGE, uri.spec, now);
1.1827 + } else {
1.1828 + UpdateTopLevel(QUERY_PAGE, pageInfo, now);
1.1829 + }
1.1830 +
1.1831 + if (!haveOrigin) {
1.1832 + AddTopLevel(QUERY_ORIGIN, uri.origin, now);
1.1833 + } else {
1.1834 + UpdateTopLevel(QUERY_ORIGIN, originInfo, now);
1.1835 + }
1.1836 +}
1.1837 +
1.1838 +// Queries to look up information about a *specific* subresource associated
1.1839 +// with a *specific* top-level load.
1.1840 +bool
1.1841 +Seer::LookupSubresource(QueryType queryType, const int32_t parentId,
1.1842 + const nsACString &key, SubresourceInfo &info)
1.1843 +{
1.1844 + MOZ_ASSERT(!NS_IsMainThread(), "LookupSubresource called on main thread.");
1.1845 +
1.1846 + nsCOMPtr<mozIStorageStatement> stmt;
1.1847 + if (queryType == QUERY_PAGE) {
1.1848 + stmt = mStatements.GetCachedStatement(
1.1849 + NS_LITERAL_CSTRING("SELECT id, hits, last_hit FROM moz_subresources "
1.1850 + "WHERE pid = :parent_id AND uri = :key;"));
1.1851 + } else {
1.1852 + stmt = mStatements.GetCachedStatement(
1.1853 + NS_LITERAL_CSTRING("SELECT id, hits, last_hit FROM moz_subhosts WHERE "
1.1854 + "hid = :parent_id AND origin = :key;"));
1.1855 + }
1.1856 + NS_ENSURE_TRUE(stmt, false);
1.1857 + mozStorageStatementScoper scope(stmt);
1.1858 +
1.1859 + nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("parent_id"),
1.1860 + parentId);
1.1861 + NS_ENSURE_SUCCESS(rv, false);
1.1862 +
1.1863 + rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);
1.1864 + NS_ENSURE_SUCCESS(rv, false);
1.1865 +
1.1866 + bool hasRows;
1.1867 + rv = stmt->ExecuteStep(&hasRows);
1.1868 + NS_ENSURE_SUCCESS(rv, false);
1.1869 + if (!hasRows) {
1.1870 + return false;
1.1871 + }
1.1872 +
1.1873 + rv = stmt->GetInt32(0, &info.id);
1.1874 + NS_ENSURE_SUCCESS(rv, false);
1.1875 +
1.1876 + rv = stmt->GetInt32(1, &info.hitCount);
1.1877 + NS_ENSURE_SUCCESS(rv, false);
1.1878 +
1.1879 + rv = stmt->GetInt64(2, &info.lastHit);
1.1880 + NS_ENSURE_SUCCESS(rv, false);
1.1881 +
1.1882 + return true;
1.1883 +}
1.1884 +
1.1885 +// Add information about a new subresource associated with a top-level load.
1.1886 +void
1.1887 +Seer::AddSubresource(QueryType queryType, const int32_t parentId,
1.1888 + const nsACString &key, const PRTime now)
1.1889 +{
1.1890 + MOZ_ASSERT(!NS_IsMainThread(), "AddSubresource called on main thread.");
1.1891 +
1.1892 + nsCOMPtr<mozIStorageStatement> stmt;
1.1893 + if (queryType == QUERY_PAGE) {
1.1894 + stmt = mStatements.GetCachedStatement(
1.1895 + NS_LITERAL_CSTRING("INSERT INTO moz_subresources "
1.1896 + "(pid, uri, hits, last_hit) VALUES "
1.1897 + "(:parent_id, :key, 1, :now);"));
1.1898 + } else {
1.1899 + stmt = mStatements.GetCachedStatement(
1.1900 + NS_LITERAL_CSTRING("INSERT INTO moz_subhosts "
1.1901 + "(hid, origin, hits, last_hit) VALUES "
1.1902 + "(:parent_id, :key, 1, :now);"));
1.1903 + }
1.1904 + if (!stmt) {
1.1905 + return;
1.1906 + }
1.1907 + mozStorageStatementScoper scope(stmt);
1.1908 +
1.1909 + nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("parent_id"),
1.1910 + parentId);
1.1911 + RETURN_IF_FAILED(rv);
1.1912 +
1.1913 + rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("key"), key);
1.1914 + RETURN_IF_FAILED(rv);
1.1915 +
1.1916 + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);
1.1917 + RETURN_IF_FAILED(rv);
1.1918 +
1.1919 + rv = stmt->Execute();
1.1920 +}
1.1921 +
1.1922 +// Update the information about a particular subresource associated with a
1.1923 +// top-level load
1.1924 +void
1.1925 +Seer::UpdateSubresource(QueryType queryType, const SubresourceInfo &info,
1.1926 + const PRTime now)
1.1927 +{
1.1928 + MOZ_ASSERT(!NS_IsMainThread(), "UpdateSubresource called on main thread.");
1.1929 +
1.1930 + nsCOMPtr<mozIStorageStatement> stmt;
1.1931 + if (queryType == QUERY_PAGE) {
1.1932 + stmt = mStatements.GetCachedStatement(
1.1933 + NS_LITERAL_CSTRING("UPDATE moz_subresources SET hits = :hit_count, "
1.1934 + "last_hit = :now WHERE id = :id;"));
1.1935 + } else {
1.1936 + stmt = mStatements.GetCachedStatement(
1.1937 + NS_LITERAL_CSTRING("UPDATE moz_subhosts SET hits = :hit_count, "
1.1938 + "last_hit = :now WHERE id = :id;"));
1.1939 + }
1.1940 + if (!stmt) {
1.1941 + return;
1.1942 + }
1.1943 + mozStorageStatementScoper scope(stmt);
1.1944 +
1.1945 + nsresult rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("hit_count"),
1.1946 + info.hitCount + 1);
1.1947 + RETURN_IF_FAILED(rv);
1.1948 +
1.1949 + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);
1.1950 + RETURN_IF_FAILED(rv);
1.1951 +
1.1952 + rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("id"), info.id);
1.1953 + RETURN_IF_FAILED(rv);
1.1954 +
1.1955 + rv = stmt->Execute();
1.1956 +}
1.1957 +
1.1958 +// Called when a subresource has been hit from a top-level load. Uses the two
1.1959 +// helper functions above to update the database appropriately.
1.1960 +void
1.1961 +Seer::LearnForSubresource(const UriInfo &targetURI, const UriInfo &sourceURI)
1.1962 +{
1.1963 + MOZ_ASSERT(!NS_IsMainThread(), "LearnForSubresource called on main thread.");
1.1964 +
1.1965 + if (NS_FAILED(EnsureInitStorage())) {
1.1966 + return;
1.1967 + }
1.1968 +
1.1969 + TopLevelInfo pageInfo, originInfo;
1.1970 + bool havePage = LookupTopLevel(QUERY_PAGE, sourceURI.spec, pageInfo);
1.1971 + bool haveOrigin = LookupTopLevel(QUERY_ORIGIN, sourceURI.origin,
1.1972 + originInfo);
1.1973 +
1.1974 + if (!havePage && !haveOrigin) {
1.1975 + // Nothing to do, since we know nothing about the top level resource
1.1976 + return;
1.1977 + }
1.1978 +
1.1979 + SubresourceInfo resourceInfo;
1.1980 + bool haveResource = false;
1.1981 + if (havePage) {
1.1982 + haveResource = LookupSubresource(QUERY_PAGE, pageInfo.id, targetURI.spec,
1.1983 + resourceInfo);
1.1984 + }
1.1985 +
1.1986 + SubresourceInfo hostInfo;
1.1987 + bool haveHost = false;
1.1988 + if (haveOrigin) {
1.1989 + haveHost = LookupSubresource(QUERY_ORIGIN, originInfo.id, targetURI.origin,
1.1990 + hostInfo);
1.1991 + }
1.1992 +
1.1993 + PRTime now = PR_Now();
1.1994 +
1.1995 + if (haveResource) {
1.1996 + UpdateSubresource(QUERY_PAGE, resourceInfo, now);
1.1997 + } else if (havePage) {
1.1998 + AddSubresource(QUERY_PAGE, pageInfo.id, targetURI.spec, now);
1.1999 + }
1.2000 + // Can't add a subresource to a page we don't have in our db.
1.2001 +
1.2002 + if (haveHost) {
1.2003 + UpdateSubresource(QUERY_ORIGIN, hostInfo, now);
1.2004 + } else if (haveOrigin) {
1.2005 + AddSubresource(QUERY_ORIGIN, originInfo.id, targetURI.origin, now);
1.2006 + }
1.2007 + // Can't add a subhost to a host we don't have in our db
1.2008 +}
1.2009 +
1.2010 +// This is called when a top-level loaded ended up redirecting to a different
1.2011 +// URI so we can keep track of that fact.
1.2012 +void
1.2013 +Seer::LearnForRedirect(const UriInfo &targetURI, const UriInfo &sourceURI)
1.2014 +{
1.2015 + MOZ_ASSERT(!NS_IsMainThread(), "LearnForRedirect called on main thread.");
1.2016 +
1.2017 + if (NS_FAILED(EnsureInitStorage())) {
1.2018 + return;
1.2019 + }
1.2020 +
1.2021 + PRTime now = PR_Now();
1.2022 + nsresult rv;
1.2023 +
1.2024 + nsCOMPtr<mozIStorageStatement> getPage = mStatements.GetCachedStatement(
1.2025 + NS_LITERAL_CSTRING("SELECT id FROM moz_pages WHERE uri = :spec;"));
1.2026 + if (!getPage) {
1.2027 + return;
1.2028 + }
1.2029 + mozStorageStatementScoper scopedPage(getPage);
1.2030 +
1.2031 + // look up source in moz_pages
1.2032 + rv = getPage->BindUTF8StringByName(NS_LITERAL_CSTRING("spec"),
1.2033 + sourceURI.spec);
1.2034 + RETURN_IF_FAILED(rv);
1.2035 +
1.2036 + bool hasRows;
1.2037 + rv = getPage->ExecuteStep(&hasRows);
1.2038 + if (NS_FAILED(rv) || !hasRows) {
1.2039 + return;
1.2040 + }
1.2041 +
1.2042 + int32_t pageId;
1.2043 + rv = getPage->GetInt32(0, &pageId);
1.2044 + RETURN_IF_FAILED(rv);
1.2045 +
1.2046 + nsCOMPtr<mozIStorageStatement> getRedirect = mStatements.GetCachedStatement(
1.2047 + NS_LITERAL_CSTRING("SELECT id, hits FROM moz_redirects WHERE "
1.2048 + "pid = :page_id AND uri = :spec;"));
1.2049 + if (!getRedirect) {
1.2050 + return;
1.2051 + }
1.2052 + mozStorageStatementScoper scopedRedirect(getRedirect);
1.2053 +
1.2054 + rv = getRedirect->BindInt32ByName(NS_LITERAL_CSTRING("page_id"), pageId);
1.2055 + RETURN_IF_FAILED(rv);
1.2056 +
1.2057 + rv = getRedirect->BindUTF8StringByName(NS_LITERAL_CSTRING("spec"),
1.2058 + targetURI.spec);
1.2059 + RETURN_IF_FAILED(rv);
1.2060 +
1.2061 + rv = getRedirect->ExecuteStep(&hasRows);
1.2062 + RETURN_IF_FAILED(rv);
1.2063 +
1.2064 + if (!hasRows) {
1.2065 + // This is the first time we've seen this top-level redirect to this URI
1.2066 + nsCOMPtr<mozIStorageStatement> addRedirect = mStatements.GetCachedStatement(
1.2067 + NS_LITERAL_CSTRING("INSERT INTO moz_redirects "
1.2068 + "(pid, uri, origin, hits, last_hit) VALUES "
1.2069 + "(:page_id, :spec, :origin, 1, :now);"));
1.2070 + if (!addRedirect) {
1.2071 + return;
1.2072 + }
1.2073 + mozStorageStatementScoper scopedAdd(addRedirect);
1.2074 +
1.2075 + rv = addRedirect->BindInt32ByName(NS_LITERAL_CSTRING("page_id"), pageId);
1.2076 + RETURN_IF_FAILED(rv);
1.2077 +
1.2078 + rv = addRedirect->BindUTF8StringByName(NS_LITERAL_CSTRING("spec"),
1.2079 + targetURI.spec);
1.2080 + RETURN_IF_FAILED(rv);
1.2081 +
1.2082 + rv = addRedirect->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"),
1.2083 + targetURI.origin);
1.2084 + RETURN_IF_FAILED(rv);
1.2085 +
1.2086 + rv = addRedirect->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);
1.2087 + RETURN_IF_FAILED(rv);
1.2088 +
1.2089 + rv = addRedirect->Execute();
1.2090 + } else {
1.2091 + // We've seen this redirect before
1.2092 + int32_t redirId, hits;
1.2093 + rv = getRedirect->GetInt32(0, &redirId);
1.2094 + RETURN_IF_FAILED(rv);
1.2095 +
1.2096 + rv = getRedirect->GetInt32(1, &hits);
1.2097 + RETURN_IF_FAILED(rv);
1.2098 +
1.2099 + nsCOMPtr<mozIStorageStatement> updateRedirect =
1.2100 + mStatements.GetCachedStatement(
1.2101 + NS_LITERAL_CSTRING("UPDATE moz_redirects SET hits = :hits, "
1.2102 + "last_hit = :now WHERE id = :redir;"));
1.2103 + if (!updateRedirect) {
1.2104 + return;
1.2105 + }
1.2106 + mozStorageStatementScoper scopedUpdate(updateRedirect);
1.2107 +
1.2108 + rv = updateRedirect->BindInt32ByName(NS_LITERAL_CSTRING("hits"), hits + 1);
1.2109 + RETURN_IF_FAILED(rv);
1.2110 +
1.2111 + rv = updateRedirect->BindInt64ByName(NS_LITERAL_CSTRING("now"), now);
1.2112 + RETURN_IF_FAILED(rv);
1.2113 +
1.2114 + rv = updateRedirect->BindInt32ByName(NS_LITERAL_CSTRING("redir"), redirId);
1.2115 + RETURN_IF_FAILED(rv);
1.2116 +
1.2117 + updateRedirect->Execute();
1.2118 + }
1.2119 +}
1.2120 +
1.2121 +// Add information about a top-level load to our list of startup pages
1.2122 +void
1.2123 +Seer::LearnForStartup(const UriInfo &uri)
1.2124 +{
1.2125 + MOZ_ASSERT(!NS_IsMainThread(), "LearnForStartup called on main thread.");
1.2126 +
1.2127 + if (NS_FAILED(EnsureInitStorage())) {
1.2128 + return;
1.2129 + }
1.2130 +
1.2131 + nsCOMPtr<mozIStorageStatement> getPage = mStatements.GetCachedStatement(
1.2132 + NS_LITERAL_CSTRING("SELECT id, hits FROM moz_startup_pages WHERE "
1.2133 + "uri = :origin;"));
1.2134 + if (!getPage) {
1.2135 + return;
1.2136 + }
1.2137 + mozStorageStatementScoper scopedPage(getPage);
1.2138 + nsresult rv;
1.2139 +
1.2140 + rv = getPage->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"), uri.origin);
1.2141 + RETURN_IF_FAILED(rv);
1.2142 +
1.2143 + bool hasRows;
1.2144 + rv = getPage->ExecuteStep(&hasRows);
1.2145 + RETURN_IF_FAILED(rv);
1.2146 +
1.2147 + if (hasRows) {
1.2148 + // We've loaded this page on startup before
1.2149 + int32_t pageId, hitCount;
1.2150 +
1.2151 + rv = getPage->GetInt32(0, &pageId);
1.2152 + RETURN_IF_FAILED(rv);
1.2153 +
1.2154 + rv = getPage->GetInt32(1, &hitCount);
1.2155 + RETURN_IF_FAILED(rv);
1.2156 +
1.2157 + nsCOMPtr<mozIStorageStatement> updatePage = mStatements.GetCachedStatement(
1.2158 + NS_LITERAL_CSTRING("UPDATE moz_startup_pages SET hits = :hit_count, "
1.2159 + "last_hit = :startup_time WHERE id = :page_id;"));
1.2160 + if (!updatePage) {
1.2161 + return;
1.2162 + }
1.2163 + mozStorageStatementScoper scopedUpdate(updatePage);
1.2164 +
1.2165 + rv = updatePage->BindInt32ByName(NS_LITERAL_CSTRING("hit_count"),
1.2166 + hitCount + 1);
1.2167 + RETURN_IF_FAILED(rv);
1.2168 +
1.2169 + rv = updatePage->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),
1.2170 + mStartupTime);
1.2171 + RETURN_IF_FAILED(rv);
1.2172 +
1.2173 + rv = updatePage->BindInt32ByName(NS_LITERAL_CSTRING("page_id"), pageId);
1.2174 + RETURN_IF_FAILED(rv);
1.2175 +
1.2176 + updatePage->Execute();
1.2177 + } else {
1.2178 + // New startup page
1.2179 + nsCOMPtr<mozIStorageStatement> addPage = mStatements.GetCachedStatement(
1.2180 + NS_LITERAL_CSTRING("INSERT INTO moz_startup_pages (uri, hits, "
1.2181 + "last_hit) VALUES (:origin, 1, :startup_time);"));
1.2182 + if (!addPage) {
1.2183 + return;
1.2184 + }
1.2185 + mozStorageStatementScoper scopedAdd(addPage);
1.2186 + rv = addPage->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"),
1.2187 + uri.origin);
1.2188 + RETURN_IF_FAILED(rv);
1.2189 +
1.2190 + rv = addPage->BindInt64ByName(NS_LITERAL_CSTRING("startup_time"),
1.2191 + mStartupTime);
1.2192 + RETURN_IF_FAILED(rv);
1.2193 +
1.2194 + addPage->Execute();
1.2195 + }
1.2196 +}
1.2197 +
1.2198 +// Called from the main thread to update the database
1.2199 +NS_IMETHODIMP
1.2200 +Seer::Learn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,
1.2201 + nsILoadContext *loadContext)
1.2202 +{
1.2203 + MOZ_ASSERT(NS_IsMainThread(),
1.2204 + "Seer interface methods must be called on the main thread");
1.2205 +
1.2206 + if (!mInitialized) {
1.2207 + return NS_ERROR_NOT_AVAILABLE;
1.2208 + }
1.2209 +
1.2210 + if (!mEnabled) {
1.2211 + return NS_ERROR_NOT_AVAILABLE;
1.2212 + }
1.2213 +
1.2214 + if (loadContext && loadContext->UsePrivateBrowsing()) {
1.2215 + // Don't want to do anything in PB mode
1.2216 + return NS_OK;
1.2217 + }
1.2218 +
1.2219 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.2220 + return NS_ERROR_INVALID_ARG;
1.2221 + }
1.2222 +
1.2223 + switch (reason) {
1.2224 + case nsINetworkSeer::LEARN_LOAD_TOPLEVEL:
1.2225 + case nsINetworkSeer::LEARN_STARTUP:
1.2226 + if (!targetURI || sourceURI) {
1.2227 + return NS_ERROR_INVALID_ARG;
1.2228 + }
1.2229 + break;
1.2230 + case nsINetworkSeer::LEARN_LOAD_REDIRECT:
1.2231 + case nsINetworkSeer::LEARN_LOAD_SUBRESOURCE:
1.2232 + if (!targetURI || !sourceURI) {
1.2233 + return NS_ERROR_INVALID_ARG;
1.2234 + }
1.2235 + break;
1.2236 + default:
1.2237 + return NS_ERROR_INVALID_ARG;
1.2238 + }
1.2239 +
1.2240 + ++mAccumulators->mLearnAttempts;
1.2241 + nsresult rv = ReserveSpaceInQueue();
1.2242 + if (NS_FAILED(rv)) {
1.2243 + ++mAccumulators->mLearnFullQueue;
1.2244 + return NS_ERROR_NOT_AVAILABLE;
1.2245 + }
1.2246 +
1.2247 + nsRefPtr<SeerLearnEvent> event = new SeerLearnEvent(targetURI, sourceURI,
1.2248 + reason);
1.2249 + return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
1.2250 +}
1.2251 +
1.2252 +// Runnable to clear out the database. Dispatched from the main thread to the
1.2253 +// seer thread
1.2254 +class SeerResetEvent : public nsRunnable
1.2255 +{
1.2256 +public:
1.2257 + SeerResetEvent()
1.2258 + { }
1.2259 +
1.2260 + NS_IMETHOD Run() MOZ_OVERRIDE
1.2261 + {
1.2262 + MOZ_ASSERT(!NS_IsMainThread(), "Running reset on main thread");
1.2263 +
1.2264 + gSeer->ResetInternal();
1.2265 +
1.2266 + return NS_OK;
1.2267 + }
1.2268 +};
1.2269 +
1.2270 +// Helper that actually does the database wipe.
1.2271 +void
1.2272 +Seer::ResetInternal()
1.2273 +{
1.2274 + MOZ_ASSERT(!NS_IsMainThread(), "Resetting db on main thread");
1.2275 +
1.2276 + nsresult rv = EnsureInitStorage();
1.2277 + RETURN_IF_FAILED(rv);
1.2278 +
1.2279 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_redirects;"));
1.2280 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages;"));
1.2281 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startups;"));
1.2282 +
1.2283 + // These cascade to moz_subresources and moz_subhosts, respectively.
1.2284 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages;"));
1.2285 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_hosts;"));
1.2286 +
1.2287 + VacuumDatabase();
1.2288 +
1.2289 + // Go ahead and ensure this is flushed to disk
1.2290 + CommitTransaction();
1.2291 + BeginTransaction();
1.2292 +}
1.2293 +
1.2294 +// Called on the main thread to clear out all our knowledge. Tabula Rasa FTW!
1.2295 +NS_IMETHODIMP
1.2296 +Seer::Reset()
1.2297 +{
1.2298 + MOZ_ASSERT(NS_IsMainThread(),
1.2299 + "Seer interface methods must be called on the main thread");
1.2300 +
1.2301 + if (!mInitialized) {
1.2302 + return NS_ERROR_NOT_AVAILABLE;
1.2303 + }
1.2304 +
1.2305 + if (!mEnabled) {
1.2306 + return NS_OK;
1.2307 + }
1.2308 +
1.2309 + nsRefPtr<SeerResetEvent> event = new SeerResetEvent();
1.2310 + return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
1.2311 +}
1.2312 +
1.2313 +class SeerCleanupEvent : public nsRunnable
1.2314 +{
1.2315 +public:
1.2316 + NS_IMETHOD Run() MOZ_OVERRIDE
1.2317 + {
1.2318 + gSeer->Cleanup();
1.2319 + gSeer->mCleanupScheduled = false;
1.2320 + return NS_OK;
1.2321 + }
1.2322 +};
1.2323 +
1.2324 +// Returns the current size (in bytes) of the db file on disk
1.2325 +int64_t
1.2326 +Seer::GetDBFileSize()
1.2327 +{
1.2328 + MOZ_ASSERT(!NS_IsMainThread(), "GetDBFileSize called on main thread!");
1.2329 +
1.2330 + nsresult rv = EnsureInitStorage();
1.2331 + if (NS_FAILED(rv)) {
1.2332 + SEER_LOG(("GetDBFileSize called without db available!"));
1.2333 + return 0;
1.2334 + }
1.2335 +
1.2336 + CommitTransaction();
1.2337 +
1.2338 + nsCOMPtr<mozIStorageStatement> countStmt = mStatements.GetCachedStatement(
1.2339 + NS_LITERAL_CSTRING("PRAGMA page_count;"));
1.2340 + if (!countStmt) {
1.2341 + return 0;
1.2342 + }
1.2343 + mozStorageStatementScoper scopedCount(countStmt);
1.2344 + bool hasRows;
1.2345 + rv = countStmt->ExecuteStep(&hasRows);
1.2346 + if (NS_FAILED(rv) || !hasRows) {
1.2347 + return 0;
1.2348 + }
1.2349 + int64_t pageCount;
1.2350 + rv = countStmt->GetInt64(0, &pageCount);
1.2351 + if (NS_FAILED(rv)) {
1.2352 + return 0;
1.2353 + }
1.2354 +
1.2355 + nsCOMPtr<mozIStorageStatement> sizeStmt = mStatements.GetCachedStatement(
1.2356 + NS_LITERAL_CSTRING("PRAGMA page_size;"));
1.2357 + if (!sizeStmt) {
1.2358 + return 0;
1.2359 + }
1.2360 + mozStorageStatementScoper scopedSize(sizeStmt);
1.2361 + rv = sizeStmt->ExecuteStep(&hasRows);
1.2362 + if (NS_FAILED(rv) || !hasRows) {
1.2363 + return 0;
1.2364 + }
1.2365 + int64_t pageSize;
1.2366 + rv = sizeStmt->GetInt64(0, &pageSize);
1.2367 + if (NS_FAILED(rv)) {
1.2368 + return 0;
1.2369 + }
1.2370 +
1.2371 + BeginTransaction();
1.2372 +
1.2373 + return pageCount * pageSize;
1.2374 +}
1.2375 +
1.2376 +// Returns the size (in bytes) that the db file will consume on disk AFTER we
1.2377 +// vacuum the db.
1.2378 +int64_t
1.2379 +Seer::GetDBFileSizeAfterVacuum()
1.2380 +{
1.2381 + MOZ_ASSERT(!NS_IsMainThread(), "GetDBFileSizeAfterVacuum called on main thread!");
1.2382 +
1.2383 + CommitTransaction();
1.2384 +
1.2385 + nsCOMPtr<mozIStorageStatement> countStmt = mStatements.GetCachedStatement(
1.2386 + NS_LITERAL_CSTRING("PRAGMA page_count;"));
1.2387 + if (!countStmt) {
1.2388 + return 0;
1.2389 + }
1.2390 + mozStorageStatementScoper scopedCount(countStmt);
1.2391 + bool hasRows;
1.2392 + nsresult rv = countStmt->ExecuteStep(&hasRows);
1.2393 + if (NS_FAILED(rv) || !hasRows) {
1.2394 + return 0;
1.2395 + }
1.2396 + int64_t pageCount;
1.2397 + rv = countStmt->GetInt64(0, &pageCount);
1.2398 + if (NS_FAILED(rv)) {
1.2399 + return 0;
1.2400 + }
1.2401 +
1.2402 + nsCOMPtr<mozIStorageStatement> sizeStmt = mStatements.GetCachedStatement(
1.2403 + NS_LITERAL_CSTRING("PRAGMA page_size;"));
1.2404 + if (!sizeStmt) {
1.2405 + return 0;
1.2406 + }
1.2407 + mozStorageStatementScoper scopedSize(sizeStmt);
1.2408 + rv = sizeStmt->ExecuteStep(&hasRows);
1.2409 + if (NS_FAILED(rv) || !hasRows) {
1.2410 + return 0;
1.2411 + }
1.2412 + int64_t pageSize;
1.2413 + rv = sizeStmt->GetInt64(0, &pageSize);
1.2414 + if (NS_FAILED(rv)) {
1.2415 + return 0;
1.2416 + }
1.2417 +
1.2418 + nsCOMPtr<mozIStorageStatement> freeStmt = mStatements.GetCachedStatement(
1.2419 + NS_LITERAL_CSTRING("PRAGMA freelist_count;"));
1.2420 + if (!freeStmt) {
1.2421 + return 0;
1.2422 + }
1.2423 + mozStorageStatementScoper scopedFree(freeStmt);
1.2424 + rv = freeStmt->ExecuteStep(&hasRows);
1.2425 + if (NS_FAILED(rv) || !hasRows) {
1.2426 + return 0;
1.2427 + }
1.2428 + int64_t freelistCount;
1.2429 + rv = freeStmt->GetInt64(0, &freelistCount);
1.2430 + if (NS_FAILED(rv)) {
1.2431 + return 0;
1.2432 + }
1.2433 +
1.2434 + BeginTransaction();
1.2435 +
1.2436 + return (pageCount - freelistCount) * pageSize;
1.2437 +}
1.2438 +
1.2439 +void
1.2440 +Seer::MaybeScheduleCleanup()
1.2441 +{
1.2442 + MOZ_ASSERT(!NS_IsMainThread(), "MaybeScheduleCleanup called on main thread!");
1.2443 +
1.2444 + // This is a little racy, but it's a nice little shutdown optimization if the
1.2445 + // race works out the right way.
1.2446 + if (!mInitialized) {
1.2447 + return;
1.2448 + }
1.2449 +
1.2450 + if (mCleanupScheduled) {
1.2451 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, false);
1.2452 + return;
1.2453 + }
1.2454 +
1.2455 + int64_t dbFileSize = GetDBFileSize();
1.2456 + if (dbFileSize < mMaxDBSize) {
1.2457 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, false);
1.2458 + return;
1.2459 + }
1.2460 +
1.2461 + mCleanupScheduled = true;
1.2462 +
1.2463 + nsRefPtr<SeerCleanupEvent> event = new SeerCleanupEvent();
1.2464 + nsresult rv = mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
1.2465 + if (NS_FAILED(rv)) {
1.2466 + mCleanupScheduled = false;
1.2467 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, false);
1.2468 + } else {
1.2469 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SCHEDULED, true);
1.2470 + }
1.2471 +}
1.2472 +
1.2473 +#ifndef ANDROID
1.2474 +static const long long CLEANUP_CUTOFF = ONE_MONTH;
1.2475 +#else
1.2476 +static const long long CLEANUP_CUTOFF = ONE_WEEK;
1.2477 +#endif
1.2478 +
1.2479 +void
1.2480 +Seer::CleanupOrigins(PRTime now)
1.2481 +{
1.2482 + PRTime cutoff = now - CLEANUP_CUTOFF;
1.2483 +
1.2484 + nsCOMPtr<mozIStorageStatement> deleteOrigins = mStatements.GetCachedStatement(
1.2485 + NS_LITERAL_CSTRING("DELETE FROM moz_hosts WHERE last_load <= :cutoff"));
1.2486 + if (!deleteOrigins) {
1.2487 + return;
1.2488 + }
1.2489 + mozStorageStatementScoper scopedOrigins(deleteOrigins);
1.2490 +
1.2491 + nsresult rv = deleteOrigins->BindInt32ByName(NS_LITERAL_CSTRING("cutoff"),
1.2492 + cutoff);
1.2493 + RETURN_IF_FAILED(rv);
1.2494 +
1.2495 + deleteOrigins->Execute();
1.2496 +}
1.2497 +
1.2498 +void
1.2499 +Seer::CleanupStartupPages(PRTime now)
1.2500 +{
1.2501 + PRTime cutoff = now - ONE_WEEK;
1.2502 +
1.2503 + nsCOMPtr<mozIStorageStatement> deletePages = mStatements.GetCachedStatement(
1.2504 + NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages WHERE "
1.2505 + "last_hit <= :cutoff"));
1.2506 + if (!deletePages) {
1.2507 + return;
1.2508 + }
1.2509 + mozStorageStatementScoper scopedPages(deletePages);
1.2510 +
1.2511 + nsresult rv = deletePages->BindInt32ByName(NS_LITERAL_CSTRING("cutoff"),
1.2512 + cutoff);
1.2513 + RETURN_IF_FAILED(rv);
1.2514 +
1.2515 + deletePages->Execute();
1.2516 +}
1.2517 +
1.2518 +int32_t
1.2519 +Seer::GetSubresourceCount()
1.2520 +{
1.2521 + nsCOMPtr<mozIStorageStatement> count = mStatements.GetCachedStatement(
1.2522 + NS_LITERAL_CSTRING("SELECT COUNT(id) FROM moz_subresources"));
1.2523 + if (!count) {
1.2524 + return 0;
1.2525 + }
1.2526 + mozStorageStatementScoper scopedCount(count);
1.2527 +
1.2528 + bool hasRows;
1.2529 + nsresult rv = count->ExecuteStep(&hasRows);
1.2530 + if (NS_FAILED(rv) || !hasRows) {
1.2531 + return 0;
1.2532 + }
1.2533 +
1.2534 + int32_t subresourceCount = 0;
1.2535 + count->GetInt32(0, &subresourceCount);
1.2536 +
1.2537 + return subresourceCount;
1.2538 +}
1.2539 +
1.2540 +void
1.2541 +Seer::Cleanup()
1.2542 +{
1.2543 + MOZ_ASSERT(!NS_IsMainThread(), "Seer::Cleanup called on main thread!");
1.2544 +
1.2545 + nsresult rv = EnsureInitStorage();
1.2546 + if (NS_FAILED(rv)) {
1.2547 + return;
1.2548 + }
1.2549 +
1.2550 + int64_t dbFileSize = GetDBFileSize();
1.2551 + float preservePercentage = static_cast<float>(mPreservePercentage) / 100.0;
1.2552 + int64_t evictionCutoff = static_cast<int64_t>(mMaxDBSize) * preservePercentage;
1.2553 + if (dbFileSize < evictionCutoff) {
1.2554 + return;
1.2555 + }
1.2556 +
1.2557 + CommitTransaction();
1.2558 + BeginTransaction();
1.2559 +
1.2560 + PRTime now = PR_Now();
1.2561 + if (mLastCleanupTime) {
1.2562 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_DELTA,
1.2563 + (now - mLastCleanupTime) / 1000);
1.2564 + }
1.2565 + mLastCleanupTime = now;
1.2566 +
1.2567 + CleanupOrigins(now);
1.2568 + CleanupStartupPages(now);
1.2569 +
1.2570 + dbFileSize = GetDBFileSizeAfterVacuum();
1.2571 + if (dbFileSize < evictionCutoff) {
1.2572 + // We've deleted enough stuff, time to free up the disk space and be on
1.2573 + // our way.
1.2574 + VacuumDatabase();
1.2575 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SUCCEEDED, true);
1.2576 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_TIME,
1.2577 + (PR_Now() - mLastCleanupTime) / 1000);
1.2578 + return;
1.2579 + }
1.2580 +
1.2581 + bool canDelete = true;
1.2582 + while (canDelete && (dbFileSize >= evictionCutoff)) {
1.2583 + int32_t subresourceCount = GetSubresourceCount();
1.2584 + if (!subresourceCount) {
1.2585 + canDelete = false;
1.2586 + break;
1.2587 + }
1.2588 +
1.2589 + // DB size scales pretty much linearly with the number of rows in
1.2590 + // moz_subresources, so we can guess how many rows we need to delete pretty
1.2591 + // accurately.
1.2592 + float percentNeeded = static_cast<float>(dbFileSize - evictionCutoff) /
1.2593 + static_cast<float>(dbFileSize);
1.2594 +
1.2595 + int32_t subresourcesToDelete = static_cast<int32_t>(percentNeeded * subresourceCount);
1.2596 + if (!subresourcesToDelete) {
1.2597 + // We're getting pretty close to nothing here, anyway, so we may as well
1.2598 + // just trash it all. This delete cascades to moz_subresources, as well.
1.2599 + rv = mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages;"));
1.2600 + if (NS_FAILED(rv)) {
1.2601 + canDelete = false;
1.2602 + break;
1.2603 + }
1.2604 + } else {
1.2605 + nsCOMPtr<mozIStorageStatement> deleteStatement = mStatements.GetCachedStatement(
1.2606 + NS_LITERAL_CSTRING("DELETE FROM moz_subresources WHERE id IN "
1.2607 + "(SELECT id FROM moz_subresources ORDER BY "
1.2608 + "last_hit ASC LIMIT :limit);"));
1.2609 + if (!deleteStatement) {
1.2610 + canDelete = false;
1.2611 + break;
1.2612 + }
1.2613 + mozStorageStatementScoper scopedDelete(deleteStatement);
1.2614 +
1.2615 + rv = deleteStatement->BindInt32ByName(NS_LITERAL_CSTRING("limit"),
1.2616 + subresourcesToDelete);
1.2617 + if (NS_FAILED(rv)) {
1.2618 + canDelete = false;
1.2619 + break;
1.2620 + }
1.2621 +
1.2622 + rv = deleteStatement->Execute();
1.2623 + if (NS_FAILED(rv)) {
1.2624 + canDelete = false;
1.2625 + break;
1.2626 + }
1.2627 +
1.2628 + // Now we clean up pages that no longer reference any subresources
1.2629 + rv = mDB->ExecuteSimpleSQL(
1.2630 + NS_LITERAL_CSTRING("DELETE FROM moz_pages WHERE id NOT IN "
1.2631 + "(SELECT DISTINCT(pid) FROM moz_subresources);"));
1.2632 + if (NS_FAILED(rv)) {
1.2633 + canDelete = false;
1.2634 + break;
1.2635 + }
1.2636 + }
1.2637 +
1.2638 + if (canDelete) {
1.2639 + dbFileSize = GetDBFileSizeAfterVacuum();
1.2640 + }
1.2641 + }
1.2642 +
1.2643 + if (!canDelete || (dbFileSize >= evictionCutoff)) {
1.2644 + // Last-ditch effort to free up space
1.2645 + ResetInternal();
1.2646 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SUCCEEDED, false);
1.2647 + } else {
1.2648 + // We do this to actually free up the space on disk
1.2649 + VacuumDatabase();
1.2650 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_SUCCEEDED, true);
1.2651 + }
1.2652 + Telemetry::Accumulate(Telemetry::SEER_CLEANUP_TIME,
1.2653 + (PR_Now() - mLastCleanupTime) / 1000);
1.2654 +}
1.2655 +
1.2656 +void
1.2657 +Seer::VacuumDatabase()
1.2658 +{
1.2659 + MOZ_ASSERT(!NS_IsMainThread(), "VacuumDatabase called on main thread!");
1.2660 +
1.2661 + CommitTransaction();
1.2662 + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM;"));
1.2663 + BeginTransaction();
1.2664 +}
1.2665 +
1.2666 +#ifdef SEER_TESTS
1.2667 +class SeerPrepareForDnsTestEvent : public nsRunnable
1.2668 +{
1.2669 +public:
1.2670 + SeerPrepareForDnsTestEvent(int64_t timestamp, const char *uri)
1.2671 + :mTimestamp(timestamp)
1.2672 + ,mUri(uri)
1.2673 + { }
1.2674 +
1.2675 + NS_IMETHOD Run() MOZ_OVERRIDE
1.2676 + {
1.2677 + MOZ_ASSERT(!NS_IsMainThread(), "Preparing for DNS Test on main thread!");
1.2678 + gSeer->PrepareForDnsTestInternal(mTimestamp, mUri);
1.2679 + return NS_OK;
1.2680 + }
1.2681 +
1.2682 +private:
1.2683 + int64_t mTimestamp;
1.2684 + nsAutoCString mUri;
1.2685 +};
1.2686 +
1.2687 +void
1.2688 +Seer::PrepareForDnsTestInternal(int64_t timestamp, const nsACString &uri)
1.2689 +{
1.2690 + nsCOMPtr<mozIStorageStatement> update = mStatements.GetCachedStatement(
1.2691 + NS_LITERAL_CSTRING("UPDATE moz_subresources SET last_hit = :timestamp, "
1.2692 + "hits = 2 WHERE uri = :uri;"));
1.2693 + if (!update) {
1.2694 + return;
1.2695 + }
1.2696 + mozStorageStatementScoper scopedUpdate(update);
1.2697 +
1.2698 + nsresult rv = update->BindInt64ByName(NS_LITERAL_CSTRING("timestamp"),
1.2699 + timestamp);
1.2700 + RETURN_IF_FAILED(rv);
1.2701 +
1.2702 + rv = update->BindUTF8StringByName(NS_LITERAL_CSTRING("uri"), uri);
1.2703 + RETURN_IF_FAILED(rv);
1.2704 +
1.2705 + update->Execute();
1.2706 +}
1.2707 +#endif
1.2708 +
1.2709 +NS_IMETHODIMP
1.2710 +Seer::PrepareForDnsTest(int64_t timestamp, const char *uri)
1.2711 +{
1.2712 +#ifdef SEER_TESTS
1.2713 + MOZ_ASSERT(NS_IsMainThread(),
1.2714 + "Seer interface methods must be called on the main thread");
1.2715 +
1.2716 + if (!mInitialized) {
1.2717 + return NS_ERROR_NOT_AVAILABLE;
1.2718 + }
1.2719 +
1.2720 + nsRefPtr<SeerPrepareForDnsTestEvent> event =
1.2721 + new SeerPrepareForDnsTestEvent(timestamp, uri);
1.2722 + return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
1.2723 +#else
1.2724 + return NS_ERROR_NOT_IMPLEMENTED;
1.2725 +#endif
1.2726 +}
1.2727 +
1.2728 +// Helper functions to make using the seer easier from native code
1.2729 +
1.2730 +static nsresult
1.2731 +EnsureGlobalSeer(nsINetworkSeer **aSeer)
1.2732 +{
1.2733 + nsresult rv;
1.2734 + nsCOMPtr<nsINetworkSeer> seer = do_GetService("@mozilla.org/network/seer;1",
1.2735 + &rv);
1.2736 + NS_ENSURE_SUCCESS(rv, rv);
1.2737 +
1.2738 + NS_IF_ADDREF(*aSeer = seer);
1.2739 + return NS_OK;
1.2740 +}
1.2741 +
1.2742 +nsresult
1.2743 +SeerPredict(nsIURI *targetURI, nsIURI *sourceURI, SeerPredictReason reason,
1.2744 + nsILoadContext *loadContext, nsINetworkSeerVerifier *verifier)
1.2745 +{
1.2746 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.2747 + return NS_OK;
1.2748 + }
1.2749 +
1.2750 + nsCOMPtr<nsINetworkSeer> seer;
1.2751 + nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));
1.2752 + NS_ENSURE_SUCCESS(rv, rv);
1.2753 +
1.2754 + return seer->Predict(targetURI, sourceURI, reason, loadContext, verifier);
1.2755 +}
1.2756 +
1.2757 +nsresult
1.2758 +SeerLearn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,
1.2759 + nsILoadContext *loadContext)
1.2760 +{
1.2761 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.2762 + return NS_OK;
1.2763 + }
1.2764 +
1.2765 + nsCOMPtr<nsINetworkSeer> seer;
1.2766 + nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));
1.2767 + NS_ENSURE_SUCCESS(rv, rv);
1.2768 +
1.2769 + return seer->Learn(targetURI, sourceURI, reason, loadContext);
1.2770 +}
1.2771 +
1.2772 +nsresult
1.2773 +SeerLearn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,
1.2774 + nsILoadGroup *loadGroup)
1.2775 +{
1.2776 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.2777 + return NS_OK;
1.2778 + }
1.2779 +
1.2780 + nsCOMPtr<nsINetworkSeer> seer;
1.2781 + nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));
1.2782 + NS_ENSURE_SUCCESS(rv, rv);
1.2783 +
1.2784 + nsCOMPtr<nsILoadContext> loadContext;
1.2785 +
1.2786 + if (loadGroup) {
1.2787 + nsCOMPtr<nsIInterfaceRequestor> callbacks;
1.2788 + loadGroup->GetNotificationCallbacks(getter_AddRefs(callbacks));
1.2789 + if (callbacks) {
1.2790 + loadContext = do_GetInterface(callbacks);
1.2791 + }
1.2792 + }
1.2793 +
1.2794 + return seer->Learn(targetURI, sourceURI, reason, loadContext);
1.2795 +}
1.2796 +
1.2797 +nsresult
1.2798 +SeerLearn(nsIURI *targetURI, nsIURI *sourceURI, SeerLearnReason reason,
1.2799 + nsIDocument *document)
1.2800 +{
1.2801 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.2802 + return NS_OK;
1.2803 + }
1.2804 +
1.2805 + nsCOMPtr<nsINetworkSeer> seer;
1.2806 + nsresult rv = EnsureGlobalSeer(getter_AddRefs(seer));
1.2807 + NS_ENSURE_SUCCESS(rv, rv);
1.2808 +
1.2809 + nsCOMPtr<nsILoadContext> loadContext;
1.2810 +
1.2811 + if (document) {
1.2812 + loadContext = document->GetLoadContext();
1.2813 + }
1.2814 +
1.2815 + return seer->Learn(targetURI, sourceURI, reason, loadContext);
1.2816 +}
1.2817 +
1.2818 +nsresult
1.2819 +SeerLearnRedirect(nsIURI *targetURI, nsIChannel *channel,
1.2820 + nsILoadContext *loadContext)
1.2821 +{
1.2822 + nsCOMPtr<nsIURI> sourceURI;
1.2823 + nsresult rv = channel->GetOriginalURI(getter_AddRefs(sourceURI));
1.2824 + NS_ENSURE_SUCCESS(rv, rv);
1.2825 +
1.2826 + bool sameUri;
1.2827 + rv = targetURI->Equals(sourceURI, &sameUri);
1.2828 + NS_ENSURE_SUCCESS(rv, rv);
1.2829 +
1.2830 + if (sameUri) {
1.2831 + return NS_OK;
1.2832 + }
1.2833 +
1.2834 + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
1.2835 + return NS_OK;
1.2836 + }
1.2837 +
1.2838 + nsCOMPtr<nsINetworkSeer> seer;
1.2839 + rv = EnsureGlobalSeer(getter_AddRefs(seer));
1.2840 + NS_ENSURE_SUCCESS(rv, rv);
1.2841 +
1.2842 + return seer->Learn(targetURI, sourceURI,
1.2843 + nsINetworkSeer::LEARN_LOAD_REDIRECT, loadContext);
1.2844 +}
1.2845 +
1.2846 +} // ::mozilla::net
1.2847 +} // ::mozilla
