1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/toolkit/components/url-classifier/content/listmanager.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,467 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +
1.9 +// This is the only implementation of nsIUrlListManager.
1.10 +// A class that manages lists, namely white and black lists for
1.11 +// phishing or malware protection. The ListManager knows how to fetch,
1.12 +// update, and store lists.
1.13 +//
1.14 +// There is a single listmanager for the whole application.
1.15 +//
1.16 +// TODO more comprehensive update tests, for example add unittest check
1.17 +// that the listmanagers tables are properly written on updates
1.18 +
1.19 +function QueryAdapter(callback) {
1.20 + this.callback_ = callback;
1.21 +};
1.22 +
1.23 +QueryAdapter.prototype.handleResponse = function(value) {
1.24 + this.callback_.handleEvent(value);
1.25 +}
1.26 +
1.27 +/**
1.28 + * A ListManager keeps track of black and white lists and knows
1.29 + * how to update them.
1.30 + *
1.31 + * @constructor
1.32 + */
1.33 +function PROT_ListManager() {
1.34 + this.debugZone = "listmanager";
1.35 + G_debugService.enableZone(this.debugZone);
1.36 +
1.37 + this.currentUpdateChecker_ = null; // set when we toggle updates
1.38 + this.prefs_ = new G_Preferences();
1.39 + this.updateInterval = this.prefs_.getPref("urlclassifier.updateinterval", 30 * 60) * 1000;
1.40 +
1.41 + this.updateserverURL_ = null;
1.42 + this.gethashURL_ = null;
1.43 +
1.44 + this.isTesting_ = false;
1.45 +
1.46 + this.tablesData = {};
1.47 +
1.48 + this.observerServiceObserver_ = new G_ObserverServiceObserver(
1.49 + 'quit-application',
1.50 + BindToObject(this.shutdown_, this),
1.51 + true /*only once*/);
1.52 +
1.53 + this.cookieObserver_ = new G_ObserverServiceObserver(
1.54 + 'cookie-changed',
1.55 + BindToObject(this.cookieChanged_, this),
1.56 + false);
1.57 +
1.58 + /* Backoff interval should be between 30 and 60 minutes. */
1.59 + var backoffInterval = 30 * 60 * 1000;
1.60 + backoffInterval += Math.floor(Math.random() * (30 * 60 * 1000));
1.61 +
1.62 + this.requestBackoff_ = new RequestBackoff(2 /* max errors */,
1.63 + 60*1000 /* retry interval, 1 min */,
1.64 + 4 /* num requests */,
1.65 + 60*60*1000 /* request time, 60 min */,
1.66 + backoffInterval /* backoff interval, 60 min */,
1.67 + 8*60*60*1000 /* max backoff, 8hr */);
1.68 +
1.69 + this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"]
1.70 + .getService(Ci.nsIUrlClassifierDBService);
1.71 +
1.72 + this.hashCompleter_ = Cc["@mozilla.org/url-classifier/hashcompleter;1"]
1.73 + .getService(Ci.nsIUrlClassifierHashCompleter);
1.74 +}
1.75 +
1.76 +/**
1.77 + * xpcom-shutdown callback
1.78 + * Delete all of our data tables which seem to leak otherwise.
1.79 + */
1.80 +PROT_ListManager.prototype.shutdown_ = function() {
1.81 + for (var name in this.tablesData) {
1.82 + delete this.tablesData[name];
1.83 + }
1.84 +}
1.85 +
1.86 +/**
1.87 + * Set the url we check for updates. If the new url is valid and different,
1.88 + * update our table list.
1.89 + *
1.90 + * After setting the update url, the caller is responsible for registering
1.91 + * tables and then toggling update checking. All the code for this logic is
1.92 + * currently in browser/components/safebrowsing. Maybe it should be part of
1.93 + * the listmanger?
1.94 + */
1.95 +PROT_ListManager.prototype.setUpdateUrl = function(url) {
1.96 + G_Debug(this, "Set update url: " + url);
1.97 + if (url != this.updateserverURL_) {
1.98 + this.updateserverURL_ = url;
1.99 + this.requestBackoff_.reset();
1.100 +
1.101 + // Remove old tables which probably aren't valid for the new provider.
1.102 + for (var name in this.tablesData) {
1.103 + delete this.tablesData[name];
1.104 + }
1.105 + }
1.106 +}
1.107 +
1.108 +/**
1.109 + * Set the gethash url.
1.110 + */
1.111 +PROT_ListManager.prototype.setGethashUrl = function(url) {
1.112 + G_Debug(this, "Set gethash url: " + url);
1.113 + if (url != this.gethashURL_) {
1.114 + this.gethashURL_ = url;
1.115 + this.hashCompleter_.gethashUrl = url;
1.116 + }
1.117 +}
1.118 +
1.119 +/**
1.120 + * Register a new table table
1.121 + * @param tableName - the name of the table
1.122 + * @param opt_requireMac true if a mac is required on update, false otherwise
1.123 + * @returns true if the table could be created; false otherwise
1.124 + */
1.125 +PROT_ListManager.prototype.registerTable = function(tableName,
1.126 + opt_requireMac) {
1.127 + this.tablesData[tableName] = {};
1.128 + this.tablesData[tableName].needsUpdate = false;
1.129 +
1.130 + return true;
1.131 +}
1.132 +
1.133 +/**
1.134 + * Enable updates for some tables
1.135 + * @param tables - an array of table names that need updating
1.136 + */
1.137 +PROT_ListManager.prototype.enableUpdate = function(tableName) {
1.138 + var changed = false;
1.139 + var table = this.tablesData[tableName];
1.140 + if (table) {
1.141 + G_Debug(this, "Enabling table updates for " + tableName);
1.142 + table.needsUpdate = true;
1.143 + changed = true;
1.144 + }
1.145 +
1.146 + if (changed === true)
1.147 + this.maybeToggleUpdateChecking();
1.148 +}
1.149 +
1.150 +/**
1.151 + * Disables updates for some tables
1.152 + * @param tables - an array of table names that no longer need updating
1.153 + */
1.154 +PROT_ListManager.prototype.disableUpdate = function(tableName) {
1.155 + var changed = false;
1.156 + var table = this.tablesData[tableName];
1.157 + if (table) {
1.158 + G_Debug(this, "Disabling table updates for " + tableName);
1.159 + table.needsUpdate = false;
1.160 + changed = true;
1.161 + }
1.162 +
1.163 + if (changed === true)
1.164 + this.maybeToggleUpdateChecking();
1.165 +}
1.166 +
1.167 +/**
1.168 + * Determine if we have some tables that need updating.
1.169 + */
1.170 +PROT_ListManager.prototype.requireTableUpdates = function() {
1.171 + for (var type in this.tablesData) {
1.172 + // Tables that need updating even if other tables dont require it
1.173 + if (this.tablesData[type].needsUpdate)
1.174 + return true;
1.175 + }
1.176 +
1.177 + return false;
1.178 +}
1.179 +
1.180 +/**
1.181 + * Start managing the lists we know about. We don't do this automatically
1.182 + * when the listmanager is instantiated because their profile directory
1.183 + * (where we store the lists) might not be available.
1.184 + */
1.185 +PROT_ListManager.prototype.maybeStartManagingUpdates = function() {
1.186 + if (this.isTesting_)
1.187 + return;
1.188 +
1.189 + // We might have been told about tables already, so see if we should be
1.190 + // actually updating.
1.191 + this.maybeToggleUpdateChecking();
1.192 +}
1.193 +
1.194 +/**
1.195 + * Acts as a nsIUrlClassifierCallback for getTables.
1.196 + */
1.197 +PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData)
1.198 +{
1.199 + this.startingUpdate_ = false;
1.200 + var initialUpdateDelay = 3000;
1.201 +
1.202 + // Check if any table registered for updates has ever been downloaded.
1.203 + var diskTablesAreUpdating = false;
1.204 + for (var tableName in this.tablesData) {
1.205 + if (this.tablesData[tableName].needsUpdate) {
1.206 + if (onDiskTableData.indexOf(tableName) != -1) {
1.207 + diskTablesAreUpdating = true;
1.208 + }
1.209 + }
1.210 + }
1.211 +
1.212 + // If the user has never downloaded tables, do the check now.
1.213 + // If the user has tables, add a fuzz of a few minutes.
1.214 + if (diskTablesAreUpdating) {
1.215 + // Add a fuzz of 0-5 minutes.
1.216 + initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
1.217 + }
1.218 +
1.219 + this.currentUpdateChecker_ =
1.220 + new G_Alarm(BindToObject(this.checkForUpdates, this),
1.221 + initialUpdateDelay);
1.222 +}
1.223 +
1.224 +/**
1.225 + * Determine if we have any tables that require updating. Different
1.226 + * Wardens may call us with new tables that need to be updated.
1.227 + */
1.228 +PROT_ListManager.prototype.maybeToggleUpdateChecking = function() {
1.229 + // If we are testing or dont have an application directory yet, we should
1.230 + // not start reading tables from disk or schedule remote updates
1.231 + if (this.isTesting_)
1.232 + return;
1.233 +
1.234 + // We update tables if we have some tables that want updates. If there
1.235 + // are no tables that want to be updated - we dont need to check anything.
1.236 + if (this.requireTableUpdates() === true) {
1.237 + G_Debug(this, "Starting managing lists");
1.238 + this.startUpdateChecker();
1.239 +
1.240 + // Multiple warden can ask us to reenable updates at the same time, but we
1.241 + // really just need to schedule a single update.
1.242 + if (!this.currentUpdateChecker && !this.startingUpdate_) {
1.243 + this.startingUpdate_ = true;
1.244 + // check the current state of tables in the database
1.245 + this.dbService_.getTables(BindToObject(this.kickoffUpdate_, this));
1.246 + }
1.247 + } else {
1.248 + G_Debug(this, "Stopping managing lists (if currently active)");
1.249 + this.stopUpdateChecker(); // Cancel pending updates
1.250 + }
1.251 +}
1.252 +
1.253 +/**
1.254 + * Start periodic checks for updates. Idempotent.
1.255 + * We want to distribute update checks evenly across the update period (an
1.256 + * hour). The first update is scheduled for a random time between 0.5 and 1.5
1.257 + * times the update interval.
1.258 + */
1.259 +PROT_ListManager.prototype.startUpdateChecker = function() {
1.260 + this.stopUpdateChecker();
1.261 +
1.262 + // Schedule the first check for between 15 and 45 minutes.
1.263 + var repeatingUpdateDelay = this.updateInterval / 2;
1.264 + repeatingUpdateDelay += Math.floor(Math.random() * this.updateInterval);
1.265 + this.updateChecker_ = new G_Alarm(BindToObject(this.initialUpdateCheck_,
1.266 + this),
1.267 + repeatingUpdateDelay);
1.268 +}
1.269 +
1.270 +/**
1.271 + * Callback for the first update check.
1.272 + * We go ahead and check for table updates, then start a regular timer (once
1.273 + * every update interval).
1.274 + */
1.275 +PROT_ListManager.prototype.initialUpdateCheck_ = function() {
1.276 + this.checkForUpdates();
1.277 + this.updateChecker_ = new G_Alarm(BindToObject(this.checkForUpdates, this),
1.278 + this.updateInterval, true /* repeat */);
1.279 +}
1.280 +
1.281 +/**
1.282 + * Stop checking for updates. Idempotent.
1.283 + */
1.284 +PROT_ListManager.prototype.stopUpdateChecker = function() {
1.285 + if (this.updateChecker_) {
1.286 + this.updateChecker_.cancel();
1.287 + this.updateChecker_ = null;
1.288 + }
1.289 + // Cancel the oneoff check from maybeToggleUpdateChecking.
1.290 + if (this.currentUpdateChecker_) {
1.291 + this.currentUpdateChecker_.cancel();
1.292 + this.currentUpdateChecker_ = null;
1.293 + }
1.294 +}
1.295 +
1.296 +/**
1.297 + * Provides an exception free way to look up the data in a table. We
1.298 + * use this because at certain points our tables might not be loaded,
1.299 + * and querying them could throw.
1.300 + *
1.301 + * @param table String Name of the table that we want to consult
1.302 + * @param key Principal being used to lookup the database
1.303 + * @param callback nsIUrlListManagerCallback (ie., Function) given false or the
1.304 + * value in the table corresponding to key. If the table name does not
1.305 + * exist, we return false, too.
1.306 + */
1.307 +PROT_ListManager.prototype.safeLookup = function(key, callback) {
1.308 + try {
1.309 + G_Debug(this, "safeLookup: " + key);
1.310 + var cb = new QueryAdapter(callback);
1.311 + this.dbService_.lookup(key,
1.312 + BindToObject(cb.handleResponse, cb),
1.313 + true);
1.314 + } catch(e) {
1.315 + G_Debug(this, "safeLookup masked failure for key " + key + ": " + e);
1.316 + callback.handleEvent("");
1.317 + }
1.318 +}
1.319 +
1.320 +/**
1.321 + * Updates our internal tables from the update server
1.322 + *
1.323 + * @returns true when a new request was scheduled, false if an old request
1.324 + * was still pending.
1.325 + */
1.326 +PROT_ListManager.prototype.checkForUpdates = function() {
1.327 + // Allow new updates to be scheduled from maybeToggleUpdateChecking()
1.328 + this.currentUpdateChecker_ = null;
1.329 +
1.330 + if (!this.updateserverURL_) {
1.331 + G_Debug(this, 'checkForUpdates: no update server url');
1.332 + return false;
1.333 + }
1.334 +
1.335 + // See if we've triggered the request backoff logic.
1.336 + if (!this.requestBackoff_.canMakeRequest())
1.337 + return false;
1.338 +
1.339 + // Grab the current state of the tables from the database
1.340 + this.dbService_.getTables(BindToObject(this.makeUpdateRequest_, this));
1.341 + return true;
1.342 +}
1.343 +
1.344 +/**
1.345 + * Method that fires the actual HTTP update request.
1.346 + * First we reset any tables that have disappeared.
1.347 + * @param tableData List of table data already in the database, in the form
1.348 + * tablename;<chunk ranges>\n
1.349 + */
1.350 +PROT_ListManager.prototype.makeUpdateRequest_ = function(tableData) {
1.351 + var tableList;
1.352 + var tableNames = {};
1.353 + for (var tableName in this.tablesData) {
1.354 + if (this.tablesData[tableName].needsUpdate)
1.355 + tableNames[tableName] = true;
1.356 + if (!tableList) {
1.357 + tableList = tableName;
1.358 + } else {
1.359 + tableList += "," + tableName;
1.360 + }
1.361 + }
1.362 +
1.363 + var request = "";
1.364 +
1.365 + // For each table already in the database, include the chunk data from
1.366 + // the database
1.367 + var lines = tableData.split("\n");
1.368 + for (var i = 0; i < lines.length; i++) {
1.369 + var fields = lines[i].split(";");
1.370 + if (tableNames[fields[0]]) {
1.371 + request += lines[i] + "\n";
1.372 + delete tableNames[fields[0]];
1.373 + }
1.374 + }
1.375 +
1.376 + // For each requested table that didn't have chunk data in the database,
1.377 + // request it fresh
1.378 + for (var tableName in tableNames) {
1.379 + request += tableName + ";\n";
1.380 + }
1.381 +
1.382 + G_Debug(this, 'checkForUpdates: scheduling request..');
1.383 + var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
1.384 + .getService(Ci.nsIUrlClassifierStreamUpdater);
1.385 + try {
1.386 + streamer.updateUrl = this.updateserverURL_;
1.387 + } catch (e) {
1.388 + G_Debug(this, 'invalid url');
1.389 + return;
1.390 + }
1.391 +
1.392 + this.requestBackoff_.noteRequest();
1.393 +
1.394 + if (!streamer.downloadUpdates(tableList,
1.395 + request,
1.396 + BindToObject(this.updateSuccess_, this),
1.397 + BindToObject(this.updateError_, this),
1.398 + BindToObject(this.downloadError_, this))) {
1.399 + G_Debug(this, "pending update, wait until later");
1.400 + }
1.401 +}
1.402 +
1.403 +/**
1.404 + * Callback function if the update request succeeded.
1.405 + * @param waitForUpdate String The number of seconds that the client should
1.406 + * wait before requesting again.
1.407 + */
1.408 +PROT_ListManager.prototype.updateSuccess_ = function(waitForUpdate) {
1.409 + G_Debug(this, "update success: " + waitForUpdate);
1.410 + if (waitForUpdate) {
1.411 + var delay = parseInt(waitForUpdate, 10);
1.412 + // As long as the delay is something sane (5 minutes or more), update
1.413 + // our delay time for requesting updates
1.414 + if (delay >= (5 * 60) && this.updateChecker_)
1.415 + this.updateChecker_.setDelay(delay * 1000);
1.416 + }
1.417 +
1.418 + // Let the backoff object know that we completed successfully.
1.419 + this.requestBackoff_.noteServerResponse(200);
1.420 +}
1.421 +
1.422 +/**
1.423 + * Callback function if the update request succeeded.
1.424 + * @param result String The error code of the failure
1.425 + */
1.426 +PROT_ListManager.prototype.updateError_ = function(result) {
1.427 + G_Debug(this, "update error: " + result);
1.428 + // XXX: there was some trouble applying the updates.
1.429 +}
1.430 +
1.431 +/**
1.432 + * Callback function when the download failed
1.433 + * @param status String http status or an empty string if connection refused.
1.434 + */
1.435 +PROT_ListManager.prototype.downloadError_ = function(status) {
1.436 + G_Debug(this, "download error: " + status);
1.437 + // If status is empty, then we assume that we got an NS_CONNECTION_REFUSED
1.438 + // error. In this case, we treat this is a http 500 error.
1.439 + if (!status) {
1.440 + status = 500;
1.441 + }
1.442 + status = parseInt(status, 10);
1.443 + this.requestBackoff_.noteServerResponse(status);
1.444 +
1.445 + if (this.requestBackoff_.isErrorStatus(status)) {
1.446 + // Schedule an update for when our backoff is complete
1.447 + this.currentUpdateChecker_ =
1.448 + new G_Alarm(BindToObject(this.checkForUpdates, this),
1.449 + this.requestBackoff_.nextRequestDelay());
1.450 + }
1.451 +}
1.452 +
1.453 +/**
1.454 + * Called when cookies are cleared
1.455 + */
1.456 +PROT_ListManager.prototype.cookieChanged_ = function(subject, topic, data) {
1.457 + if (data != "cleared")
1.458 + return;
1.459 +
1.460 + G_Debug(this, "cookies cleared");
1.461 +}
1.462 +
1.463 +PROT_ListManager.prototype.QueryInterface = function(iid) {
1.464 + if (iid.equals(Ci.nsISupports) ||
1.465 + iid.equals(Ci.nsIUrlListManager) ||
1.466 + iid.equals(Ci.nsITimerCallback))
1.467 + return this;
1.468 +
1.469 + throw Components.results.NS_ERROR_NO_INTERFACE;
1.470 +}
