toolkit/components/url-classifier/content/listmanager.js@b8a032363ba2
toolkit/components/url-classifier/content/listmanager.js
Thu, 22 Jan 2015 13:21:57 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Thu, 22 Jan 2015 13:21:57 +0100
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
- permissions
- -rw-r--r--
Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 // This is the only implementation of nsIUrlListManager.
7 // A class that manages lists, namely white and black lists for
8 // phishing or malware protection. The ListManager knows how to fetch,
9 // update, and store lists.
10 //
11 // There is a single listmanager for the whole application.
12 //
13 // TODO more comprehensive update tests, for example add unittest check
14 // that the listmanagers tables are properly written on updates
16 function QueryAdapter(callback) {
17 this.callback_ = callback;
18 };
20 QueryAdapter.prototype.handleResponse = function(value) {
21 this.callback_.handleEvent(value);
22 }
24 /**
25 * A ListManager keeps track of black and white lists and knows
26 * how to update them.
27 *
28 * @constructor
29 */
30 function PROT_ListManager() {
31 this.debugZone = "listmanager";
32 G_debugService.enableZone(this.debugZone);
34 this.currentUpdateChecker_ = null; // set when we toggle updates
35 this.prefs_ = new G_Preferences();
36 this.updateInterval = this.prefs_.getPref("urlclassifier.updateinterval", 30 * 60) * 1000;
38 this.updateserverURL_ = null;
39 this.gethashURL_ = null;
41 this.isTesting_ = false;
43 this.tablesData = {};
45 this.observerServiceObserver_ = new G_ObserverServiceObserver(
46 'quit-application',
47 BindToObject(this.shutdown_, this),
48 true /*only once*/);
50 this.cookieObserver_ = new G_ObserverServiceObserver(
51 'cookie-changed',
52 BindToObject(this.cookieChanged_, this),
53 false);
55 /* Backoff interval should be between 30 and 60 minutes. */
56 var backoffInterval = 30 * 60 * 1000;
57 backoffInterval += Math.floor(Math.random() * (30 * 60 * 1000));
59 this.requestBackoff_ = new RequestBackoff(2 /* max errors */,
60 60*1000 /* retry interval, 1 min */,
61 4 /* num requests */,
62 60*60*1000 /* request time, 60 min */,
63 backoffInterval /* backoff interval, 60 min */,
64 8*60*60*1000 /* max backoff, 8hr */);
66 this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"]
67 .getService(Ci.nsIUrlClassifierDBService);
69 this.hashCompleter_ = Cc["@mozilla.org/url-classifier/hashcompleter;1"]
70 .getService(Ci.nsIUrlClassifierHashCompleter);
71 }
73 /**
74 * xpcom-shutdown callback
75 * Delete all of our data tables which seem to leak otherwise.
76 */
77 PROT_ListManager.prototype.shutdown_ = function() {
78 for (var name in this.tablesData) {
79 delete this.tablesData[name];
80 }
81 }
83 /**
84 * Set the url we check for updates. If the new url is valid and different,
85 * update our table list.
86 *
87 * After setting the update url, the caller is responsible for registering
88 * tables and then toggling update checking. All the code for this logic is
89 * currently in browser/components/safebrowsing. Maybe it should be part of
90 * the listmanger?
91 */
92 PROT_ListManager.prototype.setUpdateUrl = function(url) {
93 G_Debug(this, "Set update url: " + url);
94 if (url != this.updateserverURL_) {
95 this.updateserverURL_ = url;
96 this.requestBackoff_.reset();
98 // Remove old tables which probably aren't valid for the new provider.
99 for (var name in this.tablesData) {
100 delete this.tablesData[name];
101 }
102 }
103 }
105 /**
106 * Set the gethash url.
107 */
108 PROT_ListManager.prototype.setGethashUrl = function(url) {
109 G_Debug(this, "Set gethash url: " + url);
110 if (url != this.gethashURL_) {
111 this.gethashURL_ = url;
112 this.hashCompleter_.gethashUrl = url;
113 }
114 }
116 /**
117 * Register a new table table
118 * @param tableName - the name of the table
119 * @param opt_requireMac true if a mac is required on update, false otherwise
120 * @returns true if the table could be created; false otherwise
121 */
122 PROT_ListManager.prototype.registerTable = function(tableName,
123 opt_requireMac) {
124 this.tablesData[tableName] = {};
125 this.tablesData[tableName].needsUpdate = false;
127 return true;
128 }
130 /**
131 * Enable updates for some tables
132 * @param tables - an array of table names that need updating
133 */
134 PROT_ListManager.prototype.enableUpdate = function(tableName) {
135 var changed = false;
136 var table = this.tablesData[tableName];
137 if (table) {
138 G_Debug(this, "Enabling table updates for " + tableName);
139 table.needsUpdate = true;
140 changed = true;
141 }
143 if (changed === true)
144 this.maybeToggleUpdateChecking();
145 }
147 /**
148 * Disables updates for some tables
149 * @param tables - an array of table names that no longer need updating
150 */
151 PROT_ListManager.prototype.disableUpdate = function(tableName) {
152 var changed = false;
153 var table = this.tablesData[tableName];
154 if (table) {
155 G_Debug(this, "Disabling table updates for " + tableName);
156 table.needsUpdate = false;
157 changed = true;
158 }
160 if (changed === true)
161 this.maybeToggleUpdateChecking();
162 }
164 /**
165 * Determine if we have some tables that need updating.
166 */
167 PROT_ListManager.prototype.requireTableUpdates = function() {
168 for (var type in this.tablesData) {
169 // Tables that need updating even if other tables dont require it
170 if (this.tablesData[type].needsUpdate)
171 return true;
172 }
174 return false;
175 }
177 /**
178 * Start managing the lists we know about. We don't do this automatically
179 * when the listmanager is instantiated because their profile directory
180 * (where we store the lists) might not be available.
181 */
182 PROT_ListManager.prototype.maybeStartManagingUpdates = function() {
183 if (this.isTesting_)
184 return;
186 // We might have been told about tables already, so see if we should be
187 // actually updating.
188 this.maybeToggleUpdateChecking();
189 }
191 /**
192 * Acts as a nsIUrlClassifierCallback for getTables.
193 */
194 PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData)
195 {
196 this.startingUpdate_ = false;
197 var initialUpdateDelay = 3000;
199 // Check if any table registered for updates has ever been downloaded.
200 var diskTablesAreUpdating = false;
201 for (var tableName in this.tablesData) {
202 if (this.tablesData[tableName].needsUpdate) {
203 if (onDiskTableData.indexOf(tableName) != -1) {
204 diskTablesAreUpdating = true;
205 }
206 }
207 }
209 // If the user has never downloaded tables, do the check now.
210 // If the user has tables, add a fuzz of a few minutes.
211 if (diskTablesAreUpdating) {
212 // Add a fuzz of 0-5 minutes.
213 initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
214 }
216 this.currentUpdateChecker_ =
217 new G_Alarm(BindToObject(this.checkForUpdates, this),
218 initialUpdateDelay);
219 }
221 /**
222 * Determine if we have any tables that require updating. Different
223 * Wardens may call us with new tables that need to be updated.
224 */
225 PROT_ListManager.prototype.maybeToggleUpdateChecking = function() {
226 // If we are testing or dont have an application directory yet, we should
227 // not start reading tables from disk or schedule remote updates
228 if (this.isTesting_)
229 return;
231 // We update tables if we have some tables that want updates. If there
232 // are no tables that want to be updated - we dont need to check anything.
233 if (this.requireTableUpdates() === true) {
234 G_Debug(this, "Starting managing lists");
235 this.startUpdateChecker();
237 // Multiple warden can ask us to reenable updates at the same time, but we
238 // really just need to schedule a single update.
239 if (!this.currentUpdateChecker && !this.startingUpdate_) {
240 this.startingUpdate_ = true;
241 // check the current state of tables in the database
242 this.dbService_.getTables(BindToObject(this.kickoffUpdate_, this));
243 }
244 } else {
245 G_Debug(this, "Stopping managing lists (if currently active)");
246 this.stopUpdateChecker(); // Cancel pending updates
247 }
248 }
250 /**
251 * Start periodic checks for updates. Idempotent.
252 * We want to distribute update checks evenly across the update period (an
253 * hour). The first update is scheduled for a random time between 0.5 and 1.5
254 * times the update interval.
255 */
256 PROT_ListManager.prototype.startUpdateChecker = function() {
257 this.stopUpdateChecker();
259 // Schedule the first check for between 15 and 45 minutes.
260 var repeatingUpdateDelay = this.updateInterval / 2;
261 repeatingUpdateDelay += Math.floor(Math.random() * this.updateInterval);
262 this.updateChecker_ = new G_Alarm(BindToObject(this.initialUpdateCheck_,
263 this),
264 repeatingUpdateDelay);
265 }
267 /**
268 * Callback for the first update check.
269 * We go ahead and check for table updates, then start a regular timer (once
270 * every update interval).
271 */
272 PROT_ListManager.prototype.initialUpdateCheck_ = function() {
273 this.checkForUpdates();
274 this.updateChecker_ = new G_Alarm(BindToObject(this.checkForUpdates, this),
275 this.updateInterval, true /* repeat */);
276 }
278 /**
279 * Stop checking for updates. Idempotent.
280 */
281 PROT_ListManager.prototype.stopUpdateChecker = function() {
282 if (this.updateChecker_) {
283 this.updateChecker_.cancel();
284 this.updateChecker_ = null;
285 }
286 // Cancel the oneoff check from maybeToggleUpdateChecking.
287 if (this.currentUpdateChecker_) {
288 this.currentUpdateChecker_.cancel();
289 this.currentUpdateChecker_ = null;
290 }
291 }
293 /**
294 * Provides an exception free way to look up the data in a table. We
295 * use this because at certain points our tables might not be loaded,
296 * and querying them could throw.
297 *
298 * @param table String Name of the table that we want to consult
299 * @param key Principal being used to lookup the database
300 * @param callback nsIUrlListManagerCallback (ie., Function) given false or the
301 * value in the table corresponding to key. If the table name does not
302 * exist, we return false, too.
303 */
304 PROT_ListManager.prototype.safeLookup = function(key, callback) {
305 try {
306 G_Debug(this, "safeLookup: " + key);
307 var cb = new QueryAdapter(callback);
308 this.dbService_.lookup(key,
309 BindToObject(cb.handleResponse, cb),
310 true);
311 } catch(e) {
312 G_Debug(this, "safeLookup masked failure for key " + key + ": " + e);
313 callback.handleEvent("");
314 }
315 }
317 /**
318 * Updates our internal tables from the update server
319 *
320 * @returns true when a new request was scheduled, false if an old request
321 * was still pending.
322 */
323 PROT_ListManager.prototype.checkForUpdates = function() {
324 // Allow new updates to be scheduled from maybeToggleUpdateChecking()
325 this.currentUpdateChecker_ = null;
327 if (!this.updateserverURL_) {
328 G_Debug(this, 'checkForUpdates: no update server url');
329 return false;
330 }
332 // See if we've triggered the request backoff logic.
333 if (!this.requestBackoff_.canMakeRequest())
334 return false;
336 // Grab the current state of the tables from the database
337 this.dbService_.getTables(BindToObject(this.makeUpdateRequest_, this));
338 return true;
339 }
341 /**
342 * Method that fires the actual HTTP update request.
343 * First we reset any tables that have disappeared.
344 * @param tableData List of table data already in the database, in the form
345 * tablename;<chunk ranges>\n
346 */
347 PROT_ListManager.prototype.makeUpdateRequest_ = function(tableData) {
348 var tableList;
349 var tableNames = {};
350 for (var tableName in this.tablesData) {
351 if (this.tablesData[tableName].needsUpdate)
352 tableNames[tableName] = true;
353 if (!tableList) {
354 tableList = tableName;
355 } else {
356 tableList += "," + tableName;
357 }
358 }
360 var request = "";
362 // For each table already in the database, include the chunk data from
363 // the database
364 var lines = tableData.split("\n");
365 for (var i = 0; i < lines.length; i++) {
366 var fields = lines[i].split(";");
367 if (tableNames[fields[0]]) {
368 request += lines[i] + "\n";
369 delete tableNames[fields[0]];
370 }
371 }
373 // For each requested table that didn't have chunk data in the database,
374 // request it fresh
375 for (var tableName in tableNames) {
376 request += tableName + ";\n";
377 }
379 G_Debug(this, 'checkForUpdates: scheduling request..');
380 var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
381 .getService(Ci.nsIUrlClassifierStreamUpdater);
382 try {
383 streamer.updateUrl = this.updateserverURL_;
384 } catch (e) {
385 G_Debug(this, 'invalid url');
386 return;
387 }
389 this.requestBackoff_.noteRequest();
391 if (!streamer.downloadUpdates(tableList,
392 request,
393 BindToObject(this.updateSuccess_, this),
394 BindToObject(this.updateError_, this),
395 BindToObject(this.downloadError_, this))) {
396 G_Debug(this, "pending update, wait until later");
397 }
398 }
400 /**
401 * Callback function if the update request succeeded.
402 * @param waitForUpdate String The number of seconds that the client should
403 * wait before requesting again.
404 */
405 PROT_ListManager.prototype.updateSuccess_ = function(waitForUpdate) {
406 G_Debug(this, "update success: " + waitForUpdate);
407 if (waitForUpdate) {
408 var delay = parseInt(waitForUpdate, 10);
409 // As long as the delay is something sane (5 minutes or more), update
410 // our delay time for requesting updates
411 if (delay >= (5 * 60) && this.updateChecker_)
412 this.updateChecker_.setDelay(delay * 1000);
413 }
415 // Let the backoff object know that we completed successfully.
416 this.requestBackoff_.noteServerResponse(200);
417 }
419 /**
420 * Callback function if the update request succeeded.
421 * @param result String The error code of the failure
422 */
423 PROT_ListManager.prototype.updateError_ = function(result) {
424 G_Debug(this, "update error: " + result);
425 // XXX: there was some trouble applying the updates.
426 }
428 /**
429 * Callback function when the download failed
430 * @param status String http status or an empty string if connection refused.
431 */
432 PROT_ListManager.prototype.downloadError_ = function(status) {
433 G_Debug(this, "download error: " + status);
434 // If status is empty, then we assume that we got an NS_CONNECTION_REFUSED
435 // error. In this case, we treat this is a http 500 error.
436 if (!status) {
437 status = 500;
438 }
439 status = parseInt(status, 10);
440 this.requestBackoff_.noteServerResponse(status);
442 if (this.requestBackoff_.isErrorStatus(status)) {
443 // Schedule an update for when our backoff is complete
444 this.currentUpdateChecker_ =
445 new G_Alarm(BindToObject(this.checkForUpdates, this),
446 this.requestBackoff_.nextRequestDelay());
447 }
448 }
450 /**
451 * Called when cookies are cleared
452 */
453 PROT_ListManager.prototype.cookieChanged_ = function(subject, topic, data) {
454 if (data != "cleared")
455 return;
457 G_Debug(this, "cookies cleared");
458 }
460 PROT_ListManager.prototype.QueryInterface = function(iid) {
461 if (iid.equals(Ci.nsISupports) ||
462 iid.equals(Ci.nsIUrlListManager) ||
463 iid.equals(Ci.nsITimerCallback))
464 return this;
466 throw Components.results.NS_ERROR_NO_INTERFACE;
467 }
