michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: const Cc = Components.classes;
michael@0: const Ci = Components.interfaces;
michael@0: const Cr = Components.results;
michael@0: const Cu = Components.utils;
michael@0: 
michael@0: // COMPLETE_LENGTH and PARTIAL_LENGTH copied from nsUrlClassifierDBService.h,
michael@0: // they correspond to the length, in bytes, of a hash prefix and the total
michael@0: // hash.
michael@0: const COMPLETE_LENGTH = 32;
michael@0: const PARTIAL_LENGTH = 4;
michael@0: 
michael@0: // These backoff related constants are taken from v2 of the Google Safe Browsing
michael@0: // API.
michael@0: // BACKOFF_ERRORS: the number of errors incurred until we start to back off.
michael@0: // BACKOFF_INTERVAL: the initial time, in seconds, to wait once we start backing
michael@0: //                   off.
michael@0: // BACKOFF_MAX: as the backoff time doubles after each failure, this is a
michael@0: //              ceiling on the time to wait, in seconds.
michael@0: // BACKOFF_TIME: length of the interval of time, in seconds, during which errors
michael@0: //               are taken into account.
michael@0: 
michael@0: const BACKOFF_ERRORS = 2;
michael@0: const BACKOFF_INTERVAL = 30 * 60;
michael@0: const BACKOFF_MAX = 8 * 60 * 60;
michael@0: const BACKOFF_TIME = 5 * 60;
michael@0: 
michael@0: Cu.import("resource://gre/modules/XPCOMUtils.jsm");
michael@0: Cu.import("resource://gre/modules/Services.jsm");
michael@0: 
michael@0: function HashCompleter() {
michael@0:   // This is a HashCompleterRequest and is used by multiple calls to |complete|
michael@0:   // in succession to avoid unnecessarily creating requests. Once it has been
michael@0:   // started, this is set to null again.
michael@0:   this._currentRequest = null;
michael@0: 
michael@0:   // Whether we have been informed of a shutdown by the xpcom-shutdown event.
michael@0:   this._shuttingDown = false;
michael@0: 
michael@0:   // All of these backoff properties are different per completer as the DB
michael@0:   // service keeps one completer per table.
michael@0:   //
michael@0:   // _backoff tells us whether we are "backing off" from making requests.
michael@0:   // It is set in |noteServerResponse| and set after a number of failures.
michael@0:   this._backoff = false;
michael@0:   // _backoffTime tells us how long we should wait (in seconds) before making
michael@0:   // another request.
michael@0:   this._backoffTime = 0;
michael@0:   // _nextRequestTime is the earliest time at which we are allowed to make
michael@0:   // another request by the backoff policy. It is measured in milliseconds.
michael@0:   this._nextRequestTime = 0;
michael@0:   // A list of the times at which a failed request was made, recorded in
michael@0:   // |noteServerResponse|. Sorted by oldest to newest and its length is clamped
michael@0:   // by BACKOFF_ERRORS.
michael@0:   this._errorTimes = [];
michael@0: 
michael@0:   Services.obs.addObserver(this, "xpcom-shutdown", true);
michael@0: }
michael@0: HashCompleter.prototype = {
michael@0:   classID: Components.ID("{9111de73-9322-4bfc-8b65-2b727f3e6ec8}"),
michael@0:   QueryInterface: XPCOMUtils.generateQI([Ci.nsIUrlClassifierHashCompleter,
michael@0:                                          Ci.nsIRunnable,
michael@0:                                          Ci.nsIObserver,
michael@0:                                          Ci.nsISupportsWeakReference,
michael@0:                                          Ci.nsISupports]),
michael@0: 
michael@0:   // This is mainly how the HashCompleter interacts with other components.
michael@0:   // Even though it only takes one partial hash and callback, subsequent
michael@0:   // calls are made into the same HTTP request by using a thread dispatch.
michael@0:   complete: function HC_complete(aPartialHash, aCallback) {
michael@0:     if (!this._currentRequest) {
michael@0:       this._currentRequest = new HashCompleterRequest(this);
michael@0: 
michael@0:       // It's possible for getHashUrl to not be set, usually at start-up.
michael@0:       if (this._getHashUrl) {
michael@0:         Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     this._currentRequest.add(aPartialHash, aCallback);
michael@0:   },
michael@0: 
michael@0:   // This is called when either the getHashUrl has been set or after a few calls
michael@0:   // to |complete|. It starts off the HTTP request by making a |begin| call
michael@0:   // to the HashCompleterRequest.
michael@0:   run: function HC_run() {
michael@0:     if (this._shuttingDown) {
michael@0:       this._currentRequest = null;
michael@0:       throw Cr.NS_ERROR_NOT_INITIALIZED;
michael@0:     }
michael@0: 
michael@0:     if (!this._currentRequest) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     if (!this._getHashUrl) {
michael@0:       throw Cr.NS_ERROR_NOT_INITIALIZED;
michael@0:     }
michael@0: 
michael@0:     let url = this._getHashUrl;
michael@0: 
michael@0:     let uri = Services.io.newURI(url, null, null);
michael@0:     this._currentRequest.setURI(uri);
michael@0: 
michael@0:     // If |begin| fails, we should get rid of our request.
michael@0:     try {
michael@0:       this._currentRequest.begin();
michael@0:     }
michael@0:     finally {
michael@0:       this._currentRequest = null;
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   get gethashUrl() {
michael@0:     return this._getHashUrl;
michael@0:   },
michael@0:   // Because we hold off on making a request until we have a valid getHashUrl,
michael@0:   // we kick off the process here.
michael@0:   set gethashUrl(aNewUrl) {
michael@0:     this._getHashUrl = aNewUrl;
michael@0: 
michael@0:     if (this._currentRequest) {
michael@0:       Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   // This handles all the logic about setting a back off time based on
michael@0:   // server responses. It should only be called once in the life time of a
michael@0:   // request.
michael@0:   // The logic behind backoffs is documented in the Google Safe Browsing API,
michael@0:   // the general idea is that a completer should go into backoff mode after
michael@0:   // BACKOFF_ERRORS errors in the last BACKOFF_TIME seconds. From there,
michael@0:   // we do not make a request for BACKOFF_INTERVAL seconds and for every failed
michael@0:   // request after that we double how long to wait, to a maximum of BACKOFF_MAX.
michael@0:   // Note that even in the case of a successful response we still keep a history
michael@0:   // of past errors.
michael@0:   noteServerResponse: function HC_noteServerResponse(aSuccess) {
michael@0:     if (aSuccess) {
michael@0:       this._backoff = false;
michael@0:       this._nextRequestTime = 0;
michael@0:       this._backoffTime = 0;
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let now = Date.now();
michael@0: 
michael@0:     // We only alter the size of |_errorTimes| here, so we can guarantee that
michael@0:     // its length is at most BACKOFF_ERRORS.
michael@0:     this._errorTimes.push(now);
michael@0:     if (this._errorTimes.length > BACKOFF_ERRORS) {
michael@0:       this._errorTimes.shift();
michael@0:     }
michael@0: 
michael@0:     if (this._backoff) {
michael@0:       this._backoffTime *= 2;
michael@0:     } else if (this._errorTimes.length == BACKOFF_ERRORS &&
michael@0:                ((now - this._errorTimes[0]) / 1000) <= BACKOFF_TIME) {
michael@0:       this._backoff = true;
michael@0:       this._backoffTime = BACKOFF_INTERVAL;
michael@0:     }
michael@0: 
michael@0:     if (this._backoff) {
michael@0:       this._backoffTime = Math.min(this._backoffTime, BACKOFF_MAX);
michael@0:       this._nextRequestTime = now + (this._backoffTime * 1000);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   // This is not included on the interface but is used to communicate to the
michael@0:   // HashCompleterRequest. It returns a time in milliseconds.
michael@0:   get nextRequestTime() {
michael@0:     return this._nextRequestTime;
michael@0:   },
michael@0: 
michael@0:   observe: function HC_observe(aSubject, aTopic, aData) {
michael@0:     if (aTopic == "xpcom-shutdown") {
michael@0:       this._shuttingDown = true;
michael@0:     }
michael@0:   },
michael@0: };
michael@0: 
michael@0: function HashCompleterRequest(aCompleter) {
michael@0:   // HashCompleter object that created this HashCompleterRequest.
michael@0:   this._completer = aCompleter;
michael@0:   // The internal set of hashes and callbacks that this request corresponds to.
michael@0:   this._requests = [];
michael@0:   // URI to query for hash completions. Largely comes from the
michael@0:   // browser.safebrowsing.gethashURL pref.
michael@0:   this._uri = null;
michael@0:   // nsIChannel that the hash completion query is transmitted over.
michael@0:   this._channel = null;
michael@0:   // Response body of hash completion. Created in onDataAvailable.
michael@0:   this._response = "";
michael@0:   // Whether we have been informed of a shutdown by the xpcom-shutdown event.
michael@0:   this._shuttingDown = false;
michael@0: }
michael@0: HashCompleterRequest.prototype = {
michael@0:   QueryInterface: XPCOMUtils.generateQI([Ci.nsIRequestObserver,
michael@0:                                          Ci.nsIStreamListener,
michael@0:                                          Ci.nsIObserver,
michael@0:                                          Ci.nsISupports]),
michael@0: 
michael@0:   // This is called by the HashCompleter to add a hash and callback to the
michael@0:   // HashCompleterRequest. It must be called before calling |begin|.
michael@0:   add: function HCR_add(aPartialHash, aCallback) {
michael@0:     this._requests.push({
michael@0:       partialHash: aPartialHash,
michael@0:       callback: aCallback,
michael@0:       responses: [],
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   // This initiates the HTTP request. It can fail due to backoff timings and
michael@0:   // will notify all callbacks as necessary.
michael@0:   begin: function HCR_begin() {
michael@0:     if (Date.now() < this._completer.nextRequestTime) {
michael@0:       this.notifyFailure(Cr.NS_ERROR_ABORT);
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     Services.obs.addObserver(this, "xpcom-shutdown", false);
michael@0: 
michael@0:     try {
michael@0:       this.openChannel();
michael@0:     }
michael@0:     catch (err) {
michael@0:       this.notifyFailure(err);
michael@0:       throw err;
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   setURI: function HCR_setURI(aURI) {
michael@0:     this._uri = aURI;
michael@0:   },
michael@0: 
michael@0:   // Creates an nsIChannel for the request and fills the body.
michael@0:   openChannel: function HCR_openChannel() {
michael@0:     let loadFlags = Ci.nsIChannel.INHIBIT_CACHING |
michael@0:                     Ci.nsIChannel.LOAD_BYPASS_CACHE;
michael@0: 
michael@0:     let channel = Services.io.newChannelFromURI(this._uri);
michael@0:     channel.loadFlags = loadFlags;
michael@0: 
michael@0:     this._channel = channel;
michael@0: 
michael@0:     let body = this.buildRequest();
michael@0:     this.addRequestBody(body);
michael@0: 
michael@0:     channel.asyncOpen(this, null);
michael@0:   },
michael@0: 
michael@0:   // Returns a string for the request body based on the contents of
michael@0:   // this._requests.
michael@0:   buildRequest: function HCR_buildRequest() {
michael@0:     // Sometimes duplicate entries are sent to HashCompleter but we do not need
michael@0:     // to propagate these to the server. (bug 633644)
michael@0:     let prefixes = [];
michael@0: 
michael@0:     for (let i = 0; i < this._requests.length; i++) {
michael@0:       let request = this._requests[i];
michael@0:       if (prefixes.indexOf(request.partialHash) == -1) {
michael@0:         prefixes.push(request.partialHash);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     // Randomize the order to obscure the original request from noise
michael@0:     // unbiased Fisher-Yates shuffle
michael@0:     let i = prefixes.length;
michael@0:     while (i--) {
michael@0:       let j = Math.floor(Math.random() * (i + 1));
michael@0:       let temp = prefixes[i];
michael@0:       prefixes[i] = prefixes[j];
michael@0:       prefixes[j] = temp;
michael@0:     }
michael@0: 
michael@0:     let body;
michael@0:     body = PARTIAL_LENGTH + ":" + (PARTIAL_LENGTH * prefixes.length) +
michael@0:            "\n" + prefixes.join("");
michael@0: 
michael@0:     return body;
michael@0:   },
michael@0: 
michael@0:   // Sets the request body of this._channel.
michael@0:   addRequestBody: function HCR_addRequestBody(aBody) {
michael@0:     let inputStream = Cc["@mozilla.org/io/string-input-stream;1"].
michael@0:                       createInstance(Ci.nsIStringInputStream);
michael@0: 
michael@0:     inputStream.setData(aBody, aBody.length);
michael@0: 
michael@0:     let uploadChannel = this._channel.QueryInterface(Ci.nsIUploadChannel);
michael@0:     uploadChannel.setUploadStream(inputStream, "text/plain", -1);
michael@0: 
michael@0:     let httpChannel = this._channel.QueryInterface(Ci.nsIHttpChannel);
michael@0:     httpChannel.requestMethod = "POST";
michael@0:   },
michael@0: 
michael@0:   // Parses the response body and eventually adds items to the |responses| array
michael@0:   // for elements of |this._requests|.
michael@0:   handleResponse: function HCR_handleResponse() {
michael@0:     if (this._response == "") {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let start = 0;
michael@0: 
michael@0:     let length = this._response.length;
michael@0:     while (start != length)
michael@0:       start = this.handleTable(start);
michael@0:   },
michael@0: 
michael@0:   // This parses a table entry in the response body and calls |handleItem|
michael@0:   // for complete hash in the table entry. 
michael@0:   handleTable: function HCR_handleTable(aStart) {
michael@0:     let body = this._response.substring(aStart);
michael@0: 
michael@0:     // deal with new line indexes as there could be
michael@0:     // new line characters in the data parts.
michael@0:     let newlineIndex = body.indexOf("\n");
michael@0:     if (newlineIndex == -1) {
michael@0:       throw errorWithStack();
michael@0:     }
michael@0:     let header = body.substring(0, newlineIndex);
michael@0:     let entries = header.split(":");
michael@0:     if (entries.length != 3) {
michael@0:       throw errorWithStack();
michael@0:     }
michael@0: 
michael@0:     let list = entries[0];
michael@0:     let addChunk = parseInt(entries[1]);
michael@0:     let dataLength = parseInt(entries[2]);
michael@0: 
michael@0:     if (dataLength % COMPLETE_LENGTH != 0 ||
michael@0:         dataLength == 0 ||
michael@0:         dataLength > body.length - (newlineIndex + 1)) {
michael@0:       throw errorWithStack();
michael@0:     }
michael@0: 
michael@0:     let data = body.substr(newlineIndex + 1, dataLength);
michael@0:     for (let i = 0; i < (dataLength / COMPLETE_LENGTH); i++) {
michael@0:       this.handleItem(data.substr(i * COMPLETE_LENGTH, COMPLETE_LENGTH), list,
michael@0:                       addChunk);
michael@0:     }
michael@0: 
michael@0:     return aStart + newlineIndex + 1 + dataLength;
michael@0:   },
michael@0: 
michael@0:   // This adds a complete hash to any entry in |this._requests| that matches
michael@0:   // the hash.
michael@0:   handleItem: function HCR_handleItem(aData, aTableName, aChunkId) {
michael@0:     for (let i = 0; i < this._requests.length; i++) {
michael@0:       let request = this._requests[i];
michael@0:       if (aData.substring(0,4) == request.partialHash) {
michael@0:         request.responses.push({
michael@0:           completeHash: aData,
michael@0:           tableName: aTableName,
michael@0:           chunkId: aChunkId,
michael@0:         });
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   // notifySuccess and notifyFailure are used to alert the callbacks with
michael@0:   // results. notifySuccess makes |completion| and |completionFinished| calls
michael@0:   // while notifyFailure only makes a |completionFinished| call with the error
michael@0:   // code.
michael@0:   notifySuccess: function HCR_notifySuccess() {
michael@0:     for (let i = 0; i < this._requests.length; i++) {
michael@0:       let request = this._requests[i];
michael@0:       for (let j = 0; j < request.responses.length; j++) {
michael@0:         let response = request.responses[j];
michael@0:         request.callback.completion(response.completeHash, response.tableName,
michael@0:                                     response.chunkId);
michael@0:       }
michael@0: 
michael@0:       request.callback.completionFinished(Cr.NS_OK);
michael@0:     }
michael@0:   },
michael@0:   notifyFailure: function HCR_notifyFailure(aStatus) {
michael@0:     for (let i = 0; i < this._requests; i++) {
michael@0:       let request = this._requests[i];
michael@0:       request.callback.completionFinished(aStatus);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   onDataAvailable: function HCR_onDataAvailable(aRequest, aContext,
michael@0:                                                 aInputStream, aOffset, aCount) {
michael@0:     let sis = Cc["@mozilla.org/scriptableinputstream;1"].
michael@0:               createInstance(Ci.nsIScriptableInputStream);
michael@0:     sis.init(aInputStream);
michael@0:     this._response += sis.readBytes(aCount);
michael@0:   },
michael@0: 
michael@0:   onStartRequest: function HCR_onStartRequest(aRequest, aContext) {
michael@0:     // At this point no data is available for us and we have no reason to
michael@0:     // terminate the connection, so we do nothing until |onStopRequest|.
michael@0:   },
michael@0: 
michael@0:   onStopRequest: function HCR_onStopRequest(aRequest, aContext, aStatusCode) {
michael@0:     Services.obs.removeObserver(this, "xpcom-shutdown");
michael@0: 
michael@0:     if (this._shuttingDown) {
michael@0:       throw Cr.NS_ERROR_ABORT;
michael@0:     }
michael@0: 
michael@0:     if (Components.isSuccessCode(aStatusCode)) {
michael@0:       let channel = aRequest.QueryInterface(Ci.nsIHttpChannel);
michael@0:       let success = channel.requestSucceeded;
michael@0:       if (!success) {
michael@0:         aStatusCode = Cr.NS_ERROR_ABORT;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     let success = Components.isSuccessCode(aStatusCode);
michael@0:     this._completer.noteServerResponse(success);
michael@0: 
michael@0:     if (success) {
michael@0:       try {
michael@0:         this.handleResponse();
michael@0:       }
michael@0:       catch (err) {
michael@0:         dump(err.stack);
michael@0:         aStatusCode = err.value;
michael@0:         success = false;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (success) {
michael@0:       this.notifySuccess();
michael@0:     } else {
michael@0:       this.notifyFailure(aStatusCode);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   observe: function HCR_observe(aSubject, aTopic, aData) {
michael@0:     if (aTopic != "xpcom-shutdown") {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     this._shuttingDown = true;
michael@0:     if (this._channel) {
michael@0:       this._channel.cancel(Cr.NS_ERROR_ABORT);
michael@0:     }
michael@0:   },
michael@0: };
michael@0: 
michael@0: // Converts a URL safe base64 string to a normal base64 string. Will not change
michael@0: // normal base64 strings. This is modelled after the same function in
michael@0: // nsUrlClassifierUtils.h.
michael@0: function unUrlsafeBase64(aStr) {
michael@0:   return !aStr ? "" : aStr.replace(/-/g, "+")
michael@0:                           .replace(/_/g, "/");
michael@0: }
michael@0: 
michael@0: function errorWithStack() {
michael@0:   let err = new Error();
michael@0:   err.value = Cr.NS_ERROR_FAILURE;
michael@0:   return err;
michael@0: }
michael@0: 
michael@0: this.NSGetFactory = XPCOMUtils.generateNSGetFactory([HashCompleter]);