michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: // How to run this file:
michael@0: // 1. [obtain firefox source code]
michael@0: // 2. [build/obtain firefox binaries]
michael@0: // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
michael@0: //                                  [path to]/getHSTSPreloadlist.js \
michael@0: //                                  [absolute path to]/nsSTSPreloadlist.inc'
michael@0: 
michael@0: // <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
michael@0: // <https://bugzilla.mozilla.org/show_bug.cgi?id=546628>
michael@0: const Cc = Components.classes;
michael@0: const Ci = Components.interfaces;
michael@0: const Cu = Components.utils;
michael@0: const Cr = Components.results;
michael@0: 
michael@0: // Register resource://app/ URI
michael@0: let ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService);
michael@0: let resHandler = ios.getProtocolHandler("resource")
michael@0:                  .QueryInterface(Ci.nsIResProtocolHandler);
michael@0: let mozDir = Cc["@mozilla.org/file/directory_service;1"]
michael@0:              .getService(Ci.nsIProperties)
michael@0:              .get("CurProcD", Ci.nsILocalFile);
michael@0: let mozDirURI = ios.newFileURI(mozDir);
michael@0: resHandler.setSubstitution("app", mozDirURI);
michael@0: 
michael@0: Cu.import("resource://gre/modules/Services.jsm");
michael@0: Cu.import("resource://gre/modules/FileUtils.jsm");
michael@0: Cu.import("resource:///modules/XPCOMUtils.jsm");
michael@0: 
michael@0: const SOURCE = "https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json";
michael@0: const OUTPUT = "nsSTSPreloadList.inc";
michael@0: const ERROR_OUTPUT = "nsSTSPreloadList.errors";
michael@0: const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
michael@0: const MAX_CONCURRENT_REQUESTS = 5;
michael@0: const MAX_RETRIES = 3;
michael@0: const REQUEST_TIMEOUT = 30 * 1000;
michael@0: const ERROR_NONE = "no error";
michael@0: const ERROR_CONNECTING_TO_HOST = "could not connect to host";
michael@0: const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
michael@0: const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
michael@0: const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
michael@0: " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
michael@0: " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
michael@0: "\n" +
michael@0: "/*****************************************************************************/\n" +
michael@0: "/* This is an automatically generated file. If you're not                    */\n" +
michael@0: "/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
michael@0: "/*****************************************************************************/\n" +
michael@0: "\n" +
michael@0: "#include <stdint.h>\n";
michael@0: const PREFIX = "\n" +
michael@0: "class nsSTSPreload\n" +
michael@0: "{\n" +
michael@0: "  public:\n" +
michael@0: "    const char *mHost;\n" +
michael@0: "    const bool mIncludeSubdomains;\n" +
michael@0: "};\n" +
michael@0: "\n" +
michael@0: "static const nsSTSPreload kSTSPreloadList[] = {\n";
michael@0: const POSTFIX =  "};\n";
michael@0: 
michael@0: function download() {
michael@0:   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
michael@0:             .createInstance(Ci.nsIXMLHttpRequest);
michael@0:   req.open("GET", SOURCE, false); // doing the request synchronously
michael@0:   try {
michael@0:     req.send();
michael@0:   }
michael@0:   catch (e) {
michael@0:     throw "ERROR: problem downloading '" + SOURCE + "': " + e;
michael@0:   }
michael@0: 
michael@0:   if (req.status != 200) {
michael@0:     throw "ERROR: problem downloading '" + SOURCE + "': status " + req.status;
michael@0:   }
michael@0: 
michael@0:   // we have to filter out '//' comments
michael@0:   var result = req.responseText.replace(/\/\/[^\n]*\n/g, "");
michael@0:   var data = null;
michael@0:   try {
michael@0:     data = JSON.parse(result);
michael@0:   }
michael@0:   catch (e) {
michael@0:     throw "ERROR: could not parse data from '" + SOURCE + "': " + e;
michael@0:   }
michael@0:   return data;
michael@0: }
michael@0: 
michael@0: function getHosts(rawdata) {
michael@0:   var hosts = [];
michael@0: 
michael@0:   if (!rawdata || !rawdata.entries) {
michael@0:     throw "ERROR: source data not formatted correctly: 'entries' not found";
michael@0:   }
michael@0: 
michael@0:   for (entry of rawdata.entries) {
michael@0:     if (entry.mode && entry.mode == "force-https") {
michael@0:       if (entry.name) {
michael@0:         entry.retries = MAX_RETRIES;
michael@0:         entry.originalIncludeSubdomains = entry.include_subdomains;
michael@0:         hosts.push(entry);
michael@0:       } else {
michael@0:         throw "ERROR: entry not formatted correctly: no name found";
michael@0:       }
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return hosts;
michael@0: }
michael@0: 
michael@0: var gSSService = Cc["@mozilla.org/ssservice;1"]
michael@0:                    .getService(Ci.nsISiteSecurityService);
michael@0: 
michael@0: function processStsHeader(host, header, status) {
michael@0:   var maxAge = { value: 0 };
michael@0:   var includeSubdomains = { value: false };
michael@0:   var error = ERROR_NONE;
michael@0:   if (header != null) {
michael@0:     try {
michael@0:       var uri = Services.io.newURI("https://" + host.name, null, null);
michael@0:       gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS,
michael@0:                                uri, header, 0, maxAge, includeSubdomains);
michael@0:     }
michael@0:     catch (e) {
michael@0:       dump("ERROR: could not process header '" + header + "' from " +
michael@0:            host.name + ": " + e + "\n");
michael@0:       error = e;
michael@0:     }
michael@0:   }
michael@0:   else {
michael@0:     if (status == 0) {
michael@0:       error = ERROR_CONNECTING_TO_HOST;
michael@0:     } else {
michael@0:       error = ERROR_NO_HSTS_HEADER;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   let forceInclude = (host.forceInclude || host.pins == "google");
michael@0: 
michael@0:   if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
michael@0:     error = ERROR_MAX_AGE_TOO_LOW;
michael@0:   }
michael@0: 
michael@0:   return { name: host.name,
michael@0:            maxAge: maxAge.value,
michael@0:            includeSubdomains: includeSubdomains.value,
michael@0:            error: error,
michael@0:            retries: host.retries - 1,
michael@0:            forceInclude: forceInclude,
michael@0:            originalIncludeSubdomains: host.originalIncludeSubdomains };
michael@0: }
michael@0: 
michael@0: function RedirectStopper() {};
michael@0: 
michael@0: RedirectStopper.prototype = {
michael@0:   // nsIChannelEventSink
michael@0:   asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) {
michael@0:     throw Cr.NS_ERROR_ENTITY_CHANGED;
michael@0:   },
michael@0: 
michael@0:   getInterface: function(iid) {
michael@0:     return this.QueryInterface(iid);
michael@0:   },
michael@0: 
michael@0:   QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink])
michael@0: };
michael@0: 
michael@0: function getHSTSStatus(host, resultList) {
michael@0:   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
michael@0:             .createInstance(Ci.nsIXMLHttpRequest);
michael@0:   var inResultList = false;
michael@0:   var uri = "https://" + host.name + "/";
michael@0:   req.open("GET", uri, true);
michael@0:   req.timeout = REQUEST_TIMEOUT;
michael@0:   req.channel.notificationCallbacks = new RedirectStopper();
michael@0:   req.onreadystatechange = function(event) {
michael@0:     if (!inResultList && req.readyState == 4) {
michael@0:       inResultList = true;
michael@0:       var header = req.getResponseHeader("strict-transport-security");
michael@0:       resultList.push(processStsHeader(host, header, req.status));
michael@0:     }
michael@0:   };
michael@0: 
michael@0:   try {
michael@0:     req.send();
michael@0:   }
michael@0:   catch (e) {
michael@0:     dump("ERROR: exception making request to " + host.name + ": " + e + "\n");
michael@0:   }
michael@0: }
michael@0: 
michael@0: function compareHSTSStatus(a, b) {
michael@0:   return (a.name > b.name ? 1 : (a.name < b.name ? -1 : 0));
michael@0: }
michael@0: 
michael@0: function writeTo(string, fos) {
michael@0:   fos.write(string, string.length);
michael@0: }
michael@0: 
michael@0: // Determines and returns a string representing a declaration of when this
michael@0: // preload list should no longer be used.
michael@0: // This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
michael@0: function getExpirationTimeString() {
michael@0:   var now = new Date();
michael@0:   var nowMillis = now.getTime();
michael@0:   // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
michael@0:   var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
michael@0:   var expirationMicros = expirationMillis * 1000;
michael@0:   return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
michael@0: }
michael@0: 
michael@0: function errorToString(status) {
michael@0:   return (status.error == ERROR_MAX_AGE_TOO_LOW
michael@0:           ? status.error + status.maxAge
michael@0:           : status.error);
michael@0: }
michael@0: 
michael@0: function writeEntry(status, outputStream) {
michael@0:   let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
michael@0:                            ? status.originalIncludeSubdomains
michael@0:                            : status.includeSubdomains);
michael@0:   let includeSubdomains = (incSubdomainsBool ? "true" : "false");
michael@0:   writeTo("  { \"" + status.name + "\", " + includeSubdomains + " },\n",
michael@0:           outputStream);
michael@0: }
michael@0: 
michael@0: function output(sortedStatuses, currentList) {
michael@0:   try {
michael@0:     var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
michael@0:     var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
michael@0:     var fos = FileUtils.openSafeFileOutputStream(file);
michael@0:     var eos = FileUtils.openSafeFileOutputStream(errorFile);
michael@0:     writeTo(HEADER, fos);
michael@0:     writeTo(getExpirationTimeString(), fos);
michael@0:     writeTo(PREFIX, fos);
michael@0:     for (var status of sortedStatuses) {
michael@0: 
michael@0:       // If we've encountered an error for this entry (other than the site not
michael@0:       // sending an HSTS header), be safe and don't remove it from the list
michael@0:       // (given that it was already on the list).
michael@0:       if (status.error != ERROR_NONE &&
michael@0:           status.error != ERROR_NO_HSTS_HEADER &&
michael@0:           status.error != ERROR_MAX_AGE_TOO_LOW &&
michael@0:           status.name in currentList) {
michael@0:         dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
michael@0:         writeTo(status.name + ": " + errorToString(status) + "\n", eos);
michael@0:         status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
michael@0:         status.includeSubdomains = currentList[status.name];
michael@0:       }
michael@0: 
michael@0:       if (status.maxAge >= MINIMUM_REQUIRED_MAX_AGE || status.forceInclude) {
michael@0:         writeEntry(status, fos);
michael@0:         dump("INFO: " + status.name + " ON the preload list\n");
michael@0:         if (status.forceInclude && status.error != ERROR_NONE) {
michael@0:           writeTo(status.name + ": " + errorToString(status) + " (error "
michael@0:                   + "ignored - included regardless)\n", eos);
michael@0:         }
michael@0:       }
michael@0:       else {
michael@0:         dump("INFO: " + status.name + " NOT ON the preload list\n");
michael@0:         writeTo(status.name + ": " + errorToString(status) + "\n", eos);
michael@0:       }
michael@0:     }
michael@0:     writeTo(POSTFIX, fos);
michael@0:     FileUtils.closeSafeFileOutputStream(fos);
michael@0:     FileUtils.closeSafeFileOutputStream(eos);
michael@0:   }
michael@0:   catch (e) {
michael@0:     dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
michael@0:   }
michael@0: }
michael@0: 
michael@0: function shouldRetry(response) {
michael@0:   return (response.error != ERROR_NO_HSTS_HEADER &&
michael@0:           response.error != ERROR_MAX_AGE_TOO_LOW &&
michael@0:           response.error != ERROR_NONE && response.retries > 0);
michael@0: }
michael@0: 
michael@0: function getHSTSStatuses(inHosts, outStatuses) {
michael@0:   var expectedOutputLength = inHosts.length;
michael@0:   var tmpOutput = [];
michael@0:   for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
michael@0:     var host = inHosts.shift();
michael@0:     dump("spinning off request to '" + host.name + "' (remaining retries: " +
michael@0:          host.retries + ")\n");
michael@0:     getHSTSStatus(host, tmpOutput);
michael@0:   }
michael@0: 
michael@0:   while (outStatuses.length != expectedOutputLength) {
michael@0:     waitForAResponse(tmpOutput);
michael@0:     var response = tmpOutput.shift();
michael@0:     dump("request to '" + response.name + "' finished\n");
michael@0:     if (shouldRetry(response))
michael@0:       inHosts.push(response);
michael@0:     else
michael@0:       outStatuses.push(response);
michael@0: 
michael@0:     if (inHosts.length > 0) {
michael@0:       var host = inHosts.shift();
michael@0:       dump("spinning off request to '" + host.name + "' (remaining retries: " +
michael@0:            host.retries + ")\n");
michael@0:       getHSTSStatus(host, tmpOutput);
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: // Since all events are processed on the main thread, and since event
michael@0: // handlers are not preemptible, there shouldn't be any concurrency issues.
michael@0: function waitForAResponse(outputList) {
michael@0:   // From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
michael@0:   var threadManager = Cc["@mozilla.org/thread-manager;1"]
michael@0:                       .getService(Ci.nsIThreadManager);
michael@0:   var mainThread = threadManager.currentThread;
michael@0:   while (outputList.length == 0) {
michael@0:     mainThread.processNextEvent(true);
michael@0:   }
michael@0: }
michael@0: 
michael@0: function readCurrentList(filename) {
michael@0:   var currentHosts = {};
michael@0:   var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
michael@0:   file.initWithPath(filename);
michael@0:   var fis = Cc["@mozilla.org/network/file-input-stream;1"]
michael@0:               .createInstance(Ci.nsILineInputStream);
michael@0:   fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
michael@0:   var line = {};
michael@0:   var entryRegex = /  { "([^"]*)", (true|false) },/;
michael@0:   while (fis.readLine(line)) {
michael@0:     var match = entryRegex.exec(line.value);
michael@0:     if (match) {
michael@0:       currentHosts[match[1]] = (match[2] == "true");
michael@0:     }
michael@0:   }
michael@0:   return currentHosts;
michael@0: }
michael@0: 
michael@0: function combineLists(newHosts, currentHosts) {
michael@0:   for (let currentHost in currentHosts) {
michael@0:     let found = false;
michael@0:     for (let newHost of newHosts) {
michael@0:       if (newHost.name == currentHost) {
michael@0:         found = true;
michael@0:         break;
michael@0:       }
michael@0:     }
michael@0:     if (!found) {
michael@0:       newHosts.push({ name: currentHost, retries: MAX_RETRIES });
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: // ****************************************************************************
michael@0: // This is where the action happens:
michael@0: if (arguments.length < 1) {
michael@0:   throw "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>";
michael@0: }
michael@0: // get the current preload list
michael@0: var currentHosts = readCurrentList(arguments[0]);
michael@0: // disable the current preload list so it won't interfere with requests we make
michael@0: Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
michael@0: // download and parse the raw json file from the Chromium source
michael@0: var rawdata = download();
michael@0: // get just the hosts with mode: "force-https"
michael@0: var hosts = getHosts(rawdata);
michael@0: // add hosts in the current list to the new list (avoiding duplicates)
michael@0: combineLists(hosts, currentHosts);
michael@0: // get the HSTS status of each host
michael@0: var hstsStatuses = [];
michael@0: getHSTSStatuses(hosts, hstsStatuses);
michael@0: // sort the hosts alphabetically
michael@0: hstsStatuses.sort(compareHSTSStatus);
michael@0: // write the results to a file (this is where we filter out hosts that we
michael@0: // either couldn't connect to, didn't receive an HSTS header from, couldn't
michael@0: // parse the header, or had a header with too short a max-age)
michael@0: output(hstsStatuses, currentHosts);
michael@0: // ****************************************************************************