The Tor Browser: comparison security/manager/tools/getHSTSPreloadList.js

comparison: security/manager/tools/getHSTSPreloadList.js

security/manager/tools/getHSTSPreloadList.js

branch: TOR_BUG_9701
changeset 15: b8a032363ba2

equal deleted inserted replaced

--1:000000000000
+:fe9d0c5d4486
+/* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+// How to run this file:
+// 1. [obtain firefox source code]
+// 2. [build/obtain firefox binaries]
+// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
+//                                  [path to]/getHSTSPreloadlist.js \
+//                                  [absolute path to]/nsSTSPreloadlist.inc'
+// <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
+// <https://bugzilla.mozilla.org/show_bug.cgi?id=546628>
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+const Cr = Components.results;
+// Register resource://app/ URI
+let ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService);
+let resHandler = ios.getProtocolHandler("resource")
+.QueryInterface(Ci.nsIResProtocolHandler);
+let mozDir = Cc["@mozilla.org/file/directory_service;1"]
+.getService(Ci.nsIProperties)
+.get("CurProcD", Ci.nsILocalFile);
+let mozDirURI = ios.newFileURI(mozDir);
+resHandler.setSubstitution("app", mozDirURI);
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+Cu.import("resource:///modules/XPCOMUtils.jsm");
+const SOURCE = "https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json";
+const OUTPUT = "nsSTSPreloadList.inc";
+const ERROR_OUTPUT = "nsSTSPreloadList.errors";
+const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
+const MAX_CONCURRENT_REQUESTS = 5;
+const MAX_RETRIES = 3;
+const REQUEST_TIMEOUT = 30 * 1000;
+const ERROR_NONE = "no error";
+const ERROR_CONNECTING_TO_HOST = "could not connect to host";
+const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
+const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
+const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
+" * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
+" * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
+"\n" +
+"/*****************************************************************************/\n" +
+"/* This is an automatically generated file. If you're not                    */\n" +
+"/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
+"/*****************************************************************************/\n" +
+"\n" +
+"#include <stdint.h>\n";
+const PREFIX = "\n" +
+"class nsSTSPreload\n" +
+"{\n" +
+"  public:\n" +
+"    const char *mHost;\n" +
+"    const bool mIncludeSubdomains;\n" +
+"};\n" +
+"\n" +
+"static const nsSTSPreload kSTSPreloadList[] = {\n";
+const POSTFIX =  "};\n";
+function download() {
+var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+.createInstance(Ci.nsIXMLHttpRequest);
+req.open("GET", SOURCE, false); // doing the request synchronously
+try {
+req.send();
+}
+catch (e) {
+throw "ERROR: problem downloading '" + SOURCE + "': " + e;
+}
+if (req.status != 200) {
+throw "ERROR: problem downloading '" + SOURCE + "': status " + req.status;
+}
+// we have to filter out '//' comments
+var result = req.responseText.replace(/\/\/[^\n]*\n/g, "");
+var data = null;
+try {
+data = JSON.parse(result);
+}
+catch (e) {
+throw "ERROR: could not parse data from '" + SOURCE + "': " + e;
+}
+return data;
+}
+function getHosts(rawdata) {
+var hosts = [];
+if (!rawdata || !rawdata.entries) {
+throw "ERROR: source data not formatted correctly: 'entries' not found";
+}
+for (entry of rawdata.entries) {
+if (entry.mode && entry.mode == "force-https") {
+if (entry.name) {
+entry.retries = MAX_RETRIES;
+entry.originalIncludeSubdomains = entry.include_subdomains;
+hosts.push(entry);
+} else {
+throw "ERROR: entry not formatted correctly: no name found";
+}
+}
+}
+return hosts;
+}
+var gSSService = Cc["@mozilla.org/ssservice;1"]
+.getService(Ci.nsISiteSecurityService);
+function processStsHeader(host, header, status) {
+var maxAge = { value: 0 };
+var includeSubdomains = { value: false };
+var error = ERROR_NONE;
+if (header != null) {
+try {
+var uri = Services.io.newURI("https://" + host.name, null, null);
+gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS,
+uri, header, 0, maxAge, includeSubdomains);
+}
+catch (e) {
+dump("ERROR: could not process header '" + header + "' from " +
+host.name + ": " + e + "\n");
+error = e;
+}
+}
+else {
+if (status == 0) {
+error = ERROR_CONNECTING_TO_HOST;
+} else {
+error = ERROR_NO_HSTS_HEADER;
+}
+}
+let forceInclude = (host.forceInclude || host.pins == "google");
+if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
+error = ERROR_MAX_AGE_TOO_LOW;
+}
+return { name: host.name,
+maxAge: maxAge.value,
+includeSubdomains: includeSubdomains.value,
+error: error,
+retries: host.retries - 1,
+forceInclude: forceInclude,
+originalIncludeSubdomains: host.originalIncludeSubdomains };
+}
+function RedirectStopper() {};
+RedirectStopper.prototype = {
+// nsIChannelEventSink
+asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) {
+throw Cr.NS_ERROR_ENTITY_CHANGED;
+},
+getInterface: function(iid) {
+return this.QueryInterface(iid);
+},
+QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink])
+};
+function getHSTSStatus(host, resultList) {
+var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+.createInstance(Ci.nsIXMLHttpRequest);
+var inResultList = false;
+var uri = "https://" + host.name + "/";
+req.open("GET", uri, true);
+req.timeout = REQUEST_TIMEOUT;
+req.channel.notificationCallbacks = new RedirectStopper();
+req.onreadystatechange = function(event) {
+if (!inResultList && req.readyState == 4) {
+inResultList = true;
+var header = req.getResponseHeader("strict-transport-security");
+resultList.push(processStsHeader(host, header, req.status));
+}
+};
+try {
+req.send();
+}
+catch (e) {
+dump("ERROR: exception making request to " + host.name + ": " + e + "\n");
+}
+}
+function compareHSTSStatus(a, b) {
+return (a.name > b.name ? 1 : (a.name < b.name ? -1 : 0));
+}
+function writeTo(string, fos) {
+fos.write(string, string.length);
+}
+// Determines and returns a string representing a declaration of when this
+// preload list should no longer be used.
+// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
+function getExpirationTimeString() {
+var now = new Date();
+var nowMillis = now.getTime();
+// MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
+var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
+var expirationMicros = expirationMillis * 1000;
+return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
+}
+function errorToString(status) {
+return (status.error == ERROR_MAX_AGE_TOO_LOW
+? status.error + status.maxAge
+: status.error);
+}
+function writeEntry(status, outputStream) {
+let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
+? status.originalIncludeSubdomains
+: status.includeSubdomains);
+let includeSubdomains = (incSubdomainsBool ? "true" : "false");
+writeTo("  { \"" + status.name + "\", " + includeSubdomains + " },\n",
+outputStream);
+}
+function output(sortedStatuses, currentList) {
+try {
+var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
+var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
+var fos = FileUtils.openSafeFileOutputStream(file);
+var eos = FileUtils.openSafeFileOutputStream(errorFile);
+writeTo(HEADER, fos);
+writeTo(getExpirationTimeString(), fos);
+writeTo(PREFIX, fos);
+for (var status of sortedStatuses) {
+// If we've encountered an error for this entry (other than the site not
+// sending an HSTS header), be safe and don't remove it from the list
+// (given that it was already on the list).
+if (status.error != ERROR_NONE &&
+status.error != ERROR_NO_HSTS_HEADER &&
+status.error != ERROR_MAX_AGE_TOO_LOW &&
+status.name in currentList) {
+dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
+writeTo(status.name + ": " + errorToString(status) + "\n", eos);
+status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
+status.includeSubdomains = currentList[status.name];
+}
+if (status.maxAge >= MINIMUM_REQUIRED_MAX_AGE || status.forceInclude) {
+writeEntry(status, fos);
+dump("INFO: " + status.name + " ON the preload list\n");
+if (status.forceInclude && status.error != ERROR_NONE) {
+writeTo(status.name + ": " + errorToString(status) + " (error "
++ "ignored - included regardless)\n", eos);
+}
+}
+else {
+dump("INFO: " + status.name + " NOT ON the preload list\n");
+writeTo(status.name + ": " + errorToString(status) + "\n", eos);
+}
+}
+writeTo(POSTFIX, fos);
+FileUtils.closeSafeFileOutputStream(fos);
+FileUtils.closeSafeFileOutputStream(eos);
+}
+catch (e) {
+dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
+}
+}
+function shouldRetry(response) {
+return (response.error != ERROR_NO_HSTS_HEADER &&
+response.error != ERROR_MAX_AGE_TOO_LOW &&
+response.error != ERROR_NONE && response.retries > 0);
+}
+function getHSTSStatuses(inHosts, outStatuses) {
+var expectedOutputLength = inHosts.length;
+var tmpOutput = [];
+for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
+var host = inHosts.shift();
+dump("spinning off request to '" + host.name + "' (remaining retries: " +
+host.retries + ")\n");
+getHSTSStatus(host, tmpOutput);
+}
+while (outStatuses.length != expectedOutputLength) {
+waitForAResponse(tmpOutput);
+var response = tmpOutput.shift();
+dump("request to '" + response.name + "' finished\n");
+if (shouldRetry(response))
+inHosts.push(response);
+else
+outStatuses.push(response);
+if (inHosts.length > 0) {
+var host = inHosts.shift();
+dump("spinning off request to '" + host.name + "' (remaining retries: " +
+host.retries + ")\n");
+getHSTSStatus(host, tmpOutput);
+}
+}
+}
+// Since all events are processed on the main thread, and since event
+// handlers are not preemptible, there shouldn't be any concurrency issues.
+function waitForAResponse(outputList) {
+// From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
+var threadManager = Cc["@mozilla.org/thread-manager;1"]
+.getService(Ci.nsIThreadManager);
+var mainThread = threadManager.currentThread;
+while (outputList.length == 0) {
+mainThread.processNextEvent(true);
+}
+}
+function readCurrentList(filename) {
+var currentHosts = {};
+var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+file.initWithPath(filename);
+var fis = Cc["@mozilla.org/network/file-input-stream;1"]
+.createInstance(Ci.nsILineInputStream);
+fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
+var line = {};
+var entryRegex = /  { "([^"]*)", (true|false) },/;
+while (fis.readLine(line)) {
+var match = entryRegex.exec(line.value);
+if (match) {
+currentHosts[match[1]] = (match[2] == "true");
+}
+}
+return currentHosts;
+}
+function combineLists(newHosts, currentHosts) {
+for (let currentHost in currentHosts) {
+let found = false;
+for (let newHost of newHosts) {
+if (newHost.name == currentHost) {
+found = true;
+break;
+}
+}
+if (!found) {
+newHosts.push({ name: currentHost, retries: MAX_RETRIES });
+}
+}
+}
+// ****************************************************************************
+// This is where the action happens:
+if (arguments.length < 1) {
+throw "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>";
+}
+// get the current preload list
+var currentHosts = readCurrentList(arguments[0]);
+// disable the current preload list so it won't interfere with requests we make
+Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
+// download and parse the raw json file from the Chromium source
+var rawdata = download();
+// get just the hosts with mode: "force-https"
+var hosts = getHosts(rawdata);
+// add hosts in the current list to the new list (avoiding duplicates)
+combineLists(hosts, currentHosts);
+// get the HSTS status of each host
+var hstsStatuses = [];
+getHSTSStatuses(hosts, hstsStatuses);
+// sort the hosts alphabetically
+hstsStatuses.sort(compareHSTSStatus);
+// write the results to a file (this is where we filter out hosts that we
+// either couldn't connect to, didn't receive an HSTS header from, couldn't
+// parse the header, or had a header with too short a max-age)
+output(hstsStatuses, currentHosts);
+// ****************************************************************************