1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/security/manager/tools/getHSTSPreloadList.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,379 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +// How to run this file:
1.9 +// 1. [obtain firefox source code]
1.10 +// 2. [build/obtain firefox binaries]
1.11 +// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
1.12 +// [path to]/getHSTSPreloadlist.js \
1.13 +// [absolute path to]/nsSTSPreloadlist.inc'
1.14 +
1.15 +// <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
1.16 +// <https://bugzilla.mozilla.org/show_bug.cgi?id=546628>
1.17 +const Cc = Components.classes;
1.18 +const Ci = Components.interfaces;
1.19 +const Cu = Components.utils;
1.20 +const Cr = Components.results;
1.21 +
1.22 +// Register resource://app/ URI
1.23 +let ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService);
1.24 +let resHandler = ios.getProtocolHandler("resource")
1.25 + .QueryInterface(Ci.nsIResProtocolHandler);
1.26 +let mozDir = Cc["@mozilla.org/file/directory_service;1"]
1.27 + .getService(Ci.nsIProperties)
1.28 + .get("CurProcD", Ci.nsILocalFile);
1.29 +let mozDirURI = ios.newFileURI(mozDir);
1.30 +resHandler.setSubstitution("app", mozDirURI);
1.31 +
1.32 +Cu.import("resource://gre/modules/Services.jsm");
1.33 +Cu.import("resource://gre/modules/FileUtils.jsm");
1.34 +Cu.import("resource:///modules/XPCOMUtils.jsm");
1.35 +
1.36 +const SOURCE = "https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json";
1.37 +const OUTPUT = "nsSTSPreloadList.inc";
1.38 +const ERROR_OUTPUT = "nsSTSPreloadList.errors";
1.39 +const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
1.40 +const MAX_CONCURRENT_REQUESTS = 5;
1.41 +const MAX_RETRIES = 3;
1.42 +const REQUEST_TIMEOUT = 30 * 1000;
1.43 +const ERROR_NONE = "no error";
1.44 +const ERROR_CONNECTING_TO_HOST = "could not connect to host";
1.45 +const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
1.46 +const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
1.47 +const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
1.48 +" * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
1.49 +" * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
1.50 +"\n" +
1.51 +"/*****************************************************************************/\n" +
1.52 +"/* This is an automatically generated file. If you're not */\n" +
1.53 +"/* nsSiteSecurityService.cpp, you shouldn't be #including it. */\n" +
1.54 +"/*****************************************************************************/\n" +
1.55 +"\n" +
1.56 +"#include <stdint.h>\n";
1.57 +const PREFIX = "\n" +
1.58 +"class nsSTSPreload\n" +
1.59 +"{\n" +
1.60 +" public:\n" +
1.61 +" const char *mHost;\n" +
1.62 +" const bool mIncludeSubdomains;\n" +
1.63 +"};\n" +
1.64 +"\n" +
1.65 +"static const nsSTSPreload kSTSPreloadList[] = {\n";
1.66 +const POSTFIX = "};\n";
1.67 +
1.68 +function download() {
1.69 + var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
1.70 + .createInstance(Ci.nsIXMLHttpRequest);
1.71 + req.open("GET", SOURCE, false); // doing the request synchronously
1.72 + try {
1.73 + req.send();
1.74 + }
1.75 + catch (e) {
1.76 + throw "ERROR: problem downloading '" + SOURCE + "': " + e;
1.77 + }
1.78 +
1.79 + if (req.status != 200) {
1.80 + throw "ERROR: problem downloading '" + SOURCE + "': status " + req.status;
1.81 + }
1.82 +
1.83 + // we have to filter out '//' comments
1.84 + var result = req.responseText.replace(/\/\/[^\n]*\n/g, "");
1.85 + var data = null;
1.86 + try {
1.87 + data = JSON.parse(result);
1.88 + }
1.89 + catch (e) {
1.90 + throw "ERROR: could not parse data from '" + SOURCE + "': " + e;
1.91 + }
1.92 + return data;
1.93 +}
1.94 +
1.95 +function getHosts(rawdata) {
1.96 + var hosts = [];
1.97 +
1.98 + if (!rawdata || !rawdata.entries) {
1.99 + throw "ERROR: source data not formatted correctly: 'entries' not found";
1.100 + }
1.101 +
1.102 + for (entry of rawdata.entries) {
1.103 + if (entry.mode && entry.mode == "force-https") {
1.104 + if (entry.name) {
1.105 + entry.retries = MAX_RETRIES;
1.106 + entry.originalIncludeSubdomains = entry.include_subdomains;
1.107 + hosts.push(entry);
1.108 + } else {
1.109 + throw "ERROR: entry not formatted correctly: no name found";
1.110 + }
1.111 + }
1.112 + }
1.113 +
1.114 + return hosts;
1.115 +}
1.116 +
1.117 +var gSSService = Cc["@mozilla.org/ssservice;1"]
1.118 + .getService(Ci.nsISiteSecurityService);
1.119 +
1.120 +function processStsHeader(host, header, status) {
1.121 + var maxAge = { value: 0 };
1.122 + var includeSubdomains = { value: false };
1.123 + var error = ERROR_NONE;
1.124 + if (header != null) {
1.125 + try {
1.126 + var uri = Services.io.newURI("https://" + host.name, null, null);
1.127 + gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS,
1.128 + uri, header, 0, maxAge, includeSubdomains);
1.129 + }
1.130 + catch (e) {
1.131 + dump("ERROR: could not process header '" + header + "' from " +
1.132 + host.name + ": " + e + "\n");
1.133 + error = e;
1.134 + }
1.135 + }
1.136 + else {
1.137 + if (status == 0) {
1.138 + error = ERROR_CONNECTING_TO_HOST;
1.139 + } else {
1.140 + error = ERROR_NO_HSTS_HEADER;
1.141 + }
1.142 + }
1.143 +
1.144 + let forceInclude = (host.forceInclude || host.pins == "google");
1.145 +
1.146 + if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
1.147 + error = ERROR_MAX_AGE_TOO_LOW;
1.148 + }
1.149 +
1.150 + return { name: host.name,
1.151 + maxAge: maxAge.value,
1.152 + includeSubdomains: includeSubdomains.value,
1.153 + error: error,
1.154 + retries: host.retries - 1,
1.155 + forceInclude: forceInclude,
1.156 + originalIncludeSubdomains: host.originalIncludeSubdomains };
1.157 +}
1.158 +
1.159 +function RedirectStopper() {};
1.160 +
1.161 +RedirectStopper.prototype = {
1.162 + // nsIChannelEventSink
1.163 + asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) {
1.164 + throw Cr.NS_ERROR_ENTITY_CHANGED;
1.165 + },
1.166 +
1.167 + getInterface: function(iid) {
1.168 + return this.QueryInterface(iid);
1.169 + },
1.170 +
1.171 + QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink])
1.172 +};
1.173 +
1.174 +function getHSTSStatus(host, resultList) {
1.175 + var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
1.176 + .createInstance(Ci.nsIXMLHttpRequest);
1.177 + var inResultList = false;
1.178 + var uri = "https://" + host.name + "/";
1.179 + req.open("GET", uri, true);
1.180 + req.timeout = REQUEST_TIMEOUT;
1.181 + req.channel.notificationCallbacks = new RedirectStopper();
1.182 + req.onreadystatechange = function(event) {
1.183 + if (!inResultList && req.readyState == 4) {
1.184 + inResultList = true;
1.185 + var header = req.getResponseHeader("strict-transport-security");
1.186 + resultList.push(processStsHeader(host, header, req.status));
1.187 + }
1.188 + };
1.189 +
1.190 + try {
1.191 + req.send();
1.192 + }
1.193 + catch (e) {
1.194 + dump("ERROR: exception making request to " + host.name + ": " + e + "\n");
1.195 + }
1.196 +}
1.197 +
1.198 +function compareHSTSStatus(a, b) {
1.199 + return (a.name > b.name ? 1 : (a.name < b.name ? -1 : 0));
1.200 +}
1.201 +
1.202 +function writeTo(string, fos) {
1.203 + fos.write(string, string.length);
1.204 +}
1.205 +
1.206 +// Determines and returns a string representing a declaration of when this
1.207 +// preload list should no longer be used.
1.208 +// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
1.209 +function getExpirationTimeString() {
1.210 + var now = new Date();
1.211 + var nowMillis = now.getTime();
1.212 + // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
1.213 + var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
1.214 + var expirationMicros = expirationMillis * 1000;
1.215 + return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
1.216 +}
1.217 +
1.218 +function errorToString(status) {
1.219 + return (status.error == ERROR_MAX_AGE_TOO_LOW
1.220 + ? status.error + status.maxAge
1.221 + : status.error);
1.222 +}
1.223 +
1.224 +function writeEntry(status, outputStream) {
1.225 + let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
1.226 + ? status.originalIncludeSubdomains
1.227 + : status.includeSubdomains);
1.228 + let includeSubdomains = (incSubdomainsBool ? "true" : "false");
1.229 + writeTo(" { \"" + status.name + "\", " + includeSubdomains + " },\n",
1.230 + outputStream);
1.231 +}
1.232 +
1.233 +function output(sortedStatuses, currentList) {
1.234 + try {
1.235 + var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
1.236 + var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
1.237 + var fos = FileUtils.openSafeFileOutputStream(file);
1.238 + var eos = FileUtils.openSafeFileOutputStream(errorFile);
1.239 + writeTo(HEADER, fos);
1.240 + writeTo(getExpirationTimeString(), fos);
1.241 + writeTo(PREFIX, fos);
1.242 + for (var status of sortedStatuses) {
1.243 +
1.244 + // If we've encountered an error for this entry (other than the site not
1.245 + // sending an HSTS header), be safe and don't remove it from the list
1.246 + // (given that it was already on the list).
1.247 + if (status.error != ERROR_NONE &&
1.248 + status.error != ERROR_NO_HSTS_HEADER &&
1.249 + status.error != ERROR_MAX_AGE_TOO_LOW &&
1.250 + status.name in currentList) {
1.251 + dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
1.252 + writeTo(status.name + ": " + errorToString(status) + "\n", eos);
1.253 + status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
1.254 + status.includeSubdomains = currentList[status.name];
1.255 + }
1.256 +
1.257 + if (status.maxAge >= MINIMUM_REQUIRED_MAX_AGE || status.forceInclude) {
1.258 + writeEntry(status, fos);
1.259 + dump("INFO: " + status.name + " ON the preload list\n");
1.260 + if (status.forceInclude && status.error != ERROR_NONE) {
1.261 + writeTo(status.name + ": " + errorToString(status) + " (error "
1.262 + + "ignored - included regardless)\n", eos);
1.263 + }
1.264 + }
1.265 + else {
1.266 + dump("INFO: " + status.name + " NOT ON the preload list\n");
1.267 + writeTo(status.name + ": " + errorToString(status) + "\n", eos);
1.268 + }
1.269 + }
1.270 + writeTo(POSTFIX, fos);
1.271 + FileUtils.closeSafeFileOutputStream(fos);
1.272 + FileUtils.closeSafeFileOutputStream(eos);
1.273 + }
1.274 + catch (e) {
1.275 + dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
1.276 + }
1.277 +}
1.278 +
1.279 +function shouldRetry(response) {
1.280 + return (response.error != ERROR_NO_HSTS_HEADER &&
1.281 + response.error != ERROR_MAX_AGE_TOO_LOW &&
1.282 + response.error != ERROR_NONE && response.retries > 0);
1.283 +}
1.284 +
1.285 +function getHSTSStatuses(inHosts, outStatuses) {
1.286 + var expectedOutputLength = inHosts.length;
1.287 + var tmpOutput = [];
1.288 + for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
1.289 + var host = inHosts.shift();
1.290 + dump("spinning off request to '" + host.name + "' (remaining retries: " +
1.291 + host.retries + ")\n");
1.292 + getHSTSStatus(host, tmpOutput);
1.293 + }
1.294 +
1.295 + while (outStatuses.length != expectedOutputLength) {
1.296 + waitForAResponse(tmpOutput);
1.297 + var response = tmpOutput.shift();
1.298 + dump("request to '" + response.name + "' finished\n");
1.299 + if (shouldRetry(response))
1.300 + inHosts.push(response);
1.301 + else
1.302 + outStatuses.push(response);
1.303 +
1.304 + if (inHosts.length > 0) {
1.305 + var host = inHosts.shift();
1.306 + dump("spinning off request to '" + host.name + "' (remaining retries: " +
1.307 + host.retries + ")\n");
1.308 + getHSTSStatus(host, tmpOutput);
1.309 + }
1.310 + }
1.311 +}
1.312 +
1.313 +// Since all events are processed on the main thread, and since event
1.314 +// handlers are not preemptible, there shouldn't be any concurrency issues.
1.315 +function waitForAResponse(outputList) {
1.316 + // From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
1.317 + var threadManager = Cc["@mozilla.org/thread-manager;1"]
1.318 + .getService(Ci.nsIThreadManager);
1.319 + var mainThread = threadManager.currentThread;
1.320 + while (outputList.length == 0) {
1.321 + mainThread.processNextEvent(true);
1.322 + }
1.323 +}
1.324 +
1.325 +function readCurrentList(filename) {
1.326 + var currentHosts = {};
1.327 + var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
1.328 + file.initWithPath(filename);
1.329 + var fis = Cc["@mozilla.org/network/file-input-stream;1"]
1.330 + .createInstance(Ci.nsILineInputStream);
1.331 + fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
1.332 + var line = {};
1.333 + var entryRegex = / { "([^"]*)", (true|false) },/;
1.334 + while (fis.readLine(line)) {
1.335 + var match = entryRegex.exec(line.value);
1.336 + if (match) {
1.337 + currentHosts[match[1]] = (match[2] == "true");
1.338 + }
1.339 + }
1.340 + return currentHosts;
1.341 +}
1.342 +
1.343 +function combineLists(newHosts, currentHosts) {
1.344 + for (let currentHost in currentHosts) {
1.345 + let found = false;
1.346 + for (let newHost of newHosts) {
1.347 + if (newHost.name == currentHost) {
1.348 + found = true;
1.349 + break;
1.350 + }
1.351 + }
1.352 + if (!found) {
1.353 + newHosts.push({ name: currentHost, retries: MAX_RETRIES });
1.354 + }
1.355 + }
1.356 +}
1.357 +
1.358 +// ****************************************************************************
1.359 +// This is where the action happens:
1.360 +if (arguments.length < 1) {
1.361 + throw "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>";
1.362 +}
1.363 +// get the current preload list
1.364 +var currentHosts = readCurrentList(arguments[0]);
1.365 +// disable the current preload list so it won't interfere with requests we make
1.366 +Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
1.367 +// download and parse the raw json file from the Chromium source
1.368 +var rawdata = download();
1.369 +// get just the hosts with mode: "force-https"
1.370 +var hosts = getHosts(rawdata);
1.371 +// add hosts in the current list to the new list (avoiding duplicates)
1.372 +combineLists(hosts, currentHosts);
1.373 +// get the HSTS status of each host
1.374 +var hstsStatuses = [];
1.375 +getHSTSStatuses(hosts, hstsStatuses);
1.376 +// sort the hosts alphabetically
1.377 +hstsStatuses.sort(compareHSTSStatus);
1.378 +// write the results to a file (this is where we filter out hosts that we
1.379 +// either couldn't connect to, didn't receive an HSTS header from, couldn't
1.380 +// parse the header, or had a header with too short a max-age)
1.381 +output(hstsStatuses, currentHosts);
1.382 +// ****************************************************************************
