The Tor Browser: security/manager/tools/getHSTSPreloadList.js@b8a032363ba2 (annotated)

security/manager/tools/getHSTSPreloadList.js@b8a032363ba2 (annotated)

security/manager/tools/getHSTSPreloadList.js

Thu, 22 Jan 2015 13:21:57 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 22 Jan 2015 13:21:57 +0100
branch: TOR_BUG_9701
changeset 15: b8a032363ba2
permissions: -rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
 //                                  [path to]/getHSTSPreloadlist.js \
 //                                  [absolute path to]/nsSTSPreloadlist.inc'
 // <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
 // <https://bugzilla.mozilla.org/show_bug.cgi?id=546628>
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cu = Components.utils;
 const Cr = Components.results;
 // Register resource://app/ URI
 let ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService);
 let resHandler = ios.getProtocolHandler("resource")
                  .QueryInterface(Ci.nsIResProtocolHandler);
 let mozDir = Cc["@mozilla.org/file/directory_service;1"]
              .getService(Ci.nsIProperties)
              .get("CurProcD", Ci.nsILocalFile);
 let mozDirURI = ios.newFileURI(mozDir);
 resHandler.setSubstitution("app", mozDirURI);
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/FileUtils.jsm");
 Cu.import("resource:///modules/XPCOMUtils.jsm");
 const SOURCE = "https://src.chromium.org/chrome/trunk/src/net/http/transport_security_state_static.json";
 const OUTPUT = "nsSTSPreloadList.inc";
 const ERROR_OUTPUT = "nsSTSPreloadList.errors";
 const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
 const MAX_CONCURRENT_REQUESTS = 5;
 const MAX_RETRIES = 3;
 const REQUEST_TIMEOUT = 30 * 1000;
 const ERROR_NONE = "no error";
 const ERROR_CONNECTING_TO_HOST = "could not connect to host";
 const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
 const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
 const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
 " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
 " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
 "\n" +
 "/*****************************************************************************/\n" +
 "/* This is an automatically generated file. If you're not                    */\n" +
 "/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
 "/*****************************************************************************/\n" +
 "\n" +
 "#include <stdint.h>\n";
 const PREFIX = "\n" +
 "class nsSTSPreload\n" +
 "{\n" +
 "  public:\n" +
 "    const char *mHost;\n" +
 "    const bool mIncludeSubdomains;\n" +
 "};\n" +
 "\n" +
 "static const nsSTSPreload kSTSPreloadList[] = {\n";
 const POSTFIX =  "};\n";
 function download() {
   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
             .createInstance(Ci.nsIXMLHttpRequest);
   req.open("GET", SOURCE, false); // doing the request synchronously
   try {
     req.send();
   }
   catch (e) {
     throw "ERROR: problem downloading '" + SOURCE + "': " + e;
   }
   if (req.status != 200) {
     throw "ERROR: problem downloading '" + SOURCE + "': status " + req.status;
   }
   // we have to filter out '//' comments
   var result = req.responseText.replace(/\/\/[^\n]*\n/g, "");
   var data = null;
   try {
     data = JSON.parse(result);
   }
   catch (e) {
     throw "ERROR: could not parse data from '" + SOURCE + "': " + e;
   }
   return data;
 }
 function getHosts(rawdata) {
   var hosts = [];
   if (!rawdata || !rawdata.entries) {
     throw "ERROR: source data not formatted correctly: 'entries' not found";
   }
   for (entry of rawdata.entries) {
     if (entry.mode && entry.mode == "force-https") {
       if (entry.name) {
         entry.retries = MAX_RETRIES;
         entry.originalIncludeSubdomains = entry.include_subdomains;
         hosts.push(entry);
       } else {
         throw "ERROR: entry not formatted correctly: no name found";
       }
     }
   }
   return hosts;
 }
 var gSSService = Cc["@mozilla.org/ssservice;1"]
                    .getService(Ci.nsISiteSecurityService);
 function processStsHeader(host, header, status) {
   var maxAge = { value: 0 };
   var includeSubdomains = { value: false };
   var error = ERROR_NONE;
   if (header != null) {
     try {
       var uri = Services.io.newURI("https://" + host.name, null, null);
       gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS,
                                uri, header, 0, maxAge, includeSubdomains);
     }
     catch (e) {
       dump("ERROR: could not process header '" + header + "' from " +
            host.name + ": " + e + "\n");
       error = e;
     }
   }
   else {
     if (status == 0) {
       error = ERROR_CONNECTING_TO_HOST;
     } else {
       error = ERROR_NO_HSTS_HEADER;
     }
   }
   let forceInclude = (host.forceInclude || host.pins == "google");
   if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
     error = ERROR_MAX_AGE_TOO_LOW;
   }
   return { name: host.name,
            maxAge: maxAge.value,
            includeSubdomains: includeSubdomains.value,
            error: error,
            retries: host.retries - 1,
            forceInclude: forceInclude,
            originalIncludeSubdomains: host.originalIncludeSubdomains };
 }
 function RedirectStopper() {};
 RedirectStopper.prototype = {
   // nsIChannelEventSink
   asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) {
     throw Cr.NS_ERROR_ENTITY_CHANGED;
   },
   getInterface: function(iid) {
     return this.QueryInterface(iid);
   },
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink])
 };
 function getHSTSStatus(host, resultList) {
   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
             .createInstance(Ci.nsIXMLHttpRequest);
   var inResultList = false;
   var uri = "https://" + host.name + "/";
   req.open("GET", uri, true);
   req.timeout = REQUEST_TIMEOUT;
   req.channel.notificationCallbacks = new RedirectStopper();
   req.onreadystatechange = function(event) {
     if (!inResultList && req.readyState == 4) {
       inResultList = true;
       var header = req.getResponseHeader("strict-transport-security");
       resultList.push(processStsHeader(host, header, req.status));
     }
   };
   try {
     req.send();
   }
   catch (e) {
     dump("ERROR: exception making request to " + host.name + ": " + e + "\n");
   }
 }
 function compareHSTSStatus(a, b) {
   return (a.name > b.name ? 1 : (a.name < b.name ? -1 : 0));
 }
 function writeTo(string, fos) {
   fos.write(string, string.length);
 }
 // Determines and returns a string representing a declaration of when this
 // preload list should no longer be used.
 // This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
 function getExpirationTimeString() {
   var now = new Date();
   var nowMillis = now.getTime();
   // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
   var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
   var expirationMicros = expirationMillis * 1000;
   return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
 }
 function errorToString(status) {
   return (status.error == ERROR_MAX_AGE_TOO_LOW
           ? status.error + status.maxAge
           : status.error);
 }
 function writeEntry(status, outputStream) {
   let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
                            ? status.originalIncludeSubdomains
                            : status.includeSubdomains);
   let includeSubdomains = (incSubdomainsBool ? "true" : "false");
   writeTo("  { \"" + status.name + "\", " + includeSubdomains + " },\n",
           outputStream);
 }
 function output(sortedStatuses, currentList) {
   try {
     var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
     var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
     var fos = FileUtils.openSafeFileOutputStream(file);
     var eos = FileUtils.openSafeFileOutputStream(errorFile);
     writeTo(HEADER, fos);
     writeTo(getExpirationTimeString(), fos);
     writeTo(PREFIX, fos);
     for (var status of sortedStatuses) {
       // If we've encountered an error for this entry (other than the site not
       // sending an HSTS header), be safe and don't remove it from the list
       // (given that it was already on the list).
       if (status.error != ERROR_NONE &&
           status.error != ERROR_NO_HSTS_HEADER &&
           status.error != ERROR_MAX_AGE_TOO_LOW &&
           status.name in currentList) {
         dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
         writeTo(status.name + ": " + errorToString(status) + "\n", eos);
         status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
         status.includeSubdomains = currentList[status.name];
       }
       if (status.maxAge >= MINIMUM_REQUIRED_MAX_AGE || status.forceInclude) {
         writeEntry(status, fos);
         dump("INFO: " + status.name + " ON the preload list\n");
         if (status.forceInclude && status.error != ERROR_NONE) {
           writeTo(status.name + ": " + errorToString(status) + " (error "
                   + "ignored - included regardless)\n", eos);
         }
       }
       else {
         dump("INFO: " + status.name + " NOT ON the preload list\n");
         writeTo(status.name + ": " + errorToString(status) + "\n", eos);
       }
     }
     writeTo(POSTFIX, fos);
     FileUtils.closeSafeFileOutputStream(fos);
     FileUtils.closeSafeFileOutputStream(eos);
   }
   catch (e) {
     dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
   }
 }
 function shouldRetry(response) {
   return (response.error != ERROR_NO_HSTS_HEADER &&
           response.error != ERROR_MAX_AGE_TOO_LOW &&
           response.error != ERROR_NONE && response.retries > 0);
 }
 function getHSTSStatuses(inHosts, outStatuses) {
   var expectedOutputLength = inHosts.length;
   var tmpOutput = [];
   for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
     var host = inHosts.shift();
     dump("spinning off request to '" + host.name + "' (remaining retries: " +
          host.retries + ")\n");
     getHSTSStatus(host, tmpOutput);
   }
   while (outStatuses.length != expectedOutputLength) {
     waitForAResponse(tmpOutput);
     var response = tmpOutput.shift();
     dump("request to '" + response.name + "' finished\n");
     if (shouldRetry(response))
       inHosts.push(response);
     else
       outStatuses.push(response);
     if (inHosts.length > 0) {
       var host = inHosts.shift();
       dump("spinning off request to '" + host.name + "' (remaining retries: " +
            host.retries + ")\n");
       getHSTSStatus(host, tmpOutput);
     }
   }
 }
 // Since all events are processed on the main thread, and since event
 // handlers are not preemptible, there shouldn't be any concurrency issues.
 function waitForAResponse(outputList) {
   // From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
   var threadManager = Cc["@mozilla.org/thread-manager;1"]
                       .getService(Ci.nsIThreadManager);
   var mainThread = threadManager.currentThread;
   while (outputList.length == 0) {
     mainThread.processNextEvent(true);
   }
 }
 function readCurrentList(filename) {
   var currentHosts = {};
   var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
   file.initWithPath(filename);
   var fis = Cc["@mozilla.org/network/file-input-stream;1"]
               .createInstance(Ci.nsILineInputStream);
   fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
   var line = {};
   var entryRegex = /  { "([^"]*)", (true|false) },/;
   while (fis.readLine(line)) {
     var match = entryRegex.exec(line.value);
     if (match) {
       currentHosts[match[1]] = (match[2] == "true");
     }
   }
   return currentHosts;
 }
 function combineLists(newHosts, currentHosts) {
   for (let currentHost in currentHosts) {
     let found = false;
     for (let newHost of newHosts) {
       if (newHost.name == currentHost) {
         found = true;
         break;
       }
     }
     if (!found) {
       newHosts.push({ name: currentHost, retries: MAX_RETRIES });
     }
   }
 }
 // ****************************************************************************
 // This is where the action happens:
 if (arguments.length < 1) {
   throw "Usage: getHSTSPreloadList.js <absolute path to current nsSTSPreloadList.inc>";
 }
 // get the current preload list
 var currentHosts = readCurrentList(arguments[0]);
 // disable the current preload list so it won't interfere with requests we make
 Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
 // download and parse the raw json file from the Chromium source
 var rawdata = download();
 // get just the hosts with mode: "force-https"
 var hosts = getHosts(rawdata);
 // add hosts in the current list to the new list (avoiding duplicates)
 combineLists(hosts, currentHosts);
 // get the HSTS status of each host
 var hstsStatuses = [];
 getHSTSStatuses(hosts, hstsStatuses);
 // sort the hosts alphabetically
 hstsStatuses.sort(compareHSTSStatus);
 // write the results to a file (this is where we filter out hosts that we
 // either couldn't connect to, didn't receive an HSTS header from, couldn't
 // parse the header, or had a header with too short a max-age)
 output(hstsStatuses, currentHosts);
 // ****************************************************************************