The Tor Browser: security/manager/tools/genHPKPStaticPins.js@141e0f1194b1 (annotated)

security/manager/tools/genHPKPStaticPins.js@141e0f1194b1 (annotated)

security/manager/tools/genHPKPStaticPins.js

Wed, 31 Dec 2014 07:16:47 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 07:16:47 +0100
branch: TOR_BUG_9701
changeset 3: 141e0f1194b1
permissions: -rw-r--r--

Revert simplistic fix pending revisit of Mozilla integration attempt.

 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
 //                                  [path to]/genHPKPStaticpins.js \
 //                                  [absolute path to]/PreloadedHPKPins.json \
 //                                  [absolute path to]/default-ee.der \
 //                                  [absolute path to]/StaticHPKPins.h
 if (arguments.length != 3) {
   throw "Usage: genHPKPStaticPins.js " +
         "<absolute path to PreloadedHPKPins.json> " +
         "<absolute path to default-ee.der> " +
         "<absolute path to StaticHPKPins.h>";
 }
 const { 'classes': Cc, 'interfaces': Ci, 'utils': Cu, 'results': Cr } = Components;
 let { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
 let { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
 let { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 let gCertDB = Cc["@mozilla.org/security/x509certdb;1"]
                 .getService(Ci.nsIX509CertDB);
 const BUILT_IN_NICK_PREFIX = "Builtin Object Token:";
 const SHA1_PREFIX = "sha1/";
 const SHA256_PREFIX = "sha256/";
 const GOOGLE_PIN_PREFIX = "GOOGLE_PIN_";
 // Pins expire in 14 weeks (6 weeks on Beta + 8 weeks on stable)
 const PINNING_MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 14;
 const FILE_HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
 " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
 " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
 "\n" +
 "/*****************************************************************************/\n" +
 "/* This is an automatically generated file. If you're not                    */\n" +
 "/* PublicKeyPinningService.cpp, you shouldn't be #including it.              */\n" +
 "/*****************************************************************************/\n" +
 "#include <stdint.h>" +
 "\n";
 const DOMAINHEADER = "/* Domainlist */\n" +
   "struct TransportSecurityPreload {\n" +
   "  const char* mHost;\n" +
   "  const bool mIncludeSubdomains;\n" +
   "  const bool mTestMode;\n" +
   "  const bool mIsMoz;\n" +
   "  const int32_t mId;\n" +
   "  const StaticPinset *pinset;\n" +
   "};\n\n";
 const PINSETDEF = "/* Pinsets are each an ordered list by the actual value of the fingerprint */\n" +
   "struct StaticFingerprints {\n" +
   "  const size_t size;\n" +
   "  const char* const* data;\n" +
   "};\n\n" +
   "struct StaticPinset {\n" +
   "  const StaticFingerprints* sha1;\n" +
   "  const StaticFingerprints* sha256;\n" +
   "};\n\n";
 // Command-line arguments
 var gStaticPins = parseJson(arguments[0]);
 var gTestCertFile = arguments[1];
 // Open the output file.
 let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
 file.initWithPath(arguments[2]);
 let gFileOutputStream = FileUtils.openSafeFileOutputStream(file);
 function writeString(string) {
   gFileOutputStream.write(string, string.length);
 }
 function readFileToString(filename) {
   let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
   file.initWithPath(filename);
   let stream = Cc["@mozilla.org/network/file-input-stream;1"]
                  .createInstance(Ci.nsIFileInputStream);
   stream.init(file, -1, 0, 0);
   let buf = NetUtil.readInputStreamToString(stream, stream.available());
   return buf;
 }
 function stripComments(buf) {
   var lines = buf.split("\n");
   let entryRegex = /^\s*\/\//;
   let data = "";
   for (let i = 0; i < lines.length; ++i) {
     let match = entryRegex.exec(lines[i]);
     if (!match) {
       data = data + lines[i];
     }
   }
   return data;
 }
 function isBuiltinToken(tokenName) {
   return tokenName == "Builtin Object Token";
 }
 function isCertBuiltIn(cert) {
   let tokenNames = cert.getAllTokenNames({});
   if (!tokenNames) {
     return false;
   }
   if (tokenNames.some(isBuiltinToken)) {
     return true;
   }
   return false;
 }
 function download(filename) {
   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
               .createInstance(Ci.nsIXMLHttpRequest);
   req.open("GET", filename, false); // doing the request synchronously
   try {
     req.send();
   }
   catch (e) {
     throw "ERROR: problem downloading '" + filename + "': " + e;
   }
   if (req.status != 200) {
     throw("ERROR: problem downloading '" + filename + "': status " +
           req.status);
   }
   return req.responseText;
 }
 function downloadAsJson(filename) {
   // we have to filter out '//' comments
   var result = download(filename).replace(/\/\/[^\n]*\n/g, "");
   var data = null;
   try {
     data = JSON.parse(result);
   }
   catch (e) {
     throw "ERROR: could not parse data from '" + filename + "': " + e;
   }
   return data;
 }
 // Returns a Subject Public Key Digest from the given pem, if it exists.
 function getSKDFromPem(pem) {
   let cert = gCertDB.constructX509FromBase64(pem, pem.length);
   return cert.sha256SubjectPublicKeyInfoDigest;
 }
 // Downloads the static certs file and tries to map Google Chrome nicknames
 // to Mozilla nicknames, as well as storing any hashes for pins for which we
 // don't have root PEMs. Each entry consists of a line containing the name of
 // the pin followed either by a hash in the format "sha1/" + base64(hash), or
 // a PEM encoded certificate. For certificates that we have in our database,
 // return a map of Google's nickname to ours. For ones that aren't return a
 // map of Google's nickname to sha1 values. This code is modeled after agl's
 // https://github.com/agl/transport-security-state-generate, which doesn't
 // live in the Chromium repo because go is not an official language in
 // Chromium.
 // For all of the entries in this file:
 // - If the entry has a hash format, find the Mozilla pin name (cert nickname)
 // and stick the hash into certSKDToName
 // - If the entry has a PEM format, parse the PEM, find the Mozilla pin name
 // and stick the hash in certSKDToName
 // We MUST be able to find a corresponding cert nickname for the Chrome names,
 // otherwise we skip all pinsets referring to that Chrome name.
 function downloadAndParseChromeCerts(filename, certSKDToName) {
   // Prefixes that we care about.
   const BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
   const END_CERT = "-----END CERTIFICATE-----";
   // Parsing states.
   const PRE_NAME = 0;
   const POST_NAME = 1;
   const IN_CERT = 2;
   let state = PRE_NAME;
   let lines = download(filename).split("\n");
   let name = "";
   let pemCert = "";
   let hash = "";
   let chromeNameToHash = {};
   let chromeNameToMozName = {}
   for (let i = 0; i < lines.length; ++i) {
     let line = lines[i];
     // Skip comments and newlines.
     if (line.length == 0 || line[0] == '#') {
       continue;
     }
     switch(state) {
       case PRE_NAME:
         chromeName = line;
         state = POST_NAME;
         break;
       case POST_NAME:
         if (line.startsWith(SHA1_PREFIX) ||
             line.startsWith(SHA256_PREFIX)) {
           if (line.startsWith(SHA1_PREFIX)) {
             hash = line.substring(SHA1_PREFIX.length);
           } else if (line.startsWith(SHA256_PREFIX)) {
             hash = line.substring(SHA256_PREFIX);
           }
           // Store the entire prefixed hash, so we can disambiguate sha1 from
           // sha256 later.
           chromeNameToHash[chromeName] = line;
           certNameToSKD[chromeName] = hash;
           certSKDToName[hash] = chromeName;
           state = PRE_NAME;
         } else if (line.startsWith(BEGIN_CERT)) {
           state = IN_CERT;
         } else {
           throw "ERROR: couldn't parse Chrome certificate file " + line;
         }
         break;
       case IN_CERT:
         if (line.startsWith(END_CERT)) {
           state = PRE_NAME;
           hash = getSKDFromPem(pemCert);
           pemCert = "";
           if (hash in certSKDToName) {
             mozName = certSKDToName[hash];
           } else {
             // Not one of our built-in certs. Prefix the name with
             // GOOGLE_PIN_.
             mozName = GOOGLE_PIN_PREFIX + chromeName;
             dump("Can't find hash in builtin certs for Chrome nickname " +
                  chromeName + ", inserting " + mozName + "\n");
             certSKDToName[hash] = mozName;
             certNameToSKD[mozName] = hash;
           }
           chromeNameToMozName[chromeName] = mozName;
         } else {
           pemCert += line;
         }
         break;
       default:
         throw "ERROR: couldn't parse Chrome certificate file " + line;
     }
   }
   return [ chromeNameToHash, chromeNameToMozName ];
 }
 // We can only import pinsets from chrome if for every name in the pinset:
 // - We have a hash from Chrome's static certificate file
 // - We have a builtin cert
 // If the pinset meets these requirements, we store a map array of pinset
 // objects:
 // {
 //   pinset_name : {
 //     // Array of names with entries in certNameToSKD
 //     sha1_hashes: [],
 //     sha256_hashes: []
 //   }
 // }
 // and an array of imported pinset entries:
 // { name: string, include_subdomains: boolean, test_mode: boolean,
 //   pins: pinset_name }
 function downloadAndParseChromePins(filename,
                                     chromeNameToHash,
                                     chromeNameToMozName,
                                     certNameToSKD,
                                     certSKDToName) {
   let chromePreloads = downloadAsJson(filename);
   let chromePins = chromePreloads.pinsets;
   let chromeImportedPinsets = {};
   let chromeImportedEntries = [];
   chromePins.forEach(function(pin) {
     let valid = true;
     let pinset = { name: pin.name, sha1_hashes: [], sha256_hashes: [] };
     // Translate the Chrome pinset format to ours
     pin.static_spki_hashes.forEach(function(name) {
       if (name in chromeNameToHash) {
         let hash = chromeNameToHash[name];
         if (hash.startsWith(SHA1_PREFIX)) {
           hash = hash.substring(SHA1_PREFIX.length);
           pinset.sha1_hashes.push(certSKDToName[hash]);
         } else if (hash.startsWith(SHA256_PREFIX)) {
           hash = hash.substring(SHA256_PREFIX.length);
           pinset.sha256_hashes.push(certSKDToName[hash]);
         } else {
           throw("Unsupported hash type: " + chromeNameToHash[name]);
         }
         // We should have already added hashes for all of these when we
         // imported the certificate file.
         if (!certNameToSKD[name]) {
           throw("No hash for name: " + name);
         }
       } else if (name in chromeNameToMozName) {
         pinset.sha256_hashes.push(chromeNameToMozName[name]);
       } else {
         dump("Skipping Chrome pinset " + pinset.name + ", couldn't find " +
              "builtin " + name + " from cert file\n");
         valid = false;
       }
     });
     if (valid) {
       chromeImportedPinsets[pinset.name] = pinset;
     }
   });
   // Grab the domain entry lists. Chrome's entry format is similar to
   // ours, except theirs includes a HSTS mode.
   const cData = gStaticPins.chromium_data;
   let entries = chromePreloads.entries;
   entries.forEach(function(entry) {
     let pinsetName = cData.substitute_pinsets[entry.pins];
     if (!pinsetName) {
       pinsetName = entry.pins;
     }
     let isProductionDomain =
       (cData.production_domains.indexOf(entry.name) != -1);
     let isProductionPinset =
       (cData.production_pinsets.indexOf(pinsetName) != -1);
     let excludeDomain =
       (cData.exclude_domains.indexOf(entry.name) != -1);
     let isTestMode = !isProductionPinset && !isProductionDomain;
     if (entry.pins && !excludeDomain && chromeImportedPinsets[entry.pins]) {
       chromeImportedEntries.push({
         name: entry.name,
         include_subdomains: entry.include_subdomains,
         test_mode: isTestMode,
         is_moz: false,
         pins: pinsetName });
     }
   });
   return [ chromeImportedPinsets, chromeImportedEntries ];
 }
 // Returns a pair of maps [certNameToSKD, certSKDToName] between cert
 // nicknames and digests of the SPKInfo for the mozilla trust store
 function loadNSSCertinfo(derTestFile, extraCertificates) {
   let allCerts = gCertDB.getCerts();
   let enumerator = allCerts.getEnumerator();
   let certNameToSKD = {};
   let certSKDToName = {};
   while (enumerator.hasMoreElements()) {
     let cert = enumerator.getNext().QueryInterface(Ci.nsIX509Cert);
     if (!isCertBuiltIn(cert)) {
       continue;
     }
     let name = cert.nickname.substr(BUILT_IN_NICK_PREFIX.length);
     let SKD = cert.sha256SubjectPublicKeyInfoDigest;
     certNameToSKD[name] = SKD;
     certSKDToName[SKD] = name;
   }
   for (let cert of extraCertificates) {
     let name = cert.commonName;
     let SKD = cert.sha256SubjectPublicKeyInfoDigest;
     certNameToSKD[name] = SKD;
     certSKDToName[SKD] = name;
   }
   {
     // A certificate for *.example.com.
     let der = readFileToString(derTestFile);
     let testCert = gCertDB.constructX509(der, der.length);
     // We can't include this cert in the previous loop, because it skips
     // non-builtin certs and the nickname is not built-in to the cert.
     let name = "End Entity Test Cert";
     let SKD  = testCert.sha256SubjectPublicKeyInfoDigest;
     certNameToSKD[name] = SKD;
     certSKDToName[SKD] = name;
   }
   return [certNameToSKD, certSKDToName];
 }
 function parseJson(filename) {
   let json = stripComments(readFileToString(filename));
   return JSON.parse(json);
 }
 function nameToAlias(certName) {
   // change the name to a string valid as a c identifier
   // remove  non-ascii characters
   certName = certName.replace( /[^[:ascii:]]/g, "_");
   // replace non word characters
   certName = certName.replace(/[^A-Za-z0-9]/g ,"_");
   return "k" + certName + "Fingerprint";
 }
 function compareByName (a, b) {
   return a.name.localeCompare(b.name);
 }
 function genExpirationTime() {
   let now = new Date();
   let nowMillis = now.getTime();
   let expirationMillis = nowMillis + (PINNING_MINIMUM_REQUIRED_MAX_AGE * 1000);
   let expirationMicros = expirationMillis * 1000;
   return "static const PRTime kPreloadPKPinsExpirationTime = INT64_C(" +
          expirationMicros +");\n";
 }
 function writeFullPinset(certNameToSKD, certSKDToName, pinset) {
   // We aren't guaranteed to have sha1 hashes in our own imported pins.
   let prefix = "kPinset_" + pinset.name;
   let sha1Name = "nullptr";
   let sha256Name = "nullptr";
   if (pinset.sha1_hashes && pinset.sha1_hashes.length > 0) {
     writeFingerprints(certNameToSKD, certSKDToName, pinset.name,
                       pinset.sha1_hashes, "sha1");
     sha1Name = "&" + prefix + "_sha1";
   }
   if (pinset.sha256_hashes && pinset.sha256_hashes.length > 0) {
     writeFingerprints(certNameToSKD, certSKDToName, pinset.name,
                       pinset.sha256_hashes, "sha256");
     sha256Name = "&" + prefix + "_sha256";
   }
   writeString("static const StaticPinset " + prefix + " = {\n" +
           "  " + sha1Name + ",\n  " + sha256Name + "\n};\n\n");
 }
 function writeFingerprints(certNameToSKD, certSKDToName, name, hashes, type) {
   let varPrefix = "kPinset_" + name + "_" + type;
   writeString("static const char* " + varPrefix + "_Data[] = {\n");
   let SKDList = [];
   for (let certName of hashes) {
     if (!(certName in certNameToSKD)) {
       throw "Can't find " + certName + " in certNameToSKD";
     }
     SKDList.push(certNameToSKD[certName]);
   }
   for (let skd of SKDList.sort()) {
     writeString("  " + nameToAlias(certSKDToName[skd]) + ",\n");
   }
   if (hashes.length == 0) {
     // ANSI C requires that an initialiser list be non-empty.
     writeString("  0\n");
   }
   writeString("};\n");
   writeString("static const StaticFingerprints " + varPrefix + " = {\n  " +
     "sizeof(" + varPrefix + "_Data) / sizeof(const char*),\n  " + varPrefix +
     "_Data\n};\n\n");
 }
 function writeEntry(entry) {
   let printVal = "  { \"" + entry.name + "\",\ ";
   if (entry.include_subdomains) {
     printVal += "true, ";
   } else {
     printVal += "false, ";
   }
   // Default to test mode if not specified.
   let testMode = true;
   if (entry.hasOwnProperty("test_mode")) {
     testMode = entry.test_mode;
   }
   if (testMode) {
     printVal += "true, ";
   } else {
     printVal += "false, ";
   }
   if (entry.is_moz || (entry.pins == "mozilla")) {
     printVal += "true, ";
   } else {
     printVal += "false, ";
   }
   if (entry.id >= 256) {
     throw("Not enough buckets in histogram");
   }
   if (entry.id >= 0) {
     printVal += entry.id + ", ";
   } else {
     printVal += "-1, ";
   }
   printVal += "&kPinset_" + entry.pins;
   printVal += " },\n";
   writeString(printVal);
 }
 function writeDomainList(chromeImportedEntries) {
   writeString("/* Sort hostnames for binary search. */\n");
   writeString("static const TransportSecurityPreload " +
           "kPublicKeyPinningPreloadList[] = {\n");
   let count = 0;
   let sortedEntries = gStaticPins.entries;
   sortedEntries.push.apply(sortedEntries, chromeImportedEntries);
   for (let entry of sortedEntries.sort(compareByName)) {
     count++;
     writeEntry(entry);
   }
   writeString("};\n");
   writeString("\n// Pinning Preload List Length = " + count + ";\n");
   writeString("\nstatic const int32_t kUnknownId = -1;\n");
 }
 function writeFile(certNameToSKD, certSKDToName,
                    chromeImportedPinsets, chromeImportedEntries) {
   // Compute used pins from both Chrome's and our pinsets, so we can output
   // them later.
   usedFingerprints = {};
   gStaticPins.pinsets.forEach(function(pinset) {
     // We aren't guaranteed to have sha1_hashes in our own JSON.
     if (pinset.sha1_hashes) {
       pinset.sha1_hashes.forEach(function(name) {
         usedFingerprints[name] = true;
       });
     }
     if (pinset.sha256_hashes) {
       pinset.sha256_hashes.forEach(function(name) {
         usedFingerprints[name] = true;
       });
     }
   });
   for (let key in chromeImportedPinsets) {
     let pinset = chromeImportedPinsets[key];
     pinset.sha1_hashes.forEach(function(name) {
       usedFingerprints[name] = true;
     });
     pinset.sha256_hashes.forEach(function(name) {
       usedFingerprints[name] = true;
     });
   }
   writeString(FILE_HEADER);
   // Write actual fingerprints.
   Object.keys(usedFingerprints).sort().forEach(function(certName) {
     if (certName) {
       writeString("/* " + certName + " */\n");
       writeString("static const char " + nameToAlias(certName) + "[] =\n");
       writeString("  \"" + certNameToSKD[certName] + "\";\n");
       writeString("\n");
     }
   });
   // Write the pinsets
   writeString(PINSETDEF);
   writeString("/* PreloadedHPKPins.json pinsets */\n");
   gStaticPins.pinsets.sort(compareByName).forEach(function(pinset) {
     writeFullPinset(certNameToSKD, certSKDToName, pinset);
   });
   writeString("/* Chrome static pinsets */\n");
   for (let key in chromeImportedPinsets) {
     writeFullPinset(certNameToSKD, certSKDToName, chromeImportedPinsets[key]);
   }
   // Write the domainlist entries.
   writeString(DOMAINHEADER);
   writeDomainList(chromeImportedEntries);
   writeString("\n");
   writeString(genExpirationTime());
 }
 function loadExtraCertificates(certStringList) {
   let constructedCerts = [];
   for (let certString of certStringList) {
     constructedCerts.push(gCertDB.constructX509FromBase64(certString));
   }
   return constructedCerts;
 }
 let extraCertificates = loadExtraCertificates(gStaticPins.extra_certificates);
 let [ certNameToSKD, certSKDToName ] = loadNSSCertinfo(gTestCertFile,
                                                        extraCertificates);
 let [ chromeNameToHash, chromeNameToMozName ] = downloadAndParseChromeCerts(
   gStaticPins.chromium_data.cert_file_url, certSKDToName);
 let [ chromeImportedPinsets, chromeImportedEntries ] =
   downloadAndParseChromePins(gStaticPins.chromium_data.json_file_url,
     chromeNameToHash, chromeNameToMozName, certNameToSKD, certSKDToName);
 writeFile(certNameToSKD, certSKDToName, chromeImportedPinsets,
           chromeImportedEntries);
 FileUtils.closeSafeFileOutputStream(gFileOutputStream);