The Tor Browser / file comparison

comparison: security/manager/tools/genHPKPStaticPins.js

security/manager/tools/genHPKPStaticPins.js

branch
TOR_BUG_9701
changeset 3
141e0f1194b1
equal deleted inserted replaced
-1:000000000000 0:ccd32e650154
1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 // How to run this file:
6 // 1. [obtain firefox source code]
7 // 2. [build/obtain firefox binaries]
8 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
9 // [path to]/genHPKPStaticpins.js \
10 // [absolute path to]/PreloadedHPKPins.json \
11 // [absolute path to]/default-ee.der \
12 // [absolute path to]/StaticHPKPins.h
13
14 if (arguments.length != 3) {
15 throw "Usage: genHPKPStaticPins.js " +
16 "<absolute path to PreloadedHPKPins.json> " +
17 "<absolute path to default-ee.der> " +
18 "<absolute path to StaticHPKPins.h>";
19 }
20
21 const { 'classes': Cc, 'interfaces': Ci, 'utils': Cu, 'results': Cr } = Components;
22
23 let { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
24 let { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
25 let { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
26
27 let gCertDB = Cc["@mozilla.org/security/x509certdb;1"]
28 .getService(Ci.nsIX509CertDB);
29
30 const BUILT_IN_NICK_PREFIX = "Builtin Object Token:";
31 const SHA1_PREFIX = "sha1/";
32 const SHA256_PREFIX = "sha256/";
33 const GOOGLE_PIN_PREFIX = "GOOGLE_PIN_";
34
35 // Pins expire in 14 weeks (6 weeks on Beta + 8 weeks on stable)
36 const PINNING_MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 14;
37
38 const FILE_HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
39 " * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
40 " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
41 "\n" +
42 "/*****************************************************************************/\n" +
43 "/* This is an automatically generated file. If you're not */\n" +
44 "/* PublicKeyPinningService.cpp, you shouldn't be #including it. */\n" +
45 "/*****************************************************************************/\n" +
46 "#include <stdint.h>" +
47 "\n";
48
49 const DOMAINHEADER = "/* Domainlist */\n" +
50 "struct TransportSecurityPreload {\n" +
51 " const char* mHost;\n" +
52 " const bool mIncludeSubdomains;\n" +
53 " const bool mTestMode;\n" +
54 " const bool mIsMoz;\n" +
55 " const int32_t mId;\n" +
56 " const StaticPinset *pinset;\n" +
57 "};\n\n";
58
59 const PINSETDEF = "/* Pinsets are each an ordered list by the actual value of the fingerprint */\n" +
60 "struct StaticFingerprints {\n" +
61 " const size_t size;\n" +
62 " const char* const* data;\n" +
63 "};\n\n" +
64 "struct StaticPinset {\n" +
65 " const StaticFingerprints* sha1;\n" +
66 " const StaticFingerprints* sha256;\n" +
67 "};\n\n";
68
69 // Command-line arguments
70 var gStaticPins = parseJson(arguments[0]);
71 var gTestCertFile = arguments[1];
72
73 // Open the output file.
74 let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
75 file.initWithPath(arguments[2]);
76 let gFileOutputStream = FileUtils.openSafeFileOutputStream(file);
77
78 function writeString(string) {
79 gFileOutputStream.write(string, string.length);
80 }
81
82 function readFileToString(filename) {
83 let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
84 file.initWithPath(filename);
85 let stream = Cc["@mozilla.org/network/file-input-stream;1"]
86 .createInstance(Ci.nsIFileInputStream);
87 stream.init(file, -1, 0, 0);
88 let buf = NetUtil.readInputStreamToString(stream, stream.available());
89 return buf;
90 }
91
92 function stripComments(buf) {
93 var lines = buf.split("\n");
94 let entryRegex = /^\s*\/\//;
95 let data = "";
96 for (let i = 0; i < lines.length; ++i) {
97 let match = entryRegex.exec(lines[i]);
98 if (!match) {
99 data = data + lines[i];
100 }
101 }
102 return data;
103 }
104
105 function isBuiltinToken(tokenName) {
106 return tokenName == "Builtin Object Token";
107 }
108
109 function isCertBuiltIn(cert) {
110 let tokenNames = cert.getAllTokenNames({});
111 if (!tokenNames) {
112 return false;
113 }
114 if (tokenNames.some(isBuiltinToken)) {
115 return true;
116 }
117 return false;
118 }
119
120 function download(filename) {
121 var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
122 .createInstance(Ci.nsIXMLHttpRequest);
123 req.open("GET", filename, false); // doing the request synchronously
124 try {
125 req.send();
126 }
127 catch (e) {
128 throw "ERROR: problem downloading '" + filename + "': " + e;
129 }
130
131 if (req.status != 200) {
132 throw("ERROR: problem downloading '" + filename + "': status " +
133 req.status);
134 }
135 return req.responseText;
136 }
137
138 function downloadAsJson(filename) {
139 // we have to filter out '//' comments
140 var result = download(filename).replace(/\/\/[^\n]*\n/g, "");
141 var data = null;
142 try {
143 data = JSON.parse(result);
144 }
145 catch (e) {
146 throw "ERROR: could not parse data from '" + filename + "': " + e;
147 }
148 return data;
149 }
150
151 // Returns a Subject Public Key Digest from the given pem, if it exists.
152 function getSKDFromPem(pem) {
153 let cert = gCertDB.constructX509FromBase64(pem, pem.length);
154 return cert.sha256SubjectPublicKeyInfoDigest;
155 }
156
157 // Downloads the static certs file and tries to map Google Chrome nicknames
158 // to Mozilla nicknames, as well as storing any hashes for pins for which we
159 // don't have root PEMs. Each entry consists of a line containing the name of
160 // the pin followed either by a hash in the format "sha1/" + base64(hash), or
161 // a PEM encoded certificate. For certificates that we have in our database,
162 // return a map of Google's nickname to ours. For ones that aren't return a
163 // map of Google's nickname to sha1 values. This code is modeled after agl's
164 // https://github.com/agl/transport-security-state-generate, which doesn't
165 // live in the Chromium repo because go is not an official language in
166 // Chromium.
167 // For all of the entries in this file:
168 // - If the entry has a hash format, find the Mozilla pin name (cert nickname)
169 // and stick the hash into certSKDToName
170 // - If the entry has a PEM format, parse the PEM, find the Mozilla pin name
171 // and stick the hash in certSKDToName
172 // We MUST be able to find a corresponding cert nickname for the Chrome names,
173 // otherwise we skip all pinsets referring to that Chrome name.
174 function downloadAndParseChromeCerts(filename, certSKDToName) {
175 // Prefixes that we care about.
176 const BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
177 const END_CERT = "-----END CERTIFICATE-----";
178
179 // Parsing states.
180 const PRE_NAME = 0;
181 const POST_NAME = 1;
182 const IN_CERT = 2;
183 let state = PRE_NAME;
184
185 let lines = download(filename).split("\n");
186 let name = "";
187 let pemCert = "";
188 let hash = "";
189 let chromeNameToHash = {};
190 let chromeNameToMozName = {}
191 for (let i = 0; i < lines.length; ++i) {
192 let line = lines[i];
193 // Skip comments and newlines.
194 if (line.length == 0 || line[0] == '#') {
195 continue;
196 }
197 switch(state) {
198 case PRE_NAME:
199 chromeName = line;
200 state = POST_NAME;
201 break;
202 case POST_NAME:
203 if (line.startsWith(SHA1_PREFIX) ||
204 line.startsWith(SHA256_PREFIX)) {
205 if (line.startsWith(SHA1_PREFIX)) {
206 hash = line.substring(SHA1_PREFIX.length);
207 } else if (line.startsWith(SHA256_PREFIX)) {
208 hash = line.substring(SHA256_PREFIX);
209 }
210 // Store the entire prefixed hash, so we can disambiguate sha1 from
211 // sha256 later.
212 chromeNameToHash[chromeName] = line;
213 certNameToSKD[chromeName] = hash;
214 certSKDToName[hash] = chromeName;
215 state = PRE_NAME;
216 } else if (line.startsWith(BEGIN_CERT)) {
217 state = IN_CERT;
218 } else {
219 throw "ERROR: couldn't parse Chrome certificate file " + line;
220 }
221 break;
222 case IN_CERT:
223 if (line.startsWith(END_CERT)) {
224 state = PRE_NAME;
225 hash = getSKDFromPem(pemCert);
226 pemCert = "";
227 if (hash in certSKDToName) {
228 mozName = certSKDToName[hash];
229 } else {
230 // Not one of our built-in certs. Prefix the name with
231 // GOOGLE_PIN_.
232 mozName = GOOGLE_PIN_PREFIX + chromeName;
233 dump("Can't find hash in builtin certs for Chrome nickname " +
234 chromeName + ", inserting " + mozName + "\n");
235 certSKDToName[hash] = mozName;
236 certNameToSKD[mozName] = hash;
237 }
238 chromeNameToMozName[chromeName] = mozName;
239 } else {
240 pemCert += line;
241 }
242 break;
243 default:
244 throw "ERROR: couldn't parse Chrome certificate file " + line;
245 }
246 }
247 return [ chromeNameToHash, chromeNameToMozName ];
248 }
249
250 // We can only import pinsets from chrome if for every name in the pinset:
251 // - We have a hash from Chrome's static certificate file
252 // - We have a builtin cert
253 // If the pinset meets these requirements, we store a map array of pinset
254 // objects:
255 // {
256 // pinset_name : {
257 // // Array of names with entries in certNameToSKD
258 // sha1_hashes: [],
259 // sha256_hashes: []
260 // }
261 // }
262 // and an array of imported pinset entries:
263 // { name: string, include_subdomains: boolean, test_mode: boolean,
264 // pins: pinset_name }
265 function downloadAndParseChromePins(filename,
266 chromeNameToHash,
267 chromeNameToMozName,
268 certNameToSKD,
269 certSKDToName) {
270 let chromePreloads = downloadAsJson(filename);
271 let chromePins = chromePreloads.pinsets;
272 let chromeImportedPinsets = {};
273 let chromeImportedEntries = [];
274
275 chromePins.forEach(function(pin) {
276 let valid = true;
277 let pinset = { name: pin.name, sha1_hashes: [], sha256_hashes: [] };
278 // Translate the Chrome pinset format to ours
279 pin.static_spki_hashes.forEach(function(name) {
280 if (name in chromeNameToHash) {
281 let hash = chromeNameToHash[name];
282 if (hash.startsWith(SHA1_PREFIX)) {
283 hash = hash.substring(SHA1_PREFIX.length);
284 pinset.sha1_hashes.push(certSKDToName[hash]);
285 } else if (hash.startsWith(SHA256_PREFIX)) {
286 hash = hash.substring(SHA256_PREFIX.length);
287 pinset.sha256_hashes.push(certSKDToName[hash]);
288 } else {
289 throw("Unsupported hash type: " + chromeNameToHash[name]);
290 }
291 // We should have already added hashes for all of these when we
292 // imported the certificate file.
293 if (!certNameToSKD[name]) {
294 throw("No hash for name: " + name);
295 }
296 } else if (name in chromeNameToMozName) {
297 pinset.sha256_hashes.push(chromeNameToMozName[name]);
298 } else {
299 dump("Skipping Chrome pinset " + pinset.name + ", couldn't find " +
300 "builtin " + name + " from cert file\n");
301 valid = false;
302 }
303 });
304 if (valid) {
305 chromeImportedPinsets[pinset.name] = pinset;
306 }
307 });
308
309 // Grab the domain entry lists. Chrome's entry format is similar to
310 // ours, except theirs includes a HSTS mode.
311 const cData = gStaticPins.chromium_data;
312 let entries = chromePreloads.entries;
313 entries.forEach(function(entry) {
314 let pinsetName = cData.substitute_pinsets[entry.pins];
315 if (!pinsetName) {
316 pinsetName = entry.pins;
317 }
318 let isProductionDomain =
319 (cData.production_domains.indexOf(entry.name) != -1);
320 let isProductionPinset =
321 (cData.production_pinsets.indexOf(pinsetName) != -1);
322 let excludeDomain =
323 (cData.exclude_domains.indexOf(entry.name) != -1);
324 let isTestMode = !isProductionPinset && !isProductionDomain;
325 if (entry.pins && !excludeDomain && chromeImportedPinsets[entry.pins]) {
326 chromeImportedEntries.push({
327 name: entry.name,
328 include_subdomains: entry.include_subdomains,
329 test_mode: isTestMode,
330 is_moz: false,
331 pins: pinsetName });
332 }
333 });
334 return [ chromeImportedPinsets, chromeImportedEntries ];
335 }
336
337 // Returns a pair of maps [certNameToSKD, certSKDToName] between cert
338 // nicknames and digests of the SPKInfo for the mozilla trust store
339 function loadNSSCertinfo(derTestFile, extraCertificates) {
340 let allCerts = gCertDB.getCerts();
341 let enumerator = allCerts.getEnumerator();
342 let certNameToSKD = {};
343 let certSKDToName = {};
344 while (enumerator.hasMoreElements()) {
345 let cert = enumerator.getNext().QueryInterface(Ci.nsIX509Cert);
346 if (!isCertBuiltIn(cert)) {
347 continue;
348 }
349 let name = cert.nickname.substr(BUILT_IN_NICK_PREFIX.length);
350 let SKD = cert.sha256SubjectPublicKeyInfoDigest;
351 certNameToSKD[name] = SKD;
352 certSKDToName[SKD] = name;
353 }
354
355 for (let cert of extraCertificates) {
356 let name = cert.commonName;
357 let SKD = cert.sha256SubjectPublicKeyInfoDigest;
358 certNameToSKD[name] = SKD;
359 certSKDToName[SKD] = name;
360 }
361
362 {
363 // A certificate for *.example.com.
364 let der = readFileToString(derTestFile);
365 let testCert = gCertDB.constructX509(der, der.length);
366 // We can't include this cert in the previous loop, because it skips
367 // non-builtin certs and the nickname is not built-in to the cert.
368 let name = "End Entity Test Cert";
369 let SKD = testCert.sha256SubjectPublicKeyInfoDigest;
370 certNameToSKD[name] = SKD;
371 certSKDToName[SKD] = name;
372 }
373 return [certNameToSKD, certSKDToName];
374 }
375
376 function parseJson(filename) {
377 let json = stripComments(readFileToString(filename));
378 return JSON.parse(json);
379 }
380
381 function nameToAlias(certName) {
382 // change the name to a string valid as a c identifier
383 // remove non-ascii characters
384 certName = certName.replace( /[^[:ascii:]]/g, "_");
385 // replace non word characters
386 certName = certName.replace(/[^A-Za-z0-9]/g ,"_");
387
388 return "k" + certName + "Fingerprint";
389 }
390
391 function compareByName (a, b) {
392 return a.name.localeCompare(b.name);
393 }
394
395 function genExpirationTime() {
396 let now = new Date();
397 let nowMillis = now.getTime();
398 let expirationMillis = nowMillis + (PINNING_MINIMUM_REQUIRED_MAX_AGE * 1000);
399 let expirationMicros = expirationMillis * 1000;
400 return "static const PRTime kPreloadPKPinsExpirationTime = INT64_C(" +
401 expirationMicros +");\n";
402 }
403
404 function writeFullPinset(certNameToSKD, certSKDToName, pinset) {
405 // We aren't guaranteed to have sha1 hashes in our own imported pins.
406 let prefix = "kPinset_" + pinset.name;
407 let sha1Name = "nullptr";
408 let sha256Name = "nullptr";
409 if (pinset.sha1_hashes && pinset.sha1_hashes.length > 0) {
410 writeFingerprints(certNameToSKD, certSKDToName, pinset.name,
411 pinset.sha1_hashes, "sha1");
412 sha1Name = "&" + prefix + "_sha1";
413 }
414 if (pinset.sha256_hashes && pinset.sha256_hashes.length > 0) {
415 writeFingerprints(certNameToSKD, certSKDToName, pinset.name,
416 pinset.sha256_hashes, "sha256");
417 sha256Name = "&" + prefix + "_sha256";
418 }
419 writeString("static const StaticPinset " + prefix + " = {\n" +
420 " " + sha1Name + ",\n " + sha256Name + "\n};\n\n");
421 }
422
423 function writeFingerprints(certNameToSKD, certSKDToName, name, hashes, type) {
424 let varPrefix = "kPinset_" + name + "_" + type;
425 writeString("static const char* " + varPrefix + "_Data[] = {\n");
426 let SKDList = [];
427 for (let certName of hashes) {
428 if (!(certName in certNameToSKD)) {
429 throw "Can't find " + certName + " in certNameToSKD";
430 }
431 SKDList.push(certNameToSKD[certName]);
432 }
433 for (let skd of SKDList.sort()) {
434 writeString(" " + nameToAlias(certSKDToName[skd]) + ",\n");
435 }
436 if (hashes.length == 0) {
437 // ANSI C requires that an initialiser list be non-empty.
438 writeString(" 0\n");
439 }
440 writeString("};\n");
441 writeString("static const StaticFingerprints " + varPrefix + " = {\n " +
442 "sizeof(" + varPrefix + "_Data) / sizeof(const char*),\n " + varPrefix +
443 "_Data\n};\n\n");
444 }
445
446 function writeEntry(entry) {
447 let printVal = " { \"" + entry.name + "\",\ ";
448 if (entry.include_subdomains) {
449 printVal += "true, ";
450 } else {
451 printVal += "false, ";
452 }
453 // Default to test mode if not specified.
454 let testMode = true;
455 if (entry.hasOwnProperty("test_mode")) {
456 testMode = entry.test_mode;
457 }
458 if (testMode) {
459 printVal += "true, ";
460 } else {
461 printVal += "false, ";
462 }
463 if (entry.is_moz || (entry.pins == "mozilla")) {
464 printVal += "true, ";
465 } else {
466 printVal += "false, ";
467 }
468 if (entry.id >= 256) {
469 throw("Not enough buckets in histogram");
470 }
471 if (entry.id >= 0) {
472 printVal += entry.id + ", ";
473 } else {
474 printVal += "-1, ";
475 }
476 printVal += "&kPinset_" + entry.pins;
477 printVal += " },\n";
478 writeString(printVal);
479 }
480
481 function writeDomainList(chromeImportedEntries) {
482 writeString("/* Sort hostnames for binary search. */\n");
483 writeString("static const TransportSecurityPreload " +
484 "kPublicKeyPinningPreloadList[] = {\n");
485 let count = 0;
486 let sortedEntries = gStaticPins.entries;
487 sortedEntries.push.apply(sortedEntries, chromeImportedEntries);
488 for (let entry of sortedEntries.sort(compareByName)) {
489 count++;
490 writeEntry(entry);
491 }
492 writeString("};\n");
493
494 writeString("\n// Pinning Preload List Length = " + count + ";\n");
495 writeString("\nstatic const int32_t kUnknownId = -1;\n");
496 }
497
498 function writeFile(certNameToSKD, certSKDToName,
499 chromeImportedPinsets, chromeImportedEntries) {
500 // Compute used pins from both Chrome's and our pinsets, so we can output
501 // them later.
502 usedFingerprints = {};
503 gStaticPins.pinsets.forEach(function(pinset) {
504 // We aren't guaranteed to have sha1_hashes in our own JSON.
505 if (pinset.sha1_hashes) {
506 pinset.sha1_hashes.forEach(function(name) {
507 usedFingerprints[name] = true;
508 });
509 }
510 if (pinset.sha256_hashes) {
511 pinset.sha256_hashes.forEach(function(name) {
512 usedFingerprints[name] = true;
513 });
514 }
515 });
516 for (let key in chromeImportedPinsets) {
517 let pinset = chromeImportedPinsets[key];
518 pinset.sha1_hashes.forEach(function(name) {
519 usedFingerprints[name] = true;
520 });
521 pinset.sha256_hashes.forEach(function(name) {
522 usedFingerprints[name] = true;
523 });
524 }
525
526 writeString(FILE_HEADER);
527
528 // Write actual fingerprints.
529 Object.keys(usedFingerprints).sort().forEach(function(certName) {
530 if (certName) {
531 writeString("/* " + certName + " */\n");
532 writeString("static const char " + nameToAlias(certName) + "[] =\n");
533 writeString(" \"" + certNameToSKD[certName] + "\";\n");
534 writeString("\n");
535 }
536 });
537
538 // Write the pinsets
539 writeString(PINSETDEF);
540 writeString("/* PreloadedHPKPins.json pinsets */\n");
541 gStaticPins.pinsets.sort(compareByName).forEach(function(pinset) {
542 writeFullPinset(certNameToSKD, certSKDToName, pinset);
543 });
544 writeString("/* Chrome static pinsets */\n");
545 for (let key in chromeImportedPinsets) {
546 writeFullPinset(certNameToSKD, certSKDToName, chromeImportedPinsets[key]);
547 }
548
549 // Write the domainlist entries.
550 writeString(DOMAINHEADER);
551 writeDomainList(chromeImportedEntries);
552 writeString("\n");
553 writeString(genExpirationTime());
554 }
555
556 function loadExtraCertificates(certStringList) {
557 let constructedCerts = [];
558 for (let certString of certStringList) {
559 constructedCerts.push(gCertDB.constructX509FromBase64(certString));
560 }
561 return constructedCerts;
562 }
563
564 let extraCertificates = loadExtraCertificates(gStaticPins.extra_certificates);
565 let [ certNameToSKD, certSKDToName ] = loadNSSCertinfo(gTestCertFile,
566 extraCertificates);
567 let [ chromeNameToHash, chromeNameToMozName ] = downloadAndParseChromeCerts(
568 gStaticPins.chromium_data.cert_file_url, certSKDToName);
569 let [ chromeImportedPinsets, chromeImportedEntries ] =
570 downloadAndParseChromePins(gStaticPins.chromium_data.json_file_url,
571 chromeNameToHash, chromeNameToMozName, certNameToSKD, certSKDToName);
572
573 writeFile(certNameToSKD, certSKDToName, chromeImportedPinsets,
574 chromeImportedEntries);
575
576 FileUtils.closeSafeFileOutputStream(gFileOutputStream);

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial