The Tor Browser: js/src/builtin/make_intl_data.py@129ffea94266 (annotated)

js/src/builtin/make_intl_data.py@129ffea94266 (annotated)

js/src/builtin/make_intl_data.py

Sat, 03 Jan 2015 20:18:00 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Sat, 03 Jan 2015 20:18:00 +0100
branch: TOR_BUG_3246
changeset 7: 129ffea94266
permissions: -rwxr-xr-x

Conditionally enable double key logic according to:
private browsing mode or privacy.thirdparty.isolate preference and
implement in GetCookieStringCommon and FindCookie where it counts...
With some reservations of how to convince FindCookie users to test
condition and pass a nullptr when disabling double key logic.

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """ Usage: make_intl_data.py [language-subtag-registry.txt]
     This script extracts information about mappings between deprecated and
     current BCP 47 language tags from the IANA Language Subtag Registry and
     converts it to JavaScript object definitions in IntlData.js. The definitions
     are used in Intl.js.
     The IANA Language Subtag Registry is imported from
     http://www.iana.org/assignments/language-subtag-registry
     and uses the syntax specified in
     http://tools.ietf.org/html/rfc5646#section-3
 """
 def readRegistryRecord(registry):
     """ Yields the records of the IANA Language Subtag Registry as dictionaries. """
     record = {}
     for line in registry:
         line = line.strip()
         if line == "":
             continue
         if line == "%%":
             yield record
             record = {}
         else:
             if ":" in line:
                 key, value = line.split(":", 1)
                 key, value = key.strip(), value.strip()
                 record[key] = value
             else:
                 # continuation line
                 record[key] += " " + line
     if record:
         yield record
     return
 def readRegistry(registry):
     """ Reads IANA Language Subtag Registry and extracts information for Intl.js.
         Information extracted:
         - langTagMappings: mappings from complete language tags to preferred
           complete language tags
         - langSubtagMappings: mappings from subtags to preferred subtags
         - extlangMappings: mappings from extlang subtags to preferred subtags,
           with prefix to be removed
         Returns these three mappings as dictionaries, along with the registry's
         file date.
         We also check that mappings for language subtags don't affect extlang
         subtags and vice versa, so that CanonicalizeLanguageTag doesn't have
         to separate them for processing. Region codes are separated by case,
         and script codes by length, so they're unproblematic.
     """
     langTagMappings = {}
     langSubtagMappings = {}
     extlangMappings = {}
     languageSubtags = set()
     extlangSubtags = set()
     for record in readRegistryRecord(registry):
         if "File-Date" in record:
             fileDate = record["File-Date"]
             continue
         if record["Type"] == "grandfathered":
             # Grandfathered tags don't use standard syntax, so
             # CanonicalizeLanguageTag expects the mapping table to provide
             # the final form for all.
             # For langTagMappings, keys must be in lower case; values in
             # the case used in the registry.
             tag = record["Tag"]
             if "Preferred-Value" in record:
                 langTagMappings[tag.lower()] = record["Preferred-Value"]
             else:
                 langTagMappings[tag.lower()] = tag
         elif record["Type"] == "redundant":
             # For langTagMappings, keys must be in lower case; values in
             # the case used in the registry.
             if "Preferred-Value" in record:
                 langTagMappings[record["Tag"].lower()] = record["Preferred-Value"]
         elif record["Type"] in ("language", "script", "region", "variant"):
             # For langSubtagMappings, keys and values must be in the case used
             # in the registry.
             subtag = record["Subtag"]
             if record["Type"] == "language":
                 languageSubtags.add(subtag)
             if "Preferred-Value" in record:
                 if subtag == "heploc":
                     # The entry for heploc is unique in its complexity; handle
                     # it as special case below.
                     continue
                 if "Prefix" in record:
                     # This might indicate another heploc-like complex case.
                     raise Exception("Please evaluate: subtag mapping with prefix value.")
                 langSubtagMappings[subtag] = record["Preferred-Value"]
         elif record["Type"] == "extlang":
             # For extlangMappings, keys must be in the case used in the
             # registry; values are records with the preferred value and the
             # prefix to be removed.
             subtag = record["Subtag"]
             extlangSubtags.add(subtag)
             if "Preferred-Value" in record:
                 preferred = record["Preferred-Value"]
                 prefix = record["Prefix"]
                 extlangMappings[subtag] = {"preferred": preferred, "prefix": prefix}
         else:
             # No other types are allowed by
             # http://tools.ietf.org/html/rfc5646#section-3.1.3
             assert False, "Unrecognized Type: {0}".format(record["Type"])
     # Check that mappings for language subtags and extlang subtags don't affect
     # each other.
     for lang in languageSubtags:
         if lang in extlangMappings and extlangMappings[lang]["preferred"] != lang:
             raise Exception("Conflict: lang with extlang mapping: " + lang)
     for extlang in extlangSubtags:
         if extlang in langSubtagMappings:
             raise Exception("Conflict: extlang with lang mapping: " + extlang)
     # Special case for heploc.
     langTagMappings["ja-latn-hepburn-heploc"] = "ja-Latn-alalc97"
     return {"fileDate": fileDate,
             "langTagMappings": langTagMappings,
             "langSubtagMappings": langSubtagMappings,
             "extlangMappings": extlangMappings}
 def writeMappingsVar(intlData, dict, name, description, fileDate, url):
     """ Writes a variable definition with a mapping table to file intlData.
         Writes the contents of dictionary dict to file intlData with the given
         variable name and a comment with description, fileDate, and URL.
     """
     intlData.write("\n")
     intlData.write("// {0}.\n".format(description))
     intlData.write("// Derived from IANA Language Subtag Registry, file date {0}.\n".format(fileDate))
     intlData.write("// {0}\n".format(url))
     intlData.write("var {0} = {{\n".format(name))
     keys = sorted(dict)
     for key in keys:
         if isinstance(dict[key], basestring):
             value = '"{0}"'.format(dict[key])
         else:
             preferred = dict[key]["preferred"]
             prefix = dict[key]["prefix"]
             value = '{{preferred: "{0}", prefix: "{1}"}}'.format(preferred, prefix)
         intlData.write('    "{0}": {1},\n'.format(key, value))
     intlData.write("};\n")
 def writeLanguageTagData(intlData, fileDate, url, langTagMappings, langSubtagMappings, extlangMappings):
     """ Writes the language tag data to the Intl data file. """
     writeMappingsVar(intlData, langTagMappings, "langTagMappings",
                      "Mappings from complete tags to preferred values", fileDate, url)
     writeMappingsVar(intlData, langSubtagMappings, "langSubtagMappings",
                      "Mappings from non-extlang subtags to preferred values", fileDate, url)
     writeMappingsVar(intlData, extlangMappings, "extlangMappings",
                      "Mappings from extlang subtags to preferred values", fileDate, url)
 if __name__ == '__main__':
     import codecs
     import sys
     import urllib2
     url = "http://www.iana.org/assignments/language-subtag-registry"
     if len(sys.argv) > 1:
         print("Always make sure you have the newest language-subtag-registry.txt!")
         registry = codecs.open(sys.argv[1], "r", encoding="utf-8")
     else:
         print("Downloading IANA Language Subtag Registry...")
         reader = urllib2.urlopen(url)
         text = reader.read().decode("utf-8")
         reader.close()
         registry = codecs.open("language-subtag-registry.txt", "w+", encoding="utf-8")
         registry.write(text)
         registry.seek(0)
     print("Processing IANA Language Subtag Registry...")
     data = readRegistry(registry)
     fileDate = data["fileDate"]
     langTagMappings = data["langTagMappings"]
     langSubtagMappings = data["langSubtagMappings"]
     extlangMappings = data["extlangMappings"]
     registry.close()
     print("Writing Intl data...")
     intlData = codecs.open("IntlData.js", "w", encoding="utf-8")
     intlData.write("// Generated by make_intl_data.py. DO NOT EDIT.\n")
     writeLanguageTagData(intlData, fileDate, url, langTagMappings, langSubtagMappings, extlangMappings)
     intlData.close()

The Tor Browser / annotate

js/src/builtin/make_intl_data.py@129ffea94266 (annotated)

js/src/builtin/make_intl_data.py