The Tor Browser: addon-sdk/source/python-lib/cuddlefish/manifest.py@ac0c01689b40 (annotated)

addon-sdk/source/python-lib/cuddlefish/manifest.py@ac0c01689b40 (annotated)

addon-sdk/source/python-lib/cuddlefish/manifest.py

Thu, 15 Jan 2015 15:59:08 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 15 Jan 2015 15:59:08 +0100
branch: TOR_BUG_9701
changeset 10: ac0c01689b40
permissions: -rw-r--r--

Implement a real Private Browsing Mode condition by changing the API/ABI;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os, sys, re, hashlib
 import simplejson as json
 SEP = os.path.sep
 from cuddlefish.util import filter_filenames, filter_dirnames
 # Load new layout mapping hashtable
 path = os.path.join(os.environ.get('CUDDLEFISH_ROOT'), "mapping.json")
 data = open(path, 'r').read()
 NEW_LAYOUT_MAPPING = json.loads(data)
 def js_zipname(packagename, modulename):
     return "%s-lib/%s.js" % (packagename, modulename)
 def docs_zipname(packagename, modulename):
     return "%s-docs/%s.md" % (packagename, modulename)
 def datamap_zipname(packagename):
     return "%s-data.json" % packagename
 def datafile_zipname(packagename, datapath):
     return "%s-data/%s" % (packagename, datapath)
 def to_json(o):
     return json.dumps(o, indent=1).encode("utf-8")+"\n"
 class ModuleNotFoundError(Exception):
     def __init__(self, requirement_type, requirement_name,
                  used_by, line_number, looked_in):
         Exception.__init__(self)
         self.requirement_type = requirement_type # "require" or "define"
         self.requirement_name = requirement_name # string, what they require()d
         self.used_by = used_by # string, full path to module which did require()
         self.line_number = line_number # int, 1-indexed line number of first require()
         self.looked_in = looked_in # list of full paths to potential .js files
     def __str__(self):
         what = "%s(%s)" % (self.requirement_type, self.requirement_name)
         where = self.used_by
         if self.line_number is not None:
             where = "%s:%d" % (self.used_by, self.line_number)
         searched = "Looked for it in:\n  %s\n" % "\n  ".join(self.looked_in)
         return ("ModuleNotFoundError: unable to satisfy: %s from\n"
                 "  %s:\n" % (what, where)) + searched
 class BadModuleIdentifier(Exception):
     pass
 class BadSection(Exception):
     pass
 class UnreachablePrefixError(Exception):
     pass
 class ManifestEntry:
     def __init__(self):
         self.docs_filename = None
         self.docs_hash = None
         self.requirements = {}
         self.datamap = None
     def get_path(self):
         name = self.moduleName
         if name.endswith(".js"):
             name = name[:-3]
         items = []
         # Only add package name for addons, so that system module paths match
         # the path from the commonjs root directory and also match the loader
         # mappings.
         if self.packageName != "addon-sdk":
             items.append(self.packageName)
         # And for the same reason, do not append `lib/`.
         if self.sectionName == "tests":
             items.append(self.sectionName)
         items.append(name)
         return "/".join(items)
     def get_entry_for_manifest(self):
         entry = { "packageName": self.packageName,
                   "sectionName": self.sectionName,
                   "moduleName": self.moduleName,
                   "jsSHA256": self.js_hash,
                   "docsSHA256": self.docs_hash,
                   "requirements": {},
                   }
         for req in self.requirements:
             if isinstance(self.requirements[req], ManifestEntry):
                 them = self.requirements[req] # this is another ManifestEntry
                 entry["requirements"][req] = them.get_path()
             else:
                 # something magic. The manifest entry indicates that they're
                 # allowed to require() it
                 entry["requirements"][req] = self.requirements[req]
             assert isinstance(entry["requirements"][req], unicode) or \
                    isinstance(entry["requirements"][req], str)
         return entry
     def add_js(self, js_filename):
         self.js_filename = js_filename
         self.js_hash = hash_file(js_filename)
     def add_docs(self, docs_filename):
         self.docs_filename = docs_filename
         self.docs_hash = hash_file(docs_filename)
     def add_requirement(self, reqname, reqdata):
         self.requirements[reqname] = reqdata
     def add_data(self, datamap):
         self.datamap = datamap
     def get_js_zipname(self):
         return js_zipname(self.packagename, self.modulename)
     def get_docs_zipname(self):
         if self.docs_hash:
             return docs_zipname(self.packagename, self.modulename)
         return None
     # self.js_filename
     # self.docs_filename
 def hash_file(fn):
     return hashlib.sha256(open(fn,"rb").read()).hexdigest()
 def get_datafiles(datadir):
     # yields pathnames relative to DATADIR, ignoring some files
     for dirpath, dirnames, filenames in os.walk(datadir):
         filenames = list(filter_filenames(filenames))
         # this tells os.walk to prune the search
         dirnames[:] = filter_dirnames(dirnames)
         for filename in filenames:
             fullname = os.path.join(dirpath, filename)
             assert fullname.startswith(datadir+SEP), "%s%s not in %s" % (datadir, SEP, fullname)
             yield fullname[len(datadir+SEP):]
 class DataMap:
     # one per package
     def __init__(self, pkg):
         self.pkg = pkg
         self.name = pkg.name
         self.files_to_copy = []
         datamap = {}
         datadir = os.path.join(pkg.root_dir, "data")
         for dataname in get_datafiles(datadir):
             absname = os.path.join(datadir, dataname)
             zipname = datafile_zipname(pkg.name, dataname)
             datamap[dataname] = hash_file(absname)
             self.files_to_copy.append( (zipname, absname) )
         self.data_manifest = to_json(datamap)
         self.data_manifest_hash = hashlib.sha256(self.data_manifest).hexdigest()
         self.data_manifest_zipname = datamap_zipname(pkg.name)
         self.data_uri_prefix = "%s/data/" % (self.name)
 class BadChromeMarkerError(Exception):
     pass
 class ModuleInfo:
     def __init__(self, package, section, name, js, docs):
         self.package = package
         self.section = section
         self.name = name
         self.js = js
         self.docs = docs
     def __hash__(self):
         return hash( (self.package.name, self.section, self.name,
                       self.js, self.docs) )
     def __eq__(self, them):
         if them.__class__ is not self.__class__:
             return False
         if ((them.package.name, them.section, them.name, them.js, them.docs) !=
             (self.package.name, self.section, self.name, self.js, self.docs) ):
             return False
         return True
     def __repr__(self):
         return "ModuleInfo [%s %s %s] (%s, %s)" % (self.package.name,
                                                    self.section,
                                                    self.name,
                                                    self.js, self.docs)
 class ManifestBuilder:
     def __init__(self, target_cfg, pkg_cfg, deps, extra_modules,
                  stderr=sys.stderr):
         self.manifest = {} # maps (package,section,module) to ManifestEntry
         self.target_cfg = target_cfg # the entry point
         self.pkg_cfg = pkg_cfg # all known packages
         self.deps = deps # list of package names to search
         self.used_packagenames = set()
         self.stderr = stderr
         self.extra_modules = extra_modules
         self.modules = {} # maps ModuleInfo to URI in self.manifest
         self.datamaps = {} # maps package name to DataMap instance
         self.files = [] # maps manifest index to (absfn,absfn) js/docs pair
         self.test_modules = [] # for runtime
     def build(self, scan_tests, test_filter_re):
         # process the top module, which recurses to process everything it
         # reaches
         if "main" in self.target_cfg:
             top_mi = self.find_top(self.target_cfg)
             top_me = self.process_module(top_mi)
             self.top_path = top_me.get_path()
             self.datamaps[self.target_cfg.name] = DataMap(self.target_cfg)
         if scan_tests:
             mi = self._find_module_in_package("addon-sdk", "lib", "sdk/test/runner", [])
             self.process_module(mi)
             # also scan all test files in all packages that we use. By making
             # a copy of self.used_packagenames first, we refrain from
             # processing tests in packages that our own tests depend upon. If
             # we're running tests for package A, and either modules in A or
             # tests in A depend upon modules from package B, we *don't* want
             # to run tests for package B.
             test_modules = []
             dirnames = self.target_cfg["tests"]
             if isinstance(dirnames, basestring):
                 dirnames = [dirnames]
             dirnames = [os.path.join(self.target_cfg.root_dir, d)
                         for d in dirnames]
             for d in dirnames:
                 for filename in os.listdir(d):
                     if filename.startswith("test-") and filename.endswith(".js"):
                         testname = filename[:-3] # require(testname)
                         if test_filter_re:
                             if not re.search(test_filter_re, testname):
                                 continue
                         tmi = ModuleInfo(self.target_cfg, "tests", testname,
                                          os.path.join(d, filename), None)
                         # scan the test's dependencies
                         tme = self.process_module(tmi)
                         test_modules.append( (testname, tme) )
             # also add it as an artificial dependency of unit-test-finder, so
             # the runtime dynamic load can work.
             test_finder = self.get_manifest_entry("addon-sdk", "lib",
                                                   "sdk/deprecated/unit-test-finder")
             for (testname,tme) in test_modules:
                 test_finder.add_requirement(testname, tme)
                 # finally, tell the runtime about it, so they won't have to
                 # search for all tests. self.test_modules will be passed
                 # through the harness-options.json file in the
                 # .allTestModules property.
                 # Pass the absolute module path.
                 self.test_modules.append(tme.get_path())
         # include files used by the loader
         for em in self.extra_modules:
             (pkgname, section, modname, js) = em
             mi = ModuleInfo(self.pkg_cfg.packages[pkgname], section, modname,
                             js, None)
             self.process_module(mi)
     def get_module_entries(self):
         return frozenset(self.manifest.values())
     def get_data_entries(self):
         return frozenset(self.datamaps.values())
     def get_used_packages(self):
         used = set()
         for index in self.manifest:
             (package, section, module) = index
             used.add(package)
         return sorted(used)
     def get_used_files(self, bundle_sdk_modules):
         # returns all .js files that we reference, plus data/ files. You will
         # need to add the loader, off-manifest files that it needs, and
         # generated metadata.
         for datamap in self.datamaps.values():
             for (zipname, absname) in datamap.files_to_copy:
                 yield absname
         for me in self.get_module_entries():
             # Only ship SDK files if we are told to do so
             if me.packageName != "addon-sdk" or bundle_sdk_modules:
                 yield me.js_filename
     def get_all_test_modules(self):
         return self.test_modules
     def get_harness_options_manifest(self, bundle_sdk_modules):
         manifest = {}
         for me in self.get_module_entries():
             path = me.get_path()
             # Do not add manifest entries for system modules.
             # Doesn't prevent from shipping modules.
             # Shipping modules is decided in `get_used_files`.
             if me.packageName != "addon-sdk" or bundle_sdk_modules:
                 manifest[path] = me.get_entry_for_manifest()
         return manifest
     def get_manifest_entry(self, package, section, module):
         index = (package, section, module)
         if index not in self.manifest:
             m = self.manifest[index] = ManifestEntry()
             m.packageName = package
             m.sectionName = section
             m.moduleName = module
             self.used_packagenames.add(package)
         return self.manifest[index]
     def uri_name_from_path(self, pkg, fn):
         # given a filename like .../pkg1/lib/bar/foo.js, and a package
         # specification (with a .root_dir like ".../pkg1" and a .lib list of
         # paths where .lib[0] is like "lib"), return the appropriate NAME
         # that can be put into a URI like resource://JID-pkg1-lib/NAME . This
         # will throw an exception if the file is outside of the lib/
         # directory, since that means we can't construct a URI that points to
         # it.
         #
         # This should be a lot easier, and shouldn't fail when the file is in
         # the root of the package. Both should become possible when the XPI
         # is rearranged and our URI scheme is simplified.
         fn = os.path.abspath(fn)
         pkglib = pkg.lib[0]
         libdir = os.path.abspath(os.path.join(pkg.root_dir, pkglib))
         # AARGH, section and name! we need to reverse-engineer a
         # ModuleInfo instance that will produce a URI (in the form
         # PREFIX/PKGNAME-SECTION/JS) that will map to the existing file.
         # Until we fix URI generation to get rid of "sections", this is
         # limited to files in the same .directories.lib as the rest of
         # the package uses. So if the package's main files are in lib/,
         # but the main.js is in the package root, there is no URI we can
         # construct that will point to it, and we must fail.
         #
         # This will become much easier (and the failure case removed)
         # when we get rid of sections and change the URIs to look like
         # (PREFIX/PKGNAME/PATH-TO-JS).
         # AARGH 2, allowing .lib to be a list is really getting in the
         # way. That needs to go away eventually too.
         if not fn.startswith(libdir):
             raise UnreachablePrefixError("Sorry, but the 'main' file (%s) in package %s is outside that package's 'lib' directory (%s), so I cannot construct a URI to reach it."
                                          % (fn, pkg.name, pkglib))
         name = fn[len(libdir):].lstrip(SEP)[:-len(".js")]
         return name
     def parse_main(self, root_dir, main, check_lib_dir=None):
         # 'main' can be like one of the following:
         #   a: ./lib/main.js  b: ./lib/main  c: lib/main
         # we require it to be a path to the file, though, and ignore the
         # .directories stuff. So just "main" is insufficient if you really
         # want something in a "lib/" subdirectory.
         if main.endswith(".js"):
             main = main[:-len(".js")]
         if main.startswith("./"):
             main = main[len("./"):]
         # package.json must always use "/", but on windows we'll replace that
         # with "\" before using it as an actual filename
         main = os.sep.join(main.split("/"))
         paths = [os.path.join(root_dir, main+".js")]
         if check_lib_dir is not None:
             paths.append(os.path.join(root_dir, check_lib_dir, main+".js"))
         return paths
     def find_top_js(self, target_cfg):
         for libdir in target_cfg.lib:
             for n in self.parse_main(target_cfg.root_dir, target_cfg.main,
                                      libdir):
                 if os.path.exists(n):
                     return n
         raise KeyError("unable to find main module '%s.js' in top-level package" % target_cfg.main)
     def find_top(self, target_cfg):
         top_js = self.find_top_js(target_cfg)
         n = os.path.join(target_cfg.root_dir, "README.md")
         if os.path.exists(n):
             top_docs = n
         else:
             top_docs = None
         name = self.uri_name_from_path(target_cfg, top_js)
         return ModuleInfo(target_cfg, "lib", name, top_js, top_docs)
     def process_module(self, mi):
         pkg = mi.package
         #print "ENTERING", pkg.name, mi.name
         # mi.name must be fully-qualified
         assert (not mi.name.startswith("./") and
                 not mi.name.startswith("../"))
         # create and claim the manifest row first
         me = self.get_manifest_entry(pkg.name, mi.section, mi.name)
         me.add_js(mi.js)
         if mi.docs:
             me.add_docs(mi.docs)
         js_lines = open(mi.js,"r").readlines()
         requires, problems, locations = scan_module(mi.js,js_lines,self.stderr)
         if problems:
             # the relevant instructions have already been written to stderr
             raise BadChromeMarkerError()
         # We update our requirements on the way out of the depth-first
         # traversal of the module graph
         for reqname in sorted(requires.keys()):
             # If requirement is chrome or a pseudo-module (starts with @) make
             # path a requirement name.
             if reqname == "chrome" or reqname.startswith("@"):
                 me.add_requirement(reqname, reqname)
             else:
                 # when two modules require() the same name, do they get a
                 # shared instance? This is a deep question. For now say yes.
                 # find_req_for() returns an entry to put in our
                 # 'requirements' dict, and will recursively process
                 # everything transitively required from here. It will also
                 # populate the self.modules[] cache. Note that we must
                 # tolerate cycles in the reference graph.
                 looked_in = [] # populated by subroutines
                 them_me = self.find_req_for(mi, reqname, looked_in, locations)
                 if them_me is None:
                     if mi.section == "tests":
                         # tolerate missing modules in tests, because
                         # test-securable-module.js, and the modules/red.js
                         # that it imports, both do that intentionally
                         continue
                     lineno = locations.get(reqname) # None means define()
                     if lineno is None:
                         reqtype = "define"
                     else:
                         reqtype = "require"
                     err = ModuleNotFoundError(reqtype, reqname,
                                               mi.js, lineno, looked_in)
                     raise err
                 else:
                     me.add_requirement(reqname, them_me)
         return me
         #print "LEAVING", pkg.name, mi.name
     def find_req_for(self, from_module, reqname, looked_in, locations):
         # handle a single require(reqname) statement from from_module .
         # Return a uri that exists in self.manifest
         # Populate looked_in with places we looked.
         def BAD(msg):
             return BadModuleIdentifier(msg + " in require(%s) from %s" %
                                        (reqname, from_module))
         if not reqname:
             raise BAD("no actual modulename")
         # Allow things in tests/*.js to require both test code and real code.
         # But things in lib/*.js can only require real code.
         if from_module.section == "tests":
             lookfor_sections = ["tests", "lib"]
         elif from_module.section == "lib":
             lookfor_sections = ["lib"]
         else:
             raise BadSection(from_module.section)
         modulename = from_module.name
         #print " %s require(%s))" % (from_module, reqname)
         if reqname.startswith("./") or reqname.startswith("../"):
             # 1: they want something relative to themselves, always from
             # their own package
             them = modulename.split("/")[:-1]
             bits = reqname.split("/")
             while bits[0] in (".", ".."):
                 if not bits:
                     raise BAD("no actual modulename")
                 if bits[0] == "..":
                     if not them:
                         raise BAD("too many ..")
                     them.pop()
                 bits.pop(0)
             bits = them+bits
             lookfor_pkg = from_module.package.name
             lookfor_mod = "/".join(bits)
             return self._get_module_from_package(lookfor_pkg,
                                                  lookfor_sections, lookfor_mod,
                                                  looked_in)
         # non-relative import. Might be a short name (requiring a search
         # through "library" packages), or a fully-qualified one.
         if "/" in reqname:
             # 2: PKG/MOD: find PKG, look inside for MOD
             bits = reqname.split("/")
             lookfor_pkg = bits[0]
             lookfor_mod = "/".join(bits[1:])
             mi = self._get_module_from_package(lookfor_pkg,
                                                lookfor_sections, lookfor_mod,
                                                looked_in)
             if mi: # caution, 0==None
                 return mi
         else:
             # 3: try finding PKG, if found, use its main.js entry point
             lookfor_pkg = reqname
             mi = self._get_entrypoint_from_package(lookfor_pkg, looked_in)
             if mi:
                 return mi
         # 4: search packages for MOD or MODPARENT/MODCHILD. We always search
         # their own package first, then the list of packages defined by their
         # .dependencies list
         from_pkg = from_module.package.name
         mi = self._search_packages_for_module(from_pkg,
                                               lookfor_sections, reqname,
                                               looked_in)
         if mi:
             return mi
         # Only after we look for module in the addon itself, search for a module
         # in new layout.
         # First normalize require argument in order to easily find a mapping
         normalized = reqname
         if normalized.endswith(".js"):
             normalized = normalized[:-len(".js")]
         if normalized.startswith("addon-kit/"):
             normalized = normalized[len("addon-kit/"):]
         if normalized.startswith("api-utils/"):
             normalized = normalized[len("api-utils/"):]
         if normalized in NEW_LAYOUT_MAPPING:
             # get the new absolute path for this module
             original_reqname = reqname
             reqname = NEW_LAYOUT_MAPPING[normalized]
             from_pkg = from_module.package.name
             # If the addon didn't explicitely told us to ignore deprecated
             # require path, warn the developer:
             # (target_cfg is the package.json file)
             if not "ignore-deprecated-path" in self.target_cfg:
                 lineno = locations.get(original_reqname)
                 print >>self.stderr, "Warning: Use of deprecated require path:"
                 print >>self.stderr, "  In %s:%d:" % (from_module.js, lineno)
                 print >>self.stderr, "    require('%s')." % original_reqname
                 print >>self.stderr, "  New path should be:"
                 print >>self.stderr, "    require('%s')" % reqname
             return self._search_packages_for_module(from_pkg,
                                                     lookfor_sections, reqname,
                                                     looked_in)
         else:
             # We weren't able to find this module, really.
             return None
     def _handle_module(self, mi):
         if not mi:
             return None
         # we tolerate cycles in the reference graph, which means we need to
         # populate the self.modules cache before recursing into
         # process_module() . We must also check the cache first, so recursion
         # can terminate.
         if mi in self.modules:
             return self.modules[mi]
         # this creates the entry
         new_entry = self.get_manifest_entry(mi.package.name, mi.section, mi.name)
         # and populates the cache
         self.modules[mi] = new_entry
         self.process_module(mi)
         return new_entry
     def _get_module_from_package(self, pkgname, sections, modname, looked_in):
         if pkgname not in self.pkg_cfg.packages:
             return None
         mi = self._find_module_in_package(pkgname, sections, modname,
                                           looked_in)
         return self._handle_module(mi)
     def _get_entrypoint_from_package(self, pkgname, looked_in):
         if pkgname not in self.pkg_cfg.packages:
             return None
         pkg = self.pkg_cfg.packages[pkgname]
         main = pkg.get("main", None)
         if not main:
             return None
         for js in self.parse_main(pkg.root_dir, main):
             looked_in.append(js)
             if os.path.exists(js):
                 section = "lib"
                 name = self.uri_name_from_path(pkg, js)
                 docs = None
                 mi = ModuleInfo(pkg, section, name, js, docs)
                 return self._handle_module(mi)
         return None
     def _search_packages_for_module(self, from_pkg, sections, reqname,
                                     looked_in):
         searchpath = [] # list of package names
         searchpath.append(from_pkg) # search self first
         us = self.pkg_cfg.packages[from_pkg]
         if 'dependencies' in us:
             # only look in dependencies
             searchpath.extend(us['dependencies'])
         else:
             # they didn't declare any dependencies (or they declared an empty
             # list, but we'll treat that as not declaring one, because it's
             # easier), so look in all deps, sorted alphabetically, so
             # addon-kit comes first. Note that self.deps includes all
             # packages found by traversing the ".dependencies" lists in each
             # package.json, starting from the main addon package, plus
             # everything added by --extra-packages
             searchpath.extend(sorted(self.deps))
         for pkgname in searchpath:
             mi = self._find_module_in_package(pkgname, sections, reqname,
                                               looked_in)
             if mi:
                 return self._handle_module(mi)
         return None
     def _find_module_in_package(self, pkgname, sections, name, looked_in):
         # require("a/b/c") should look at ...\a\b\c.js on windows
         filename = os.sep.join(name.split("/"))
         # normalize filename, make sure that we do not add .js if it already has
         # it.
         if not filename.endswith(".js") and not filename.endswith(".json"):
           filename += ".js"
         if filename.endswith(".js"):
           basename = filename[:-3]
         if filename.endswith(".json"):
           basename = filename[:-5]
         pkg = self.pkg_cfg.packages[pkgname]
         if isinstance(sections, basestring):
             sections = [sections]
         for section in sections:
             for sdir in pkg.get(section, []):
                 js = os.path.join(pkg.root_dir, sdir, filename)
                 looked_in.append(js)
                 if os.path.exists(js):
                     docs = None
                     maybe_docs = os.path.join(pkg.root_dir, "docs",
                                               basename+".md")
                     if section == "lib" and os.path.exists(maybe_docs):
                         docs = maybe_docs
                     return ModuleInfo(pkg, section, name, js, docs)
         return None
 def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
                    test_filter_re=None, extra_modules=[]):
     """
     Perform recursive dependency analysis starting from entry_point,
     building up a manifest of modules that need to be included in the XPI.
     Each entry will map require() names to the URL of the module that will
     be used to satisfy that dependency. The manifest will be used by the
     runtime's require() code.
     This returns a ManifestBuilder object, with two public methods. The
     first, get_module_entries(), returns a set of ManifestEntry objects, each
     of which can be asked for the following:
      * its contribution to the harness-options.json '.manifest'
      * the local disk name
      * the name in the XPI at which it should be placed
     The second is get_data_entries(), which returns a set of DataEntry
     objects, each of which has:
      * local disk name
      * name in the XPI
     note: we don't build the XPI here, but our manifest is passed to the
     code which does, so it knows what to copy into the XPI.
     """
     mxt = ManifestBuilder(target_cfg, pkg_cfg, deps, extra_modules)
     mxt.build(scan_tests, test_filter_re)
     return mxt
 COMMENT_PREFIXES = ["//", "/*", "*", "dump("]
 REQUIRE_RE = r"(?<![\'\"])require\s*\(\s*[\'\"]([^\'\"]+?)[\'\"]\s*\)"
 # detect the define idiom of the form:
 #   define("module name", ["dep1", "dep2", "dep3"], function() {})
 # by capturing the contents of the list in a group.
 DEF_RE = re.compile(r"(require|define)\s*\(\s*([\'\"][^\'\"]+[\'\"]\s*,)?\s*\[([^\]]+)\]")
 # Out of the async dependencies, do not allow quotes in them.
 DEF_RE_ALLOWED = re.compile(r"^[\'\"][^\'\"]+[\'\"]$")
 def scan_requirements_with_grep(fn, lines):
     requires = {}
     first_location = {}
     for (lineno0, line) in enumerate(lines):
         for clause in line.split(";"):
             clause = clause.strip()
             iscomment = False
             for commentprefix in COMMENT_PREFIXES:
                 if clause.startswith(commentprefix):
                     iscomment = True
             if iscomment:
                 continue
             mo = re.finditer(REQUIRE_RE, clause)
             if mo:
                 for mod in mo:
                     modname = mod.group(1)
                     requires[modname] = {}
                     if modname not in first_location:
                         first_location[modname] = lineno0 + 1
     # define() can happen across multiple lines, so join everyone up.
     wholeshebang = "\n".join(lines)
     for match in DEF_RE.finditer(wholeshebang):
         # this should net us a list of string literals separated by commas
         for strbit in match.group(3).split(","):
             strbit = strbit.strip()
             # There could be a trailing comma netting us just whitespace, so
             # filter that out. Make sure that only string values with
             # quotes around them are allowed, and no quotes are inside
             # the quoted value.
             if strbit and DEF_RE_ALLOWED.match(strbit):
                 modname = strbit[1:-1]
                 if modname not in ["exports"]:
                     requires[modname] = {}
                     # joining all the lines means we lose line numbers, so we
                     # can't fill first_location[]
     return requires, first_location
 CHROME_ALIASES = [
     (re.compile(r"Components\.classes"), "Cc"),
     (re.compile(r"Components\.interfaces"), "Ci"),
     (re.compile(r"Components\.utils"), "Cu"),
     (re.compile(r"Components\.results"), "Cr"),
     (re.compile(r"Components\.manager"), "Cm"),
     ]
 OTHER_CHROME = re.compile(r"Components\.[a-zA-Z]")
 def scan_for_bad_chrome(fn, lines, stderr):
     problems = False
     old_chrome = set() # i.e. "Cc" when we see "Components.classes"
     old_chrome_lines = [] # list of (lineno, line.strip()) tuples
     for lineno,line in enumerate(lines):
         # note: this scanner is not obligated to spot all possible forms of
         # chrome access. The scanner is detecting voluntary requests for
         # chrome. Runtime tools will enforce allowance or denial of access.
         line = line.strip()
         iscomment = False
         for commentprefix in COMMENT_PREFIXES:
             if line.startswith(commentprefix):
                 iscomment = True
                 break
         if iscomment:
             continue
         old_chrome_in_this_line = set()
         for (regexp,alias) in CHROME_ALIASES:
             if regexp.search(line):
                 old_chrome_in_this_line.add(alias)
         if not old_chrome_in_this_line:
             if OTHER_CHROME.search(line):
                 old_chrome_in_this_line.add("components")
         old_chrome.update(old_chrome_in_this_line)
         if old_chrome_in_this_line:
             old_chrome_lines.append( (lineno+1, line) )
     if old_chrome:
         print >>stderr, """
 The following lines from file %(fn)s:
 %(lines)s
 use 'Components' to access chrome authority. To do so, you need to add a
 line somewhat like the following:
   const {%(needs)s} = require("chrome");
 Then you can use any shortcuts to its properties that you import from the
 'chrome' module ('Cc', 'Ci', 'Cm', 'Cr', and 'Cu' for the 'classes',
 'interfaces', 'manager', 'results', and 'utils' properties, respectively. And
 `components` for `Components` object itself).
 """ % { "fn": fn, "needs": ",".join(sorted(old_chrome)),
         "lines": "\n".join([" %3d: %s" % (lineno,line)
                             for (lineno, line) in old_chrome_lines]),
         }
         problems = True
     return problems
 def scan_module(fn, lines, stderr=sys.stderr):
     filename = os.path.basename(fn)
     requires, locations = scan_requirements_with_grep(fn, lines)
     if filename == "cuddlefish.js":
         # this is the loader: don't scan for chrome
         problems = False
     else:
         problems = scan_for_bad_chrome(fn, lines, stderr)
     return requires, problems, locations
 if __name__ == '__main__':
     for fn in sys.argv[1:]:
         requires, problems, locations = scan_module(fn, open(fn).readlines())
         print
         print "---", fn
         if problems:
             print "PROBLEMS"
             sys.exit(1)
         print "requires: %s" % (",".join(sorted(requires.keys())))
         print "locations: %s" % locations

The Tor Browser / annotate

addon-sdk/source/python-lib/cuddlefish/manifest.py@ac0c01689b40 (annotated)

addon-sdk/source/python-lib/cuddlefish/manifest.py