The Tor Browser: comparison addon-sdk/source/python-lib/cuddlefish/manifest.py

--1:000000000000
+:a05db170e549
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os, sys, re, hashlib
+import simplejson as json
+SEP = os.path.sep
+from cuddlefish.util import filter_filenames, filter_dirnames
+# Load new layout mapping hashtable
+path = os.path.join(os.environ.get('CUDDLEFISH_ROOT'), "mapping.json")
+data = open(path, 'r').read()
+NEW_LAYOUT_MAPPING = json.loads(data)
+def js_zipname(packagename, modulename):
+return "%s-lib/%s.js" % (packagename, modulename)
+def docs_zipname(packagename, modulename):
+return "%s-docs/%s.md" % (packagename, modulename)
+def datamap_zipname(packagename):
+return "%s-data.json" % packagename
+def datafile_zipname(packagename, datapath):
+return "%s-data/%s" % (packagename, datapath)
+def to_json(o):
+return json.dumps(o, indent=1).encode("utf-8")+"\n"
+class ModuleNotFoundError(Exception):
+def __init__(self, requirement_type, requirement_name,
+used_by, line_number, looked_in):
+Exception.__init__(self)
+self.requirement_type = requirement_type # "require" or "define"
+self.requirement_name = requirement_name # string, what they require()d
+self.used_by = used_by # string, full path to module which did require()
+self.line_number = line_number # int, 1-indexed line number of first require()
+self.looked_in = looked_in # list of full paths to potential .js files
+def __str__(self):
+what = "%s(%s)" % (self.requirement_type, self.requirement_name)
+where = self.used_by
+if self.line_number is not None:
+where = "%s:%d" % (self.used_by, self.line_number)
+searched = "Looked for it in:\n  %s\n" % "\n  ".join(self.looked_in)
+return ("ModuleNotFoundError: unable to satisfy: %s from\n"
+"  %s:\n" % (what, where)) + searched
+class BadModuleIdentifier(Exception):
+pass
+class BadSection(Exception):
+pass
+class UnreachablePrefixError(Exception):
+pass
+class ManifestEntry:
+def __init__(self):
+self.docs_filename = None
+self.docs_hash = None
+self.requirements = {}
+self.datamap = None
+def get_path(self):
+name = self.moduleName
+if name.endswith(".js"):
+name = name[:-3]
+items = []
+# Only add package name for addons, so that system module paths match
+# the path from the commonjs root directory and also match the loader
+# mappings.
+if self.packageName != "addon-sdk":
+items.append(self.packageName)
+# And for the same reason, do not append `lib/`.
+if self.sectionName == "tests":
+items.append(self.sectionName)
+items.append(name)
+return "/".join(items)
+def get_entry_for_manifest(self):
+entry = { "packageName": self.packageName,
+"sectionName": self.sectionName,
+"moduleName": self.moduleName,
+"jsSHA256": self.js_hash,
+"docsSHA256": self.docs_hash,
+"requirements": {},
+}
+for req in self.requirements:
+if isinstance(self.requirements[req], ManifestEntry):
+them = self.requirements[req] # this is another ManifestEntry
+entry["requirements"][req] = them.get_path()
+else:
+# something magic. The manifest entry indicates that they're
+# allowed to require() it
+entry["requirements"][req] = self.requirements[req]
+assert isinstance(entry["requirements"][req], unicode) or \
+isinstance(entry["requirements"][req], str)
+return entry
+def add_js(self, js_filename):
+self.js_filename = js_filename
+self.js_hash = hash_file(js_filename)
+def add_docs(self, docs_filename):
+self.docs_filename = docs_filename
+self.docs_hash = hash_file(docs_filename)
+def add_requirement(self, reqname, reqdata):
+self.requirements[reqname] = reqdata
+def add_data(self, datamap):
+self.datamap = datamap
+def get_js_zipname(self):
+return js_zipname(self.packagename, self.modulename)
+def get_docs_zipname(self):
+if self.docs_hash:
+return docs_zipname(self.packagename, self.modulename)
+return None
+# self.js_filename
+# self.docs_filename
+def hash_file(fn):
+return hashlib.sha256(open(fn,"rb").read()).hexdigest()
+def get_datafiles(datadir):
+# yields pathnames relative to DATADIR, ignoring some files
+for dirpath, dirnames, filenames in os.walk(datadir):
+filenames = list(filter_filenames(filenames))
+# this tells os.walk to prune the search
+dirnames[:] = filter_dirnames(dirnames)
+for filename in filenames:
+fullname = os.path.join(dirpath, filename)
+assert fullname.startswith(datadir+SEP), "%s%s not in %s" % (datadir, SEP, fullname)
+yield fullname[len(datadir+SEP):]
+class DataMap:
+# one per package
+def __init__(self, pkg):
+self.pkg = pkg
+self.name = pkg.name
+self.files_to_copy = []
+datamap = {}
+datadir = os.path.join(pkg.root_dir, "data")
+for dataname in get_datafiles(datadir):
+absname = os.path.join(datadir, dataname)
+zipname = datafile_zipname(pkg.name, dataname)
+datamap[dataname] = hash_file(absname)
+self.files_to_copy.append( (zipname, absname) )
+self.data_manifest = to_json(datamap)
+self.data_manifest_hash = hashlib.sha256(self.data_manifest).hexdigest()
+self.data_manifest_zipname = datamap_zipname(pkg.name)
+self.data_uri_prefix = "%s/data/" % (self.name)
+class BadChromeMarkerError(Exception):
+pass
+class ModuleInfo:
+def __init__(self, package, section, name, js, docs):
+self.package = package
+self.section = section
+self.name = name
+self.js = js
+self.docs = docs
+def __hash__(self):
+return hash( (self.package.name, self.section, self.name,
+self.js, self.docs) )
+def __eq__(self, them):
+if them.__class__ is not self.__class__:
+return False
+if ((them.package.name, them.section, them.name, them.js, them.docs) !=
+(self.package.name, self.section, self.name, self.js, self.docs) ):
+return False
+return True
+def __repr__(self):
+return "ModuleInfo [%s %s %s] (%s, %s)" % (self.package.name,
+self.section,
+self.name,
+self.js, self.docs)
+class ManifestBuilder:
+def __init__(self, target_cfg, pkg_cfg, deps, extra_modules,
+stderr=sys.stderr):
+self.manifest = {} # maps (package,section,module) to ManifestEntry
+self.target_cfg = target_cfg # the entry point
+self.pkg_cfg = pkg_cfg # all known packages
+self.deps = deps # list of package names to search
+self.used_packagenames = set()
+self.stderr = stderr
+self.extra_modules = extra_modules
+self.modules = {} # maps ModuleInfo to URI in self.manifest
+self.datamaps = {} # maps package name to DataMap instance
+self.files = [] # maps manifest index to (absfn,absfn) js/docs pair
+self.test_modules = [] # for runtime
+def build(self, scan_tests, test_filter_re):
+# process the top module, which recurses to process everything it
+# reaches
+if "main" in self.target_cfg:
+top_mi = self.find_top(self.target_cfg)
+top_me = self.process_module(top_mi)
+self.top_path = top_me.get_path()
+self.datamaps[self.target_cfg.name] = DataMap(self.target_cfg)
+if scan_tests:
+mi = self._find_module_in_package("addon-sdk", "lib", "sdk/test/runner", [])
+self.process_module(mi)
+# also scan all test files in all packages that we use. By making
+# a copy of self.used_packagenames first, we refrain from
+# processing tests in packages that our own tests depend upon. If
+# we're running tests for package A, and either modules in A or
+# tests in A depend upon modules from package B, we *don't* want
+# to run tests for package B.
+test_modules = []
+dirnames = self.target_cfg["tests"]
+if isinstance(dirnames, basestring):
+dirnames = [dirnames]
+dirnames = [os.path.join(self.target_cfg.root_dir, d)
+for d in dirnames]
+for d in dirnames:
+for filename in os.listdir(d):
+if filename.startswith("test-") and filename.endswith(".js"):
+testname = filename[:-3] # require(testname)
+if test_filter_re:
+if not re.search(test_filter_re, testname):
+continue
+tmi = ModuleInfo(self.target_cfg, "tests", testname,
+os.path.join(d, filename), None)
+# scan the test's dependencies
+tme = self.process_module(tmi)
+test_modules.append( (testname, tme) )
+# also add it as an artificial dependency of unit-test-finder, so
+# the runtime dynamic load can work.
+test_finder = self.get_manifest_entry("addon-sdk", "lib",
+"sdk/deprecated/unit-test-finder")
+for (testname,tme) in test_modules:
+test_finder.add_requirement(testname, tme)
+# finally, tell the runtime about it, so they won't have to
+# search for all tests. self.test_modules will be passed
+# through the harness-options.json file in the
+# .allTestModules property.
+# Pass the absolute module path.
+self.test_modules.append(tme.get_path())
+# include files used by the loader
+for em in self.extra_modules:
+(pkgname, section, modname, js) = em
+mi = ModuleInfo(self.pkg_cfg.packages[pkgname], section, modname,
+js, None)
+self.process_module(mi)
+def get_module_entries(self):
+return frozenset(self.manifest.values())
+def get_data_entries(self):
+return frozenset(self.datamaps.values())
+def get_used_packages(self):
+used = set()
+for index in self.manifest:
+(package, section, module) = index
+used.add(package)
+return sorted(used)
+def get_used_files(self, bundle_sdk_modules):
+# returns all .js files that we reference, plus data/ files. You will
+# need to add the loader, off-manifest files that it needs, and
+# generated metadata.
+for datamap in self.datamaps.values():
+for (zipname, absname) in datamap.files_to_copy:
+yield absname
+for me in self.get_module_entries():
+# Only ship SDK files if we are told to do so
+if me.packageName != "addon-sdk" or bundle_sdk_modules:
+yield me.js_filename
+def get_all_test_modules(self):
+return self.test_modules
+def get_harness_options_manifest(self, bundle_sdk_modules):
+manifest = {}
+for me in self.get_module_entries():
+path = me.get_path()
+# Do not add manifest entries for system modules.
+# Doesn't prevent from shipping modules.
+# Shipping modules is decided in `get_used_files`.
+if me.packageName != "addon-sdk" or bundle_sdk_modules:
+manifest[path] = me.get_entry_for_manifest()
+return manifest
+def get_manifest_entry(self, package, section, module):
+index = (package, section, module)
+if index not in self.manifest:
+m = self.manifest[index] = ManifestEntry()
+m.packageName = package
+m.sectionName = section
+m.moduleName = module
+self.used_packagenames.add(package)
+return self.manifest[index]
+def uri_name_from_path(self, pkg, fn):
+# given a filename like .../pkg1/lib/bar/foo.js, and a package
+# specification (with a .root_dir like ".../pkg1" and a .lib list of
+# paths where .lib[0] is like "lib"), return the appropriate NAME
+# that can be put into a URI like resource://JID-pkg1-lib/NAME . This
+# will throw an exception if the file is outside of the lib/
+# directory, since that means we can't construct a URI that points to
+# it.
+#
+# This should be a lot easier, and shouldn't fail when the file is in
+# the root of the package. Both should become possible when the XPI
+# is rearranged and our URI scheme is simplified.
+fn = os.path.abspath(fn)
+pkglib = pkg.lib[0]
+libdir = os.path.abspath(os.path.join(pkg.root_dir, pkglib))
+# AARGH, section and name! we need to reverse-engineer a
+# ModuleInfo instance that will produce a URI (in the form
+# PREFIX/PKGNAME-SECTION/JS) that will map to the existing file.
+# Until we fix URI generation to get rid of "sections", this is
+# limited to files in the same .directories.lib as the rest of
+# the package uses. So if the package's main files are in lib/,
+# but the main.js is in the package root, there is no URI we can
+# construct that will point to it, and we must fail.
+#
+# This will become much easier (and the failure case removed)
+# when we get rid of sections and change the URIs to look like
+# (PREFIX/PKGNAME/PATH-TO-JS).
+# AARGH 2, allowing .lib to be a list is really getting in the
+# way. That needs to go away eventually too.
+if not fn.startswith(libdir):
+raise UnreachablePrefixError("Sorry, but the 'main' file (%s) in package %s is outside that package's 'lib' directory (%s), so I cannot construct a URI to reach it."
+% (fn, pkg.name, pkglib))
+name = fn[len(libdir):].lstrip(SEP)[:-len(".js")]
+return name
+def parse_main(self, root_dir, main, check_lib_dir=None):
+# 'main' can be like one of the following:
+#   a: ./lib/main.js  b: ./lib/main  c: lib/main
+# we require it to be a path to the file, though, and ignore the
+# .directories stuff. So just "main" is insufficient if you really
+# want something in a "lib/" subdirectory.
+if main.endswith(".js"):
+main = main[:-len(".js")]
+if main.startswith("./"):
+main = main[len("./"):]
+# package.json must always use "/", but on windows we'll replace that
+# with "\" before using it as an actual filename
+main = os.sep.join(main.split("/"))
+paths = [os.path.join(root_dir, main+".js")]
+if check_lib_dir is not None:
+paths.append(os.path.join(root_dir, check_lib_dir, main+".js"))
+return paths
+def find_top_js(self, target_cfg):
+for libdir in target_cfg.lib:
+for n in self.parse_main(target_cfg.root_dir, target_cfg.main,
+libdir):
+if os.path.exists(n):
+return n
+raise KeyError("unable to find main module '%s.js' in top-level package" % target_cfg.main)
+def find_top(self, target_cfg):
+top_js = self.find_top_js(target_cfg)
+n = os.path.join(target_cfg.root_dir, "README.md")
+if os.path.exists(n):
+top_docs = n
+else:
+top_docs = None
+name = self.uri_name_from_path(target_cfg, top_js)
+return ModuleInfo(target_cfg, "lib", name, top_js, top_docs)
+def process_module(self, mi):
+pkg = mi.package
+#print "ENTERING", pkg.name, mi.name
+# mi.name must be fully-qualified
+assert (not mi.name.startswith("./") and
+not mi.name.startswith("../"))
+# create and claim the manifest row first
+me = self.get_manifest_entry(pkg.name, mi.section, mi.name)
+me.add_js(mi.js)
+if mi.docs:
+me.add_docs(mi.docs)
+js_lines = open(mi.js,"r").readlines()
+requires, problems, locations = scan_module(mi.js,js_lines,self.stderr)
+if problems:
+# the relevant instructions have already been written to stderr
+raise BadChromeMarkerError()
+# We update our requirements on the way out of the depth-first
+# traversal of the module graph
+for reqname in sorted(requires.keys()):
+# If requirement is chrome or a pseudo-module (starts with @) make
+# path a requirement name.
+if reqname == "chrome" or reqname.startswith("@"):
+me.add_requirement(reqname, reqname)
+else:
+# when two modules require() the same name, do they get a
+# shared instance? This is a deep question. For now say yes.
+# find_req_for() returns an entry to put in our
+# 'requirements' dict, and will recursively process
+# everything transitively required from here. It will also
+# populate the self.modules[] cache. Note that we must
+# tolerate cycles in the reference graph.
+looked_in = [] # populated by subroutines
+them_me = self.find_req_for(mi, reqname, looked_in, locations)
+if them_me is None:
+if mi.section == "tests":
+# tolerate missing modules in tests, because
+# test-securable-module.js, and the modules/red.js
+# that it imports, both do that intentionally
+continue
+lineno = locations.get(reqname) # None means define()
+if lineno is None:
+reqtype = "define"
+else:
+reqtype = "require"
+err = ModuleNotFoundError(reqtype, reqname,
+mi.js, lineno, looked_in)
+raise err
+else:
+me.add_requirement(reqname, them_me)
+return me
+#print "LEAVING", pkg.name, mi.name
+def find_req_for(self, from_module, reqname, looked_in, locations):
+# handle a single require(reqname) statement from from_module .
+# Return a uri that exists in self.manifest
+# Populate looked_in with places we looked.
+def BAD(msg):
+return BadModuleIdentifier(msg + " in require(%s) from %s" %
+(reqname, from_module))
+if not reqname:
+raise BAD("no actual modulename")
+# Allow things in tests/*.js to require both test code and real code.
+# But things in lib/*.js can only require real code.
+if from_module.section == "tests":
+lookfor_sections = ["tests", "lib"]
+elif from_module.section == "lib":
+lookfor_sections = ["lib"]
+else:
+raise BadSection(from_module.section)
+modulename = from_module.name
+#print " %s require(%s))" % (from_module, reqname)
+if reqname.startswith("./") or reqname.startswith("../"):
+# 1: they want something relative to themselves, always from
+# their own package
+them = modulename.split("/")[:-1]
+bits = reqname.split("/")
+while bits[0] in (".", ".."):
+if not bits:
+raise BAD("no actual modulename")
+if bits[0] == "..":
+if not them:
+raise BAD("too many ..")
+them.pop()
+bits.pop(0)
+bits = them+bits
+lookfor_pkg = from_module.package.name
+lookfor_mod = "/".join(bits)
+return self._get_module_from_package(lookfor_pkg,
+lookfor_sections, lookfor_mod,
+looked_in)
+# non-relative import. Might be a short name (requiring a search
+# through "library" packages), or a fully-qualified one.
+if "/" in reqname:
+# 2: PKG/MOD: find PKG, look inside for MOD
+bits = reqname.split("/")
+lookfor_pkg = bits[0]
+lookfor_mod = "/".join(bits[1:])
+mi = self._get_module_from_package(lookfor_pkg,
+lookfor_sections, lookfor_mod,
+looked_in)
+if mi: # caution, 0==None
+return mi
+else:
+# 3: try finding PKG, if found, use its main.js entry point
+lookfor_pkg = reqname
+mi = self._get_entrypoint_from_package(lookfor_pkg, looked_in)
+if mi:
+return mi
+# 4: search packages for MOD or MODPARENT/MODCHILD. We always search
+# their own package first, then the list of packages defined by their
+# .dependencies list
+from_pkg = from_module.package.name
+mi = self._search_packages_for_module(from_pkg,
+lookfor_sections, reqname,
+looked_in)
+if mi:
+return mi
+# Only after we look for module in the addon itself, search for a module
+# in new layout.
+# First normalize require argument in order to easily find a mapping
+normalized = reqname
+if normalized.endswith(".js"):
+normalized = normalized[:-len(".js")]
+if normalized.startswith("addon-kit/"):
+normalized = normalized[len("addon-kit/"):]
+if normalized.startswith("api-utils/"):
+normalized = normalized[len("api-utils/"):]
+if normalized in NEW_LAYOUT_MAPPING:
+# get the new absolute path for this module
+original_reqname = reqname
+reqname = NEW_LAYOUT_MAPPING[normalized]
+from_pkg = from_module.package.name
+# If the addon didn't explicitely told us to ignore deprecated
+# require path, warn the developer:
+# (target_cfg is the package.json file)
+if not "ignore-deprecated-path" in self.target_cfg:
+lineno = locations.get(original_reqname)
+print >>self.stderr, "Warning: Use of deprecated require path:"
+print >>self.stderr, "  In %s:%d:" % (from_module.js, lineno)
+print >>self.stderr, "    require('%s')." % original_reqname
+print >>self.stderr, "  New path should be:"
+print >>self.stderr, "    require('%s')" % reqname
+return self._search_packages_for_module(from_pkg,
+lookfor_sections, reqname,
+looked_in)
+else:
+# We weren't able to find this module, really.
+return None
+def _handle_module(self, mi):
+if not mi:
+return None
+# we tolerate cycles in the reference graph, which means we need to
+# populate the self.modules cache before recursing into
+# process_module() . We must also check the cache first, so recursion
+# can terminate.
+if mi in self.modules:
+return self.modules[mi]
+# this creates the entry
+new_entry = self.get_manifest_entry(mi.package.name, mi.section, mi.name)
+# and populates the cache
+self.modules[mi] = new_entry
+self.process_module(mi)
+return new_entry
+def _get_module_from_package(self, pkgname, sections, modname, looked_in):
+if pkgname not in self.pkg_cfg.packages:
+return None
+mi = self._find_module_in_package(pkgname, sections, modname,
+looked_in)
+return self._handle_module(mi)
+def _get_entrypoint_from_package(self, pkgname, looked_in):
+if pkgname not in self.pkg_cfg.packages:
+return None
+pkg = self.pkg_cfg.packages[pkgname]
+main = pkg.get("main", None)
+if not main:
+return None
+for js in self.parse_main(pkg.root_dir, main):
+looked_in.append(js)
+if os.path.exists(js):
+section = "lib"
+name = self.uri_name_from_path(pkg, js)
+docs = None
+mi = ModuleInfo(pkg, section, name, js, docs)
+return self._handle_module(mi)
+return None
+def _search_packages_for_module(self, from_pkg, sections, reqname,
+looked_in):
+searchpath = [] # list of package names
+searchpath.append(from_pkg) # search self first
+us = self.pkg_cfg.packages[from_pkg]
+if 'dependencies' in us:
+# only look in dependencies
+searchpath.extend(us['dependencies'])
+else:
+# they didn't declare any dependencies (or they declared an empty
+# list, but we'll treat that as not declaring one, because it's
+# easier), so look in all deps, sorted alphabetically, so
+# addon-kit comes first. Note that self.deps includes all
+# packages found by traversing the ".dependencies" lists in each
+# package.json, starting from the main addon package, plus
+# everything added by --extra-packages
+searchpath.extend(sorted(self.deps))
+for pkgname in searchpath:
+mi = self._find_module_in_package(pkgname, sections, reqname,
+looked_in)
+if mi:
+return self._handle_module(mi)
+return None
+def _find_module_in_package(self, pkgname, sections, name, looked_in):
+# require("a/b/c") should look at ...\a\b\c.js on windows
+filename = os.sep.join(name.split("/"))
+# normalize filename, make sure that we do not add .js if it already has
+# it.
+if not filename.endswith(".js") and not filename.endswith(".json"):
+filename += ".js"
+if filename.endswith(".js"):
+basename = filename[:-3]
+if filename.endswith(".json"):
+basename = filename[:-5]
+pkg = self.pkg_cfg.packages[pkgname]
+if isinstance(sections, basestring):
+sections = [sections]
+for section in sections:
+for sdir in pkg.get(section, []):
+js = os.path.join(pkg.root_dir, sdir, filename)
+looked_in.append(js)
+if os.path.exists(js):
+docs = None
+maybe_docs = os.path.join(pkg.root_dir, "docs",
+basename+".md")
+if section == "lib" and os.path.exists(maybe_docs):
+docs = maybe_docs
+return ModuleInfo(pkg, section, name, js, docs)
+return None
+def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
+test_filter_re=None, extra_modules=[]):
+"""
+Perform recursive dependency analysis starting from entry_point,
+building up a manifest of modules that need to be included in the XPI.
+Each entry will map require() names to the URL of the module that will
+be used to satisfy that dependency. The manifest will be used by the
+runtime's require() code.
+This returns a ManifestBuilder object, with two public methods. The
+first, get_module_entries(), returns a set of ManifestEntry objects, each
+of which can be asked for the following:
+* its contribution to the harness-options.json '.manifest'
+* the local disk name
+* the name in the XPI at which it should be placed
+The second is get_data_entries(), which returns a set of DataEntry
+objects, each of which has:
+* local disk name
+* name in the XPI
+note: we don't build the XPI here, but our manifest is passed to the
+code which does, so it knows what to copy into the XPI.
+"""
+mxt = ManifestBuilder(target_cfg, pkg_cfg, deps, extra_modules)
+mxt.build(scan_tests, test_filter_re)
+return mxt
+COMMENT_PREFIXES = ["//", "/*", "*", "dump("]
+REQUIRE_RE = r"(?<![\'\"])require\s*\(\s*[\'\"]([^\'\"]+?)[\'\"]\s*\)"
+# detect the define idiom of the form:
+#   define("module name", ["dep1", "dep2", "dep3"], function() {})
+# by capturing the contents of the list in a group.
+DEF_RE = re.compile(r"(require|define)\s*\(\s*([\'\"][^\'\"]+[\'\"]\s*,)?\s*\[([^\]]+)\]")
+# Out of the async dependencies, do not allow quotes in them.
+DEF_RE_ALLOWED = re.compile(r"^[\'\"][^\'\"]+[\'\"]$")
+def scan_requirements_with_grep(fn, lines):
+requires = {}
+first_location = {}
+for (lineno0, line) in enumerate(lines):
+for clause in line.split(";"):
+clause = clause.strip()
+iscomment = False
+for commentprefix in COMMENT_PREFIXES:
+if clause.startswith(commentprefix):
+iscomment = True
+if iscomment:
+continue
+mo = re.finditer(REQUIRE_RE, clause)
+if mo:
+for mod in mo:
+modname = mod.group(1)
+requires[modname] = {}
+if modname not in first_location:
+first_location[modname] = lineno0 + 1
+# define() can happen across multiple lines, so join everyone up.
+wholeshebang = "\n".join(lines)
+for match in DEF_RE.finditer(wholeshebang):
+# this should net us a list of string literals separated by commas
+for strbit in match.group(3).split(","):
+strbit = strbit.strip()
+# There could be a trailing comma netting us just whitespace, so
+# filter that out. Make sure that only string values with
+# quotes around them are allowed, and no quotes are inside
+# the quoted value.
+if strbit and DEF_RE_ALLOWED.match(strbit):
+modname = strbit[1:-1]
+if modname not in ["exports"]:
+requires[modname] = {}
+# joining all the lines means we lose line numbers, so we
+# can't fill first_location[]
+return requires, first_location
+CHROME_ALIASES = [
+(re.compile(r"Components\.classes"), "Cc"),
+(re.compile(r"Components\.interfaces"), "Ci"),
+(re.compile(r"Components\.utils"), "Cu"),
+(re.compile(r"Components\.results"), "Cr"),
+(re.compile(r"Components\.manager"), "Cm"),
+]
+OTHER_CHROME = re.compile(r"Components\.[a-zA-Z]")
+def scan_for_bad_chrome(fn, lines, stderr):
+problems = False
+old_chrome = set() # i.e. "Cc" when we see "Components.classes"
+old_chrome_lines = [] # list of (lineno, line.strip()) tuples
+for lineno,line in enumerate(lines):
+# note: this scanner is not obligated to spot all possible forms of
+# chrome access. The scanner is detecting voluntary requests for
+# chrome. Runtime tools will enforce allowance or denial of access.
+line = line.strip()
+iscomment = False
+for commentprefix in COMMENT_PREFIXES:
+if line.startswith(commentprefix):
+iscomment = True
+break
+if iscomment:
+continue
+old_chrome_in_this_line = set()
+for (regexp,alias) in CHROME_ALIASES:
+if regexp.search(line):
+old_chrome_in_this_line.add(alias)
+if not old_chrome_in_this_line:
+if OTHER_CHROME.search(line):
+old_chrome_in_this_line.add("components")
+old_chrome.update(old_chrome_in_this_line)
+if old_chrome_in_this_line:
+old_chrome_lines.append( (lineno+1, line) )
+if old_chrome:
+print >>stderr, """
+The following lines from file %(fn)s:
+%(lines)s
+use 'Components' to access chrome authority. To do so, you need to add a
+line somewhat like the following:
+const {%(needs)s} = require("chrome");
+Then you can use any shortcuts to its properties that you import from the
+'chrome' module ('Cc', 'Ci', 'Cm', 'Cr', and 'Cu' for the 'classes',
+'interfaces', 'manager', 'results', and 'utils' properties, respectively. And
+`components` for `Components` object itself).
+""" % { "fn": fn, "needs": ",".join(sorted(old_chrome)),
+"lines": "\n".join([" %3d: %s" % (lineno,line)
+for (lineno, line) in old_chrome_lines]),
+}
+problems = True
+return problems
+def scan_module(fn, lines, stderr=sys.stderr):
+filename = os.path.basename(fn)
+requires, locations = scan_requirements_with_grep(fn, lines)
+if filename == "cuddlefish.js":
+# this is the loader: don't scan for chrome
+problems = False
+else:
+problems = scan_for_bad_chrome(fn, lines, stderr)
+return requires, problems, locations
+if __name__ == '__main__':
+for fn in sys.argv[1:]:
+requires, problems, locations = scan_module(fn, open(fn).readlines())
+print
+print "---", fn
+if problems:
+print "PROBLEMS"
+sys.exit(1)
+print "requires: %s" % (",".join(sorted(requires.keys())))
+print "locations: %s" % locations

The Tor Browser / file comparison

comparison: addon-sdk/source/python-lib/cuddlefish/manifest.py

addon-sdk/source/python-lib/cuddlefish/manifest.py