1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/addon-sdk/source/python-lib/cuddlefish/manifest.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,795 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +
1.9 +import os, sys, re, hashlib
1.10 +import simplejson as json
1.11 +SEP = os.path.sep
1.12 +from cuddlefish.util import filter_filenames, filter_dirnames
1.13 +
1.14 +# Load new layout mapping hashtable
1.15 +path = os.path.join(os.environ.get('CUDDLEFISH_ROOT'), "mapping.json")
1.16 +data = open(path, 'r').read()
1.17 +NEW_LAYOUT_MAPPING = json.loads(data)
1.18 +
1.19 +def js_zipname(packagename, modulename):
1.20 + return "%s-lib/%s.js" % (packagename, modulename)
1.21 +def docs_zipname(packagename, modulename):
1.22 + return "%s-docs/%s.md" % (packagename, modulename)
1.23 +def datamap_zipname(packagename):
1.24 + return "%s-data.json" % packagename
1.25 +def datafile_zipname(packagename, datapath):
1.26 + return "%s-data/%s" % (packagename, datapath)
1.27 +
1.28 +def to_json(o):
1.29 + return json.dumps(o, indent=1).encode("utf-8")+"\n"
1.30 +
1.31 +class ModuleNotFoundError(Exception):
1.32 + def __init__(self, requirement_type, requirement_name,
1.33 + used_by, line_number, looked_in):
1.34 + Exception.__init__(self)
1.35 + self.requirement_type = requirement_type # "require" or "define"
1.36 + self.requirement_name = requirement_name # string, what they require()d
1.37 + self.used_by = used_by # string, full path to module which did require()
1.38 + self.line_number = line_number # int, 1-indexed line number of first require()
1.39 + self.looked_in = looked_in # list of full paths to potential .js files
1.40 + def __str__(self):
1.41 + what = "%s(%s)" % (self.requirement_type, self.requirement_name)
1.42 + where = self.used_by
1.43 + if self.line_number is not None:
1.44 + where = "%s:%d" % (self.used_by, self.line_number)
1.45 + searched = "Looked for it in:\n %s\n" % "\n ".join(self.looked_in)
1.46 + return ("ModuleNotFoundError: unable to satisfy: %s from\n"
1.47 + " %s:\n" % (what, where)) + searched
1.48 +
1.49 +class BadModuleIdentifier(Exception):
1.50 + pass
1.51 +class BadSection(Exception):
1.52 + pass
1.53 +class UnreachablePrefixError(Exception):
1.54 + pass
1.55 +
1.56 +class ManifestEntry:
1.57 + def __init__(self):
1.58 + self.docs_filename = None
1.59 + self.docs_hash = None
1.60 + self.requirements = {}
1.61 + self.datamap = None
1.62 +
1.63 + def get_path(self):
1.64 + name = self.moduleName
1.65 +
1.66 + if name.endswith(".js"):
1.67 + name = name[:-3]
1.68 + items = []
1.69 + # Only add package name for addons, so that system module paths match
1.70 + # the path from the commonjs root directory and also match the loader
1.71 + # mappings.
1.72 + if self.packageName != "addon-sdk":
1.73 + items.append(self.packageName)
1.74 + # And for the same reason, do not append `lib/`.
1.75 + if self.sectionName == "tests":
1.76 + items.append(self.sectionName)
1.77 + items.append(name)
1.78 +
1.79 + return "/".join(items)
1.80 +
1.81 + def get_entry_for_manifest(self):
1.82 + entry = { "packageName": self.packageName,
1.83 + "sectionName": self.sectionName,
1.84 + "moduleName": self.moduleName,
1.85 + "jsSHA256": self.js_hash,
1.86 + "docsSHA256": self.docs_hash,
1.87 + "requirements": {},
1.88 + }
1.89 + for req in self.requirements:
1.90 + if isinstance(self.requirements[req], ManifestEntry):
1.91 + them = self.requirements[req] # this is another ManifestEntry
1.92 + entry["requirements"][req] = them.get_path()
1.93 + else:
1.94 + # something magic. The manifest entry indicates that they're
1.95 + # allowed to require() it
1.96 + entry["requirements"][req] = self.requirements[req]
1.97 + assert isinstance(entry["requirements"][req], unicode) or \
1.98 + isinstance(entry["requirements"][req], str)
1.99 + return entry
1.100 +
1.101 + def add_js(self, js_filename):
1.102 + self.js_filename = js_filename
1.103 + self.js_hash = hash_file(js_filename)
1.104 + def add_docs(self, docs_filename):
1.105 + self.docs_filename = docs_filename
1.106 + self.docs_hash = hash_file(docs_filename)
1.107 + def add_requirement(self, reqname, reqdata):
1.108 + self.requirements[reqname] = reqdata
1.109 + def add_data(self, datamap):
1.110 + self.datamap = datamap
1.111 +
1.112 + def get_js_zipname(self):
1.113 + return js_zipname(self.packagename, self.modulename)
1.114 + def get_docs_zipname(self):
1.115 + if self.docs_hash:
1.116 + return docs_zipname(self.packagename, self.modulename)
1.117 + return None
1.118 + # self.js_filename
1.119 + # self.docs_filename
1.120 +
1.121 +
1.122 +def hash_file(fn):
1.123 + return hashlib.sha256(open(fn,"rb").read()).hexdigest()
1.124 +
1.125 +def get_datafiles(datadir):
1.126 + # yields pathnames relative to DATADIR, ignoring some files
1.127 + for dirpath, dirnames, filenames in os.walk(datadir):
1.128 + filenames = list(filter_filenames(filenames))
1.129 + # this tells os.walk to prune the search
1.130 + dirnames[:] = filter_dirnames(dirnames)
1.131 + for filename in filenames:
1.132 + fullname = os.path.join(dirpath, filename)
1.133 + assert fullname.startswith(datadir+SEP), "%s%s not in %s" % (datadir, SEP, fullname)
1.134 + yield fullname[len(datadir+SEP):]
1.135 +
1.136 +
1.137 +class DataMap:
1.138 + # one per package
1.139 + def __init__(self, pkg):
1.140 + self.pkg = pkg
1.141 + self.name = pkg.name
1.142 + self.files_to_copy = []
1.143 + datamap = {}
1.144 + datadir = os.path.join(pkg.root_dir, "data")
1.145 + for dataname in get_datafiles(datadir):
1.146 + absname = os.path.join(datadir, dataname)
1.147 + zipname = datafile_zipname(pkg.name, dataname)
1.148 + datamap[dataname] = hash_file(absname)
1.149 + self.files_to_copy.append( (zipname, absname) )
1.150 + self.data_manifest = to_json(datamap)
1.151 + self.data_manifest_hash = hashlib.sha256(self.data_manifest).hexdigest()
1.152 + self.data_manifest_zipname = datamap_zipname(pkg.name)
1.153 + self.data_uri_prefix = "%s/data/" % (self.name)
1.154 +
1.155 +class BadChromeMarkerError(Exception):
1.156 + pass
1.157 +
1.158 +class ModuleInfo:
1.159 + def __init__(self, package, section, name, js, docs):
1.160 + self.package = package
1.161 + self.section = section
1.162 + self.name = name
1.163 + self.js = js
1.164 + self.docs = docs
1.165 +
1.166 + def __hash__(self):
1.167 + return hash( (self.package.name, self.section, self.name,
1.168 + self.js, self.docs) )
1.169 + def __eq__(self, them):
1.170 + if them.__class__ is not self.__class__:
1.171 + return False
1.172 + if ((them.package.name, them.section, them.name, them.js, them.docs) !=
1.173 + (self.package.name, self.section, self.name, self.js, self.docs) ):
1.174 + return False
1.175 + return True
1.176 +
1.177 + def __repr__(self):
1.178 + return "ModuleInfo [%s %s %s] (%s, %s)" % (self.package.name,
1.179 + self.section,
1.180 + self.name,
1.181 + self.js, self.docs)
1.182 +
1.183 +class ManifestBuilder:
1.184 + def __init__(self, target_cfg, pkg_cfg, deps, extra_modules,
1.185 + stderr=sys.stderr):
1.186 + self.manifest = {} # maps (package,section,module) to ManifestEntry
1.187 + self.target_cfg = target_cfg # the entry point
1.188 + self.pkg_cfg = pkg_cfg # all known packages
1.189 + self.deps = deps # list of package names to search
1.190 + self.used_packagenames = set()
1.191 + self.stderr = stderr
1.192 + self.extra_modules = extra_modules
1.193 + self.modules = {} # maps ModuleInfo to URI in self.manifest
1.194 + self.datamaps = {} # maps package name to DataMap instance
1.195 + self.files = [] # maps manifest index to (absfn,absfn) js/docs pair
1.196 + self.test_modules = [] # for runtime
1.197 +
1.198 + def build(self, scan_tests, test_filter_re):
1.199 + # process the top module, which recurses to process everything it
1.200 + # reaches
1.201 + if "main" in self.target_cfg:
1.202 + top_mi = self.find_top(self.target_cfg)
1.203 + top_me = self.process_module(top_mi)
1.204 + self.top_path = top_me.get_path()
1.205 + self.datamaps[self.target_cfg.name] = DataMap(self.target_cfg)
1.206 + if scan_tests:
1.207 + mi = self._find_module_in_package("addon-sdk", "lib", "sdk/test/runner", [])
1.208 + self.process_module(mi)
1.209 + # also scan all test files in all packages that we use. By making
1.210 + # a copy of self.used_packagenames first, we refrain from
1.211 + # processing tests in packages that our own tests depend upon. If
1.212 + # we're running tests for package A, and either modules in A or
1.213 + # tests in A depend upon modules from package B, we *don't* want
1.214 + # to run tests for package B.
1.215 + test_modules = []
1.216 + dirnames = self.target_cfg["tests"]
1.217 + if isinstance(dirnames, basestring):
1.218 + dirnames = [dirnames]
1.219 + dirnames = [os.path.join(self.target_cfg.root_dir, d)
1.220 + for d in dirnames]
1.221 + for d in dirnames:
1.222 + for filename in os.listdir(d):
1.223 + if filename.startswith("test-") and filename.endswith(".js"):
1.224 + testname = filename[:-3] # require(testname)
1.225 + if test_filter_re:
1.226 + if not re.search(test_filter_re, testname):
1.227 + continue
1.228 + tmi = ModuleInfo(self.target_cfg, "tests", testname,
1.229 + os.path.join(d, filename), None)
1.230 + # scan the test's dependencies
1.231 + tme = self.process_module(tmi)
1.232 + test_modules.append( (testname, tme) )
1.233 + # also add it as an artificial dependency of unit-test-finder, so
1.234 + # the runtime dynamic load can work.
1.235 + test_finder = self.get_manifest_entry("addon-sdk", "lib",
1.236 + "sdk/deprecated/unit-test-finder")
1.237 + for (testname,tme) in test_modules:
1.238 + test_finder.add_requirement(testname, tme)
1.239 + # finally, tell the runtime about it, so they won't have to
1.240 + # search for all tests. self.test_modules will be passed
1.241 + # through the harness-options.json file in the
1.242 + # .allTestModules property.
1.243 + # Pass the absolute module path.
1.244 + self.test_modules.append(tme.get_path())
1.245 +
1.246 + # include files used by the loader
1.247 + for em in self.extra_modules:
1.248 + (pkgname, section, modname, js) = em
1.249 + mi = ModuleInfo(self.pkg_cfg.packages[pkgname], section, modname,
1.250 + js, None)
1.251 + self.process_module(mi)
1.252 +
1.253 +
1.254 + def get_module_entries(self):
1.255 + return frozenset(self.manifest.values())
1.256 + def get_data_entries(self):
1.257 + return frozenset(self.datamaps.values())
1.258 +
1.259 + def get_used_packages(self):
1.260 + used = set()
1.261 + for index in self.manifest:
1.262 + (package, section, module) = index
1.263 + used.add(package)
1.264 + return sorted(used)
1.265 +
1.266 + def get_used_files(self, bundle_sdk_modules):
1.267 + # returns all .js files that we reference, plus data/ files. You will
1.268 + # need to add the loader, off-manifest files that it needs, and
1.269 + # generated metadata.
1.270 + for datamap in self.datamaps.values():
1.271 + for (zipname, absname) in datamap.files_to_copy:
1.272 + yield absname
1.273 +
1.274 + for me in self.get_module_entries():
1.275 + # Only ship SDK files if we are told to do so
1.276 + if me.packageName != "addon-sdk" or bundle_sdk_modules:
1.277 + yield me.js_filename
1.278 +
1.279 + def get_all_test_modules(self):
1.280 + return self.test_modules
1.281 +
1.282 + def get_harness_options_manifest(self, bundle_sdk_modules):
1.283 + manifest = {}
1.284 + for me in self.get_module_entries():
1.285 + path = me.get_path()
1.286 + # Do not add manifest entries for system modules.
1.287 + # Doesn't prevent from shipping modules.
1.288 + # Shipping modules is decided in `get_used_files`.
1.289 + if me.packageName != "addon-sdk" or bundle_sdk_modules:
1.290 + manifest[path] = me.get_entry_for_manifest()
1.291 + return manifest
1.292 +
1.293 + def get_manifest_entry(self, package, section, module):
1.294 + index = (package, section, module)
1.295 + if index not in self.manifest:
1.296 + m = self.manifest[index] = ManifestEntry()
1.297 + m.packageName = package
1.298 + m.sectionName = section
1.299 + m.moduleName = module
1.300 + self.used_packagenames.add(package)
1.301 + return self.manifest[index]
1.302 +
1.303 + def uri_name_from_path(self, pkg, fn):
1.304 + # given a filename like .../pkg1/lib/bar/foo.js, and a package
1.305 + # specification (with a .root_dir like ".../pkg1" and a .lib list of
1.306 + # paths where .lib[0] is like "lib"), return the appropriate NAME
1.307 + # that can be put into a URI like resource://JID-pkg1-lib/NAME . This
1.308 + # will throw an exception if the file is outside of the lib/
1.309 + # directory, since that means we can't construct a URI that points to
1.310 + # it.
1.311 + #
1.312 + # This should be a lot easier, and shouldn't fail when the file is in
1.313 + # the root of the package. Both should become possible when the XPI
1.314 + # is rearranged and our URI scheme is simplified.
1.315 + fn = os.path.abspath(fn)
1.316 + pkglib = pkg.lib[0]
1.317 + libdir = os.path.abspath(os.path.join(pkg.root_dir, pkglib))
1.318 + # AARGH, section and name! we need to reverse-engineer a
1.319 + # ModuleInfo instance that will produce a URI (in the form
1.320 + # PREFIX/PKGNAME-SECTION/JS) that will map to the existing file.
1.321 + # Until we fix URI generation to get rid of "sections", this is
1.322 + # limited to files in the same .directories.lib as the rest of
1.323 + # the package uses. So if the package's main files are in lib/,
1.324 + # but the main.js is in the package root, there is no URI we can
1.325 + # construct that will point to it, and we must fail.
1.326 + #
1.327 + # This will become much easier (and the failure case removed)
1.328 + # when we get rid of sections and change the URIs to look like
1.329 + # (PREFIX/PKGNAME/PATH-TO-JS).
1.330 +
1.331 + # AARGH 2, allowing .lib to be a list is really getting in the
1.332 + # way. That needs to go away eventually too.
1.333 + if not fn.startswith(libdir):
1.334 + raise UnreachablePrefixError("Sorry, but the 'main' file (%s) in package %s is outside that package's 'lib' directory (%s), so I cannot construct a URI to reach it."
1.335 + % (fn, pkg.name, pkglib))
1.336 + name = fn[len(libdir):].lstrip(SEP)[:-len(".js")]
1.337 + return name
1.338 +
1.339 +
1.340 + def parse_main(self, root_dir, main, check_lib_dir=None):
1.341 + # 'main' can be like one of the following:
1.342 + # a: ./lib/main.js b: ./lib/main c: lib/main
1.343 + # we require it to be a path to the file, though, and ignore the
1.344 + # .directories stuff. So just "main" is insufficient if you really
1.345 + # want something in a "lib/" subdirectory.
1.346 + if main.endswith(".js"):
1.347 + main = main[:-len(".js")]
1.348 + if main.startswith("./"):
1.349 + main = main[len("./"):]
1.350 + # package.json must always use "/", but on windows we'll replace that
1.351 + # with "\" before using it as an actual filename
1.352 + main = os.sep.join(main.split("/"))
1.353 + paths = [os.path.join(root_dir, main+".js")]
1.354 + if check_lib_dir is not None:
1.355 + paths.append(os.path.join(root_dir, check_lib_dir, main+".js"))
1.356 + return paths
1.357 +
1.358 + def find_top_js(self, target_cfg):
1.359 + for libdir in target_cfg.lib:
1.360 + for n in self.parse_main(target_cfg.root_dir, target_cfg.main,
1.361 + libdir):
1.362 + if os.path.exists(n):
1.363 + return n
1.364 + raise KeyError("unable to find main module '%s.js' in top-level package" % target_cfg.main)
1.365 +
1.366 + def find_top(self, target_cfg):
1.367 + top_js = self.find_top_js(target_cfg)
1.368 + n = os.path.join(target_cfg.root_dir, "README.md")
1.369 + if os.path.exists(n):
1.370 + top_docs = n
1.371 + else:
1.372 + top_docs = None
1.373 + name = self.uri_name_from_path(target_cfg, top_js)
1.374 + return ModuleInfo(target_cfg, "lib", name, top_js, top_docs)
1.375 +
1.376 + def process_module(self, mi):
1.377 + pkg = mi.package
1.378 + #print "ENTERING", pkg.name, mi.name
1.379 + # mi.name must be fully-qualified
1.380 + assert (not mi.name.startswith("./") and
1.381 + not mi.name.startswith("../"))
1.382 + # create and claim the manifest row first
1.383 + me = self.get_manifest_entry(pkg.name, mi.section, mi.name)
1.384 +
1.385 + me.add_js(mi.js)
1.386 + if mi.docs:
1.387 + me.add_docs(mi.docs)
1.388 +
1.389 + js_lines = open(mi.js,"r").readlines()
1.390 + requires, problems, locations = scan_module(mi.js,js_lines,self.stderr)
1.391 + if problems:
1.392 + # the relevant instructions have already been written to stderr
1.393 + raise BadChromeMarkerError()
1.394 +
1.395 + # We update our requirements on the way out of the depth-first
1.396 + # traversal of the module graph
1.397 +
1.398 + for reqname in sorted(requires.keys()):
1.399 + # If requirement is chrome or a pseudo-module (starts with @) make
1.400 + # path a requirement name.
1.401 + if reqname == "chrome" or reqname.startswith("@"):
1.402 + me.add_requirement(reqname, reqname)
1.403 + else:
1.404 + # when two modules require() the same name, do they get a
1.405 + # shared instance? This is a deep question. For now say yes.
1.406 +
1.407 + # find_req_for() returns an entry to put in our
1.408 + # 'requirements' dict, and will recursively process
1.409 + # everything transitively required from here. It will also
1.410 + # populate the self.modules[] cache. Note that we must
1.411 + # tolerate cycles in the reference graph.
1.412 + looked_in = [] # populated by subroutines
1.413 + them_me = self.find_req_for(mi, reqname, looked_in, locations)
1.414 + if them_me is None:
1.415 + if mi.section == "tests":
1.416 + # tolerate missing modules in tests, because
1.417 + # test-securable-module.js, and the modules/red.js
1.418 + # that it imports, both do that intentionally
1.419 + continue
1.420 + lineno = locations.get(reqname) # None means define()
1.421 + if lineno is None:
1.422 + reqtype = "define"
1.423 + else:
1.424 + reqtype = "require"
1.425 + err = ModuleNotFoundError(reqtype, reqname,
1.426 + mi.js, lineno, looked_in)
1.427 + raise err
1.428 + else:
1.429 + me.add_requirement(reqname, them_me)
1.430 +
1.431 + return me
1.432 + #print "LEAVING", pkg.name, mi.name
1.433 +
1.434 + def find_req_for(self, from_module, reqname, looked_in, locations):
1.435 + # handle a single require(reqname) statement from from_module .
1.436 + # Return a uri that exists in self.manifest
1.437 + # Populate looked_in with places we looked.
1.438 + def BAD(msg):
1.439 + return BadModuleIdentifier(msg + " in require(%s) from %s" %
1.440 + (reqname, from_module))
1.441 +
1.442 + if not reqname:
1.443 + raise BAD("no actual modulename")
1.444 +
1.445 + # Allow things in tests/*.js to require both test code and real code.
1.446 + # But things in lib/*.js can only require real code.
1.447 + if from_module.section == "tests":
1.448 + lookfor_sections = ["tests", "lib"]
1.449 + elif from_module.section == "lib":
1.450 + lookfor_sections = ["lib"]
1.451 + else:
1.452 + raise BadSection(from_module.section)
1.453 + modulename = from_module.name
1.454 +
1.455 + #print " %s require(%s))" % (from_module, reqname)
1.456 +
1.457 + if reqname.startswith("./") or reqname.startswith("../"):
1.458 + # 1: they want something relative to themselves, always from
1.459 + # their own package
1.460 + them = modulename.split("/")[:-1]
1.461 + bits = reqname.split("/")
1.462 + while bits[0] in (".", ".."):
1.463 + if not bits:
1.464 + raise BAD("no actual modulename")
1.465 + if bits[0] == "..":
1.466 + if not them:
1.467 + raise BAD("too many ..")
1.468 + them.pop()
1.469 + bits.pop(0)
1.470 + bits = them+bits
1.471 + lookfor_pkg = from_module.package.name
1.472 + lookfor_mod = "/".join(bits)
1.473 + return self._get_module_from_package(lookfor_pkg,
1.474 + lookfor_sections, lookfor_mod,
1.475 + looked_in)
1.476 +
1.477 + # non-relative import. Might be a short name (requiring a search
1.478 + # through "library" packages), or a fully-qualified one.
1.479 +
1.480 + if "/" in reqname:
1.481 + # 2: PKG/MOD: find PKG, look inside for MOD
1.482 + bits = reqname.split("/")
1.483 + lookfor_pkg = bits[0]
1.484 + lookfor_mod = "/".join(bits[1:])
1.485 + mi = self._get_module_from_package(lookfor_pkg,
1.486 + lookfor_sections, lookfor_mod,
1.487 + looked_in)
1.488 + if mi: # caution, 0==None
1.489 + return mi
1.490 + else:
1.491 + # 3: try finding PKG, if found, use its main.js entry point
1.492 + lookfor_pkg = reqname
1.493 + mi = self._get_entrypoint_from_package(lookfor_pkg, looked_in)
1.494 + if mi:
1.495 + return mi
1.496 +
1.497 + # 4: search packages for MOD or MODPARENT/MODCHILD. We always search
1.498 + # their own package first, then the list of packages defined by their
1.499 + # .dependencies list
1.500 + from_pkg = from_module.package.name
1.501 + mi = self._search_packages_for_module(from_pkg,
1.502 + lookfor_sections, reqname,
1.503 + looked_in)
1.504 + if mi:
1.505 + return mi
1.506 +
1.507 + # Only after we look for module in the addon itself, search for a module
1.508 + # in new layout.
1.509 + # First normalize require argument in order to easily find a mapping
1.510 + normalized = reqname
1.511 + if normalized.endswith(".js"):
1.512 + normalized = normalized[:-len(".js")]
1.513 + if normalized.startswith("addon-kit/"):
1.514 + normalized = normalized[len("addon-kit/"):]
1.515 + if normalized.startswith("api-utils/"):
1.516 + normalized = normalized[len("api-utils/"):]
1.517 + if normalized in NEW_LAYOUT_MAPPING:
1.518 + # get the new absolute path for this module
1.519 + original_reqname = reqname
1.520 + reqname = NEW_LAYOUT_MAPPING[normalized]
1.521 + from_pkg = from_module.package.name
1.522 +
1.523 + # If the addon didn't explicitely told us to ignore deprecated
1.524 + # require path, warn the developer:
1.525 + # (target_cfg is the package.json file)
1.526 + if not "ignore-deprecated-path" in self.target_cfg:
1.527 + lineno = locations.get(original_reqname)
1.528 + print >>self.stderr, "Warning: Use of deprecated require path:"
1.529 + print >>self.stderr, " In %s:%d:" % (from_module.js, lineno)
1.530 + print >>self.stderr, " require('%s')." % original_reqname
1.531 + print >>self.stderr, " New path should be:"
1.532 + print >>self.stderr, " require('%s')" % reqname
1.533 +
1.534 + return self._search_packages_for_module(from_pkg,
1.535 + lookfor_sections, reqname,
1.536 + looked_in)
1.537 + else:
1.538 + # We weren't able to find this module, really.
1.539 + return None
1.540 +
1.541 + def _handle_module(self, mi):
1.542 + if not mi:
1.543 + return None
1.544 +
1.545 + # we tolerate cycles in the reference graph, which means we need to
1.546 + # populate the self.modules cache before recursing into
1.547 + # process_module() . We must also check the cache first, so recursion
1.548 + # can terminate.
1.549 + if mi in self.modules:
1.550 + return self.modules[mi]
1.551 +
1.552 + # this creates the entry
1.553 + new_entry = self.get_manifest_entry(mi.package.name, mi.section, mi.name)
1.554 + # and populates the cache
1.555 + self.modules[mi] = new_entry
1.556 + self.process_module(mi)
1.557 + return new_entry
1.558 +
1.559 + def _get_module_from_package(self, pkgname, sections, modname, looked_in):
1.560 + if pkgname not in self.pkg_cfg.packages:
1.561 + return None
1.562 + mi = self._find_module_in_package(pkgname, sections, modname,
1.563 + looked_in)
1.564 + return self._handle_module(mi)
1.565 +
1.566 + def _get_entrypoint_from_package(self, pkgname, looked_in):
1.567 + if pkgname not in self.pkg_cfg.packages:
1.568 + return None
1.569 + pkg = self.pkg_cfg.packages[pkgname]
1.570 + main = pkg.get("main", None)
1.571 + if not main:
1.572 + return None
1.573 + for js in self.parse_main(pkg.root_dir, main):
1.574 + looked_in.append(js)
1.575 + if os.path.exists(js):
1.576 + section = "lib"
1.577 + name = self.uri_name_from_path(pkg, js)
1.578 + docs = None
1.579 + mi = ModuleInfo(pkg, section, name, js, docs)
1.580 + return self._handle_module(mi)
1.581 + return None
1.582 +
1.583 + def _search_packages_for_module(self, from_pkg, sections, reqname,
1.584 + looked_in):
1.585 + searchpath = [] # list of package names
1.586 + searchpath.append(from_pkg) # search self first
1.587 + us = self.pkg_cfg.packages[from_pkg]
1.588 + if 'dependencies' in us:
1.589 + # only look in dependencies
1.590 + searchpath.extend(us['dependencies'])
1.591 + else:
1.592 + # they didn't declare any dependencies (or they declared an empty
1.593 + # list, but we'll treat that as not declaring one, because it's
1.594 + # easier), so look in all deps, sorted alphabetically, so
1.595 + # addon-kit comes first. Note that self.deps includes all
1.596 + # packages found by traversing the ".dependencies" lists in each
1.597 + # package.json, starting from the main addon package, plus
1.598 + # everything added by --extra-packages
1.599 + searchpath.extend(sorted(self.deps))
1.600 + for pkgname in searchpath:
1.601 + mi = self._find_module_in_package(pkgname, sections, reqname,
1.602 + looked_in)
1.603 + if mi:
1.604 + return self._handle_module(mi)
1.605 + return None
1.606 +
1.607 + def _find_module_in_package(self, pkgname, sections, name, looked_in):
1.608 + # require("a/b/c") should look at ...\a\b\c.js on windows
1.609 + filename = os.sep.join(name.split("/"))
1.610 + # normalize filename, make sure that we do not add .js if it already has
1.611 + # it.
1.612 + if not filename.endswith(".js") and not filename.endswith(".json"):
1.613 + filename += ".js"
1.614 +
1.615 + if filename.endswith(".js"):
1.616 + basename = filename[:-3]
1.617 + if filename.endswith(".json"):
1.618 + basename = filename[:-5]
1.619 +
1.620 + pkg = self.pkg_cfg.packages[pkgname]
1.621 + if isinstance(sections, basestring):
1.622 + sections = [sections]
1.623 + for section in sections:
1.624 + for sdir in pkg.get(section, []):
1.625 + js = os.path.join(pkg.root_dir, sdir, filename)
1.626 + looked_in.append(js)
1.627 + if os.path.exists(js):
1.628 + docs = None
1.629 + maybe_docs = os.path.join(pkg.root_dir, "docs",
1.630 + basename+".md")
1.631 + if section == "lib" and os.path.exists(maybe_docs):
1.632 + docs = maybe_docs
1.633 + return ModuleInfo(pkg, section, name, js, docs)
1.634 + return None
1.635 +
1.636 +def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
1.637 + test_filter_re=None, extra_modules=[]):
1.638 + """
1.639 + Perform recursive dependency analysis starting from entry_point,
1.640 + building up a manifest of modules that need to be included in the XPI.
1.641 + Each entry will map require() names to the URL of the module that will
1.642 + be used to satisfy that dependency. The manifest will be used by the
1.643 + runtime's require() code.
1.644 +
1.645 + This returns a ManifestBuilder object, with two public methods. The
1.646 + first, get_module_entries(), returns a set of ManifestEntry objects, each
1.647 + of which can be asked for the following:
1.648 +
1.649 + * its contribution to the harness-options.json '.manifest'
1.650 + * the local disk name
1.651 + * the name in the XPI at which it should be placed
1.652 +
1.653 + The second is get_data_entries(), which returns a set of DataEntry
1.654 + objects, each of which has:
1.655 +
1.656 + * local disk name
1.657 + * name in the XPI
1.658 +
1.659 + note: we don't build the XPI here, but our manifest is passed to the
1.660 + code which does, so it knows what to copy into the XPI.
1.661 + """
1.662 +
1.663 + mxt = ManifestBuilder(target_cfg, pkg_cfg, deps, extra_modules)
1.664 + mxt.build(scan_tests, test_filter_re)
1.665 + return mxt
1.666 +
1.667 +
1.668 +
1.669 +COMMENT_PREFIXES = ["//", "/*", "*", "dump("]
1.670 +
1.671 +REQUIRE_RE = r"(?<![\'\"])require\s*\(\s*[\'\"]([^\'\"]+?)[\'\"]\s*\)"
1.672 +
1.673 +# detect the define idiom of the form:
1.674 +# define("module name", ["dep1", "dep2", "dep3"], function() {})
1.675 +# by capturing the contents of the list in a group.
1.676 +DEF_RE = re.compile(r"(require|define)\s*\(\s*([\'\"][^\'\"]+[\'\"]\s*,)?\s*\[([^\]]+)\]")
1.677 +
1.678 +# Out of the async dependencies, do not allow quotes in them.
1.679 +DEF_RE_ALLOWED = re.compile(r"^[\'\"][^\'\"]+[\'\"]$")
1.680 +
1.681 +def scan_requirements_with_grep(fn, lines):
1.682 + requires = {}
1.683 + first_location = {}
1.684 + for (lineno0, line) in enumerate(lines):
1.685 + for clause in line.split(";"):
1.686 + clause = clause.strip()
1.687 + iscomment = False
1.688 + for commentprefix in COMMENT_PREFIXES:
1.689 + if clause.startswith(commentprefix):
1.690 + iscomment = True
1.691 + if iscomment:
1.692 + continue
1.693 + mo = re.finditer(REQUIRE_RE, clause)
1.694 + if mo:
1.695 + for mod in mo:
1.696 + modname = mod.group(1)
1.697 + requires[modname] = {}
1.698 + if modname not in first_location:
1.699 + first_location[modname] = lineno0 + 1
1.700 +
1.701 + # define() can happen across multiple lines, so join everyone up.
1.702 + wholeshebang = "\n".join(lines)
1.703 + for match in DEF_RE.finditer(wholeshebang):
1.704 + # this should net us a list of string literals separated by commas
1.705 + for strbit in match.group(3).split(","):
1.706 + strbit = strbit.strip()
1.707 + # There could be a trailing comma netting us just whitespace, so
1.708 + # filter that out. Make sure that only string values with
1.709 + # quotes around them are allowed, and no quotes are inside
1.710 + # the quoted value.
1.711 + if strbit and DEF_RE_ALLOWED.match(strbit):
1.712 + modname = strbit[1:-1]
1.713 + if modname not in ["exports"]:
1.714 + requires[modname] = {}
1.715 + # joining all the lines means we lose line numbers, so we
1.716 + # can't fill first_location[]
1.717 +
1.718 + return requires, first_location
1.719 +
1.720 +CHROME_ALIASES = [
1.721 + (re.compile(r"Components\.classes"), "Cc"),
1.722 + (re.compile(r"Components\.interfaces"), "Ci"),
1.723 + (re.compile(r"Components\.utils"), "Cu"),
1.724 + (re.compile(r"Components\.results"), "Cr"),
1.725 + (re.compile(r"Components\.manager"), "Cm"),
1.726 + ]
1.727 +OTHER_CHROME = re.compile(r"Components\.[a-zA-Z]")
1.728 +
1.729 +def scan_for_bad_chrome(fn, lines, stderr):
1.730 + problems = False
1.731 + old_chrome = set() # i.e. "Cc" when we see "Components.classes"
1.732 + old_chrome_lines = [] # list of (lineno, line.strip()) tuples
1.733 + for lineno,line in enumerate(lines):
1.734 + # note: this scanner is not obligated to spot all possible forms of
1.735 + # chrome access. The scanner is detecting voluntary requests for
1.736 + # chrome. Runtime tools will enforce allowance or denial of access.
1.737 + line = line.strip()
1.738 + iscomment = False
1.739 + for commentprefix in COMMENT_PREFIXES:
1.740 + if line.startswith(commentprefix):
1.741 + iscomment = True
1.742 + break
1.743 + if iscomment:
1.744 + continue
1.745 + old_chrome_in_this_line = set()
1.746 + for (regexp,alias) in CHROME_ALIASES:
1.747 + if regexp.search(line):
1.748 + old_chrome_in_this_line.add(alias)
1.749 + if not old_chrome_in_this_line:
1.750 + if OTHER_CHROME.search(line):
1.751 + old_chrome_in_this_line.add("components")
1.752 + old_chrome.update(old_chrome_in_this_line)
1.753 + if old_chrome_in_this_line:
1.754 + old_chrome_lines.append( (lineno+1, line) )
1.755 +
1.756 + if old_chrome:
1.757 + print >>stderr, """
1.758 +The following lines from file %(fn)s:
1.759 +%(lines)s
1.760 +use 'Components' to access chrome authority. To do so, you need to add a
1.761 +line somewhat like the following:
1.762 +
1.763 + const {%(needs)s} = require("chrome");
1.764 +
1.765 +Then you can use any shortcuts to its properties that you import from the
1.766 +'chrome' module ('Cc', 'Ci', 'Cm', 'Cr', and 'Cu' for the 'classes',
1.767 +'interfaces', 'manager', 'results', and 'utils' properties, respectively. And
1.768 +`components` for `Components` object itself).
1.769 +""" % { "fn": fn, "needs": ",".join(sorted(old_chrome)),
1.770 + "lines": "\n".join([" %3d: %s" % (lineno,line)
1.771 + for (lineno, line) in old_chrome_lines]),
1.772 + }
1.773 + problems = True
1.774 + return problems
1.775 +
1.776 +def scan_module(fn, lines, stderr=sys.stderr):
1.777 + filename = os.path.basename(fn)
1.778 + requires, locations = scan_requirements_with_grep(fn, lines)
1.779 + if filename == "cuddlefish.js":
1.780 + # this is the loader: don't scan for chrome
1.781 + problems = False
1.782 + else:
1.783 + problems = scan_for_bad_chrome(fn, lines, stderr)
1.784 + return requires, problems, locations
1.785 +
1.786 +
1.787 +
1.788 +if __name__ == '__main__':
1.789 + for fn in sys.argv[1:]:
1.790 + requires, problems, locations = scan_module(fn, open(fn).readlines())
1.791 + print
1.792 + print "---", fn
1.793 + if problems:
1.794 + print "PROBLEMS"
1.795 + sys.exit(1)
1.796 + print "requires: %s" % (",".join(sorted(requires.keys())))
1.797 + print "locations: %s" % locations
1.798 +
