michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: 
michael@0: import os, sys, re, hashlib
michael@0: import simplejson as json
michael@0: SEP = os.path.sep
michael@0: from cuddlefish.util import filter_filenames, filter_dirnames
michael@0: 
michael@0: # Load new layout mapping hashtable
michael@0: path = os.path.join(os.environ.get('CUDDLEFISH_ROOT'), "mapping.json")
michael@0: data = open(path, 'r').read()
michael@0: NEW_LAYOUT_MAPPING = json.loads(data)
michael@0: 
michael@0: def js_zipname(packagename, modulename):
michael@0:     return "%s-lib/%s.js" % (packagename, modulename)
michael@0: def docs_zipname(packagename, modulename):
michael@0:     return "%s-docs/%s.md" % (packagename, modulename)
michael@0: def datamap_zipname(packagename):
michael@0:     return "%s-data.json" % packagename
michael@0: def datafile_zipname(packagename, datapath):
michael@0:     return "%s-data/%s" % (packagename, datapath)
michael@0: 
michael@0: def to_json(o):
michael@0:     return json.dumps(o, indent=1).encode("utf-8")+"\n"
michael@0: 
michael@0: class ModuleNotFoundError(Exception):
michael@0:     def __init__(self, requirement_type, requirement_name,
michael@0:                  used_by, line_number, looked_in):
michael@0:         Exception.__init__(self)
michael@0:         self.requirement_type = requirement_type # "require" or "define"
michael@0:         self.requirement_name = requirement_name # string, what they require()d
michael@0:         self.used_by = used_by # string, full path to module which did require()
michael@0:         self.line_number = line_number # int, 1-indexed line number of first require()
michael@0:         self.looked_in = looked_in # list of full paths to potential .js files
michael@0:     def __str__(self):
michael@0:         what = "%s(%s)" % (self.requirement_type, self.requirement_name)
michael@0:         where = self.used_by
michael@0:         if self.line_number is not None:
michael@0:             where = "%s:%d" % (self.used_by, self.line_number)
michael@0:         searched = "Looked for it in:\n  %s\n" % "\n  ".join(self.looked_in)
michael@0:         return ("ModuleNotFoundError: unable to satisfy: %s from\n"
michael@0:                 "  %s:\n" % (what, where)) + searched
michael@0: 
michael@0: class BadModuleIdentifier(Exception):
michael@0:     pass
michael@0: class BadSection(Exception):
michael@0:     pass
michael@0: class UnreachablePrefixError(Exception):
michael@0:     pass
michael@0: 
michael@0: class ManifestEntry:
michael@0:     def __init__(self):
michael@0:         self.docs_filename = None
michael@0:         self.docs_hash = None
michael@0:         self.requirements = {}
michael@0:         self.datamap = None
michael@0: 
michael@0:     def get_path(self):
michael@0:         name = self.moduleName
michael@0: 
michael@0:         if name.endswith(".js"):
michael@0:             name = name[:-3]
michael@0:         items = []
michael@0:         # Only add package name for addons, so that system module paths match
michael@0:         # the path from the commonjs root directory and also match the loader
michael@0:         # mappings.
michael@0:         if self.packageName != "addon-sdk":
michael@0:             items.append(self.packageName)
michael@0:         # And for the same reason, do not append `lib/`.
michael@0:         if self.sectionName == "tests":
michael@0:             items.append(self.sectionName)
michael@0:         items.append(name)
michael@0: 
michael@0:         return "/".join(items)
michael@0: 
michael@0:     def get_entry_for_manifest(self):
michael@0:         entry = { "packageName": self.packageName,
michael@0:                   "sectionName": self.sectionName,
michael@0:                   "moduleName": self.moduleName,
michael@0:                   "jsSHA256": self.js_hash,
michael@0:                   "docsSHA256": self.docs_hash,
michael@0:                   "requirements": {},
michael@0:                   }
michael@0:         for req in self.requirements:
michael@0:             if isinstance(self.requirements[req], ManifestEntry):
michael@0:                 them = self.requirements[req] # this is another ManifestEntry
michael@0:                 entry["requirements"][req] = them.get_path()
michael@0:             else:
michael@0:                 # something magic. The manifest entry indicates that they're
michael@0:                 # allowed to require() it
michael@0:                 entry["requirements"][req] = self.requirements[req]
michael@0:             assert isinstance(entry["requirements"][req], unicode) or \
michael@0:                    isinstance(entry["requirements"][req], str)
michael@0:         return entry
michael@0: 
michael@0:     def add_js(self, js_filename):
michael@0:         self.js_filename = js_filename
michael@0:         self.js_hash = hash_file(js_filename)
michael@0:     def add_docs(self, docs_filename):
michael@0:         self.docs_filename = docs_filename
michael@0:         self.docs_hash = hash_file(docs_filename)
michael@0:     def add_requirement(self, reqname, reqdata):
michael@0:         self.requirements[reqname] = reqdata
michael@0:     def add_data(self, datamap):
michael@0:         self.datamap = datamap
michael@0: 
michael@0:     def get_js_zipname(self):
michael@0:         return js_zipname(self.packagename, self.modulename)
michael@0:     def get_docs_zipname(self):
michael@0:         if self.docs_hash:
michael@0:             return docs_zipname(self.packagename, self.modulename)
michael@0:         return None
michael@0:     # self.js_filename
michael@0:     # self.docs_filename
michael@0: 
michael@0: 
michael@0: def hash_file(fn):
michael@0:     return hashlib.sha256(open(fn,"rb").read()).hexdigest()
michael@0: 
michael@0: def get_datafiles(datadir):
michael@0:     # yields pathnames relative to DATADIR, ignoring some files
michael@0:     for dirpath, dirnames, filenames in os.walk(datadir):
michael@0:         filenames = list(filter_filenames(filenames))
michael@0:         # this tells os.walk to prune the search
michael@0:         dirnames[:] = filter_dirnames(dirnames)
michael@0:         for filename in filenames:
michael@0:             fullname = os.path.join(dirpath, filename)
michael@0:             assert fullname.startswith(datadir+SEP), "%s%s not in %s" % (datadir, SEP, fullname)
michael@0:             yield fullname[len(datadir+SEP):]
michael@0: 
michael@0: 
michael@0: class DataMap:
michael@0:     # one per package
michael@0:     def __init__(self, pkg):
michael@0:         self.pkg = pkg
michael@0:         self.name = pkg.name
michael@0:         self.files_to_copy = []
michael@0:         datamap = {}
michael@0:         datadir = os.path.join(pkg.root_dir, "data")
michael@0:         for dataname in get_datafiles(datadir):
michael@0:             absname = os.path.join(datadir, dataname)
michael@0:             zipname = datafile_zipname(pkg.name, dataname)
michael@0:             datamap[dataname] = hash_file(absname)
michael@0:             self.files_to_copy.append( (zipname, absname) )
michael@0:         self.data_manifest = to_json(datamap)
michael@0:         self.data_manifest_hash = hashlib.sha256(self.data_manifest).hexdigest()
michael@0:         self.data_manifest_zipname = datamap_zipname(pkg.name)
michael@0:         self.data_uri_prefix = "%s/data/" % (self.name)
michael@0: 
michael@0: class BadChromeMarkerError(Exception):
michael@0:     pass
michael@0: 
michael@0: class ModuleInfo:
michael@0:     def __init__(self, package, section, name, js, docs):
michael@0:         self.package = package
michael@0:         self.section = section
michael@0:         self.name = name
michael@0:         self.js = js
michael@0:         self.docs = docs
michael@0: 
michael@0:     def __hash__(self):
michael@0:         return hash( (self.package.name, self.section, self.name,
michael@0:                       self.js, self.docs) )
michael@0:     def __eq__(self, them):
michael@0:         if them.__class__ is not self.__class__:
michael@0:             return False
michael@0:         if ((them.package.name, them.section, them.name, them.js, them.docs) !=
michael@0:             (self.package.name, self.section, self.name, self.js, self.docs) ):
michael@0:             return False
michael@0:         return True
michael@0: 
michael@0:     def __repr__(self):
michael@0:         return "ModuleInfo [%s %s %s] (%s, %s)" % (self.package.name,
michael@0:                                                    self.section,
michael@0:                                                    self.name,
michael@0:                                                    self.js, self.docs)
michael@0: 
michael@0: class ManifestBuilder:
michael@0:     def __init__(self, target_cfg, pkg_cfg, deps, extra_modules,
michael@0:                  stderr=sys.stderr):
michael@0:         self.manifest = {} # maps (package,section,module) to ManifestEntry
michael@0:         self.target_cfg = target_cfg # the entry point
michael@0:         self.pkg_cfg = pkg_cfg # all known packages
michael@0:         self.deps = deps # list of package names to search
michael@0:         self.used_packagenames = set()
michael@0:         self.stderr = stderr
michael@0:         self.extra_modules = extra_modules
michael@0:         self.modules = {} # maps ModuleInfo to URI in self.manifest
michael@0:         self.datamaps = {} # maps package name to DataMap instance
michael@0:         self.files = [] # maps manifest index to (absfn,absfn) js/docs pair
michael@0:         self.test_modules = [] # for runtime
michael@0: 
michael@0:     def build(self, scan_tests, test_filter_re):
michael@0:         # process the top module, which recurses to process everything it
michael@0:         # reaches
michael@0:         if "main" in self.target_cfg:
michael@0:             top_mi = self.find_top(self.target_cfg)
michael@0:             top_me = self.process_module(top_mi)
michael@0:             self.top_path = top_me.get_path()
michael@0:             self.datamaps[self.target_cfg.name] = DataMap(self.target_cfg)
michael@0:         if scan_tests:
michael@0:             mi = self._find_module_in_package("addon-sdk", "lib", "sdk/test/runner", [])
michael@0:             self.process_module(mi)
michael@0:             # also scan all test files in all packages that we use. By making
michael@0:             # a copy of self.used_packagenames first, we refrain from
michael@0:             # processing tests in packages that our own tests depend upon. If
michael@0:             # we're running tests for package A, and either modules in A or
michael@0:             # tests in A depend upon modules from package B, we *don't* want
michael@0:             # to run tests for package B.
michael@0:             test_modules = []
michael@0:             dirnames = self.target_cfg["tests"]
michael@0:             if isinstance(dirnames, basestring):
michael@0:                 dirnames = [dirnames]
michael@0:             dirnames = [os.path.join(self.target_cfg.root_dir, d)
michael@0:                         for d in dirnames]
michael@0:             for d in dirnames:
michael@0:                 for filename in os.listdir(d):
michael@0:                     if filename.startswith("test-") and filename.endswith(".js"):
michael@0:                         testname = filename[:-3] # require(testname)
michael@0:                         if test_filter_re:
michael@0:                             if not re.search(test_filter_re, testname):
michael@0:                                 continue
michael@0:                         tmi = ModuleInfo(self.target_cfg, "tests", testname,
michael@0:                                          os.path.join(d, filename), None)
michael@0:                         # scan the test's dependencies
michael@0:                         tme = self.process_module(tmi)
michael@0:                         test_modules.append( (testname, tme) )
michael@0:             # also add it as an artificial dependency of unit-test-finder, so
michael@0:             # the runtime dynamic load can work.
michael@0:             test_finder = self.get_manifest_entry("addon-sdk", "lib",
michael@0:                                                   "sdk/deprecated/unit-test-finder")
michael@0:             for (testname,tme) in test_modules:
michael@0:                 test_finder.add_requirement(testname, tme)
michael@0:                 # finally, tell the runtime about it, so they won't have to
michael@0:                 # search for all tests. self.test_modules will be passed
michael@0:                 # through the harness-options.json file in the
michael@0:                 # .allTestModules property.
michael@0:                 # Pass the absolute module path.
michael@0:                 self.test_modules.append(tme.get_path())
michael@0: 
michael@0:         # include files used by the loader
michael@0:         for em in self.extra_modules:
michael@0:             (pkgname, section, modname, js) = em
michael@0:             mi = ModuleInfo(self.pkg_cfg.packages[pkgname], section, modname,
michael@0:                             js, None)
michael@0:             self.process_module(mi)
michael@0: 
michael@0: 
michael@0:     def get_module_entries(self):
michael@0:         return frozenset(self.manifest.values())
michael@0:     def get_data_entries(self):
michael@0:         return frozenset(self.datamaps.values())
michael@0: 
michael@0:     def get_used_packages(self):
michael@0:         used = set()
michael@0:         for index in self.manifest:
michael@0:             (package, section, module) = index
michael@0:             used.add(package)
michael@0:         return sorted(used)
michael@0: 
michael@0:     def get_used_files(self, bundle_sdk_modules):
michael@0:         # returns all .js files that we reference, plus data/ files. You will
michael@0:         # need to add the loader, off-manifest files that it needs, and
michael@0:         # generated metadata.
michael@0:         for datamap in self.datamaps.values():
michael@0:             for (zipname, absname) in datamap.files_to_copy:
michael@0:                 yield absname
michael@0: 
michael@0:         for me in self.get_module_entries():
michael@0:             # Only ship SDK files if we are told to do so
michael@0:             if me.packageName != "addon-sdk" or bundle_sdk_modules:
michael@0:                 yield me.js_filename
michael@0: 
michael@0:     def get_all_test_modules(self):
michael@0:         return self.test_modules
michael@0: 
michael@0:     def get_harness_options_manifest(self, bundle_sdk_modules):
michael@0:         manifest = {}
michael@0:         for me in self.get_module_entries():
michael@0:             path = me.get_path()
michael@0:             # Do not add manifest entries for system modules.
michael@0:             # Doesn't prevent from shipping modules.
michael@0:             # Shipping modules is decided in `get_used_files`.
michael@0:             if me.packageName != "addon-sdk" or bundle_sdk_modules:
michael@0:                 manifest[path] = me.get_entry_for_manifest()
michael@0:         return manifest
michael@0: 
michael@0:     def get_manifest_entry(self, package, section, module):
michael@0:         index = (package, section, module)
michael@0:         if index not in self.manifest:
michael@0:             m = self.manifest[index] = ManifestEntry()
michael@0:             m.packageName = package
michael@0:             m.sectionName = section
michael@0:             m.moduleName = module
michael@0:             self.used_packagenames.add(package)
michael@0:         return self.manifest[index]
michael@0: 
michael@0:     def uri_name_from_path(self, pkg, fn):
michael@0:         # given a filename like .../pkg1/lib/bar/foo.js, and a package
michael@0:         # specification (with a .root_dir like ".../pkg1" and a .lib list of
michael@0:         # paths where .lib[0] is like "lib"), return the appropriate NAME
michael@0:         # that can be put into a URI like resource://JID-pkg1-lib/NAME . This
michael@0:         # will throw an exception if the file is outside of the lib/
michael@0:         # directory, since that means we can't construct a URI that points to
michael@0:         # it.
michael@0:         #
michael@0:         # This should be a lot easier, and shouldn't fail when the file is in
michael@0:         # the root of the package. Both should become possible when the XPI
michael@0:         # is rearranged and our URI scheme is simplified.
michael@0:         fn = os.path.abspath(fn)
michael@0:         pkglib = pkg.lib[0]
michael@0:         libdir = os.path.abspath(os.path.join(pkg.root_dir, pkglib))
michael@0:         # AARGH, section and name! we need to reverse-engineer a
michael@0:         # ModuleInfo instance that will produce a URI (in the form
michael@0:         # PREFIX/PKGNAME-SECTION/JS) that will map to the existing file.
michael@0:         # Until we fix URI generation to get rid of "sections", this is
michael@0:         # limited to files in the same .directories.lib as the rest of
michael@0:         # the package uses. So if the package's main files are in lib/,
michael@0:         # but the main.js is in the package root, there is no URI we can
michael@0:         # construct that will point to it, and we must fail.
michael@0:         #
michael@0:         # This will become much easier (and the failure case removed)
michael@0:         # when we get rid of sections and change the URIs to look like
michael@0:         # (PREFIX/PKGNAME/PATH-TO-JS).
michael@0: 
michael@0:         # AARGH 2, allowing .lib to be a list is really getting in the
michael@0:         # way. That needs to go away eventually too.
michael@0:         if not fn.startswith(libdir):
michael@0:             raise UnreachablePrefixError("Sorry, but the 'main' file (%s) in package %s is outside that package's 'lib' directory (%s), so I cannot construct a URI to reach it."
michael@0:                                          % (fn, pkg.name, pkglib))
michael@0:         name = fn[len(libdir):].lstrip(SEP)[:-len(".js")]
michael@0:         return name
michael@0: 
michael@0: 
michael@0:     def parse_main(self, root_dir, main, check_lib_dir=None):
michael@0:         # 'main' can be like one of the following:
michael@0:         #   a: ./lib/main.js  b: ./lib/main  c: lib/main
michael@0:         # we require it to be a path to the file, though, and ignore the
michael@0:         # .directories stuff. So just "main" is insufficient if you really
michael@0:         # want something in a "lib/" subdirectory.
michael@0:         if main.endswith(".js"):
michael@0:             main = main[:-len(".js")]
michael@0:         if main.startswith("./"):
michael@0:             main = main[len("./"):]
michael@0:         # package.json must always use "/", but on windows we'll replace that
michael@0:         # with "\" before using it as an actual filename
michael@0:         main = os.sep.join(main.split("/"))
michael@0:         paths = [os.path.join(root_dir, main+".js")]
michael@0:         if check_lib_dir is not None:
michael@0:             paths.append(os.path.join(root_dir, check_lib_dir, main+".js"))
michael@0:         return paths
michael@0: 
michael@0:     def find_top_js(self, target_cfg):
michael@0:         for libdir in target_cfg.lib:
michael@0:             for n in self.parse_main(target_cfg.root_dir, target_cfg.main,
michael@0:                                      libdir):
michael@0:                 if os.path.exists(n):
michael@0:                     return n
michael@0:         raise KeyError("unable to find main module '%s.js' in top-level package" % target_cfg.main)
michael@0: 
michael@0:     def find_top(self, target_cfg):
michael@0:         top_js = self.find_top_js(target_cfg)
michael@0:         n = os.path.join(target_cfg.root_dir, "README.md")
michael@0:         if os.path.exists(n):
michael@0:             top_docs = n
michael@0:         else:
michael@0:             top_docs = None
michael@0:         name = self.uri_name_from_path(target_cfg, top_js)
michael@0:         return ModuleInfo(target_cfg, "lib", name, top_js, top_docs)
michael@0: 
michael@0:     def process_module(self, mi):
michael@0:         pkg = mi.package
michael@0:         #print "ENTERING", pkg.name, mi.name
michael@0:         # mi.name must be fully-qualified
michael@0:         assert (not mi.name.startswith("./") and
michael@0:                 not mi.name.startswith("../"))
michael@0:         # create and claim the manifest row first
michael@0:         me = self.get_manifest_entry(pkg.name, mi.section, mi.name)
michael@0: 
michael@0:         me.add_js(mi.js)
michael@0:         if mi.docs:
michael@0:             me.add_docs(mi.docs)
michael@0: 
michael@0:         js_lines = open(mi.js,"r").readlines()
michael@0:         requires, problems, locations = scan_module(mi.js,js_lines,self.stderr)
michael@0:         if problems:
michael@0:             # the relevant instructions have already been written to stderr
michael@0:             raise BadChromeMarkerError()
michael@0: 
michael@0:         # We update our requirements on the way out of the depth-first
michael@0:         # traversal of the module graph
michael@0: 
michael@0:         for reqname in sorted(requires.keys()):
michael@0:             # If requirement is chrome or a pseudo-module (starts with @) make
michael@0:             # path a requirement name.
michael@0:             if reqname == "chrome" or reqname.startswith("@"):
michael@0:                 me.add_requirement(reqname, reqname)
michael@0:             else:
michael@0:                 # when two modules require() the same name, do they get a
michael@0:                 # shared instance? This is a deep question. For now say yes.
michael@0: 
michael@0:                 # find_req_for() returns an entry to put in our
michael@0:                 # 'requirements' dict, and will recursively process
michael@0:                 # everything transitively required from here. It will also
michael@0:                 # populate the self.modules[] cache. Note that we must
michael@0:                 # tolerate cycles in the reference graph.
michael@0:                 looked_in = [] # populated by subroutines
michael@0:                 them_me = self.find_req_for(mi, reqname, looked_in, locations)
michael@0:                 if them_me is None:
michael@0:                     if mi.section == "tests":
michael@0:                         # tolerate missing modules in tests, because
michael@0:                         # test-securable-module.js, and the modules/red.js
michael@0:                         # that it imports, both do that intentionally
michael@0:                         continue
michael@0:                     lineno = locations.get(reqname) # None means define()
michael@0:                     if lineno is None:
michael@0:                         reqtype = "define"
michael@0:                     else:
michael@0:                         reqtype = "require"
michael@0:                     err = ModuleNotFoundError(reqtype, reqname,
michael@0:                                               mi.js, lineno, looked_in)
michael@0:                     raise err
michael@0:                 else:
michael@0:                     me.add_requirement(reqname, them_me)
michael@0: 
michael@0:         return me
michael@0:         #print "LEAVING", pkg.name, mi.name
michael@0: 
michael@0:     def find_req_for(self, from_module, reqname, looked_in, locations):
michael@0:         # handle a single require(reqname) statement from from_module .
michael@0:         # Return a uri that exists in self.manifest
michael@0:         # Populate looked_in with places we looked.
michael@0:         def BAD(msg):
michael@0:             return BadModuleIdentifier(msg + " in require(%s) from %s" %
michael@0:                                        (reqname, from_module))
michael@0: 
michael@0:         if not reqname:
michael@0:             raise BAD("no actual modulename")
michael@0: 
michael@0:         # Allow things in tests/*.js to require both test code and real code.
michael@0:         # But things in lib/*.js can only require real code.
michael@0:         if from_module.section == "tests":
michael@0:             lookfor_sections = ["tests", "lib"]
michael@0:         elif from_module.section == "lib":
michael@0:             lookfor_sections = ["lib"]
michael@0:         else:
michael@0:             raise BadSection(from_module.section)
michael@0:         modulename = from_module.name
michael@0: 
michael@0:         #print " %s require(%s))" % (from_module, reqname)
michael@0: 
michael@0:         if reqname.startswith("./") or reqname.startswith("../"):
michael@0:             # 1: they want something relative to themselves, always from
michael@0:             # their own package
michael@0:             them = modulename.split("/")[:-1]
michael@0:             bits = reqname.split("/")
michael@0:             while bits[0] in (".", ".."):
michael@0:                 if not bits:
michael@0:                     raise BAD("no actual modulename")
michael@0:                 if bits[0] == "..":
michael@0:                     if not them:
michael@0:                         raise BAD("too many ..")
michael@0:                     them.pop()
michael@0:                 bits.pop(0)
michael@0:             bits = them+bits
michael@0:             lookfor_pkg = from_module.package.name
michael@0:             lookfor_mod = "/".join(bits)
michael@0:             return self._get_module_from_package(lookfor_pkg,
michael@0:                                                  lookfor_sections, lookfor_mod,
michael@0:                                                  looked_in)
michael@0: 
michael@0:         # non-relative import. Might be a short name (requiring a search
michael@0:         # through "library" packages), or a fully-qualified one.
michael@0: 
michael@0:         if "/" in reqname:
michael@0:             # 2: PKG/MOD: find PKG, look inside for MOD
michael@0:             bits = reqname.split("/")
michael@0:             lookfor_pkg = bits[0]
michael@0:             lookfor_mod = "/".join(bits[1:])
michael@0:             mi = self._get_module_from_package(lookfor_pkg,
michael@0:                                                lookfor_sections, lookfor_mod,
michael@0:                                                looked_in)
michael@0:             if mi: # caution, 0==None
michael@0:                 return mi
michael@0:         else:
michael@0:             # 3: try finding PKG, if found, use its main.js entry point
michael@0:             lookfor_pkg = reqname
michael@0:             mi = self._get_entrypoint_from_package(lookfor_pkg, looked_in)
michael@0:             if mi:
michael@0:                 return mi
michael@0: 
michael@0:         # 4: search packages for MOD or MODPARENT/MODCHILD. We always search
michael@0:         # their own package first, then the list of packages defined by their
michael@0:         # .dependencies list
michael@0:         from_pkg = from_module.package.name
michael@0:         mi = self._search_packages_for_module(from_pkg,
michael@0:                                               lookfor_sections, reqname,
michael@0:                                               looked_in)
michael@0:         if mi:
michael@0:             return mi
michael@0: 
michael@0:         # Only after we look for module in the addon itself, search for a module
michael@0:         # in new layout.
michael@0:         # First normalize require argument in order to easily find a mapping
michael@0:         normalized = reqname
michael@0:         if normalized.endswith(".js"):
michael@0:             normalized = normalized[:-len(".js")]
michael@0:         if normalized.startswith("addon-kit/"):
michael@0:             normalized = normalized[len("addon-kit/"):]
michael@0:         if normalized.startswith("api-utils/"):
michael@0:             normalized = normalized[len("api-utils/"):]
michael@0:         if normalized in NEW_LAYOUT_MAPPING:
michael@0:             # get the new absolute path for this module
michael@0:             original_reqname = reqname
michael@0:             reqname = NEW_LAYOUT_MAPPING[normalized]
michael@0:             from_pkg = from_module.package.name
michael@0: 
michael@0:             # If the addon didn't explicitely told us to ignore deprecated
michael@0:             # require path, warn the developer:
michael@0:             # (target_cfg is the package.json file)
michael@0:             if not "ignore-deprecated-path" in self.target_cfg:
michael@0:                 lineno = locations.get(original_reqname)
michael@0:                 print >>self.stderr, "Warning: Use of deprecated require path:"
michael@0:                 print >>self.stderr, "  In %s:%d:" % (from_module.js, lineno)
michael@0:                 print >>self.stderr, "    require('%s')." % original_reqname
michael@0:                 print >>self.stderr, "  New path should be:"
michael@0:                 print >>self.stderr, "    require('%s')" % reqname
michael@0: 
michael@0:             return self._search_packages_for_module(from_pkg,
michael@0:                                                     lookfor_sections, reqname,
michael@0:                                                     looked_in)
michael@0:         else:
michael@0:             # We weren't able to find this module, really.
michael@0:             return None
michael@0: 
michael@0:     def _handle_module(self, mi):
michael@0:         if not mi:
michael@0:             return None
michael@0: 
michael@0:         # we tolerate cycles in the reference graph, which means we need to
michael@0:         # populate the self.modules cache before recursing into
michael@0:         # process_module() . We must also check the cache first, so recursion
michael@0:         # can terminate.
michael@0:         if mi in self.modules:
michael@0:             return self.modules[mi]
michael@0: 
michael@0:         # this creates the entry
michael@0:         new_entry = self.get_manifest_entry(mi.package.name, mi.section, mi.name)
michael@0:         # and populates the cache
michael@0:         self.modules[mi] = new_entry
michael@0:         self.process_module(mi)
michael@0:         return new_entry
michael@0: 
michael@0:     def _get_module_from_package(self, pkgname, sections, modname, looked_in):
michael@0:         if pkgname not in self.pkg_cfg.packages:
michael@0:             return None
michael@0:         mi = self._find_module_in_package(pkgname, sections, modname,
michael@0:                                           looked_in)
michael@0:         return self._handle_module(mi)
michael@0: 
michael@0:     def _get_entrypoint_from_package(self, pkgname, looked_in):
michael@0:         if pkgname not in self.pkg_cfg.packages:
michael@0:             return None
michael@0:         pkg = self.pkg_cfg.packages[pkgname]
michael@0:         main = pkg.get("main", None)
michael@0:         if not main:
michael@0:             return None
michael@0:         for js in self.parse_main(pkg.root_dir, main):
michael@0:             looked_in.append(js)
michael@0:             if os.path.exists(js):
michael@0:                 section = "lib"
michael@0:                 name = self.uri_name_from_path(pkg, js)
michael@0:                 docs = None
michael@0:                 mi = ModuleInfo(pkg, section, name, js, docs)
michael@0:                 return self._handle_module(mi)
michael@0:         return None
michael@0: 
michael@0:     def _search_packages_for_module(self, from_pkg, sections, reqname,
michael@0:                                     looked_in):
michael@0:         searchpath = [] # list of package names
michael@0:         searchpath.append(from_pkg) # search self first
michael@0:         us = self.pkg_cfg.packages[from_pkg]
michael@0:         if 'dependencies' in us:
michael@0:             # only look in dependencies
michael@0:             searchpath.extend(us['dependencies'])
michael@0:         else:
michael@0:             # they didn't declare any dependencies (or they declared an empty
michael@0:             # list, but we'll treat that as not declaring one, because it's
michael@0:             # easier), so look in all deps, sorted alphabetically, so
michael@0:             # addon-kit comes first. Note that self.deps includes all
michael@0:             # packages found by traversing the ".dependencies" lists in each
michael@0:             # package.json, starting from the main addon package, plus
michael@0:             # everything added by --extra-packages
michael@0:             searchpath.extend(sorted(self.deps))
michael@0:         for pkgname in searchpath:
michael@0:             mi = self._find_module_in_package(pkgname, sections, reqname,
michael@0:                                               looked_in)
michael@0:             if mi:
michael@0:                 return self._handle_module(mi)
michael@0:         return None
michael@0: 
michael@0:     def _find_module_in_package(self, pkgname, sections, name, looked_in):
michael@0:         # require("a/b/c") should look at ...\a\b\c.js on windows
michael@0:         filename = os.sep.join(name.split("/"))
michael@0:         # normalize filename, make sure that we do not add .js if it already has
michael@0:         # it.
michael@0:         if not filename.endswith(".js") and not filename.endswith(".json"):
michael@0:           filename += ".js"
michael@0: 
michael@0:         if filename.endswith(".js"):
michael@0:           basename = filename[:-3]
michael@0:         if filename.endswith(".json"):
michael@0:           basename = filename[:-5]
michael@0: 
michael@0:         pkg = self.pkg_cfg.packages[pkgname]
michael@0:         if isinstance(sections, basestring):
michael@0:             sections = [sections]
michael@0:         for section in sections:
michael@0:             for sdir in pkg.get(section, []):
michael@0:                 js = os.path.join(pkg.root_dir, sdir, filename)
michael@0:                 looked_in.append(js)
michael@0:                 if os.path.exists(js):
michael@0:                     docs = None
michael@0:                     maybe_docs = os.path.join(pkg.root_dir, "docs",
michael@0:                                               basename+".md")
michael@0:                     if section == "lib" and os.path.exists(maybe_docs):
michael@0:                         docs = maybe_docs
michael@0:                     return ModuleInfo(pkg, section, name, js, docs)
michael@0:         return None
michael@0: 
michael@0: def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
michael@0:                    test_filter_re=None, extra_modules=[]):
michael@0:     """
michael@0:     Perform recursive dependency analysis starting from entry_point,
michael@0:     building up a manifest of modules that need to be included in the XPI.
michael@0:     Each entry will map require() names to the URL of the module that will
michael@0:     be used to satisfy that dependency. The manifest will be used by the
michael@0:     runtime's require() code.
michael@0: 
michael@0:     This returns a ManifestBuilder object, with two public methods. The
michael@0:     first, get_module_entries(), returns a set of ManifestEntry objects, each
michael@0:     of which can be asked for the following:
michael@0: 
michael@0:      * its contribution to the harness-options.json '.manifest'
michael@0:      * the local disk name
michael@0:      * the name in the XPI at which it should be placed
michael@0: 
michael@0:     The second is get_data_entries(), which returns a set of DataEntry
michael@0:     objects, each of which has:
michael@0: 
michael@0:      * local disk name
michael@0:      * name in the XPI
michael@0: 
michael@0:     note: we don't build the XPI here, but our manifest is passed to the
michael@0:     code which does, so it knows what to copy into the XPI.
michael@0:     """
michael@0: 
michael@0:     mxt = ManifestBuilder(target_cfg, pkg_cfg, deps, extra_modules)
michael@0:     mxt.build(scan_tests, test_filter_re)
michael@0:     return mxt
michael@0: 
michael@0: 
michael@0: 
michael@0: COMMENT_PREFIXES = ["//", "/*", "*", "dump("]
michael@0: 
michael@0: REQUIRE_RE = r"(?<![\'\"])require\s*\(\s*[\'\"]([^\'\"]+?)[\'\"]\s*\)"
michael@0: 
michael@0: # detect the define idiom of the form:
michael@0: #   define("module name", ["dep1", "dep2", "dep3"], function() {})
michael@0: # by capturing the contents of the list in a group.
michael@0: DEF_RE = re.compile(r"(require|define)\s*\(\s*([\'\"][^\'\"]+[\'\"]\s*,)?\s*\[([^\]]+)\]")
michael@0: 
michael@0: # Out of the async dependencies, do not allow quotes in them.
michael@0: DEF_RE_ALLOWED = re.compile(r"^[\'\"][^\'\"]+[\'\"]$")
michael@0: 
michael@0: def scan_requirements_with_grep(fn, lines):
michael@0:     requires = {}
michael@0:     first_location = {}
michael@0:     for (lineno0, line) in enumerate(lines):
michael@0:         for clause in line.split(";"):
michael@0:             clause = clause.strip()
michael@0:             iscomment = False
michael@0:             for commentprefix in COMMENT_PREFIXES:
michael@0:                 if clause.startswith(commentprefix):
michael@0:                     iscomment = True
michael@0:             if iscomment:
michael@0:                 continue
michael@0:             mo = re.finditer(REQUIRE_RE, clause)
michael@0:             if mo:
michael@0:                 for mod in mo:
michael@0:                     modname = mod.group(1)
michael@0:                     requires[modname] = {}
michael@0:                     if modname not in first_location:
michael@0:                         first_location[modname] = lineno0 + 1
michael@0: 
michael@0:     # define() can happen across multiple lines, so join everyone up.
michael@0:     wholeshebang = "\n".join(lines)
michael@0:     for match in DEF_RE.finditer(wholeshebang):
michael@0:         # this should net us a list of string literals separated by commas
michael@0:         for strbit in match.group(3).split(","):
michael@0:             strbit = strbit.strip()
michael@0:             # There could be a trailing comma netting us just whitespace, so
michael@0:             # filter that out. Make sure that only string values with
michael@0:             # quotes around them are allowed, and no quotes are inside
michael@0:             # the quoted value.
michael@0:             if strbit and DEF_RE_ALLOWED.match(strbit):
michael@0:                 modname = strbit[1:-1]
michael@0:                 if modname not in ["exports"]:
michael@0:                     requires[modname] = {}
michael@0:                     # joining all the lines means we lose line numbers, so we
michael@0:                     # can't fill first_location[]
michael@0: 
michael@0:     return requires, first_location
michael@0: 
michael@0: CHROME_ALIASES = [
michael@0:     (re.compile(r"Components\.classes"), "Cc"),
michael@0:     (re.compile(r"Components\.interfaces"), "Ci"),
michael@0:     (re.compile(r"Components\.utils"), "Cu"),
michael@0:     (re.compile(r"Components\.results"), "Cr"),
michael@0:     (re.compile(r"Components\.manager"), "Cm"),
michael@0:     ]
michael@0: OTHER_CHROME = re.compile(r"Components\.[a-zA-Z]")
michael@0: 
michael@0: def scan_for_bad_chrome(fn, lines, stderr):
michael@0:     problems = False
michael@0:     old_chrome = set() # i.e. "Cc" when we see "Components.classes"
michael@0:     old_chrome_lines = [] # list of (lineno, line.strip()) tuples
michael@0:     for lineno,line in enumerate(lines):
michael@0:         # note: this scanner is not obligated to spot all possible forms of
michael@0:         # chrome access. The scanner is detecting voluntary requests for
michael@0:         # chrome. Runtime tools will enforce allowance or denial of access.
michael@0:         line = line.strip()
michael@0:         iscomment = False
michael@0:         for commentprefix in COMMENT_PREFIXES:
michael@0:             if line.startswith(commentprefix):
michael@0:                 iscomment = True
michael@0:                 break
michael@0:         if iscomment:
michael@0:             continue
michael@0:         old_chrome_in_this_line = set()
michael@0:         for (regexp,alias) in CHROME_ALIASES:
michael@0:             if regexp.search(line):
michael@0:                 old_chrome_in_this_line.add(alias)
michael@0:         if not old_chrome_in_this_line:
michael@0:             if OTHER_CHROME.search(line):
michael@0:                 old_chrome_in_this_line.add("components")
michael@0:         old_chrome.update(old_chrome_in_this_line)
michael@0:         if old_chrome_in_this_line:
michael@0:             old_chrome_lines.append( (lineno+1, line) )
michael@0: 
michael@0:     if old_chrome:
michael@0:         print >>stderr, """
michael@0: The following lines from file %(fn)s:
michael@0: %(lines)s
michael@0: use 'Components' to access chrome authority. To do so, you need to add a
michael@0: line somewhat like the following:
michael@0: 
michael@0:   const {%(needs)s} = require("chrome");
michael@0: 
michael@0: Then you can use any shortcuts to its properties that you import from the
michael@0: 'chrome' module ('Cc', 'Ci', 'Cm', 'Cr', and 'Cu' for the 'classes',
michael@0: 'interfaces', 'manager', 'results', and 'utils' properties, respectively. And
michael@0: `components` for `Components` object itself).
michael@0: """ % { "fn": fn, "needs": ",".join(sorted(old_chrome)),
michael@0:         "lines": "\n".join([" %3d: %s" % (lineno,line)
michael@0:                             for (lineno, line) in old_chrome_lines]),
michael@0:         }
michael@0:         problems = True
michael@0:     return problems
michael@0: 
michael@0: def scan_module(fn, lines, stderr=sys.stderr):
michael@0:     filename = os.path.basename(fn)
michael@0:     requires, locations = scan_requirements_with_grep(fn, lines)
michael@0:     if filename == "cuddlefish.js":
michael@0:         # this is the loader: don't scan for chrome
michael@0:         problems = False
michael@0:     else:
michael@0:         problems = scan_for_bad_chrome(fn, lines, stderr)
michael@0:     return requires, problems, locations
michael@0: 
michael@0: 
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     for fn in sys.argv[1:]:
michael@0:         requires, problems, locations = scan_module(fn, open(fn).readlines())
michael@0:         print
michael@0:         print "---", fn
michael@0:         if problems:
michael@0:             print "PROBLEMS"
michael@0:             sys.exit(1)
michael@0:         print "requires: %s" % (",".join(sorted(requires.keys())))
michael@0:         print "locations: %s" % locations
michael@0: