The Tor Browser / file comparison

comparison: addon-sdk/source/python-lib/cuddlefish/manifest.py

addon-sdk/source/python-lib/cuddlefish/manifest.py

changeset 0
6474c204b198
equal deleted inserted replaced
-1:000000000000 0:a05db170e549
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5
6 import os, sys, re, hashlib
7 import simplejson as json
8 SEP = os.path.sep
9 from cuddlefish.util import filter_filenames, filter_dirnames
10
11 # Load new layout mapping hashtable
12 path = os.path.join(os.environ.get('CUDDLEFISH_ROOT'), "mapping.json")
13 data = open(path, 'r').read()
14 NEW_LAYOUT_MAPPING = json.loads(data)
15
16 def js_zipname(packagename, modulename):
17 return "%s-lib/%s.js" % (packagename, modulename)
18 def docs_zipname(packagename, modulename):
19 return "%s-docs/%s.md" % (packagename, modulename)
20 def datamap_zipname(packagename):
21 return "%s-data.json" % packagename
22 def datafile_zipname(packagename, datapath):
23 return "%s-data/%s" % (packagename, datapath)
24
25 def to_json(o):
26 return json.dumps(o, indent=1).encode("utf-8")+"\n"
27
28 class ModuleNotFoundError(Exception):
29 def __init__(self, requirement_type, requirement_name,
30 used_by, line_number, looked_in):
31 Exception.__init__(self)
32 self.requirement_type = requirement_type # "require" or "define"
33 self.requirement_name = requirement_name # string, what they require()d
34 self.used_by = used_by # string, full path to module which did require()
35 self.line_number = line_number # int, 1-indexed line number of first require()
36 self.looked_in = looked_in # list of full paths to potential .js files
37 def __str__(self):
38 what = "%s(%s)" % (self.requirement_type, self.requirement_name)
39 where = self.used_by
40 if self.line_number is not None:
41 where = "%s:%d" % (self.used_by, self.line_number)
42 searched = "Looked for it in:\n %s\n" % "\n ".join(self.looked_in)
43 return ("ModuleNotFoundError: unable to satisfy: %s from\n"
44 " %s:\n" % (what, where)) + searched
45
46 class BadModuleIdentifier(Exception):
47 pass
48 class BadSection(Exception):
49 pass
50 class UnreachablePrefixError(Exception):
51 pass
52
53 class ManifestEntry:
54 def __init__(self):
55 self.docs_filename = None
56 self.docs_hash = None
57 self.requirements = {}
58 self.datamap = None
59
60 def get_path(self):
61 name = self.moduleName
62
63 if name.endswith(".js"):
64 name = name[:-3]
65 items = []
66 # Only add package name for addons, so that system module paths match
67 # the path from the commonjs root directory and also match the loader
68 # mappings.
69 if self.packageName != "addon-sdk":
70 items.append(self.packageName)
71 # And for the same reason, do not append `lib/`.
72 if self.sectionName == "tests":
73 items.append(self.sectionName)
74 items.append(name)
75
76 return "/".join(items)
77
78 def get_entry_for_manifest(self):
79 entry = { "packageName": self.packageName,
80 "sectionName": self.sectionName,
81 "moduleName": self.moduleName,
82 "jsSHA256": self.js_hash,
83 "docsSHA256": self.docs_hash,
84 "requirements": {},
85 }
86 for req in self.requirements:
87 if isinstance(self.requirements[req], ManifestEntry):
88 them = self.requirements[req] # this is another ManifestEntry
89 entry["requirements"][req] = them.get_path()
90 else:
91 # something magic. The manifest entry indicates that they're
92 # allowed to require() it
93 entry["requirements"][req] = self.requirements[req]
94 assert isinstance(entry["requirements"][req], unicode) or \
95 isinstance(entry["requirements"][req], str)
96 return entry
97
98 def add_js(self, js_filename):
99 self.js_filename = js_filename
100 self.js_hash = hash_file(js_filename)
101 def add_docs(self, docs_filename):
102 self.docs_filename = docs_filename
103 self.docs_hash = hash_file(docs_filename)
104 def add_requirement(self, reqname, reqdata):
105 self.requirements[reqname] = reqdata
106 def add_data(self, datamap):
107 self.datamap = datamap
108
109 def get_js_zipname(self):
110 return js_zipname(self.packagename, self.modulename)
111 def get_docs_zipname(self):
112 if self.docs_hash:
113 return docs_zipname(self.packagename, self.modulename)
114 return None
115 # self.js_filename
116 # self.docs_filename
117
118
119 def hash_file(fn):
120 return hashlib.sha256(open(fn,"rb").read()).hexdigest()
121
122 def get_datafiles(datadir):
123 # yields pathnames relative to DATADIR, ignoring some files
124 for dirpath, dirnames, filenames in os.walk(datadir):
125 filenames = list(filter_filenames(filenames))
126 # this tells os.walk to prune the search
127 dirnames[:] = filter_dirnames(dirnames)
128 for filename in filenames:
129 fullname = os.path.join(dirpath, filename)
130 assert fullname.startswith(datadir+SEP), "%s%s not in %s" % (datadir, SEP, fullname)
131 yield fullname[len(datadir+SEP):]
132
133
134 class DataMap:
135 # one per package
136 def __init__(self, pkg):
137 self.pkg = pkg
138 self.name = pkg.name
139 self.files_to_copy = []
140 datamap = {}
141 datadir = os.path.join(pkg.root_dir, "data")
142 for dataname in get_datafiles(datadir):
143 absname = os.path.join(datadir, dataname)
144 zipname = datafile_zipname(pkg.name, dataname)
145 datamap[dataname] = hash_file(absname)
146 self.files_to_copy.append( (zipname, absname) )
147 self.data_manifest = to_json(datamap)
148 self.data_manifest_hash = hashlib.sha256(self.data_manifest).hexdigest()
149 self.data_manifest_zipname = datamap_zipname(pkg.name)
150 self.data_uri_prefix = "%s/data/" % (self.name)
151
152 class BadChromeMarkerError(Exception):
153 pass
154
155 class ModuleInfo:
156 def __init__(self, package, section, name, js, docs):
157 self.package = package
158 self.section = section
159 self.name = name
160 self.js = js
161 self.docs = docs
162
163 def __hash__(self):
164 return hash( (self.package.name, self.section, self.name,
165 self.js, self.docs) )
166 def __eq__(self, them):
167 if them.__class__ is not self.__class__:
168 return False
169 if ((them.package.name, them.section, them.name, them.js, them.docs) !=
170 (self.package.name, self.section, self.name, self.js, self.docs) ):
171 return False
172 return True
173
174 def __repr__(self):
175 return "ModuleInfo [%s %s %s] (%s, %s)" % (self.package.name,
176 self.section,
177 self.name,
178 self.js, self.docs)
179
180 class ManifestBuilder:
181 def __init__(self, target_cfg, pkg_cfg, deps, extra_modules,
182 stderr=sys.stderr):
183 self.manifest = {} # maps (package,section,module) to ManifestEntry
184 self.target_cfg = target_cfg # the entry point
185 self.pkg_cfg = pkg_cfg # all known packages
186 self.deps = deps # list of package names to search
187 self.used_packagenames = set()
188 self.stderr = stderr
189 self.extra_modules = extra_modules
190 self.modules = {} # maps ModuleInfo to URI in self.manifest
191 self.datamaps = {} # maps package name to DataMap instance
192 self.files = [] # maps manifest index to (absfn,absfn) js/docs pair
193 self.test_modules = [] # for runtime
194
195 def build(self, scan_tests, test_filter_re):
196 # process the top module, which recurses to process everything it
197 # reaches
198 if "main" in self.target_cfg:
199 top_mi = self.find_top(self.target_cfg)
200 top_me = self.process_module(top_mi)
201 self.top_path = top_me.get_path()
202 self.datamaps[self.target_cfg.name] = DataMap(self.target_cfg)
203 if scan_tests:
204 mi = self._find_module_in_package("addon-sdk", "lib", "sdk/test/runner", [])
205 self.process_module(mi)
206 # also scan all test files in all packages that we use. By making
207 # a copy of self.used_packagenames first, we refrain from
208 # processing tests in packages that our own tests depend upon. If
209 # we're running tests for package A, and either modules in A or
210 # tests in A depend upon modules from package B, we *don't* want
211 # to run tests for package B.
212 test_modules = []
213 dirnames = self.target_cfg["tests"]
214 if isinstance(dirnames, basestring):
215 dirnames = [dirnames]
216 dirnames = [os.path.join(self.target_cfg.root_dir, d)
217 for d in dirnames]
218 for d in dirnames:
219 for filename in os.listdir(d):
220 if filename.startswith("test-") and filename.endswith(".js"):
221 testname = filename[:-3] # require(testname)
222 if test_filter_re:
223 if not re.search(test_filter_re, testname):
224 continue
225 tmi = ModuleInfo(self.target_cfg, "tests", testname,
226 os.path.join(d, filename), None)
227 # scan the test's dependencies
228 tme = self.process_module(tmi)
229 test_modules.append( (testname, tme) )
230 # also add it as an artificial dependency of unit-test-finder, so
231 # the runtime dynamic load can work.
232 test_finder = self.get_manifest_entry("addon-sdk", "lib",
233 "sdk/deprecated/unit-test-finder")
234 for (testname,tme) in test_modules:
235 test_finder.add_requirement(testname, tme)
236 # finally, tell the runtime about it, so they won't have to
237 # search for all tests. self.test_modules will be passed
238 # through the harness-options.json file in the
239 # .allTestModules property.
240 # Pass the absolute module path.
241 self.test_modules.append(tme.get_path())
242
243 # include files used by the loader
244 for em in self.extra_modules:
245 (pkgname, section, modname, js) = em
246 mi = ModuleInfo(self.pkg_cfg.packages[pkgname], section, modname,
247 js, None)
248 self.process_module(mi)
249
250
251 def get_module_entries(self):
252 return frozenset(self.manifest.values())
253 def get_data_entries(self):
254 return frozenset(self.datamaps.values())
255
256 def get_used_packages(self):
257 used = set()
258 for index in self.manifest:
259 (package, section, module) = index
260 used.add(package)
261 return sorted(used)
262
263 def get_used_files(self, bundle_sdk_modules):
264 # returns all .js files that we reference, plus data/ files. You will
265 # need to add the loader, off-manifest files that it needs, and
266 # generated metadata.
267 for datamap in self.datamaps.values():
268 for (zipname, absname) in datamap.files_to_copy:
269 yield absname
270
271 for me in self.get_module_entries():
272 # Only ship SDK files if we are told to do so
273 if me.packageName != "addon-sdk" or bundle_sdk_modules:
274 yield me.js_filename
275
276 def get_all_test_modules(self):
277 return self.test_modules
278
279 def get_harness_options_manifest(self, bundle_sdk_modules):
280 manifest = {}
281 for me in self.get_module_entries():
282 path = me.get_path()
283 # Do not add manifest entries for system modules.
284 # Doesn't prevent from shipping modules.
285 # Shipping modules is decided in `get_used_files`.
286 if me.packageName != "addon-sdk" or bundle_sdk_modules:
287 manifest[path] = me.get_entry_for_manifest()
288 return manifest
289
290 def get_manifest_entry(self, package, section, module):
291 index = (package, section, module)
292 if index not in self.manifest:
293 m = self.manifest[index] = ManifestEntry()
294 m.packageName = package
295 m.sectionName = section
296 m.moduleName = module
297 self.used_packagenames.add(package)
298 return self.manifest[index]
299
300 def uri_name_from_path(self, pkg, fn):
301 # given a filename like .../pkg1/lib/bar/foo.js, and a package
302 # specification (with a .root_dir like ".../pkg1" and a .lib list of
303 # paths where .lib[0] is like "lib"), return the appropriate NAME
304 # that can be put into a URI like resource://JID-pkg1-lib/NAME . This
305 # will throw an exception if the file is outside of the lib/
306 # directory, since that means we can't construct a URI that points to
307 # it.
308 #
309 # This should be a lot easier, and shouldn't fail when the file is in
310 # the root of the package. Both should become possible when the XPI
311 # is rearranged and our URI scheme is simplified.
312 fn = os.path.abspath(fn)
313 pkglib = pkg.lib[0]
314 libdir = os.path.abspath(os.path.join(pkg.root_dir, pkglib))
315 # AARGH, section and name! we need to reverse-engineer a
316 # ModuleInfo instance that will produce a URI (in the form
317 # PREFIX/PKGNAME-SECTION/JS) that will map to the existing file.
318 # Until we fix URI generation to get rid of "sections", this is
319 # limited to files in the same .directories.lib as the rest of
320 # the package uses. So if the package's main files are in lib/,
321 # but the main.js is in the package root, there is no URI we can
322 # construct that will point to it, and we must fail.
323 #
324 # This will become much easier (and the failure case removed)
325 # when we get rid of sections and change the URIs to look like
326 # (PREFIX/PKGNAME/PATH-TO-JS).
327
328 # AARGH 2, allowing .lib to be a list is really getting in the
329 # way. That needs to go away eventually too.
330 if not fn.startswith(libdir):
331 raise UnreachablePrefixError("Sorry, but the 'main' file (%s) in package %s is outside that package's 'lib' directory (%s), so I cannot construct a URI to reach it."
332 % (fn, pkg.name, pkglib))
333 name = fn[len(libdir):].lstrip(SEP)[:-len(".js")]
334 return name
335
336
337 def parse_main(self, root_dir, main, check_lib_dir=None):
338 # 'main' can be like one of the following:
339 # a: ./lib/main.js b: ./lib/main c: lib/main
340 # we require it to be a path to the file, though, and ignore the
341 # .directories stuff. So just "main" is insufficient if you really
342 # want something in a "lib/" subdirectory.
343 if main.endswith(".js"):
344 main = main[:-len(".js")]
345 if main.startswith("./"):
346 main = main[len("./"):]
347 # package.json must always use "/", but on windows we'll replace that
348 # with "\" before using it as an actual filename
349 main = os.sep.join(main.split("/"))
350 paths = [os.path.join(root_dir, main+".js")]
351 if check_lib_dir is not None:
352 paths.append(os.path.join(root_dir, check_lib_dir, main+".js"))
353 return paths
354
355 def find_top_js(self, target_cfg):
356 for libdir in target_cfg.lib:
357 for n in self.parse_main(target_cfg.root_dir, target_cfg.main,
358 libdir):
359 if os.path.exists(n):
360 return n
361 raise KeyError("unable to find main module '%s.js' in top-level package" % target_cfg.main)
362
363 def find_top(self, target_cfg):
364 top_js = self.find_top_js(target_cfg)
365 n = os.path.join(target_cfg.root_dir, "README.md")
366 if os.path.exists(n):
367 top_docs = n
368 else:
369 top_docs = None
370 name = self.uri_name_from_path(target_cfg, top_js)
371 return ModuleInfo(target_cfg, "lib", name, top_js, top_docs)
372
373 def process_module(self, mi):
374 pkg = mi.package
375 #print "ENTERING", pkg.name, mi.name
376 # mi.name must be fully-qualified
377 assert (not mi.name.startswith("./") and
378 not mi.name.startswith("../"))
379 # create and claim the manifest row first
380 me = self.get_manifest_entry(pkg.name, mi.section, mi.name)
381
382 me.add_js(mi.js)
383 if mi.docs:
384 me.add_docs(mi.docs)
385
386 js_lines = open(mi.js,"r").readlines()
387 requires, problems, locations = scan_module(mi.js,js_lines,self.stderr)
388 if problems:
389 # the relevant instructions have already been written to stderr
390 raise BadChromeMarkerError()
391
392 # We update our requirements on the way out of the depth-first
393 # traversal of the module graph
394
395 for reqname in sorted(requires.keys()):
396 # If requirement is chrome or a pseudo-module (starts with @) make
397 # path a requirement name.
398 if reqname == "chrome" or reqname.startswith("@"):
399 me.add_requirement(reqname, reqname)
400 else:
401 # when two modules require() the same name, do they get a
402 # shared instance? This is a deep question. For now say yes.
403
404 # find_req_for() returns an entry to put in our
405 # 'requirements' dict, and will recursively process
406 # everything transitively required from here. It will also
407 # populate the self.modules[] cache. Note that we must
408 # tolerate cycles in the reference graph.
409 looked_in = [] # populated by subroutines
410 them_me = self.find_req_for(mi, reqname, looked_in, locations)
411 if them_me is None:
412 if mi.section == "tests":
413 # tolerate missing modules in tests, because
414 # test-securable-module.js, and the modules/red.js
415 # that it imports, both do that intentionally
416 continue
417 lineno = locations.get(reqname) # None means define()
418 if lineno is None:
419 reqtype = "define"
420 else:
421 reqtype = "require"
422 err = ModuleNotFoundError(reqtype, reqname,
423 mi.js, lineno, looked_in)
424 raise err
425 else:
426 me.add_requirement(reqname, them_me)
427
428 return me
429 #print "LEAVING", pkg.name, mi.name
430
431 def find_req_for(self, from_module, reqname, looked_in, locations):
432 # handle a single require(reqname) statement from from_module .
433 # Return a uri that exists in self.manifest
434 # Populate looked_in with places we looked.
435 def BAD(msg):
436 return BadModuleIdentifier(msg + " in require(%s) from %s" %
437 (reqname, from_module))
438
439 if not reqname:
440 raise BAD("no actual modulename")
441
442 # Allow things in tests/*.js to require both test code and real code.
443 # But things in lib/*.js can only require real code.
444 if from_module.section == "tests":
445 lookfor_sections = ["tests", "lib"]
446 elif from_module.section == "lib":
447 lookfor_sections = ["lib"]
448 else:
449 raise BadSection(from_module.section)
450 modulename = from_module.name
451
452 #print " %s require(%s))" % (from_module, reqname)
453
454 if reqname.startswith("./") or reqname.startswith("../"):
455 # 1: they want something relative to themselves, always from
456 # their own package
457 them = modulename.split("/")[:-1]
458 bits = reqname.split("/")
459 while bits[0] in (".", ".."):
460 if not bits:
461 raise BAD("no actual modulename")
462 if bits[0] == "..":
463 if not them:
464 raise BAD("too many ..")
465 them.pop()
466 bits.pop(0)
467 bits = them+bits
468 lookfor_pkg = from_module.package.name
469 lookfor_mod = "/".join(bits)
470 return self._get_module_from_package(lookfor_pkg,
471 lookfor_sections, lookfor_mod,
472 looked_in)
473
474 # non-relative import. Might be a short name (requiring a search
475 # through "library" packages), or a fully-qualified one.
476
477 if "/" in reqname:
478 # 2: PKG/MOD: find PKG, look inside for MOD
479 bits = reqname.split("/")
480 lookfor_pkg = bits[0]
481 lookfor_mod = "/".join(bits[1:])
482 mi = self._get_module_from_package(lookfor_pkg,
483 lookfor_sections, lookfor_mod,
484 looked_in)
485 if mi: # caution, 0==None
486 return mi
487 else:
488 # 3: try finding PKG, if found, use its main.js entry point
489 lookfor_pkg = reqname
490 mi = self._get_entrypoint_from_package(lookfor_pkg, looked_in)
491 if mi:
492 return mi
493
494 # 4: search packages for MOD or MODPARENT/MODCHILD. We always search
495 # their own package first, then the list of packages defined by their
496 # .dependencies list
497 from_pkg = from_module.package.name
498 mi = self._search_packages_for_module(from_pkg,
499 lookfor_sections, reqname,
500 looked_in)
501 if mi:
502 return mi
503
504 # Only after we look for module in the addon itself, search for a module
505 # in new layout.
506 # First normalize require argument in order to easily find a mapping
507 normalized = reqname
508 if normalized.endswith(".js"):
509 normalized = normalized[:-len(".js")]
510 if normalized.startswith("addon-kit/"):
511 normalized = normalized[len("addon-kit/"):]
512 if normalized.startswith("api-utils/"):
513 normalized = normalized[len("api-utils/"):]
514 if normalized in NEW_LAYOUT_MAPPING:
515 # get the new absolute path for this module
516 original_reqname = reqname
517 reqname = NEW_LAYOUT_MAPPING[normalized]
518 from_pkg = from_module.package.name
519
520 # If the addon didn't explicitely told us to ignore deprecated
521 # require path, warn the developer:
522 # (target_cfg is the package.json file)
523 if not "ignore-deprecated-path" in self.target_cfg:
524 lineno = locations.get(original_reqname)
525 print >>self.stderr, "Warning: Use of deprecated require path:"
526 print >>self.stderr, " In %s:%d:" % (from_module.js, lineno)
527 print >>self.stderr, " require('%s')." % original_reqname
528 print >>self.stderr, " New path should be:"
529 print >>self.stderr, " require('%s')" % reqname
530
531 return self._search_packages_for_module(from_pkg,
532 lookfor_sections, reqname,
533 looked_in)
534 else:
535 # We weren't able to find this module, really.
536 return None
537
538 def _handle_module(self, mi):
539 if not mi:
540 return None
541
542 # we tolerate cycles in the reference graph, which means we need to
543 # populate the self.modules cache before recursing into
544 # process_module() . We must also check the cache first, so recursion
545 # can terminate.
546 if mi in self.modules:
547 return self.modules[mi]
548
549 # this creates the entry
550 new_entry = self.get_manifest_entry(mi.package.name, mi.section, mi.name)
551 # and populates the cache
552 self.modules[mi] = new_entry
553 self.process_module(mi)
554 return new_entry
555
556 def _get_module_from_package(self, pkgname, sections, modname, looked_in):
557 if pkgname not in self.pkg_cfg.packages:
558 return None
559 mi = self._find_module_in_package(pkgname, sections, modname,
560 looked_in)
561 return self._handle_module(mi)
562
563 def _get_entrypoint_from_package(self, pkgname, looked_in):
564 if pkgname not in self.pkg_cfg.packages:
565 return None
566 pkg = self.pkg_cfg.packages[pkgname]
567 main = pkg.get("main", None)
568 if not main:
569 return None
570 for js in self.parse_main(pkg.root_dir, main):
571 looked_in.append(js)
572 if os.path.exists(js):
573 section = "lib"
574 name = self.uri_name_from_path(pkg, js)
575 docs = None
576 mi = ModuleInfo(pkg, section, name, js, docs)
577 return self._handle_module(mi)
578 return None
579
580 def _search_packages_for_module(self, from_pkg, sections, reqname,
581 looked_in):
582 searchpath = [] # list of package names
583 searchpath.append(from_pkg) # search self first
584 us = self.pkg_cfg.packages[from_pkg]
585 if 'dependencies' in us:
586 # only look in dependencies
587 searchpath.extend(us['dependencies'])
588 else:
589 # they didn't declare any dependencies (or they declared an empty
590 # list, but we'll treat that as not declaring one, because it's
591 # easier), so look in all deps, sorted alphabetically, so
592 # addon-kit comes first. Note that self.deps includes all
593 # packages found by traversing the ".dependencies" lists in each
594 # package.json, starting from the main addon package, plus
595 # everything added by --extra-packages
596 searchpath.extend(sorted(self.deps))
597 for pkgname in searchpath:
598 mi = self._find_module_in_package(pkgname, sections, reqname,
599 looked_in)
600 if mi:
601 return self._handle_module(mi)
602 return None
603
604 def _find_module_in_package(self, pkgname, sections, name, looked_in):
605 # require("a/b/c") should look at ...\a\b\c.js on windows
606 filename = os.sep.join(name.split("/"))
607 # normalize filename, make sure that we do not add .js if it already has
608 # it.
609 if not filename.endswith(".js") and not filename.endswith(".json"):
610 filename += ".js"
611
612 if filename.endswith(".js"):
613 basename = filename[:-3]
614 if filename.endswith(".json"):
615 basename = filename[:-5]
616
617 pkg = self.pkg_cfg.packages[pkgname]
618 if isinstance(sections, basestring):
619 sections = [sections]
620 for section in sections:
621 for sdir in pkg.get(section, []):
622 js = os.path.join(pkg.root_dir, sdir, filename)
623 looked_in.append(js)
624 if os.path.exists(js):
625 docs = None
626 maybe_docs = os.path.join(pkg.root_dir, "docs",
627 basename+".md")
628 if section == "lib" and os.path.exists(maybe_docs):
629 docs = maybe_docs
630 return ModuleInfo(pkg, section, name, js, docs)
631 return None
632
633 def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
634 test_filter_re=None, extra_modules=[]):
635 """
636 Perform recursive dependency analysis starting from entry_point,
637 building up a manifest of modules that need to be included in the XPI.
638 Each entry will map require() names to the URL of the module that will
639 be used to satisfy that dependency. The manifest will be used by the
640 runtime's require() code.
641
642 This returns a ManifestBuilder object, with two public methods. The
643 first, get_module_entries(), returns a set of ManifestEntry objects, each
644 of which can be asked for the following:
645
646 * its contribution to the harness-options.json '.manifest'
647 * the local disk name
648 * the name in the XPI at which it should be placed
649
650 The second is get_data_entries(), which returns a set of DataEntry
651 objects, each of which has:
652
653 * local disk name
654 * name in the XPI
655
656 note: we don't build the XPI here, but our manifest is passed to the
657 code which does, so it knows what to copy into the XPI.
658 """
659
660 mxt = ManifestBuilder(target_cfg, pkg_cfg, deps, extra_modules)
661 mxt.build(scan_tests, test_filter_re)
662 return mxt
663
664
665
666 COMMENT_PREFIXES = ["//", "/*", "*", "dump("]
667
668 REQUIRE_RE = r"(?<![\'\"])require\s*\(\s*[\'\"]([^\'\"]+?)[\'\"]\s*\)"
669
670 # detect the define idiom of the form:
671 # define("module name", ["dep1", "dep2", "dep3"], function() {})
672 # by capturing the contents of the list in a group.
673 DEF_RE = re.compile(r"(require|define)\s*\(\s*([\'\"][^\'\"]+[\'\"]\s*,)?\s*\[([^\]]+)\]")
674
675 # Out of the async dependencies, do not allow quotes in them.
676 DEF_RE_ALLOWED = re.compile(r"^[\'\"][^\'\"]+[\'\"]$")
677
678 def scan_requirements_with_grep(fn, lines):
679 requires = {}
680 first_location = {}
681 for (lineno0, line) in enumerate(lines):
682 for clause in line.split(";"):
683 clause = clause.strip()
684 iscomment = False
685 for commentprefix in COMMENT_PREFIXES:
686 if clause.startswith(commentprefix):
687 iscomment = True
688 if iscomment:
689 continue
690 mo = re.finditer(REQUIRE_RE, clause)
691 if mo:
692 for mod in mo:
693 modname = mod.group(1)
694 requires[modname] = {}
695 if modname not in first_location:
696 first_location[modname] = lineno0 + 1
697
698 # define() can happen across multiple lines, so join everyone up.
699 wholeshebang = "\n".join(lines)
700 for match in DEF_RE.finditer(wholeshebang):
701 # this should net us a list of string literals separated by commas
702 for strbit in match.group(3).split(","):
703 strbit = strbit.strip()
704 # There could be a trailing comma netting us just whitespace, so
705 # filter that out. Make sure that only string values with
706 # quotes around them are allowed, and no quotes are inside
707 # the quoted value.
708 if strbit and DEF_RE_ALLOWED.match(strbit):
709 modname = strbit[1:-1]
710 if modname not in ["exports"]:
711 requires[modname] = {}
712 # joining all the lines means we lose line numbers, so we
713 # can't fill first_location[]
714
715 return requires, first_location
716
717 CHROME_ALIASES = [
718 (re.compile(r"Components\.classes"), "Cc"),
719 (re.compile(r"Components\.interfaces"), "Ci"),
720 (re.compile(r"Components\.utils"), "Cu"),
721 (re.compile(r"Components\.results"), "Cr"),
722 (re.compile(r"Components\.manager"), "Cm"),
723 ]
724 OTHER_CHROME = re.compile(r"Components\.[a-zA-Z]")
725
726 def scan_for_bad_chrome(fn, lines, stderr):
727 problems = False
728 old_chrome = set() # i.e. "Cc" when we see "Components.classes"
729 old_chrome_lines = [] # list of (lineno, line.strip()) tuples
730 for lineno,line in enumerate(lines):
731 # note: this scanner is not obligated to spot all possible forms of
732 # chrome access. The scanner is detecting voluntary requests for
733 # chrome. Runtime tools will enforce allowance or denial of access.
734 line = line.strip()
735 iscomment = False
736 for commentprefix in COMMENT_PREFIXES:
737 if line.startswith(commentprefix):
738 iscomment = True
739 break
740 if iscomment:
741 continue
742 old_chrome_in_this_line = set()
743 for (regexp,alias) in CHROME_ALIASES:
744 if regexp.search(line):
745 old_chrome_in_this_line.add(alias)
746 if not old_chrome_in_this_line:
747 if OTHER_CHROME.search(line):
748 old_chrome_in_this_line.add("components")
749 old_chrome.update(old_chrome_in_this_line)
750 if old_chrome_in_this_line:
751 old_chrome_lines.append( (lineno+1, line) )
752
753 if old_chrome:
754 print >>stderr, """
755 The following lines from file %(fn)s:
756 %(lines)s
757 use 'Components' to access chrome authority. To do so, you need to add a
758 line somewhat like the following:
759
760 const {%(needs)s} = require("chrome");
761
762 Then you can use any shortcuts to its properties that you import from the
763 'chrome' module ('Cc', 'Ci', 'Cm', 'Cr', and 'Cu' for the 'classes',
764 'interfaces', 'manager', 'results', and 'utils' properties, respectively. And
765 `components` for `Components` object itself).
766 """ % { "fn": fn, "needs": ",".join(sorted(old_chrome)),
767 "lines": "\n".join([" %3d: %s" % (lineno,line)
768 for (lineno, line) in old_chrome_lines]),
769 }
770 problems = True
771 return problems
772
773 def scan_module(fn, lines, stderr=sys.stderr):
774 filename = os.path.basename(fn)
775 requires, locations = scan_requirements_with_grep(fn, lines)
776 if filename == "cuddlefish.js":
777 # this is the loader: don't scan for chrome
778 problems = False
779 else:
780 problems = scan_for_bad_chrome(fn, lines, stderr)
781 return requires, problems, locations
782
783
784
785 if __name__ == '__main__':
786 for fn in sys.argv[1:]:
787 requires, problems, locations = scan_module(fn, open(fn).readlines())
788 print
789 print "---", fn
790 if problems:
791 print "PROBLEMS"
792 sys.exit(1)
793 print "requires: %s" % (",".join(sorted(requires.keys())))
794 print "locations: %s" % locations
795

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial