1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/dom/bindings/mozwebidlcodegen/__init__.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,568 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +# This module contains code for managing WebIDL files and bindings for
1.9 +# the build system.
1.10 +
1.11 +from __future__ import unicode_literals
1.12 +
1.13 +import errno
1.14 +import hashlib
1.15 +import json
1.16 +import logging
1.17 +import os
1.18 +
1.19 +from copy import deepcopy
1.20 +
1.21 +from mach.mixin.logging import LoggingMixin
1.22 +
1.23 +from mozbuild.base import MozbuildObject
1.24 +from mozbuild.makeutil import Makefile
1.25 +from mozbuild.pythonutil import iter_modules_in_path
1.26 +from mozbuild.util import FileAvoidWrite
1.27 +
1.28 +import mozpack.path as mozpath
1.29 +
1.30 +# There are various imports in this file in functions to avoid adding
1.31 +# dependencies to config.status. See bug 949875.
1.32 +
1.33 +
1.34 +class BuildResult(object):
1.35 + """Represents the result of processing WebIDL files.
1.36 +
1.37 + This holds a summary of output file generation during code generation.
1.38 + """
1.39 +
1.40 + def __init__(self):
1.41 + # The .webidl files that had their outputs regenerated.
1.42 + self.inputs = set()
1.43 +
1.44 + # The output files that were created.
1.45 + self.created = set()
1.46 +
1.47 + # The output files that changed.
1.48 + self.updated = set()
1.49 +
1.50 + # The output files that didn't change.
1.51 + self.unchanged = set()
1.52 +
1.53 +
1.54 +class WebIDLCodegenManagerState(dict):
1.55 + """Holds state for the WebIDL code generation manager.
1.56 +
1.57 + State is currently just an extended dict. The internal implementation of
1.58 + state should be considered a black box to everyone except
1.59 + WebIDLCodegenManager. But we'll still document it.
1.60 +
1.61 + Fields:
1.62 +
1.63 + version
1.64 + The integer version of the format. This is to detect incompatible
1.65 + changes between state. It should be bumped whenever the format
1.66 + changes or semantics change.
1.67 +
1.68 + webidls
1.69 + A dictionary holding information about every known WebIDL input.
1.70 + Keys are the basenames of input WebIDL files. Values are dicts of
1.71 + metadata. Keys in those dicts are:
1.72 +
1.73 + * filename - The full path to the input filename.
1.74 + * inputs - A set of full paths to other webidl files this webidl
1.75 + depends on.
1.76 + * outputs - Set of full output paths that are created/derived from
1.77 + this file.
1.78 + * sha1 - The hexidecimal SHA-1 of the input filename from the last
1.79 + processing time.
1.80 +
1.81 + global_inputs
1.82 + A dictionary defining files that influence all processing. Keys
1.83 + are full filenames. Values are hexidecimal SHA-1 from the last
1.84 + processing time.
1.85 + """
1.86 +
1.87 + VERSION = 1
1.88 +
1.89 + def __init__(self, fh=None):
1.90 + self['version'] = self.VERSION
1.91 + self['webidls'] = {}
1.92 + self['global_depends'] = {}
1.93 +
1.94 + if not fh:
1.95 + return
1.96 +
1.97 + state = json.load(fh)
1.98 + if state['version'] != self.VERSION:
1.99 + raise Exception('Unknown state version: %s' % state['version'])
1.100 +
1.101 + self['version'] = state['version']
1.102 + self['global_depends'] = state['global_depends']
1.103 +
1.104 + for k, v in state['webidls'].items():
1.105 + self['webidls'][k] = v
1.106 +
1.107 + # Sets are converted to lists for serialization because JSON
1.108 + # doesn't support sets.
1.109 + self['webidls'][k]['inputs'] = set(v['inputs'])
1.110 + self['webidls'][k]['outputs'] = set(v['outputs'])
1.111 +
1.112 + def dump(self, fh):
1.113 + """Dump serialized state to a file handle."""
1.114 + normalized = deepcopy(self)
1.115 +
1.116 + for k, v in self['webidls'].items():
1.117 + # Convert sets to lists because JSON doesn't support sets.
1.118 + normalized['webidls'][k]['outputs'] = sorted(v['outputs'])
1.119 + normalized['webidls'][k]['inputs'] = sorted(v['inputs'])
1.120 +
1.121 + json.dump(normalized, fh, sort_keys=True)
1.122 +
1.123 +
1.124 +class WebIDLCodegenManager(LoggingMixin):
1.125 + """Manages all code generation around WebIDL.
1.126 +
1.127 + To facilitate testing, this object is meant to be generic and reusable.
1.128 + Paths, etc should be parameters and not hardcoded.
1.129 + """
1.130 +
1.131 + # Global parser derived declaration files.
1.132 + GLOBAL_DECLARE_FILES = {
1.133 + 'GeneratedAtomList.h',
1.134 + 'PrototypeList.h',
1.135 + 'RegisterBindings.h',
1.136 + 'UnionConversions.h',
1.137 + 'UnionTypes.h',
1.138 + }
1.139 +
1.140 + # Global parser derived definition files.
1.141 + GLOBAL_DEFINE_FILES = {
1.142 + 'RegisterBindings.cpp',
1.143 + 'UnionTypes.cpp',
1.144 + 'PrototypeList.cpp',
1.145 + }
1.146 +
1.147 + def __init__(self, config_path, inputs, exported_header_dir,
1.148 + codegen_dir, state_path, cache_dir=None, make_deps_path=None,
1.149 + make_deps_target=None):
1.150 + """Create an instance that manages WebIDLs in the build system.
1.151 +
1.152 + config_path refers to a WebIDL config file (e.g. Bindings.conf).
1.153 + inputs is a 4-tuple describing the input .webidl files and how to
1.154 + process them. Members are:
1.155 + (set(.webidl files), set(basenames of exported files),
1.156 + set(basenames of generated events files),
1.157 + set(example interface names))
1.158 +
1.159 + exported_header_dir and codegen_dir are directories where generated
1.160 + files will be written to.
1.161 + state_path is the path to a file that will receive JSON state from our
1.162 + actions.
1.163 + make_deps_path is the path to a make dependency file that we can
1.164 + optionally write.
1.165 + make_deps_target is the target that receives the make dependencies. It
1.166 + must be defined if using make_deps_path.
1.167 + """
1.168 + self.populate_logger()
1.169 +
1.170 + input_paths, exported_stems, generated_events_stems, example_interfaces = inputs
1.171 +
1.172 + self._config_path = config_path
1.173 + self._input_paths = set(input_paths)
1.174 + self._exported_stems = set(exported_stems)
1.175 + self._generated_events_stems = set(generated_events_stems)
1.176 + self._example_interfaces = set(example_interfaces)
1.177 + self._exported_header_dir = exported_header_dir
1.178 + self._codegen_dir = codegen_dir
1.179 + self._state_path = state_path
1.180 + self._cache_dir = cache_dir
1.181 + self._make_deps_path = make_deps_path
1.182 + self._make_deps_target = make_deps_target
1.183 +
1.184 + if (make_deps_path and not make_deps_target) or (not make_deps_path and
1.185 + make_deps_target):
1.186 + raise Exception('Must define both make_deps_path and make_deps_target '
1.187 + 'if one is defined.')
1.188 +
1.189 + self._parser_results = None
1.190 + self._config = None
1.191 + self._state = WebIDLCodegenManagerState()
1.192 +
1.193 + if os.path.exists(state_path):
1.194 + with open(state_path, 'rb') as fh:
1.195 + try:
1.196 + self._state = WebIDLCodegenManagerState(fh=fh)
1.197 + except Exception as e:
1.198 + self.log(logging.WARN, 'webidl_bad_state', {'msg': str(e)},
1.199 + 'Bad WebIDL state: {msg}')
1.200 +
1.201 + @property
1.202 + def config(self):
1.203 + if not self._config:
1.204 + self._parse_webidl()
1.205 +
1.206 + return self._config
1.207 +
1.208 + def generate_build_files(self):
1.209 + """Generate files required for the build.
1.210 +
1.211 + This function is in charge of generating all the .h/.cpp files derived
1.212 + from input .webidl files. Please note that there are build actions
1.213 + required to produce .webidl files and these build actions are
1.214 + explicitly not captured here: this function assumes all .webidl files
1.215 + are present and up to date.
1.216 +
1.217 + This routine is called as part of the build to ensure files that need
1.218 + to exist are present and up to date. This routine may not be called if
1.219 + the build dependencies (generated as a result of calling this the first
1.220 + time) say everything is up to date.
1.221 +
1.222 + Because reprocessing outputs for every .webidl on every invocation
1.223 + is expensive, we only regenerate the minimal set of files on every
1.224 + invocation. The rules for deciding what needs done are roughly as
1.225 + follows:
1.226 +
1.227 + 1. If any .webidl changes, reparse all .webidl files and regenerate
1.228 + the global derived files. Only regenerate output files (.h/.cpp)
1.229 + impacted by the modified .webidl files.
1.230 + 2. If an non-.webidl dependency (Python files, config file) changes,
1.231 + assume everything is out of date and regenerate the world. This
1.232 + is because changes in those could globally impact every output
1.233 + file.
1.234 + 3. If an output file is missing, ensure it is present by performing
1.235 + necessary regeneration.
1.236 + """
1.237 + # Despite #1 above, we assume the build system is smart enough to not
1.238 + # invoke us if nothing has changed. Therefore, any invocation means
1.239 + # something has changed. And, if anything has changed, we need to
1.240 + # parse the WebIDL.
1.241 + self._parse_webidl()
1.242 +
1.243 + result = BuildResult()
1.244 +
1.245 + # If we parse, we always update globals - they are cheap and it is
1.246 + # easier that way.
1.247 + created, updated, unchanged = self._write_global_derived()
1.248 + result.created |= created
1.249 + result.updated |= updated
1.250 + result.unchanged |= unchanged
1.251 +
1.252 + # If any of the extra dependencies changed, regenerate the world.
1.253 + global_changed, global_hashes = self._global_dependencies_changed()
1.254 + if global_changed:
1.255 + # Make a copy because we may modify.
1.256 + changed_inputs = set(self._input_paths)
1.257 + else:
1.258 + changed_inputs = self._compute_changed_inputs()
1.259 +
1.260 + self._state['global_depends'] = global_hashes
1.261 +
1.262 + # Generate bindings from .webidl files.
1.263 + for filename in sorted(changed_inputs):
1.264 + basename = mozpath.basename(filename)
1.265 + result.inputs.add(filename)
1.266 + written, deps = self._generate_build_files_for_webidl(filename)
1.267 + result.created |= written[0]
1.268 + result.updated |= written[1]
1.269 + result.unchanged |= written[2]
1.270 +
1.271 + self._state['webidls'][basename] = dict(
1.272 + filename=filename,
1.273 + outputs=written[0] | written[1] | written[2],
1.274 + inputs=set(deps),
1.275 + sha1=self._input_hashes[filename],
1.276 + )
1.277 +
1.278 + # Process some special interfaces required for testing.
1.279 + for interface in self._example_interfaces:
1.280 + written = self.generate_example_files(interface)
1.281 + result.created |= written[0]
1.282 + result.updated |= written[1]
1.283 + result.unchanged |= written[2]
1.284 +
1.285 + # Generate a make dependency file.
1.286 + if self._make_deps_path:
1.287 + mk = Makefile()
1.288 + codegen_rule = mk.create_rule([self._make_deps_target])
1.289 + codegen_rule.add_dependencies(global_hashes.keys())
1.290 + codegen_rule.add_dependencies(self._input_paths)
1.291 +
1.292 + with FileAvoidWrite(self._make_deps_path) as fh:
1.293 + mk.dump(fh)
1.294 +
1.295 + self._save_state()
1.296 +
1.297 + return result
1.298 +
1.299 + def generate_example_files(self, interface):
1.300 + """Generates example files for a given interface."""
1.301 + from Codegen import CGExampleRoot
1.302 +
1.303 + root = CGExampleRoot(self.config, interface)
1.304 +
1.305 + return self._maybe_write_codegen(root, *self._example_paths(interface))
1.306 +
1.307 + def _parse_webidl(self):
1.308 + import WebIDL
1.309 + from Configuration import Configuration
1.310 +
1.311 + self.log(logging.INFO, 'webidl_parse',
1.312 + {'count': len(self._input_paths)},
1.313 + 'Parsing {count} WebIDL files.')
1.314 +
1.315 + hashes = {}
1.316 + parser = WebIDL.Parser(self._cache_dir)
1.317 +
1.318 + for path in sorted(self._input_paths):
1.319 + with open(path, 'rb') as fh:
1.320 + data = fh.read()
1.321 + hashes[path] = hashlib.sha1(data).hexdigest()
1.322 + parser.parse(data, path)
1.323 +
1.324 + self._parser_results = parser.finish()
1.325 + self._config = Configuration(self._config_path, self._parser_results)
1.326 + self._input_hashes = hashes
1.327 +
1.328 + def _write_global_derived(self):
1.329 + from Codegen import GlobalGenRoots
1.330 +
1.331 + things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES]
1.332 + things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES)
1.333 +
1.334 + result = (set(), set(), set())
1.335 +
1.336 + for what, filename in things:
1.337 + stem = mozpath.splitext(filename)[0]
1.338 + root = getattr(GlobalGenRoots, stem)(self._config)
1.339 +
1.340 + if what == 'declare':
1.341 + code = root.declare()
1.342 + output_root = self._exported_header_dir
1.343 + elif what == 'define':
1.344 + code = root.define()
1.345 + output_root = self._codegen_dir
1.346 + else:
1.347 + raise Exception('Unknown global gen type: %s' % what)
1.348 +
1.349 + output_path = mozpath.join(output_root, filename)
1.350 + self._maybe_write_file(output_path, code, result)
1.351 +
1.352 + return result
1.353 +
1.354 + def _compute_changed_inputs(self):
1.355 + """Compute the set of input files that need to be regenerated."""
1.356 + changed_inputs = set()
1.357 + expected_outputs = self.expected_build_output_files()
1.358 +
1.359 + # Look for missing output files.
1.360 + if any(not os.path.exists(f) for f in expected_outputs):
1.361 + # FUTURE Bug 940469 Only regenerate minimum set.
1.362 + changed_inputs |= self._input_paths
1.363 +
1.364 + # That's it for examining output files. We /could/ examine SHA-1's of
1.365 + # output files from a previous run to detect modifications. But that's
1.366 + # a lot of extra work and most build systems don't do that anyway.
1.367 +
1.368 + # Now we move on to the input files.
1.369 + old_hashes = {v['filename']: v['sha1']
1.370 + for v in self._state['webidls'].values()}
1.371 +
1.372 + old_filenames = set(old_hashes.keys())
1.373 + new_filenames = self._input_paths
1.374 +
1.375 + # If an old file has disappeared or a new file has arrived, mark
1.376 + # it.
1.377 + changed_inputs |= old_filenames ^ new_filenames
1.378 +
1.379 + # For the files in common between runs, compare content. If the file
1.380 + # has changed, mark it. We don't need to perform mtime comparisons
1.381 + # because content is a stronger validator.
1.382 + for filename in old_filenames & new_filenames:
1.383 + if old_hashes[filename] != self._input_hashes[filename]:
1.384 + changed_inputs.add(filename)
1.385 +
1.386 + # We've now populated the base set of inputs that have changed.
1.387 +
1.388 + # Inherit dependencies from previous run. The full set of dependencies
1.389 + # is associated with each record, so we don't need to perform any fancy
1.390 + # graph traversal.
1.391 + for v in self._state['webidls'].values():
1.392 + if any(dep for dep in v['inputs'] if dep in changed_inputs):
1.393 + changed_inputs.add(v['filename'])
1.394 +
1.395 + # Only use paths that are known to our current state.
1.396 + # This filters out files that were deleted or changed type (e.g. from
1.397 + # static to preprocessed).
1.398 + return changed_inputs & self._input_paths
1.399 +
1.400 + def _binding_info(self, p):
1.401 + """Compute binding metadata for an input path.
1.402 +
1.403 + Returns a tuple of:
1.404 +
1.405 + (stem, binding_stem, is_event, output_files)
1.406 +
1.407 + output_files is itself a tuple. The first two items are the binding
1.408 + header and C++ paths, respectively. The 2nd pair are the event header
1.409 + and C++ paths or None if this isn't an event binding.
1.410 + """
1.411 + basename = mozpath.basename(p)
1.412 + stem = mozpath.splitext(basename)[0]
1.413 + binding_stem = '%sBinding' % stem
1.414 +
1.415 + if stem in self._exported_stems:
1.416 + header_dir = self._exported_header_dir
1.417 + else:
1.418 + header_dir = self._codegen_dir
1.419 +
1.420 + is_event = stem in self._generated_events_stems
1.421 +
1.422 + files = (
1.423 + mozpath.join(header_dir, '%s.h' % binding_stem),
1.424 + mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem),
1.425 + mozpath.join(header_dir, '%s.h' % stem) if is_event else None,
1.426 + mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None,
1.427 + )
1.428 +
1.429 + return stem, binding_stem, is_event, header_dir, files
1.430 +
1.431 + def _example_paths(self, interface):
1.432 + return (
1.433 + mozpath.join(self._codegen_dir, '%s-example.h' % interface),
1.434 + mozpath.join(self._codegen_dir, '%s-example.cpp' % interface))
1.435 +
1.436 + def expected_build_output_files(self):
1.437 + """Obtain the set of files generate_build_files() should write."""
1.438 + paths = set()
1.439 +
1.440 + # Account for global generation.
1.441 + for p in self.GLOBAL_DECLARE_FILES:
1.442 + paths.add(mozpath.join(self._exported_header_dir, p))
1.443 + for p in self.GLOBAL_DEFINE_FILES:
1.444 + paths.add(mozpath.join(self._codegen_dir, p))
1.445 +
1.446 + for p in self._input_paths:
1.447 + stem, binding_stem, is_event, header_dir, files = self._binding_info(p)
1.448 + paths |= {f for f in files if f}
1.449 +
1.450 + for interface in self._example_interfaces:
1.451 + for p in self._example_paths(interface):
1.452 + paths.add(p)
1.453 +
1.454 + return paths
1.455 +
1.456 + def _generate_build_files_for_webidl(self, filename):
1.457 + from Codegen import (
1.458 + CGBindingRoot,
1.459 + CGEventRoot,
1.460 + )
1.461 +
1.462 + self.log(logging.INFO, 'webidl_generate_build_for_input',
1.463 + {'filename': filename},
1.464 + 'Generating WebIDL files derived from {filename}')
1.465 +
1.466 + stem, binding_stem, is_event, header_dir, files = self._binding_info(filename)
1.467 + root = CGBindingRoot(self._config, binding_stem, filename)
1.468 +
1.469 + result = self._maybe_write_codegen(root, files[0], files[1])
1.470 +
1.471 + if is_event:
1.472 + generated_event = CGEventRoot(self._config, stem)
1.473 + result = self._maybe_write_codegen(generated_event, files[2],
1.474 + files[3], result)
1.475 +
1.476 + return result, root.deps()
1.477 +
1.478 + def _global_dependencies_changed(self):
1.479 + """Determine whether the global dependencies have changed."""
1.480 + current_files = set(iter_modules_in_path(mozpath.dirname(__file__)))
1.481 +
1.482 + # We need to catch other .py files from /dom/bindings. We assume these
1.483 + # are in the same directory as the config file.
1.484 + current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path)))
1.485 +
1.486 + current_files.add(self._config_path)
1.487 +
1.488 + current_hashes = {}
1.489 + for f in current_files:
1.490 + # This will fail if the file doesn't exist. If a current global
1.491 + # dependency doesn't exist, something else is wrong.
1.492 + with open(f, 'rb') as fh:
1.493 + current_hashes[f] = hashlib.sha1(fh.read()).hexdigest()
1.494 +
1.495 + # The set of files has changed.
1.496 + if current_files ^ set(self._state['global_depends'].keys()):
1.497 + return True, current_hashes
1.498 +
1.499 + # Compare hashes.
1.500 + for f, sha1 in current_hashes.items():
1.501 + if sha1 != self._state['global_depends'][f]:
1.502 + return True, current_hashes
1.503 +
1.504 + return False, current_hashes
1.505 +
1.506 + def _save_state(self):
1.507 + with open(self._state_path, 'wb') as fh:
1.508 + self._state.dump(fh)
1.509 +
1.510 + def _maybe_write_codegen(self, obj, declare_path, define_path, result=None):
1.511 + assert declare_path and define_path
1.512 + if not result:
1.513 + result = (set(), set(), set())
1.514 +
1.515 + self._maybe_write_file(declare_path, obj.declare(), result)
1.516 + self._maybe_write_file(define_path, obj.define(), result)
1.517 +
1.518 + return result
1.519 +
1.520 + def _maybe_write_file(self, path, content, result):
1.521 + fh = FileAvoidWrite(path)
1.522 + fh.write(content)
1.523 + existed, updated = fh.close()
1.524 +
1.525 + if not existed:
1.526 + result[0].add(path)
1.527 + elif updated:
1.528 + result[1].add(path)
1.529 + else:
1.530 + result[2].add(path)
1.531 +
1.532 +
1.533 +def create_build_system_manager(topsrcdir, topobjdir, dist_dir):
1.534 + """Create a WebIDLCodegenManager for use by the build system."""
1.535 + src_dir = os.path.join(topsrcdir, 'dom', 'bindings')
1.536 + obj_dir = os.path.join(topobjdir, 'dom', 'bindings')
1.537 +
1.538 + with open(os.path.join(obj_dir, 'file-lists.json'), 'rb') as fh:
1.539 + files = json.load(fh)
1.540 +
1.541 + inputs = (files['webidls'], files['exported_stems'],
1.542 + files['generated_events_stems'], files['example_interfaces'])
1.543 +
1.544 + cache_dir = os.path.join(obj_dir, '_cache')
1.545 + try:
1.546 + os.makedirs(cache_dir)
1.547 + except OSError as e:
1.548 + if e.errno != errno.EEXIST:
1.549 + raise
1.550 +
1.551 + return WebIDLCodegenManager(
1.552 + os.path.join(src_dir, 'Bindings.conf'),
1.553 + inputs,
1.554 + os.path.join(dist_dir, 'include', 'mozilla', 'dom'),
1.555 + obj_dir,
1.556 + os.path.join(obj_dir, 'codegen.json'),
1.557 + cache_dir=cache_dir,
1.558 + # The make rules include a codegen.pp file containing dependencies.
1.559 + make_deps_path=os.path.join(obj_dir, 'codegen.pp'),
1.560 + make_deps_target='codegen.pp',
1.561 + )
1.562 +
1.563 +
1.564 +class BuildSystemWebIDL(MozbuildObject):
1.565 + @property
1.566 + def manager(self):
1.567 + if not hasattr(self, '_webidl_manager'):
1.568 + self._webidl_manager = create_build_system_manager(
1.569 + self.topsrcdir, self.topobjdir, self.distdir)
1.570 +
1.571 + return self._webidl_manager
