michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: # This module contains code for managing WebIDL files and bindings for
michael@0: # the build system.
michael@0: 
michael@0: from __future__ import unicode_literals
michael@0: 
michael@0: import errno
michael@0: import hashlib
michael@0: import json
michael@0: import logging
michael@0: import os
michael@0: 
michael@0: from copy import deepcopy
michael@0: 
michael@0: from mach.mixin.logging import LoggingMixin
michael@0: 
michael@0: from mozbuild.base import MozbuildObject
michael@0: from mozbuild.makeutil import Makefile
michael@0: from mozbuild.pythonutil import iter_modules_in_path
michael@0: from mozbuild.util import FileAvoidWrite
michael@0: 
michael@0: import mozpack.path as mozpath
michael@0: 
michael@0: # There are various imports in this file in functions to avoid adding
michael@0: # dependencies to config.status. See bug 949875.
michael@0: 
michael@0: 
michael@0: class BuildResult(object):
michael@0:     """Represents the result of processing WebIDL files.
michael@0: 
michael@0:     This holds a summary of output file generation during code generation.
michael@0:     """
michael@0: 
michael@0:     def __init__(self):
michael@0:         # The .webidl files that had their outputs regenerated.
michael@0:         self.inputs = set()
michael@0: 
michael@0:         # The output files that were created.
michael@0:         self.created = set()
michael@0: 
michael@0:         # The output files that changed.
michael@0:         self.updated = set()
michael@0: 
michael@0:         # The output files that didn't change.
michael@0:         self.unchanged = set()
michael@0: 
michael@0: 
michael@0: class WebIDLCodegenManagerState(dict):
michael@0:     """Holds state for the WebIDL code generation manager.
michael@0: 
michael@0:     State is currently just an extended dict. The internal implementation of
michael@0:     state should be considered a black box to everyone except
michael@0:     WebIDLCodegenManager. But we'll still document it.
michael@0: 
michael@0:     Fields:
michael@0: 
michael@0:     version
michael@0:        The integer version of the format. This is to detect incompatible
michael@0:        changes between state. It should be bumped whenever the format
michael@0:        changes or semantics change.
michael@0: 
michael@0:     webidls
michael@0:        A dictionary holding information about every known WebIDL input.
michael@0:        Keys are the basenames of input WebIDL files. Values are dicts of
michael@0:        metadata. Keys in those dicts are:
michael@0: 
michael@0:        * filename - The full path to the input filename.
michael@0:        * inputs - A set of full paths to other webidl files this webidl
michael@0:          depends on.
michael@0:        * outputs - Set of full output paths that are created/derived from
michael@0:          this file.
michael@0:        * sha1 - The hexidecimal SHA-1 of the input filename from the last
michael@0:          processing time.
michael@0: 
michael@0:     global_inputs
michael@0:        A dictionary defining files that influence all processing. Keys
michael@0:        are full filenames. Values are hexidecimal SHA-1 from the last
michael@0:        processing time.
michael@0:     """
michael@0: 
michael@0:     VERSION = 1
michael@0: 
michael@0:     def __init__(self, fh=None):
michael@0:         self['version'] = self.VERSION
michael@0:         self['webidls'] = {}
michael@0:         self['global_depends'] = {}
michael@0: 
michael@0:         if not fh:
michael@0:             return
michael@0: 
michael@0:         state = json.load(fh)
michael@0:         if state['version'] != self.VERSION:
michael@0:             raise Exception('Unknown state version: %s' % state['version'])
michael@0: 
michael@0:         self['version'] = state['version']
michael@0:         self['global_depends'] = state['global_depends']
michael@0: 
michael@0:         for k, v in state['webidls'].items():
michael@0:             self['webidls'][k] = v
michael@0: 
michael@0:             # Sets are converted to lists for serialization because JSON
michael@0:             # doesn't support sets.
michael@0:             self['webidls'][k]['inputs'] = set(v['inputs'])
michael@0:             self['webidls'][k]['outputs'] = set(v['outputs'])
michael@0: 
michael@0:     def dump(self, fh):
michael@0:         """Dump serialized state to a file handle."""
michael@0:         normalized = deepcopy(self)
michael@0: 
michael@0:         for k, v in self['webidls'].items():
michael@0:             # Convert sets to lists because JSON doesn't support sets.
michael@0:             normalized['webidls'][k]['outputs'] = sorted(v['outputs'])
michael@0:             normalized['webidls'][k]['inputs'] = sorted(v['inputs'])
michael@0: 
michael@0:         json.dump(normalized, fh, sort_keys=True)
michael@0: 
michael@0: 
michael@0: class WebIDLCodegenManager(LoggingMixin):
michael@0:     """Manages all code generation around WebIDL.
michael@0: 
michael@0:     To facilitate testing, this object is meant to be generic and reusable.
michael@0:     Paths, etc should be parameters and not hardcoded.
michael@0:     """
michael@0: 
michael@0:     # Global parser derived declaration files.
michael@0:     GLOBAL_DECLARE_FILES = {
michael@0:         'GeneratedAtomList.h',
michael@0:         'PrototypeList.h',
michael@0:         'RegisterBindings.h',
michael@0:         'UnionConversions.h',
michael@0:         'UnionTypes.h',
michael@0:     }
michael@0: 
michael@0:     # Global parser derived definition files.
michael@0:     GLOBAL_DEFINE_FILES = {
michael@0:         'RegisterBindings.cpp',
michael@0:         'UnionTypes.cpp',
michael@0:         'PrototypeList.cpp',
michael@0:     }
michael@0: 
michael@0:     def __init__(self, config_path, inputs, exported_header_dir,
michael@0:         codegen_dir, state_path, cache_dir=None, make_deps_path=None,
michael@0:         make_deps_target=None):
michael@0:         """Create an instance that manages WebIDLs in the build system.
michael@0: 
michael@0:         config_path refers to a WebIDL config file (e.g. Bindings.conf).
michael@0:         inputs is a 4-tuple describing the input .webidl files and how to
michael@0:         process them. Members are:
michael@0:             (set(.webidl files), set(basenames of exported files),
michael@0:                 set(basenames of generated events files),
michael@0:                 set(example interface names))
michael@0: 
michael@0:         exported_header_dir and codegen_dir are directories where generated
michael@0:         files will be written to.
michael@0:         state_path is the path to a file that will receive JSON state from our
michael@0:         actions.
michael@0:         make_deps_path is the path to a make dependency file that we can
michael@0:         optionally write.
michael@0:         make_deps_target is the target that receives the make dependencies. It
michael@0:         must be defined if using make_deps_path.
michael@0:         """
michael@0:         self.populate_logger()
michael@0: 
michael@0:         input_paths, exported_stems, generated_events_stems, example_interfaces = inputs
michael@0: 
michael@0:         self._config_path = config_path
michael@0:         self._input_paths = set(input_paths)
michael@0:         self._exported_stems = set(exported_stems)
michael@0:         self._generated_events_stems = set(generated_events_stems)
michael@0:         self._example_interfaces = set(example_interfaces)
michael@0:         self._exported_header_dir = exported_header_dir
michael@0:         self._codegen_dir = codegen_dir
michael@0:         self._state_path = state_path
michael@0:         self._cache_dir = cache_dir
michael@0:         self._make_deps_path = make_deps_path
michael@0:         self._make_deps_target = make_deps_target
michael@0: 
michael@0:         if (make_deps_path and not make_deps_target) or (not make_deps_path and
michael@0:             make_deps_target):
michael@0:             raise Exception('Must define both make_deps_path and make_deps_target '
michael@0:                 'if one is defined.')
michael@0: 
michael@0:         self._parser_results = None
michael@0:         self._config = None
michael@0:         self._state = WebIDLCodegenManagerState()
michael@0: 
michael@0:         if os.path.exists(state_path):
michael@0:             with open(state_path, 'rb') as fh:
michael@0:                 try:
michael@0:                     self._state = WebIDLCodegenManagerState(fh=fh)
michael@0:                 except Exception as e:
michael@0:                     self.log(logging.WARN, 'webidl_bad_state', {'msg': str(e)},
michael@0:                         'Bad WebIDL state: {msg}')
michael@0: 
michael@0:     @property
michael@0:     def config(self):
michael@0:         if not self._config:
michael@0:             self._parse_webidl()
michael@0: 
michael@0:         return self._config
michael@0: 
michael@0:     def generate_build_files(self):
michael@0:         """Generate files required for the build.
michael@0: 
michael@0:         This function is in charge of generating all the .h/.cpp files derived
michael@0:         from input .webidl files. Please note that there are build actions
michael@0:         required to produce .webidl files and these build actions are
michael@0:         explicitly not captured here: this function assumes all .webidl files
michael@0:         are present and up to date.
michael@0: 
michael@0:         This routine is called as part of the build to ensure files that need
michael@0:         to exist are present and up to date. This routine may not be called if
michael@0:         the build dependencies (generated as a result of calling this the first
michael@0:         time) say everything is up to date.
michael@0: 
michael@0:         Because reprocessing outputs for every .webidl on every invocation
michael@0:         is expensive, we only regenerate the minimal set of files on every
michael@0:         invocation. The rules for deciding what needs done are roughly as
michael@0:         follows:
michael@0: 
michael@0:         1. If any .webidl changes, reparse all .webidl files and regenerate
michael@0:            the global derived files. Only regenerate output files (.h/.cpp)
michael@0:            impacted by the modified .webidl files.
michael@0:         2. If an non-.webidl dependency (Python files, config file) changes,
michael@0:            assume everything is out of date and regenerate the world. This
michael@0:            is because changes in those could globally impact every output
michael@0:            file.
michael@0:         3. If an output file is missing, ensure it is present by performing
michael@0:            necessary regeneration.
michael@0:         """
michael@0:         # Despite #1 above, we assume the build system is smart enough to not
michael@0:         # invoke us if nothing has changed. Therefore, any invocation means
michael@0:         # something has changed. And, if anything has changed, we need to
michael@0:         # parse the WebIDL.
michael@0:         self._parse_webidl()
michael@0: 
michael@0:         result = BuildResult()
michael@0: 
michael@0:         # If we parse, we always update globals - they are cheap and it is
michael@0:         # easier that way.
michael@0:         created, updated, unchanged = self._write_global_derived()
michael@0:         result.created |= created
michael@0:         result.updated |= updated
michael@0:         result.unchanged |= unchanged
michael@0: 
michael@0:         # If any of the extra dependencies changed, regenerate the world.
michael@0:         global_changed, global_hashes = self._global_dependencies_changed()
michael@0:         if global_changed:
michael@0:             # Make a copy because we may modify.
michael@0:             changed_inputs = set(self._input_paths)
michael@0:         else:
michael@0:             changed_inputs = self._compute_changed_inputs()
michael@0: 
michael@0:         self._state['global_depends'] = global_hashes
michael@0: 
michael@0:         # Generate bindings from .webidl files.
michael@0:         for filename in sorted(changed_inputs):
michael@0:             basename = mozpath.basename(filename)
michael@0:             result.inputs.add(filename)
michael@0:             written, deps = self._generate_build_files_for_webidl(filename)
michael@0:             result.created |= written[0]
michael@0:             result.updated |= written[1]
michael@0:             result.unchanged |= written[2]
michael@0: 
michael@0:             self._state['webidls'][basename] = dict(
michael@0:                 filename=filename,
michael@0:                 outputs=written[0] | written[1] | written[2],
michael@0:                 inputs=set(deps),
michael@0:                 sha1=self._input_hashes[filename],
michael@0:             )
michael@0: 
michael@0:         # Process some special interfaces required for testing.
michael@0:         for interface in self._example_interfaces:
michael@0:             written = self.generate_example_files(interface)
michael@0:             result.created |= written[0]
michael@0:             result.updated |= written[1]
michael@0:             result.unchanged |= written[2]
michael@0: 
michael@0:         # Generate a make dependency file.
michael@0:         if self._make_deps_path:
michael@0:             mk = Makefile()
michael@0:             codegen_rule = mk.create_rule([self._make_deps_target])
michael@0:             codegen_rule.add_dependencies(global_hashes.keys())
michael@0:             codegen_rule.add_dependencies(self._input_paths)
michael@0: 
michael@0:             with FileAvoidWrite(self._make_deps_path) as fh:
michael@0:                 mk.dump(fh)
michael@0: 
michael@0:         self._save_state()
michael@0: 
michael@0:         return result
michael@0: 
michael@0:     def generate_example_files(self, interface):
michael@0:         """Generates example files for a given interface."""
michael@0:         from Codegen import CGExampleRoot
michael@0: 
michael@0:         root = CGExampleRoot(self.config, interface)
michael@0: 
michael@0:         return self._maybe_write_codegen(root, *self._example_paths(interface))
michael@0: 
michael@0:     def _parse_webidl(self):
michael@0:         import WebIDL
michael@0:         from Configuration import Configuration
michael@0: 
michael@0:         self.log(logging.INFO, 'webidl_parse',
michael@0:             {'count': len(self._input_paths)},
michael@0:             'Parsing {count} WebIDL files.')
michael@0: 
michael@0:         hashes = {}
michael@0:         parser = WebIDL.Parser(self._cache_dir)
michael@0: 
michael@0:         for path in sorted(self._input_paths):
michael@0:             with open(path, 'rb') as fh:
michael@0:                 data = fh.read()
michael@0:                 hashes[path] = hashlib.sha1(data).hexdigest()
michael@0:                 parser.parse(data, path)
michael@0: 
michael@0:         self._parser_results = parser.finish()
michael@0:         self._config = Configuration(self._config_path, self._parser_results)
michael@0:         self._input_hashes = hashes
michael@0: 
michael@0:     def _write_global_derived(self):
michael@0:         from Codegen import GlobalGenRoots
michael@0: 
michael@0:         things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES]
michael@0:         things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES)
michael@0: 
michael@0:         result = (set(), set(), set())
michael@0: 
michael@0:         for what, filename in things:
michael@0:             stem = mozpath.splitext(filename)[0]
michael@0:             root = getattr(GlobalGenRoots, stem)(self._config)
michael@0: 
michael@0:             if what == 'declare':
michael@0:                 code = root.declare()
michael@0:                 output_root = self._exported_header_dir
michael@0:             elif what == 'define':
michael@0:                 code = root.define()
michael@0:                 output_root = self._codegen_dir
michael@0:             else:
michael@0:                 raise Exception('Unknown global gen type: %s' % what)
michael@0: 
michael@0:             output_path = mozpath.join(output_root, filename)
michael@0:             self._maybe_write_file(output_path, code, result)
michael@0: 
michael@0:         return result
michael@0: 
michael@0:     def _compute_changed_inputs(self):
michael@0:         """Compute the set of input files that need to be regenerated."""
michael@0:         changed_inputs = set()
michael@0:         expected_outputs = self.expected_build_output_files()
michael@0: 
michael@0:         # Look for missing output files.
michael@0:         if any(not os.path.exists(f) for f in expected_outputs):
michael@0:             # FUTURE Bug 940469 Only regenerate minimum set.
michael@0:             changed_inputs |= self._input_paths
michael@0: 
michael@0:         # That's it for examining output files. We /could/ examine SHA-1's of
michael@0:         # output files from a previous run to detect modifications. But that's
michael@0:         # a lot of extra work and most build systems don't do that anyway.
michael@0: 
michael@0:         # Now we move on to the input files.
michael@0:         old_hashes = {v['filename']: v['sha1']
michael@0:             for v in self._state['webidls'].values()}
michael@0: 
michael@0:         old_filenames = set(old_hashes.keys())
michael@0:         new_filenames = self._input_paths
michael@0: 
michael@0:         # If an old file has disappeared or a new file has arrived, mark
michael@0:         # it.
michael@0:         changed_inputs |= old_filenames ^ new_filenames
michael@0: 
michael@0:         # For the files in common between runs, compare content. If the file
michael@0:         # has changed, mark it. We don't need to perform mtime comparisons
michael@0:         # because content is a stronger validator.
michael@0:         for filename in old_filenames & new_filenames:
michael@0:             if old_hashes[filename] != self._input_hashes[filename]:
michael@0:                 changed_inputs.add(filename)
michael@0: 
michael@0:         # We've now populated the base set of inputs that have changed.
michael@0: 
michael@0:         # Inherit dependencies from previous run. The full set of dependencies
michael@0:         # is associated with each record, so we don't need to perform any fancy
michael@0:         # graph traversal.
michael@0:         for v in self._state['webidls'].values():
michael@0:             if any(dep for dep in v['inputs'] if dep in changed_inputs):
michael@0:                 changed_inputs.add(v['filename'])
michael@0: 
michael@0:         # Only use paths that are known to our current state.
michael@0:         # This filters out files that were deleted or changed type (e.g. from
michael@0:         # static to preprocessed).
michael@0:         return changed_inputs & self._input_paths
michael@0: 
michael@0:     def _binding_info(self, p):
michael@0:         """Compute binding metadata for an input path.
michael@0: 
michael@0:         Returns a tuple of:
michael@0: 
michael@0:           (stem, binding_stem, is_event, output_files)
michael@0: 
michael@0:         output_files is itself a tuple. The first two items are the binding
michael@0:         header and C++ paths, respectively. The 2nd pair are the event header
michael@0:         and C++ paths or None if this isn't an event binding.
michael@0:         """
michael@0:         basename = mozpath.basename(p)
michael@0:         stem = mozpath.splitext(basename)[0]
michael@0:         binding_stem = '%sBinding' % stem
michael@0: 
michael@0:         if stem in self._exported_stems:
michael@0:             header_dir = self._exported_header_dir
michael@0:         else:
michael@0:             header_dir = self._codegen_dir
michael@0: 
michael@0:         is_event = stem in self._generated_events_stems
michael@0: 
michael@0:         files = (
michael@0:             mozpath.join(header_dir, '%s.h' % binding_stem),
michael@0:             mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem),
michael@0:             mozpath.join(header_dir, '%s.h' % stem) if is_event else None,
michael@0:             mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None,
michael@0:         )
michael@0: 
michael@0:         return stem, binding_stem, is_event, header_dir, files
michael@0: 
michael@0:     def _example_paths(self, interface):
michael@0:         return (
michael@0:             mozpath.join(self._codegen_dir, '%s-example.h' % interface),
michael@0:             mozpath.join(self._codegen_dir, '%s-example.cpp' % interface))
michael@0: 
michael@0:     def expected_build_output_files(self):
michael@0:         """Obtain the set of files generate_build_files() should write."""
michael@0:         paths = set()
michael@0: 
michael@0:         # Account for global generation.
michael@0:         for p in self.GLOBAL_DECLARE_FILES:
michael@0:             paths.add(mozpath.join(self._exported_header_dir, p))
michael@0:         for p in self.GLOBAL_DEFINE_FILES:
michael@0:             paths.add(mozpath.join(self._codegen_dir, p))
michael@0: 
michael@0:         for p in self._input_paths:
michael@0:             stem, binding_stem, is_event, header_dir, files = self._binding_info(p)
michael@0:             paths |= {f for f in files if f}
michael@0: 
michael@0:         for interface in self._example_interfaces:
michael@0:             for p in self._example_paths(interface):
michael@0:                 paths.add(p)
michael@0: 
michael@0:         return paths
michael@0: 
michael@0:     def _generate_build_files_for_webidl(self, filename):
michael@0:         from Codegen import (
michael@0:             CGBindingRoot,
michael@0:             CGEventRoot,
michael@0:         )
michael@0: 
michael@0:         self.log(logging.INFO, 'webidl_generate_build_for_input',
michael@0:             {'filename': filename},
michael@0:             'Generating WebIDL files derived from {filename}')
michael@0: 
michael@0:         stem, binding_stem, is_event, header_dir, files = self._binding_info(filename)
michael@0:         root = CGBindingRoot(self._config, binding_stem, filename)
michael@0: 
michael@0:         result = self._maybe_write_codegen(root, files[0], files[1])
michael@0: 
michael@0:         if is_event:
michael@0:             generated_event = CGEventRoot(self._config, stem)
michael@0:             result = self._maybe_write_codegen(generated_event, files[2],
michael@0:                 files[3], result)
michael@0: 
michael@0:         return result, root.deps()
michael@0: 
michael@0:     def _global_dependencies_changed(self):
michael@0:         """Determine whether the global dependencies have changed."""
michael@0:         current_files = set(iter_modules_in_path(mozpath.dirname(__file__)))
michael@0: 
michael@0:         # We need to catch other .py files from /dom/bindings. We assume these
michael@0:         # are in the same directory as the config file.
michael@0:         current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path)))
michael@0: 
michael@0:         current_files.add(self._config_path)
michael@0: 
michael@0:         current_hashes = {}
michael@0:         for f in current_files:
michael@0:             # This will fail if the file doesn't exist. If a current global
michael@0:             # dependency doesn't exist, something else is wrong.
michael@0:             with open(f, 'rb') as fh:
michael@0:                 current_hashes[f] = hashlib.sha1(fh.read()).hexdigest()
michael@0: 
michael@0:         # The set of files has changed.
michael@0:         if current_files ^ set(self._state['global_depends'].keys()):
michael@0:             return True, current_hashes
michael@0: 
michael@0:         # Compare hashes.
michael@0:         for f, sha1 in current_hashes.items():
michael@0:             if sha1 != self._state['global_depends'][f]:
michael@0:                 return True, current_hashes
michael@0: 
michael@0:         return False, current_hashes
michael@0: 
michael@0:     def _save_state(self):
michael@0:         with open(self._state_path, 'wb') as fh:
michael@0:             self._state.dump(fh)
michael@0: 
michael@0:     def _maybe_write_codegen(self, obj, declare_path, define_path, result=None):
michael@0:         assert declare_path and define_path
michael@0:         if not result:
michael@0:             result = (set(), set(), set())
michael@0: 
michael@0:         self._maybe_write_file(declare_path, obj.declare(), result)
michael@0:         self._maybe_write_file(define_path, obj.define(), result)
michael@0: 
michael@0:         return result
michael@0: 
michael@0:     def _maybe_write_file(self, path, content, result):
michael@0:         fh = FileAvoidWrite(path)
michael@0:         fh.write(content)
michael@0:         existed, updated = fh.close()
michael@0: 
michael@0:         if not existed:
michael@0:             result[0].add(path)
michael@0:         elif updated:
michael@0:             result[1].add(path)
michael@0:         else:
michael@0:             result[2].add(path)
michael@0: 
michael@0: 
michael@0: def create_build_system_manager(topsrcdir, topobjdir, dist_dir):
michael@0:     """Create a WebIDLCodegenManager for use by the build system."""
michael@0:     src_dir = os.path.join(topsrcdir, 'dom', 'bindings')
michael@0:     obj_dir = os.path.join(topobjdir, 'dom', 'bindings')
michael@0: 
michael@0:     with open(os.path.join(obj_dir, 'file-lists.json'), 'rb') as fh:
michael@0:         files = json.load(fh)
michael@0: 
michael@0:     inputs = (files['webidls'], files['exported_stems'],
michael@0:         files['generated_events_stems'], files['example_interfaces'])
michael@0: 
michael@0:     cache_dir = os.path.join(obj_dir, '_cache')
michael@0:     try:
michael@0:         os.makedirs(cache_dir)
michael@0:     except OSError as e:
michael@0:         if e.errno != errno.EEXIST:
michael@0:             raise
michael@0: 
michael@0:     return WebIDLCodegenManager(
michael@0:         os.path.join(src_dir, 'Bindings.conf'),
michael@0:         inputs,
michael@0:         os.path.join(dist_dir, 'include', 'mozilla', 'dom'),
michael@0:         obj_dir,
michael@0:         os.path.join(obj_dir, 'codegen.json'),
michael@0:         cache_dir=cache_dir,
michael@0:         # The make rules include a codegen.pp file containing dependencies.
michael@0:         make_deps_path=os.path.join(obj_dir, 'codegen.pp'),
michael@0:         make_deps_target='codegen.pp',
michael@0:     )
michael@0: 
michael@0: 
michael@0: class BuildSystemWebIDL(MozbuildObject):
michael@0:     @property
michael@0:     def manager(self):
michael@0:         if not hasattr(self, '_webidl_manager'):
michael@0:             self._webidl_manager = create_build_system_manager(
michael@0:                 self.topsrcdir, self.topobjdir, self.distdir)
michael@0: 
michael@0:         return self._webidl_manager