michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: from mozbuild.preprocessor import Preprocessor
michael@0: import re
michael@0: import os
michael@0: from mozpack.errors import errors
michael@0: from mozpack.chrome.manifest import (
michael@0:     Manifest,
michael@0:     ManifestChrome,
michael@0:     ManifestInterfaces,
michael@0:     is_manifest,
michael@0:     parse_manifest,
michael@0: )
michael@0: import mozpack.path
michael@0: from collections import deque
michael@0: 
michael@0: 
michael@0: class Component(object):
michael@0:     '''
michael@0:     Class that represents a component in a package manifest.
michael@0:     '''
michael@0:     def __init__(self, name, destdir=''):
michael@0:         if name.find(' ') > 0:
michael@0:             errors.fatal('Malformed manifest: space in component name "%s"'
michael@0:                          % component)
michael@0:         self._name = name
michael@0:         self._destdir = destdir
michael@0: 
michael@0:     def __repr__(self):
michael@0:         s = self.name
michael@0:         if self.destdir:
michael@0:             s += ' destdir="%s"' % self.destdir
michael@0:         return s
michael@0: 
michael@0:     @property
michael@0:     def name(self):
michael@0:         return self._name
michael@0: 
michael@0:     @property
michael@0:     def destdir(self):
michael@0:         return self._destdir
michael@0: 
michael@0:     @staticmethod
michael@0:     def _triples(lst):
michael@0:         '''
michael@0:         Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
michael@0:         '''
michael@0:         return zip(*[iter(lst)] * 3)
michael@0: 
michael@0:     KEY_VALUE_RE = re.compile(r'''
michael@0:         \s*                 # optional whitespace.
michael@0:         ([a-zA-Z0-9_]+)     # key.
michael@0:         \s*=\s*             # optional space around =.
michael@0:         "([^"]*)"           # value without surrounding quotes.
michael@0:         (?:\s+|$)
michael@0:         ''', re.VERBOSE)
michael@0: 
michael@0:     @staticmethod
michael@0:     def _split_options(string):
michael@0:         '''
michael@0:         Split 'key1="value1" key2="value2"' into
michael@0:         {'key1':'value1', 'key2':'value2'}.
michael@0: 
michael@0:         Returned keys and values are all strings.
michael@0: 
michael@0:         Throws ValueError if the input is malformed.
michael@0:         '''
michael@0:         options = {}
michael@0:         splits = Component.KEY_VALUE_RE.split(string)
michael@0:         if len(splits) % 3 != 1:
michael@0:             # This should never happen -- we expect to always split
michael@0:             # into ['', ('key', 'val', '')*].
michael@0:             raise ValueError("Bad input")
michael@0:         if splits[0]:
michael@0:             raise ValueError('Unrecognized input ' + splits[0])
michael@0:         for key, val, no_match in Component._triples(splits[1:]):
michael@0:             if no_match:
michael@0:                 raise ValueError('Unrecognized input ' + no_match)
michael@0:             options[key] = val
michael@0:         return options
michael@0: 
michael@0:     @staticmethod
michael@0:     def _split_component_and_options(string):
michael@0:         '''
michael@0:         Split 'name key1="value1" key2="value2"' into
michael@0:         ('name', {'key1':'value1', 'key2':'value2'}).
michael@0: 
michael@0:         Returned name, keys and values are all strings.
michael@0: 
michael@0:         Raises ValueError if the input is malformed.
michael@0:         '''
michael@0:         splits = string.strip().split(None, 1)
michael@0:         if not splits:
michael@0:             raise ValueError('No component found')
michael@0:         component = splits[0].strip()
michael@0:         if not component:
michael@0:             raise ValueError('No component found')
michael@0:         if not re.match('[a-zA-Z0-9_\-]+$', component):
michael@0:             raise ValueError('Bad component name ' + component)
michael@0:         options = Component._split_options(splits[1]) if len(splits) > 1 else {}
michael@0:         return component, options
michael@0: 
michael@0:     @staticmethod
michael@0:     def from_string(string):
michael@0:         '''
michael@0:         Create a component from a string.
michael@0:         '''
michael@0:         try:
michael@0:             name, options = Component._split_component_and_options(string)
michael@0:         except ValueError as e:
michael@0:             errors.fatal('Malformed manifest: %s' % e)
michael@0:             return
michael@0:         destdir = options.pop('destdir', '')
michael@0:         if options:
michael@0:             errors.fatal('Malformed manifest: options %s not recognized'
michael@0:                          % options.keys())
michael@0:         return Component(name, destdir=destdir)
michael@0: 
michael@0: 
michael@0: class PackageManifestParser(object):
michael@0:     '''
michael@0:     Class for parsing of a package manifest, after preprocessing.
michael@0: 
michael@0:     A package manifest is a list of file paths, with some syntaxic sugar:
michael@0:         [] designates a toplevel component. Example: [xpcom]
michael@0:         - in front of a file specifies it to be removed
michael@0:         * wildcard support
michael@0:         ** expands to all files and zero or more directories
michael@0:         ; file comment
michael@0: 
michael@0:     The parser takes input from the preprocessor line by line, and pushes
michael@0:     parsed information to a sink object.
michael@0: 
michael@0:     The add and remove methods of the sink object are called with the
michael@0:     current Component instance and a path.
michael@0:     '''
michael@0:     def __init__(self, sink):
michael@0:         '''
michael@0:         Initialize the package manifest parser with the given sink.
michael@0:         '''
michael@0:         self._component = Component('')
michael@0:         self._sink = sink
michael@0: 
michael@0:     def handle_line(self, str):
michael@0:         '''
michael@0:         Handle a line of input and push the parsed information to the sink
michael@0:         object.
michael@0:         '''
michael@0:         # Remove comments.
michael@0:         str = str.strip()
michael@0:         if not str or str.startswith(';'):
michael@0:             return
michael@0:         if str.startswith('[') and str.endswith(']'):
michael@0:             self._component = Component.from_string(str[1:-1])
michael@0:         elif str.startswith('-'):
michael@0:             str = str[1:]
michael@0:             self._sink.remove(self._component, str)
michael@0:         elif ',' in str:
michael@0:             errors.fatal('Incompatible syntax')
michael@0:         else:
michael@0:             self._sink.add(self._component, str)
michael@0: 
michael@0: 
michael@0: class PreprocessorOutputWrapper(object):
michael@0:     '''
michael@0:     File-like helper to handle the preprocessor output and send it to a parser.
michael@0:     The parser's handle_line method is called in the relevant errors.context.
michael@0:     '''
michael@0:     def __init__(self, preprocessor, parser):
michael@0:         self._parser = parser
michael@0:         self._pp = preprocessor
michael@0: 
michael@0:     def write(self, str):
michael@0:         file = os.path.normpath(os.path.abspath(self._pp.context['FILE']))
michael@0:         with errors.context(file, self._pp.context['LINE']):
michael@0:             self._parser.handle_line(str)
michael@0: 
michael@0: 
michael@0: def preprocess(input, parser, defines={}):
michael@0:     '''
michael@0:     Preprocess the file-like input with the given defines, and send the
michael@0:     preprocessed output line by line to the given parser.
michael@0:     '''
michael@0:     pp = Preprocessor()
michael@0:     pp.context.update(defines)
michael@0:     pp.do_filter('substitution')
michael@0:     pp.out = PreprocessorOutputWrapper(pp, parser)
michael@0:     pp.do_include(input)
michael@0: 
michael@0: 
michael@0: def preprocess_manifest(sink, manifest, defines={}):
michael@0:     '''
michael@0:     Preprocess the given file-like manifest with the given defines, and push
michael@0:     the parsed information to a sink. See PackageManifestParser documentation
michael@0:     for more details on the sink.
michael@0:     '''
michael@0:     preprocess(manifest, PackageManifestParser(sink), defines)
michael@0: 
michael@0: 
michael@0: class CallDeque(deque):
michael@0:     '''
michael@0:     Queue of function calls to make.
michael@0:     '''
michael@0:     def append(self, function, *args):
michael@0:         deque.append(self, (errors.get_context(), function, args))
michael@0: 
michael@0:     def execute(self):
michael@0:         while True:
michael@0:             try:
michael@0:                 context, function, args = self.popleft()
michael@0:             except IndexError:
michael@0:                 return
michael@0:             if context:
michael@0:                 with errors.context(context[0], context[1]):
michael@0:                     function(*args)
michael@0:             else:
michael@0:                 function(*args)
michael@0: 
michael@0: 
michael@0: class SimplePackager(object):
michael@0:     '''
michael@0:     Helper used to translate and buffer instructions from the
michael@0:     SimpleManifestSink to a formatter. Formatters expect some information to be
michael@0:     given first that the simple manifest contents can't guarantee before the
michael@0:     end of the input.
michael@0:     '''
michael@0:     def __init__(self, formatter):
michael@0:         self.formatter = formatter
michael@0:         # Queue for formatter.add_interfaces()/add_manifest() calls.
michael@0:         self._queue = CallDeque()
michael@0:         # Queue for formatter.add_manifest() calls for ManifestChrome.
michael@0:         self._chrome_queue = CallDeque()
michael@0:         # Queue for formatter.add() calls.
michael@0:         self._file_queue = CallDeque()
michael@0:         # All manifest paths imported.
michael@0:         self._manifests = set()
michael@0:         # All manifest paths included from some other manifest.
michael@0:         self._included_manifests = set()
michael@0:         self._closed = False
michael@0: 
michael@0:     def add(self, path, file):
michael@0:         '''
michael@0:         Add the given BaseFile instance with the given path.
michael@0:         '''
michael@0:         assert not self._closed
michael@0:         if is_manifest(path):
michael@0:             self._add_manifest_file(path, file)
michael@0:         elif path.endswith('.xpt'):
michael@0:             self._queue.append(self.formatter.add_interfaces, path, file)
michael@0:         else:
michael@0:             self._file_queue.append(self.formatter.add, path, file)
michael@0: 
michael@0:     def _add_manifest_file(self, path, file):
michael@0:         '''
michael@0:         Add the given BaseFile with manifest file contents with the given path.
michael@0:         '''
michael@0:         self._manifests.add(path)
michael@0:         base = ''
michael@0:         if hasattr(file, 'path'):
michael@0:             # Find the directory the given path is relative to.
michael@0:             b = mozpack.path.normsep(file.path)
michael@0:             if b.endswith('/' + path) or b == path:
michael@0:                 base = os.path.normpath(b[:-len(path)])
michael@0:         for e in parse_manifest(base, path, file.open()):
michael@0:             # ManifestResources need to be given after ManifestChrome, so just
michael@0:             # put all ManifestChrome in a separate queue to make them first.
michael@0:             if isinstance(e, ManifestChrome):
michael@0:                 # e.move(e.base) just returns a clone of the entry.
michael@0:                 self._chrome_queue.append(self.formatter.add_manifest,
michael@0:                                           e.move(e.base))
michael@0:             elif not isinstance(e, (Manifest, ManifestInterfaces)):
michael@0:                 self._queue.append(self.formatter.add_manifest, e.move(e.base))
michael@0:             if isinstance(e, Manifest):
michael@0:                 if e.flags:
michael@0:                     errors.fatal('Flags are not supported on ' +
michael@0:                                  '"manifest" entries')
michael@0:                 self._included_manifests.add(e.path)
michael@0: 
michael@0:     def get_bases(self):
michael@0:         '''
michael@0:         Return all paths under which root manifests have been found. Root
michael@0:         manifests are manifests that are included in no other manifest.
michael@0:         '''
michael@0:         return set(mozpack.path.dirname(m)
michael@0:                    for m in self._manifests - self._included_manifests)
michael@0: 
michael@0:     def close(self):
michael@0:         '''
michael@0:         Push all instructions to the formatter.
michael@0:         '''
michael@0:         self._closed = True
michael@0:         for base in self.get_bases():
michael@0:             if base:
michael@0:                 self.formatter.add_base(base)
michael@0:         self._chrome_queue.execute()
michael@0:         self._queue.execute()
michael@0:         self._file_queue.execute()
michael@0: 
michael@0: 
michael@0: class SimpleManifestSink(object):
michael@0:     '''
michael@0:     Parser sink for "simple" package manifests. Simple package manifests use
michael@0:     the format described in the PackageManifestParser documentation, but don't
michael@0:     support file removals, and require manifests, interfaces and chrome data to
michael@0:     be explicitely listed.
michael@0:     Entries starting with bin/ are searched under bin/ in the FileFinder, but
michael@0:     are packaged without the bin/ prefix.
michael@0:     '''
michael@0:     def __init__(self, finder, formatter):
michael@0:         '''
michael@0:         Initialize the SimpleManifestSink. The given FileFinder is used to
michael@0:         get files matching the patterns given in the manifest. The given
michael@0:         formatter does the packaging job.
michael@0:         '''
michael@0:         self._finder = finder
michael@0:         self.packager = SimplePackager(formatter)
michael@0:         self._closed = False
michael@0:         self._manifests = set()
michael@0: 
michael@0:     @staticmethod
michael@0:     def normalize_path(path):
michael@0:         '''
michael@0:         Remove any bin/ prefix.
michael@0:         '''
michael@0:         if mozpack.path.basedir(path, ['bin']) == 'bin':
michael@0:             return mozpack.path.relpath(path, 'bin')
michael@0:         return path
michael@0: 
michael@0:     def add(self, component, pattern):
michael@0:         '''
michael@0:         Add files with the given pattern in the given component.
michael@0:         '''
michael@0:         assert not self._closed
michael@0:         added = False
michael@0:         for p, f in self._finder.find(pattern):
michael@0:             added = True
michael@0:             if is_manifest(p):
michael@0:                 self._manifests.add(p)
michael@0:             dest = mozpack.path.join(component.destdir, SimpleManifestSink.normalize_path(p))
michael@0:             self.packager.add(dest, f)
michael@0:         if not added:
michael@0:             errors.error('Missing file(s): %s' % pattern)
michael@0: 
michael@0:     def remove(self, component, pattern):
michael@0:         '''
michael@0:         Remove files with the given pattern in the given component.
michael@0:         '''
michael@0:         assert not self._closed
michael@0:         errors.fatal('Removal is unsupported')
michael@0: 
michael@0:     def close(self, auto_root_manifest=True):
michael@0:         '''
michael@0:         Add possibly missing bits and push all instructions to the formatter.
michael@0:         '''
michael@0:         if auto_root_manifest:
michael@0:             # Simple package manifests don't contain the root manifests, so
michael@0:             # find and add them.
michael@0:             paths = [mozpack.path.dirname(m) for m in self._manifests]
michael@0:             path = mozpack.path.dirname(mozpack.path.commonprefix(paths))
michael@0:             for p, f in self._finder.find(mozpack.path.join(path,
michael@0:                                           'chrome.manifest')):
michael@0:                 if not p in self._manifests:
michael@0:                     self.packager.add(SimpleManifestSink.normalize_path(p), f)
michael@0:         self.packager.close()