1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python/mozbuild/mozpack/packager/__init__.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,366 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +from mozbuild.preprocessor import Preprocessor
1.9 +import re
1.10 +import os
1.11 +from mozpack.errors import errors
1.12 +from mozpack.chrome.manifest import (
1.13 + Manifest,
1.14 + ManifestChrome,
1.15 + ManifestInterfaces,
1.16 + is_manifest,
1.17 + parse_manifest,
1.18 +)
1.19 +import mozpack.path
1.20 +from collections import deque
1.21 +
1.22 +
1.23 +class Component(object):
1.24 + '''
1.25 + Class that represents a component in a package manifest.
1.26 + '''
1.27 + def __init__(self, name, destdir=''):
1.28 + if name.find(' ') > 0:
1.29 + errors.fatal('Malformed manifest: space in component name "%s"'
1.30 + % component)
1.31 + self._name = name
1.32 + self._destdir = destdir
1.33 +
1.34 + def __repr__(self):
1.35 + s = self.name
1.36 + if self.destdir:
1.37 + s += ' destdir="%s"' % self.destdir
1.38 + return s
1.39 +
1.40 + @property
1.41 + def name(self):
1.42 + return self._name
1.43 +
1.44 + @property
1.45 + def destdir(self):
1.46 + return self._destdir
1.47 +
1.48 + @staticmethod
1.49 + def _triples(lst):
1.50 + '''
1.51 + Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
1.52 + '''
1.53 + return zip(*[iter(lst)] * 3)
1.54 +
1.55 + KEY_VALUE_RE = re.compile(r'''
1.56 + \s* # optional whitespace.
1.57 + ([a-zA-Z0-9_]+) # key.
1.58 + \s*=\s* # optional space around =.
1.59 + "([^"]*)" # value without surrounding quotes.
1.60 + (?:\s+|$)
1.61 + ''', re.VERBOSE)
1.62 +
1.63 + @staticmethod
1.64 + def _split_options(string):
1.65 + '''
1.66 + Split 'key1="value1" key2="value2"' into
1.67 + {'key1':'value1', 'key2':'value2'}.
1.68 +
1.69 + Returned keys and values are all strings.
1.70 +
1.71 + Throws ValueError if the input is malformed.
1.72 + '''
1.73 + options = {}
1.74 + splits = Component.KEY_VALUE_RE.split(string)
1.75 + if len(splits) % 3 != 1:
1.76 + # This should never happen -- we expect to always split
1.77 + # into ['', ('key', 'val', '')*].
1.78 + raise ValueError("Bad input")
1.79 + if splits[0]:
1.80 + raise ValueError('Unrecognized input ' + splits[0])
1.81 + for key, val, no_match in Component._triples(splits[1:]):
1.82 + if no_match:
1.83 + raise ValueError('Unrecognized input ' + no_match)
1.84 + options[key] = val
1.85 + return options
1.86 +
1.87 + @staticmethod
1.88 + def _split_component_and_options(string):
1.89 + '''
1.90 + Split 'name key1="value1" key2="value2"' into
1.91 + ('name', {'key1':'value1', 'key2':'value2'}).
1.92 +
1.93 + Returned name, keys and values are all strings.
1.94 +
1.95 + Raises ValueError if the input is malformed.
1.96 + '''
1.97 + splits = string.strip().split(None, 1)
1.98 + if not splits:
1.99 + raise ValueError('No component found')
1.100 + component = splits[0].strip()
1.101 + if not component:
1.102 + raise ValueError('No component found')
1.103 + if not re.match('[a-zA-Z0-9_\-]+$', component):
1.104 + raise ValueError('Bad component name ' + component)
1.105 + options = Component._split_options(splits[1]) if len(splits) > 1 else {}
1.106 + return component, options
1.107 +
1.108 + @staticmethod
1.109 + def from_string(string):
1.110 + '''
1.111 + Create a component from a string.
1.112 + '''
1.113 + try:
1.114 + name, options = Component._split_component_and_options(string)
1.115 + except ValueError as e:
1.116 + errors.fatal('Malformed manifest: %s' % e)
1.117 + return
1.118 + destdir = options.pop('destdir', '')
1.119 + if options:
1.120 + errors.fatal('Malformed manifest: options %s not recognized'
1.121 + % options.keys())
1.122 + return Component(name, destdir=destdir)
1.123 +
1.124 +
1.125 +class PackageManifestParser(object):
1.126 + '''
1.127 + Class for parsing of a package manifest, after preprocessing.
1.128 +
1.129 + A package manifest is a list of file paths, with some syntaxic sugar:
1.130 + [] designates a toplevel component. Example: [xpcom]
1.131 + - in front of a file specifies it to be removed
1.132 + * wildcard support
1.133 + ** expands to all files and zero or more directories
1.134 + ; file comment
1.135 +
1.136 + The parser takes input from the preprocessor line by line, and pushes
1.137 + parsed information to a sink object.
1.138 +
1.139 + The add and remove methods of the sink object are called with the
1.140 + current Component instance and a path.
1.141 + '''
1.142 + def __init__(self, sink):
1.143 + '''
1.144 + Initialize the package manifest parser with the given sink.
1.145 + '''
1.146 + self._component = Component('')
1.147 + self._sink = sink
1.148 +
1.149 + def handle_line(self, str):
1.150 + '''
1.151 + Handle a line of input and push the parsed information to the sink
1.152 + object.
1.153 + '''
1.154 + # Remove comments.
1.155 + str = str.strip()
1.156 + if not str or str.startswith(';'):
1.157 + return
1.158 + if str.startswith('[') and str.endswith(']'):
1.159 + self._component = Component.from_string(str[1:-1])
1.160 + elif str.startswith('-'):
1.161 + str = str[1:]
1.162 + self._sink.remove(self._component, str)
1.163 + elif ',' in str:
1.164 + errors.fatal('Incompatible syntax')
1.165 + else:
1.166 + self._sink.add(self._component, str)
1.167 +
1.168 +
1.169 +class PreprocessorOutputWrapper(object):
1.170 + '''
1.171 + File-like helper to handle the preprocessor output and send it to a parser.
1.172 + The parser's handle_line method is called in the relevant errors.context.
1.173 + '''
1.174 + def __init__(self, preprocessor, parser):
1.175 + self._parser = parser
1.176 + self._pp = preprocessor
1.177 +
1.178 + def write(self, str):
1.179 + file = os.path.normpath(os.path.abspath(self._pp.context['FILE']))
1.180 + with errors.context(file, self._pp.context['LINE']):
1.181 + self._parser.handle_line(str)
1.182 +
1.183 +
1.184 +def preprocess(input, parser, defines={}):
1.185 + '''
1.186 + Preprocess the file-like input with the given defines, and send the
1.187 + preprocessed output line by line to the given parser.
1.188 + '''
1.189 + pp = Preprocessor()
1.190 + pp.context.update(defines)
1.191 + pp.do_filter('substitution')
1.192 + pp.out = PreprocessorOutputWrapper(pp, parser)
1.193 + pp.do_include(input)
1.194 +
1.195 +
1.196 +def preprocess_manifest(sink, manifest, defines={}):
1.197 + '''
1.198 + Preprocess the given file-like manifest with the given defines, and push
1.199 + the parsed information to a sink. See PackageManifestParser documentation
1.200 + for more details on the sink.
1.201 + '''
1.202 + preprocess(manifest, PackageManifestParser(sink), defines)
1.203 +
1.204 +
1.205 +class CallDeque(deque):
1.206 + '''
1.207 + Queue of function calls to make.
1.208 + '''
1.209 + def append(self, function, *args):
1.210 + deque.append(self, (errors.get_context(), function, args))
1.211 +
1.212 + def execute(self):
1.213 + while True:
1.214 + try:
1.215 + context, function, args = self.popleft()
1.216 + except IndexError:
1.217 + return
1.218 + if context:
1.219 + with errors.context(context[0], context[1]):
1.220 + function(*args)
1.221 + else:
1.222 + function(*args)
1.223 +
1.224 +
1.225 +class SimplePackager(object):
1.226 + '''
1.227 + Helper used to translate and buffer instructions from the
1.228 + SimpleManifestSink to a formatter. Formatters expect some information to be
1.229 + given first that the simple manifest contents can't guarantee before the
1.230 + end of the input.
1.231 + '''
1.232 + def __init__(self, formatter):
1.233 + self.formatter = formatter
1.234 + # Queue for formatter.add_interfaces()/add_manifest() calls.
1.235 + self._queue = CallDeque()
1.236 + # Queue for formatter.add_manifest() calls for ManifestChrome.
1.237 + self._chrome_queue = CallDeque()
1.238 + # Queue for formatter.add() calls.
1.239 + self._file_queue = CallDeque()
1.240 + # All manifest paths imported.
1.241 + self._manifests = set()
1.242 + # All manifest paths included from some other manifest.
1.243 + self._included_manifests = set()
1.244 + self._closed = False
1.245 +
1.246 + def add(self, path, file):
1.247 + '''
1.248 + Add the given BaseFile instance with the given path.
1.249 + '''
1.250 + assert not self._closed
1.251 + if is_manifest(path):
1.252 + self._add_manifest_file(path, file)
1.253 + elif path.endswith('.xpt'):
1.254 + self._queue.append(self.formatter.add_interfaces, path, file)
1.255 + else:
1.256 + self._file_queue.append(self.formatter.add, path, file)
1.257 +
1.258 + def _add_manifest_file(self, path, file):
1.259 + '''
1.260 + Add the given BaseFile with manifest file contents with the given path.
1.261 + '''
1.262 + self._manifests.add(path)
1.263 + base = ''
1.264 + if hasattr(file, 'path'):
1.265 + # Find the directory the given path is relative to.
1.266 + b = mozpack.path.normsep(file.path)
1.267 + if b.endswith('/' + path) or b == path:
1.268 + base = os.path.normpath(b[:-len(path)])
1.269 + for e in parse_manifest(base, path, file.open()):
1.270 + # ManifestResources need to be given after ManifestChrome, so just
1.271 + # put all ManifestChrome in a separate queue to make them first.
1.272 + if isinstance(e, ManifestChrome):
1.273 + # e.move(e.base) just returns a clone of the entry.
1.274 + self._chrome_queue.append(self.formatter.add_manifest,
1.275 + e.move(e.base))
1.276 + elif not isinstance(e, (Manifest, ManifestInterfaces)):
1.277 + self._queue.append(self.formatter.add_manifest, e.move(e.base))
1.278 + if isinstance(e, Manifest):
1.279 + if e.flags:
1.280 + errors.fatal('Flags are not supported on ' +
1.281 + '"manifest" entries')
1.282 + self._included_manifests.add(e.path)
1.283 +
1.284 + def get_bases(self):
1.285 + '''
1.286 + Return all paths under which root manifests have been found. Root
1.287 + manifests are manifests that are included in no other manifest.
1.288 + '''
1.289 + return set(mozpack.path.dirname(m)
1.290 + for m in self._manifests - self._included_manifests)
1.291 +
1.292 + def close(self):
1.293 + '''
1.294 + Push all instructions to the formatter.
1.295 + '''
1.296 + self._closed = True
1.297 + for base in self.get_bases():
1.298 + if base:
1.299 + self.formatter.add_base(base)
1.300 + self._chrome_queue.execute()
1.301 + self._queue.execute()
1.302 + self._file_queue.execute()
1.303 +
1.304 +
1.305 +class SimpleManifestSink(object):
1.306 + '''
1.307 + Parser sink for "simple" package manifests. Simple package manifests use
1.308 + the format described in the PackageManifestParser documentation, but don't
1.309 + support file removals, and require manifests, interfaces and chrome data to
1.310 + be explicitely listed.
1.311 + Entries starting with bin/ are searched under bin/ in the FileFinder, but
1.312 + are packaged without the bin/ prefix.
1.313 + '''
1.314 + def __init__(self, finder, formatter):
1.315 + '''
1.316 + Initialize the SimpleManifestSink. The given FileFinder is used to
1.317 + get files matching the patterns given in the manifest. The given
1.318 + formatter does the packaging job.
1.319 + '''
1.320 + self._finder = finder
1.321 + self.packager = SimplePackager(formatter)
1.322 + self._closed = False
1.323 + self._manifests = set()
1.324 +
1.325 + @staticmethod
1.326 + def normalize_path(path):
1.327 + '''
1.328 + Remove any bin/ prefix.
1.329 + '''
1.330 + if mozpack.path.basedir(path, ['bin']) == 'bin':
1.331 + return mozpack.path.relpath(path, 'bin')
1.332 + return path
1.333 +
1.334 + def add(self, component, pattern):
1.335 + '''
1.336 + Add files with the given pattern in the given component.
1.337 + '''
1.338 + assert not self._closed
1.339 + added = False
1.340 + for p, f in self._finder.find(pattern):
1.341 + added = True
1.342 + if is_manifest(p):
1.343 + self._manifests.add(p)
1.344 + dest = mozpack.path.join(component.destdir, SimpleManifestSink.normalize_path(p))
1.345 + self.packager.add(dest, f)
1.346 + if not added:
1.347 + errors.error('Missing file(s): %s' % pattern)
1.348 +
1.349 + def remove(self, component, pattern):
1.350 + '''
1.351 + Remove files with the given pattern in the given component.
1.352 + '''
1.353 + assert not self._closed
1.354 + errors.fatal('Removal is unsupported')
1.355 +
1.356 + def close(self, auto_root_manifest=True):
1.357 + '''
1.358 + Add possibly missing bits and push all instructions to the formatter.
1.359 + '''
1.360 + if auto_root_manifest:
1.361 + # Simple package manifests don't contain the root manifests, so
1.362 + # find and add them.
1.363 + paths = [mozpack.path.dirname(m) for m in self._manifests]
1.364 + path = mozpack.path.dirname(mozpack.path.commonprefix(paths))
1.365 + for p, f in self._finder.find(mozpack.path.join(path,
1.366 + 'chrome.manifest')):
1.367 + if not p in self._manifests:
1.368 + self.packager.add(SimpleManifestSink.normalize_path(p), f)
1.369 + self.packager.close()
