1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python/mozbuild/mozpack/manifests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,347 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +from __future__ import unicode_literals
1.9 +
1.10 +from contextlib import contextmanager
1.11 +import json
1.12 +
1.13 +from .files import (
1.14 + AbsoluteSymlinkFile,
1.15 + ExistingFile,
1.16 + File,
1.17 + FileFinder,
1.18 + PreprocessedFile,
1.19 +)
1.20 +import mozpack.path as mozpath
1.21 +
1.22 +
1.23 +# This probably belongs in a more generic module. Where?
1.24 +@contextmanager
1.25 +def _auto_fileobj(path, fileobj, mode='r'):
1.26 + if path and fileobj:
1.27 + raise AssertionError('Only 1 of path or fileobj may be defined.')
1.28 +
1.29 + if not path and not fileobj:
1.30 + raise AssertionError('Must specified 1 of path or fileobj.')
1.31 +
1.32 + if path:
1.33 + fileobj = open(path, mode)
1.34 +
1.35 + try:
1.36 + yield fileobj
1.37 + finally:
1.38 + if path:
1.39 + fileobj.close()
1.40 +
1.41 +
1.42 +class UnreadableInstallManifest(Exception):
1.43 + """Raised when an invalid install manifest is parsed."""
1.44 +
1.45 +
1.46 +class InstallManifest(object):
1.47 + """Describes actions to be used with a copier.FileCopier instance.
1.48 +
1.49 + This class facilitates serialization and deserialization of data used to
1.50 + construct a copier.FileCopier and to perform copy operations.
1.51 +
1.52 + The manifest defines source paths, destination paths, and a mechanism by
1.53 + which the destination file should come into existence.
1.54 +
1.55 + Entries in the manifest correspond to the following types:
1.56 +
1.57 + copy -- The file specified as the source path will be copied to the
1.58 + destination path.
1.59 +
1.60 + symlink -- The destination path will be a symlink to the source path.
1.61 + If symlinks are not supported, a copy will be performed.
1.62 +
1.63 + exists -- The destination path is accounted for and won't be deleted by
1.64 + the FileCopier. If the destination path doesn't exist, an error is
1.65 + raised.
1.66 +
1.67 + optional -- The destination path is accounted for and won't be deleted by
1.68 + the FileCopier. No error is raised if the destination path does not
1.69 + exist.
1.70 +
1.71 + patternsymlink -- Paths matched by the expression in the source path
1.72 + will be symlinked to the destination directory.
1.73 +
1.74 + patterncopy -- Similar to patternsymlink except files are copied, not
1.75 + symlinked.
1.76 +
1.77 + preprocess -- The file specified at the source path will be run through
1.78 + the preprocessor, and the output will be written to the destination
1.79 + path.
1.80 +
1.81 + Version 1 of the manifest was the initial version.
1.82 + Version 2 added optional path support
1.83 + Version 3 added support for pattern entries.
1.84 + Version 4 added preprocessed file support.
1.85 + """
1.86 +
1.87 + CURRENT_VERSION = 4
1.88 +
1.89 + FIELD_SEPARATOR = '\x1f'
1.90 +
1.91 + SYMLINK = 1
1.92 + COPY = 2
1.93 + REQUIRED_EXISTS = 3
1.94 + OPTIONAL_EXISTS = 4
1.95 + PATTERN_SYMLINK = 5
1.96 + PATTERN_COPY = 6
1.97 + PREPROCESS = 7
1.98 +
1.99 + def __init__(self, path=None, fileobj=None):
1.100 + """Create a new InstallManifest entry.
1.101 +
1.102 + If path is defined, the manifest will be populated with data from the
1.103 + file path.
1.104 +
1.105 + If fileobj is defined, the manifest will be populated with data read
1.106 + from the specified file object.
1.107 +
1.108 + Both path and fileobj cannot be defined.
1.109 + """
1.110 + self._dests = {}
1.111 + self._source_file = None
1.112 +
1.113 + if path or fileobj:
1.114 + with _auto_fileobj(path, fileobj, 'rb') as fh:
1.115 + self._source_file = fh.name
1.116 + self._load_from_fileobj(fh)
1.117 +
1.118 + def _load_from_fileobj(self, fileobj):
1.119 + version = fileobj.readline().rstrip()
1.120 + if version not in ('1', '2', '3', '4'):
1.121 + raise UnreadableInstallManifest('Unknown manifest version: ' %
1.122 + version)
1.123 +
1.124 + for line in fileobj:
1.125 + line = line.rstrip()
1.126 +
1.127 + fields = line.split(self.FIELD_SEPARATOR)
1.128 +
1.129 + record_type = int(fields[0])
1.130 +
1.131 + if record_type == self.SYMLINK:
1.132 + dest, source = fields[1:]
1.133 + self.add_symlink(source, dest)
1.134 + continue
1.135 +
1.136 + if record_type == self.COPY:
1.137 + dest, source = fields[1:]
1.138 + self.add_copy(source, dest)
1.139 + continue
1.140 +
1.141 + if record_type == self.REQUIRED_EXISTS:
1.142 + _, path = fields
1.143 + self.add_required_exists(path)
1.144 + continue
1.145 +
1.146 + if record_type == self.OPTIONAL_EXISTS:
1.147 + _, path = fields
1.148 + self.add_optional_exists(path)
1.149 + continue
1.150 +
1.151 + if record_type == self.PATTERN_SYMLINK:
1.152 + _, base, pattern, dest = fields[1:]
1.153 + self.add_pattern_symlink(base, pattern, dest)
1.154 + continue
1.155 +
1.156 + if record_type == self.PATTERN_COPY:
1.157 + _, base, pattern, dest = fields[1:]
1.158 + self.add_pattern_copy(base, pattern, dest)
1.159 + continue
1.160 +
1.161 + if record_type == self.PREPROCESS:
1.162 + dest, source, deps, marker, defines = fields[1:]
1.163 + self.add_preprocess(source, dest, deps, marker,
1.164 + self._decode_field_entry(defines))
1.165 + continue
1.166 +
1.167 + raise UnreadableInstallManifest('Unknown record type: %d' %
1.168 + record_type)
1.169 +
1.170 + def __len__(self):
1.171 + return len(self._dests)
1.172 +
1.173 + def __contains__(self, item):
1.174 + return item in self._dests
1.175 +
1.176 + def __eq__(self, other):
1.177 + return isinstance(other, InstallManifest) and self._dests == other._dests
1.178 +
1.179 + def __neq__(self, other):
1.180 + return not self.__eq__(other)
1.181 +
1.182 + def __ior__(self, other):
1.183 + if not isinstance(other, InstallManifest):
1.184 + raise ValueError('Can only | with another instance of InstallManifest.')
1.185 +
1.186 + for dest in sorted(other._dests):
1.187 + self._add_entry(dest, other._dests[dest])
1.188 +
1.189 + return self
1.190 +
1.191 + def _encode_field_entry(self, data):
1.192 + """Converts an object into a format that can be stored in the manifest file.
1.193 +
1.194 + Complex data types, such as ``dict``, need to be converted into a text
1.195 + representation before they can be written to a file.
1.196 + """
1.197 + return json.dumps(data, sort_keys=True)
1.198 +
1.199 + def _decode_field_entry(self, data):
1.200 + """Restores an object from a format that can be stored in the manifest file.
1.201 +
1.202 + Complex data types, such as ``dict``, need to be converted into a text
1.203 + representation before they can be written to a file.
1.204 + """
1.205 + return json.loads(data)
1.206 +
1.207 + def write(self, path=None, fileobj=None):
1.208 + """Serialize this manifest to a file or file object.
1.209 +
1.210 + If path is specified, that file will be written to. If fileobj is specified,
1.211 + the serialized content will be written to that file object.
1.212 +
1.213 + It is an error if both are specified.
1.214 + """
1.215 + with _auto_fileobj(path, fileobj, 'wb') as fh:
1.216 + fh.write('%d\n' % self.CURRENT_VERSION)
1.217 +
1.218 + for dest in sorted(self._dests):
1.219 + entry = self._dests[dest]
1.220 +
1.221 + parts = ['%d' % entry[0], dest]
1.222 + parts.extend(entry[1:])
1.223 + fh.write('%s\n' % self.FIELD_SEPARATOR.join(
1.224 + p.encode('utf-8') for p in parts))
1.225 +
1.226 + def add_symlink(self, source, dest):
1.227 + """Add a symlink to this manifest.
1.228 +
1.229 + dest will be a symlink to source.
1.230 + """
1.231 + self._add_entry(dest, (self.SYMLINK, source))
1.232 +
1.233 + def add_copy(self, source, dest):
1.234 + """Add a copy to this manifest.
1.235 +
1.236 + source will be copied to dest.
1.237 + """
1.238 + self._add_entry(dest, (self.COPY, source))
1.239 +
1.240 + def add_required_exists(self, dest):
1.241 + """Record that a destination file must exist.
1.242 +
1.243 + This effectively prevents the listed file from being deleted.
1.244 + """
1.245 + self._add_entry(dest, (self.REQUIRED_EXISTS,))
1.246 +
1.247 + def add_optional_exists(self, dest):
1.248 + """Record that a destination file may exist.
1.249 +
1.250 + This effectively prevents the listed file from being deleted. Unlike a
1.251 + "required exists" file, files of this type do not raise errors if the
1.252 + destination file does not exist.
1.253 + """
1.254 + self._add_entry(dest, (self.OPTIONAL_EXISTS,))
1.255 +
1.256 + def add_pattern_symlink(self, base, pattern, dest):
1.257 + """Add a pattern match that results in symlinks being created.
1.258 +
1.259 + A ``FileFinder`` will be created with its base set to ``base``
1.260 + and ``FileFinder.find()`` will be called with ``pattern`` to discover
1.261 + source files. Each source file will be symlinked under ``dest``.
1.262 +
1.263 + Filenames under ``dest`` are constructed by taking the path fragment
1.264 + after ``base`` and concatenating it with ``dest``. e.g.
1.265 +
1.266 + <base>/foo/bar.h -> <dest>/foo/bar.h
1.267 + """
1.268 + self._add_entry(mozpath.join(base, pattern, dest),
1.269 + (self.PATTERN_SYMLINK, base, pattern, dest))
1.270 +
1.271 + def add_pattern_copy(self, base, pattern, dest):
1.272 + """Add a pattern match that results in copies.
1.273 +
1.274 + See ``add_pattern_symlink()`` for usage.
1.275 + """
1.276 + self._add_entry(mozpath.join(base, pattern, dest),
1.277 + (self.PATTERN_COPY, base, pattern, dest))
1.278 +
1.279 + def add_preprocess(self, source, dest, deps, marker='#', defines={}):
1.280 + """Add a preprocessed file to this manifest.
1.281 +
1.282 + ``source`` will be passed through preprocessor.py, and the output will be
1.283 + written to ``dest``.
1.284 + """
1.285 + self._add_entry(dest,
1.286 + (self.PREPROCESS, source, deps, marker, self._encode_field_entry(defines)))
1.287 +
1.288 + def _add_entry(self, dest, entry):
1.289 + if dest in self._dests:
1.290 + raise ValueError('Item already in manifest: %s' % dest)
1.291 +
1.292 + self._dests[dest] = entry
1.293 +
1.294 + def _get_deps(self, dest):
1.295 + return {self._source_file} if self._source_file else set()
1.296 +
1.297 + def populate_registry(self, registry):
1.298 + """Populate a mozpack.copier.FileRegistry instance with data from us.
1.299 +
1.300 + The caller supplied a FileRegistry instance (or at least something that
1.301 + conforms to its interface) and that instance is populated with data
1.302 + from this manifest.
1.303 + """
1.304 + for dest in sorted(self._dests):
1.305 + entry = self._dests[dest]
1.306 + install_type = entry[0]
1.307 +
1.308 + if install_type == self.SYMLINK:
1.309 + registry.add(dest, AbsoluteSymlinkFile(entry[1]))
1.310 + continue
1.311 +
1.312 + if install_type == self.COPY:
1.313 + registry.add(dest, File(entry[1]))
1.314 + continue
1.315 +
1.316 + if install_type == self.REQUIRED_EXISTS:
1.317 + registry.add(dest, ExistingFile(required=True))
1.318 + continue
1.319 +
1.320 + if install_type == self.OPTIONAL_EXISTS:
1.321 + registry.add(dest, ExistingFile(required=False))
1.322 + continue
1.323 +
1.324 + if install_type in (self.PATTERN_SYMLINK, self.PATTERN_COPY):
1.325 + _, base, pattern, dest = entry
1.326 + finder = FileFinder(base, find_executables=False)
1.327 + paths = [f[0] for f in finder.find(pattern)]
1.328 +
1.329 + if install_type == self.PATTERN_SYMLINK:
1.330 + cls = AbsoluteSymlinkFile
1.331 + else:
1.332 + cls = File
1.333 +
1.334 + for path in paths:
1.335 + source = mozpath.join(base, path)
1.336 + registry.add(mozpath.join(dest, path), cls(source))
1.337 +
1.338 + continue
1.339 +
1.340 + if install_type == self.PREPROCESS:
1.341 + registry.add(dest, PreprocessedFile(entry[1],
1.342 + depfile_path=entry[2],
1.343 + marker=entry[3],
1.344 + defines=self._decode_field_entry(entry[4]),
1.345 + extra_depends=self._get_deps(dest)))
1.346 +
1.347 + continue
1.348 +
1.349 + raise Exception('Unknown install type defined in manifest: %d' %
1.350 + install_type)
