1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python/mozbuild/mozpack/files.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,868 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +import errno
1.9 +import os
1.10 +import platform
1.11 +import shutil
1.12 +import stat
1.13 +import subprocess
1.14 +import uuid
1.15 +import mozbuild.makeutil as makeutil
1.16 +from mozbuild.preprocessor import Preprocessor
1.17 +from mozbuild.util import FileAvoidWrite
1.18 +from mozpack.executables import (
1.19 + is_executable,
1.20 + may_strip,
1.21 + strip,
1.22 + may_elfhack,
1.23 + elfhack,
1.24 +)
1.25 +from mozpack.chrome.manifest import ManifestEntry
1.26 +from io import BytesIO
1.27 +from mozpack.errors import (
1.28 + ErrorMessage,
1.29 + errors,
1.30 +)
1.31 +from mozpack.mozjar import JarReader
1.32 +import mozpack.path
1.33 +from collections import OrderedDict
1.34 +from jsmin import JavascriptMinify
1.35 +from tempfile import (
1.36 + mkstemp,
1.37 + NamedTemporaryFile,
1.38 +)
1.39 +
1.40 +
1.41 +class Dest(object):
1.42 + '''
1.43 + Helper interface for BaseFile.copy. The interface works as follows:
1.44 + - read() and write() can be used to sequentially read/write from the
1.45 + underlying file.
1.46 + - a call to read() after a write() will re-open the underlying file and
1.47 + read from it.
1.48 + - a call to write() after a read() will re-open the underlying file,
1.49 + emptying it, and write to it.
1.50 + '''
1.51 + def __init__(self, path):
1.52 + self.path = path
1.53 + self.mode = None
1.54 +
1.55 + @property
1.56 + def name(self):
1.57 + return self.path
1.58 +
1.59 + def read(self, length=-1):
1.60 + if self.mode != 'r':
1.61 + self.file = open(self.path, 'rb')
1.62 + self.mode = 'r'
1.63 + return self.file.read(length)
1.64 +
1.65 + def write(self, data):
1.66 + if self.mode != 'w':
1.67 + self.file = open(self.path, 'wb')
1.68 + self.mode = 'w'
1.69 + return self.file.write(data)
1.70 +
1.71 + def exists(self):
1.72 + return os.path.exists(self.path)
1.73 +
1.74 + def close(self):
1.75 + if self.mode:
1.76 + self.mode = None
1.77 + self.file.close()
1.78 +
1.79 +
1.80 +class BaseFile(object):
1.81 + '''
1.82 + Base interface and helper for file copying. Derived class may implement
1.83 + their own copy function, or rely on BaseFile.copy using the open() member
1.84 + function and/or the path property.
1.85 + '''
1.86 + @staticmethod
1.87 + def is_older(first, second):
1.88 + '''
1.89 + Compares the modification time of two files, and returns whether the
1.90 + ``first`` file is older than the ``second`` file.
1.91 + '''
1.92 + # os.path.getmtime returns a result in seconds with precision up to
1.93 + # the microsecond. But microsecond is too precise because
1.94 + # shutil.copystat only copies milliseconds, and seconds is not
1.95 + # enough precision.
1.96 + return int(os.path.getmtime(first) * 1000) \
1.97 + <= int(os.path.getmtime(second) * 1000)
1.98 +
1.99 + @staticmethod
1.100 + def any_newer(dest, inputs):
1.101 + '''
1.102 + Compares the modification time of ``dest`` to multiple input files, and
1.103 + returns whether any of the ``inputs`` is newer (has a later mtime) than
1.104 + ``dest``.
1.105 + '''
1.106 + # os.path.getmtime returns a result in seconds with precision up to
1.107 + # the microsecond. But microsecond is too precise because
1.108 + # shutil.copystat only copies milliseconds, and seconds is not
1.109 + # enough precision.
1.110 + dest_mtime = int(os.path.getmtime(dest) * 1000)
1.111 + for input in inputs:
1.112 + if dest_mtime < int(os.path.getmtime(input) * 1000):
1.113 + return True
1.114 + return False
1.115 +
1.116 + def copy(self, dest, skip_if_older=True):
1.117 + '''
1.118 + Copy the BaseFile content to the destination given as a string or a
1.119 + Dest instance. Avoids replacing existing files if the BaseFile content
1.120 + matches that of the destination, or in case of plain files, if the
1.121 + destination is newer than the original file. This latter behaviour is
1.122 + disabled when skip_if_older is False.
1.123 + Returns whether a copy was actually performed (True) or not (False).
1.124 + '''
1.125 + if isinstance(dest, basestring):
1.126 + dest = Dest(dest)
1.127 + else:
1.128 + assert isinstance(dest, Dest)
1.129 +
1.130 + can_skip_content_check = False
1.131 + if not dest.exists():
1.132 + can_skip_content_check = True
1.133 + elif getattr(self, 'path', None) and getattr(dest, 'path', None):
1.134 + if skip_if_older and BaseFile.is_older(self.path, dest.path):
1.135 + return False
1.136 + elif os.path.getsize(self.path) != os.path.getsize(dest.path):
1.137 + can_skip_content_check = True
1.138 +
1.139 + if can_skip_content_check:
1.140 + if getattr(self, 'path', None) and getattr(dest, 'path', None):
1.141 + shutil.copy2(self.path, dest.path)
1.142 + else:
1.143 + # Ensure the file is always created
1.144 + if not dest.exists():
1.145 + dest.write('')
1.146 + shutil.copyfileobj(self.open(), dest)
1.147 + return True
1.148 +
1.149 + src = self.open()
1.150 + copy_content = ''
1.151 + while True:
1.152 + dest_content = dest.read(32768)
1.153 + src_content = src.read(32768)
1.154 + copy_content += src_content
1.155 + if len(dest_content) == len(src_content) == 0:
1.156 + break
1.157 + # If the read content differs between origin and destination,
1.158 + # write what was read up to now, and copy the remainder.
1.159 + if dest_content != src_content:
1.160 + dest.write(copy_content)
1.161 + shutil.copyfileobj(src, dest)
1.162 + break
1.163 + if hasattr(self, 'path') and hasattr(dest, 'path'):
1.164 + shutil.copystat(self.path, dest.path)
1.165 + return True
1.166 +
1.167 + def open(self):
1.168 + '''
1.169 + Return a file-like object allowing to read() the content of the
1.170 + associated file. This is meant to be overloaded in subclasses to return
1.171 + a custom file-like object.
1.172 + '''
1.173 + assert self.path is not None
1.174 + return open(self.path, 'rb')
1.175 +
1.176 + @property
1.177 + def mode(self):
1.178 + '''
1.179 + Return the file's unix mode, or None if it has no meaning.
1.180 + '''
1.181 + return None
1.182 +
1.183 +
1.184 +class File(BaseFile):
1.185 + '''
1.186 + File class for plain files.
1.187 + '''
1.188 + def __init__(self, path):
1.189 + self.path = path
1.190 +
1.191 + @property
1.192 + def mode(self):
1.193 + '''
1.194 + Return the file's unix mode, as returned by os.stat().st_mode.
1.195 + '''
1.196 + if platform.system() == 'Windows':
1.197 + return None
1.198 + assert self.path is not None
1.199 + return os.stat(self.path).st_mode
1.200 +
1.201 +class ExecutableFile(File):
1.202 + '''
1.203 + File class for executable and library files on OS/2, OS/X and ELF systems.
1.204 + (see mozpack.executables.is_executable documentation).
1.205 + '''
1.206 + def copy(self, dest, skip_if_older=True):
1.207 + real_dest = dest
1.208 + if not isinstance(dest, basestring):
1.209 + fd, dest = mkstemp()
1.210 + os.close(fd)
1.211 + os.remove(dest)
1.212 + assert isinstance(dest, basestring)
1.213 + # If File.copy didn't actually copy because dest is newer, check the
1.214 + # file sizes. If dest is smaller, it means it is already stripped and
1.215 + # elfhacked, so we can skip.
1.216 + if not File.copy(self, dest, skip_if_older) and \
1.217 + os.path.getsize(self.path) > os.path.getsize(dest):
1.218 + return False
1.219 + try:
1.220 + if may_strip(dest):
1.221 + strip(dest)
1.222 + if may_elfhack(dest):
1.223 + elfhack(dest)
1.224 + except ErrorMessage:
1.225 + os.remove(dest)
1.226 + raise
1.227 +
1.228 + if real_dest != dest:
1.229 + f = File(dest)
1.230 + ret = f.copy(real_dest, skip_if_older)
1.231 + os.remove(dest)
1.232 + return ret
1.233 + return True
1.234 +
1.235 +
1.236 +class AbsoluteSymlinkFile(File):
1.237 + '''File class that is copied by symlinking (if available).
1.238 +
1.239 + This class only works if the target path is absolute.
1.240 + '''
1.241 +
1.242 + def __init__(self, path):
1.243 + if not os.path.isabs(path):
1.244 + raise ValueError('Symlink target not absolute: %s' % path)
1.245 +
1.246 + File.__init__(self, path)
1.247 +
1.248 + def copy(self, dest, skip_if_older=True):
1.249 + assert isinstance(dest, basestring)
1.250 +
1.251 + # The logic in this function is complicated by the fact that symlinks
1.252 + # aren't universally supported. So, where symlinks aren't supported, we
1.253 + # fall back to file copying. Keep in mind that symlink support is
1.254 + # per-filesystem, not per-OS.
1.255 +
1.256 + # Handle the simple case where symlinks are definitely not supported by
1.257 + # falling back to file copy.
1.258 + if not hasattr(os, 'symlink'):
1.259 + return File.copy(self, dest, skip_if_older=skip_if_older)
1.260 +
1.261 + # Always verify the symlink target path exists.
1.262 + if not os.path.exists(self.path):
1.263 + raise ErrorMessage('Symlink target path does not exist: %s' % self.path)
1.264 +
1.265 + st = None
1.266 +
1.267 + try:
1.268 + st = os.lstat(dest)
1.269 + except OSError as ose:
1.270 + if ose.errno != errno.ENOENT:
1.271 + raise
1.272 +
1.273 + # If the dest is a symlink pointing to us, we have nothing to do.
1.274 + # If it's the wrong symlink, the filesystem must support symlinks,
1.275 + # so we replace with a proper symlink.
1.276 + if st and stat.S_ISLNK(st.st_mode):
1.277 + link = os.readlink(dest)
1.278 + if link == self.path:
1.279 + return False
1.280 +
1.281 + os.remove(dest)
1.282 + os.symlink(self.path, dest)
1.283 + return True
1.284 +
1.285 + # If the destination doesn't exist, we try to create a symlink. If that
1.286 + # fails, we fall back to copy code.
1.287 + if not st:
1.288 + try:
1.289 + os.symlink(self.path, dest)
1.290 + return True
1.291 + except OSError:
1.292 + return File.copy(self, dest, skip_if_older=skip_if_older)
1.293 +
1.294 + # Now the complicated part. If the destination exists, we could be
1.295 + # replacing a file with a symlink. Or, the filesystem may not support
1.296 + # symlinks. We want to minimize I/O overhead for performance reasons,
1.297 + # so we keep the existing destination file around as long as possible.
1.298 + # A lot of the system calls would be eliminated if we cached whether
1.299 + # symlinks are supported. However, even if we performed a single
1.300 + # up-front test of whether the root of the destination directory
1.301 + # supports symlinks, there's no guarantee that all operations for that
1.302 + # dest (or source) would be on the same filesystem and would support
1.303 + # symlinks.
1.304 + #
1.305 + # Our strategy is to attempt to create a new symlink with a random
1.306 + # name. If that fails, we fall back to copy mode. If that works, we
1.307 + # remove the old destination and move the newly-created symlink into
1.308 + # its place.
1.309 +
1.310 + temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4()))
1.311 + try:
1.312 + os.symlink(self.path, temp_dest)
1.313 + # TODO Figure out exactly how symlink creation fails and only trap
1.314 + # that.
1.315 + except EnvironmentError:
1.316 + return File.copy(self, dest, skip_if_older=skip_if_older)
1.317 +
1.318 + # If removing the original file fails, don't forget to clean up the
1.319 + # temporary symlink.
1.320 + try:
1.321 + os.remove(dest)
1.322 + except EnvironmentError:
1.323 + os.remove(temp_dest)
1.324 + raise
1.325 +
1.326 + os.rename(temp_dest, dest)
1.327 + return True
1.328 +
1.329 +
1.330 +class ExistingFile(BaseFile):
1.331 + '''
1.332 + File class that represents a file that may exist but whose content comes
1.333 + from elsewhere.
1.334 +
1.335 + This purpose of this class is to account for files that are installed via
1.336 + external means. It is typically only used in manifests or in registries to
1.337 + account for files.
1.338 +
1.339 + When asked to copy, this class does nothing because nothing is known about
1.340 + the source file/data.
1.341 +
1.342 + Instances of this class come in two flavors: required and optional. If an
1.343 + existing file is required, it must exist during copy() or an error is
1.344 + raised.
1.345 + '''
1.346 + def __init__(self, required):
1.347 + self.required = required
1.348 +
1.349 + def copy(self, dest, skip_if_older=True):
1.350 + if isinstance(dest, basestring):
1.351 + dest = Dest(dest)
1.352 + else:
1.353 + assert isinstance(dest, Dest)
1.354 +
1.355 + if not self.required:
1.356 + return
1.357 +
1.358 + if not dest.exists():
1.359 + errors.fatal("Required existing file doesn't exist: %s" %
1.360 + dest.path)
1.361 +
1.362 +
1.363 +class PreprocessedFile(BaseFile):
1.364 + '''
1.365 + File class for a file that is preprocessed. PreprocessedFile.copy() runs
1.366 + the preprocessor on the file to create the output.
1.367 + '''
1.368 + def __init__(self, path, depfile_path, marker, defines, extra_depends=None):
1.369 + self.path = path
1.370 + self.depfile = depfile_path
1.371 + self.marker = marker
1.372 + self.defines = defines
1.373 + self.extra_depends = list(extra_depends or [])
1.374 +
1.375 + def copy(self, dest, skip_if_older=True):
1.376 + '''
1.377 + Invokes the preprocessor to create the destination file.
1.378 + '''
1.379 + if isinstance(dest, basestring):
1.380 + dest = Dest(dest)
1.381 + else:
1.382 + assert isinstance(dest, Dest)
1.383 +
1.384 + # We have to account for the case where the destination exists and is a
1.385 + # symlink to something. Since we know the preprocessor is certainly not
1.386 + # going to create a symlink, we can just remove the existing one. If the
1.387 + # destination is not a symlink, we leave it alone, since we're going to
1.388 + # overwrite its contents anyway.
1.389 + # If symlinks aren't supported at all, we can skip this step.
1.390 + if hasattr(os, 'symlink'):
1.391 + if os.path.islink(dest.path):
1.392 + os.remove(dest.path)
1.393 +
1.394 + pp_deps = set(self.extra_depends)
1.395 +
1.396 + # If a dependency file was specified, and it exists, add any
1.397 + # dependencies from that file to our list.
1.398 + if self.depfile and os.path.exists(self.depfile):
1.399 + target = mozpack.path.normpath(dest.name)
1.400 + with open(self.depfile, 'rb') as fileobj:
1.401 + for rule in makeutil.read_dep_makefile(fileobj):
1.402 + if target in rule.targets():
1.403 + pp_deps.update(rule.dependencies())
1.404 +
1.405 + skip = False
1.406 + if dest.exists() and skip_if_older:
1.407 + # If a dependency file was specified, and it doesn't exist,
1.408 + # assume that the preprocessor needs to be rerun. That will
1.409 + # regenerate the dependency file.
1.410 + if self.depfile and not os.path.exists(self.depfile):
1.411 + skip = False
1.412 + else:
1.413 + skip = not BaseFile.any_newer(dest.path, pp_deps)
1.414 +
1.415 + if skip:
1.416 + return False
1.417 +
1.418 + deps_out = None
1.419 + if self.depfile:
1.420 + deps_out = FileAvoidWrite(self.depfile)
1.421 + pp = Preprocessor(defines=self.defines, marker=self.marker)
1.422 +
1.423 + with open(self.path, 'rU') as input:
1.424 + pp.processFile(input=input, output=dest, depfile=deps_out)
1.425 +
1.426 + dest.close()
1.427 + if self.depfile:
1.428 + deps_out.close()
1.429 +
1.430 + return True
1.431 +
1.432 +
1.433 +class GeneratedFile(BaseFile):
1.434 + '''
1.435 + File class for content with no previous existence on the filesystem.
1.436 + '''
1.437 + def __init__(self, content):
1.438 + self.content = content
1.439 +
1.440 + def open(self):
1.441 + return BytesIO(self.content)
1.442 +
1.443 +
1.444 +class DeflatedFile(BaseFile):
1.445 + '''
1.446 + File class for members of a jar archive. DeflatedFile.copy() effectively
1.447 + extracts the file from the jar archive.
1.448 + '''
1.449 + def __init__(self, file):
1.450 + from mozpack.mozjar import JarFileReader
1.451 + assert isinstance(file, JarFileReader)
1.452 + self.file = file
1.453 +
1.454 + def open(self):
1.455 + self.file.seek(0)
1.456 + return self.file
1.457 +
1.458 +
1.459 +class XPTFile(GeneratedFile):
1.460 + '''
1.461 + File class for a linked XPT file. It takes several XPT files as input
1.462 + (using the add() and remove() member functions), and links them at copy()
1.463 + time.
1.464 + '''
1.465 + def __init__(self):
1.466 + self._files = set()
1.467 +
1.468 + def add(self, xpt):
1.469 + '''
1.470 + Add the given XPT file (as a BaseFile instance) to the list of XPTs
1.471 + to link.
1.472 + '''
1.473 + assert isinstance(xpt, BaseFile)
1.474 + self._files.add(xpt)
1.475 +
1.476 + def remove(self, xpt):
1.477 + '''
1.478 + Remove the given XPT file (as a BaseFile instance) from the list of
1.479 + XPTs to link.
1.480 + '''
1.481 + assert isinstance(xpt, BaseFile)
1.482 + self._files.remove(xpt)
1.483 +
1.484 + def copy(self, dest, skip_if_older=True):
1.485 + '''
1.486 + Link the registered XPTs and place the resulting linked XPT at the
1.487 + destination given as a string or a Dest instance. Avoids an expensive
1.488 + XPT linking if the interfaces in an existing destination match those of
1.489 + the individual XPTs to link.
1.490 + skip_if_older is ignored.
1.491 + '''
1.492 + if isinstance(dest, basestring):
1.493 + dest = Dest(dest)
1.494 + assert isinstance(dest, Dest)
1.495 +
1.496 + from xpt import xpt_link, Typelib, Interface
1.497 + all_typelibs = [Typelib.read(f.open()) for f in self._files]
1.498 + if dest.exists():
1.499 + # Typelib.read() needs to seek(), so use a BytesIO for dest
1.500 + # content.
1.501 + dest_interfaces = \
1.502 + dict((i.name, i)
1.503 + for i in Typelib.read(BytesIO(dest.read())).interfaces
1.504 + if i.iid != Interface.UNRESOLVED_IID)
1.505 + identical = True
1.506 + for f in self._files:
1.507 + typelib = Typelib.read(f.open())
1.508 + for i in typelib.interfaces:
1.509 + if i.iid != Interface.UNRESOLVED_IID and \
1.510 + not (i.name in dest_interfaces and
1.511 + i == dest_interfaces[i.name]):
1.512 + identical = False
1.513 + break
1.514 + if identical:
1.515 + return False
1.516 + s = BytesIO()
1.517 + xpt_link(all_typelibs).write(s)
1.518 + dest.write(s.getvalue())
1.519 + return True
1.520 +
1.521 + def open(self):
1.522 + raise RuntimeError("Unsupported")
1.523 +
1.524 + def isempty(self):
1.525 + '''
1.526 + Return whether there are XPT files to link.
1.527 + '''
1.528 + return len(self._files) == 0
1.529 +
1.530 +
1.531 +class ManifestFile(BaseFile):
1.532 + '''
1.533 + File class for a manifest file. It takes individual manifest entries (using
1.534 + the add() and remove() member functions), and adjusts them to be relative
1.535 + to the base path for the manifest, given at creation.
1.536 + Example:
1.537 + There is a manifest entry "content webapprt webapprt/content/" relative
1.538 + to "webapprt/chrome". When packaging, the entry will be stored in
1.539 + jar:webapprt/omni.ja!/chrome/chrome.manifest, which means the entry
1.540 + will have to be relative to "chrome" instead of "webapprt/chrome". This
1.541 + doesn't really matter when serializing the entry, since this base path
1.542 + is not written out, but it matters when moving the entry at the same
1.543 + time, e.g. to jar:webapprt/omni.ja!/chrome.manifest, which we don't do
1.544 + currently but could in the future.
1.545 + '''
1.546 + def __init__(self, base, entries=None):
1.547 + self._entries = entries if entries else []
1.548 + self._base = base
1.549 +
1.550 + def add(self, entry):
1.551 + '''
1.552 + Add the given entry to the manifest. Entries are rebased at open() time
1.553 + instead of add() time so that they can be more easily remove()d.
1.554 + '''
1.555 + assert isinstance(entry, ManifestEntry)
1.556 + self._entries.append(entry)
1.557 +
1.558 + def remove(self, entry):
1.559 + '''
1.560 + Remove the given entry from the manifest.
1.561 + '''
1.562 + assert isinstance(entry, ManifestEntry)
1.563 + self._entries.remove(entry)
1.564 +
1.565 + def open(self):
1.566 + '''
1.567 + Return a file-like object allowing to read() the serialized content of
1.568 + the manifest.
1.569 + '''
1.570 + return BytesIO(''.join('%s\n' % e.rebase(self._base)
1.571 + for e in self._entries))
1.572 +
1.573 + def __iter__(self):
1.574 + '''
1.575 + Iterate over entries in the manifest file.
1.576 + '''
1.577 + return iter(self._entries)
1.578 +
1.579 + def isempty(self):
1.580 + '''
1.581 + Return whether there are manifest entries to write
1.582 + '''
1.583 + return len(self._entries) == 0
1.584 +
1.585 +
1.586 +class MinifiedProperties(BaseFile):
1.587 + '''
1.588 + File class for minified properties. This wraps around a BaseFile instance,
1.589 + and removes lines starting with a # from its content.
1.590 + '''
1.591 + def __init__(self, file):
1.592 + assert isinstance(file, BaseFile)
1.593 + self._file = file
1.594 +
1.595 + def open(self):
1.596 + '''
1.597 + Return a file-like object allowing to read() the minified content of
1.598 + the properties file.
1.599 + '''
1.600 + return BytesIO(''.join(l for l in self._file.open().readlines()
1.601 + if not l.startswith('#')))
1.602 +
1.603 +
1.604 +class MinifiedJavaScript(BaseFile):
1.605 + '''
1.606 + File class for minifying JavaScript files.
1.607 + '''
1.608 + def __init__(self, file, verify_command=None):
1.609 + assert isinstance(file, BaseFile)
1.610 + self._file = file
1.611 + self._verify_command = verify_command
1.612 +
1.613 + def open(self):
1.614 + output = BytesIO()
1.615 + minify = JavascriptMinify(self._file.open(), output)
1.616 + minify.minify()
1.617 + output.seek(0)
1.618 +
1.619 + if not self._verify_command:
1.620 + return output
1.621 +
1.622 + input_source = self._file.open().read()
1.623 + output_source = output.getvalue()
1.624 +
1.625 + with NamedTemporaryFile() as fh1, NamedTemporaryFile() as fh2:
1.626 + fh1.write(input_source)
1.627 + fh2.write(output_source)
1.628 + fh1.flush()
1.629 + fh2.flush()
1.630 +
1.631 + try:
1.632 + args = list(self._verify_command)
1.633 + args.extend([fh1.name, fh2.name])
1.634 + subprocess.check_output(args, stderr=subprocess.STDOUT)
1.635 + except subprocess.CalledProcessError as e:
1.636 + errors.warn('JS minification verification failed for %s:' %
1.637 + (getattr(self._file, 'path', '<unknown>')))
1.638 + # Prefix each line with "Warning:" so mozharness doesn't
1.639 + # think these error messages are real errors.
1.640 + for line in e.output.splitlines():
1.641 + errors.warn(line)
1.642 +
1.643 + return self._file.open()
1.644 +
1.645 + return output
1.646 +
1.647 +
1.648 +class BaseFinder(object):
1.649 + def __init__(self, base, minify=False, minify_js=False,
1.650 + minify_js_verify_command=None):
1.651 + '''
1.652 + Initializes the instance with a reference base directory.
1.653 +
1.654 + The optional minify argument specifies whether minification of code
1.655 + should occur. minify_js is an additional option to control minification
1.656 + of JavaScript. It requires minify to be True.
1.657 +
1.658 + minify_js_verify_command can be used to optionally verify the results
1.659 + of JavaScript minification. If defined, it is expected to be an iterable
1.660 + that will constitute the first arguments to a called process which will
1.661 + receive the filenames of the original and minified JavaScript files.
1.662 + The invoked process can then verify the results. If minification is
1.663 + rejected, the process exits with a non-0 exit code and the original
1.664 + JavaScript source is used. An example value for this argument is
1.665 + ('/path/to/js', '/path/to/verify/script.js').
1.666 + '''
1.667 + if minify_js and not minify:
1.668 + raise ValueError('minify_js requires minify.')
1.669 +
1.670 + self.base = base
1.671 + self._minify = minify
1.672 + self._minify_js = minify_js
1.673 + self._minify_js_verify_command = minify_js_verify_command
1.674 +
1.675 + def find(self, pattern):
1.676 + '''
1.677 + Yield path, BaseFile_instance pairs for all files under the base
1.678 + directory and its subdirectories that match the given pattern. See the
1.679 + mozpack.path.match documentation for a description of the handled
1.680 + patterns.
1.681 + '''
1.682 + while pattern.startswith('/'):
1.683 + pattern = pattern[1:]
1.684 + for p, f in self._find(pattern):
1.685 + yield p, self._minify_file(p, f)
1.686 +
1.687 + def __iter__(self):
1.688 + '''
1.689 + Iterates over all files under the base directory (excluding files
1.690 + starting with a '.' and files at any level under a directory starting
1.691 + with a '.').
1.692 + for path, file in finder:
1.693 + ...
1.694 + '''
1.695 + return self.find('')
1.696 +
1.697 + def __contains__(self, pattern):
1.698 + raise RuntimeError("'in' operator forbidden for %s. Use contains()." %
1.699 + self.__class__.__name__)
1.700 +
1.701 + def contains(self, pattern):
1.702 + '''
1.703 + Return whether some files under the base directory match the given
1.704 + pattern. See the mozpack.path.match documentation for a description of
1.705 + the handled patterns.
1.706 + '''
1.707 + return any(self.find(pattern))
1.708 +
1.709 + def _minify_file(self, path, file):
1.710 + '''
1.711 + Return an appropriate MinifiedSomething wrapper for the given BaseFile
1.712 + instance (file), according to the file type (determined by the given
1.713 + path), if the FileFinder was created with minification enabled.
1.714 + Otherwise, just return the given BaseFile instance.
1.715 + '''
1.716 + if not self._minify or isinstance(file, ExecutableFile):
1.717 + return file
1.718 +
1.719 + if path.endswith('.properties'):
1.720 + return MinifiedProperties(file)
1.721 +
1.722 + if self._minify_js and path.endswith(('.js', '.jsm')):
1.723 + return MinifiedJavaScript(file, self._minify_js_verify_command)
1.724 +
1.725 + return file
1.726 +
1.727 +
1.728 +class FileFinder(BaseFinder):
1.729 + '''
1.730 + Helper to get appropriate BaseFile instances from the file system.
1.731 + '''
1.732 + def __init__(self, base, find_executables=True, ignore=(), **kargs):
1.733 + '''
1.734 + Create a FileFinder for files under the given base directory.
1.735 +
1.736 + The find_executables argument determines whether the finder needs to
1.737 + try to guess whether files are executables. Disabling this guessing
1.738 + when not necessary can speed up the finder significantly.
1.739 +
1.740 + ``ignore`` accepts an iterable of patterns to ignore. Entries are
1.741 + strings that match paths relative to ``base`` using
1.742 + ``mozpack.path.match()``. This means if an entry corresponds
1.743 + to a directory, all files under that directory will be ignored. If
1.744 + an entry corresponds to a file, that particular file will be ignored.
1.745 + '''
1.746 + BaseFinder.__init__(self, base, **kargs)
1.747 + self.find_executables = find_executables
1.748 + self.ignore = ignore
1.749 +
1.750 + def _find(self, pattern):
1.751 + '''
1.752 + Actual implementation of FileFinder.find(), dispatching to specialized
1.753 + member functions depending on what kind of pattern was given.
1.754 + Note all files with a name starting with a '.' are ignored when
1.755 + scanning directories, but are not ignored when explicitely requested.
1.756 + '''
1.757 + if '*' in pattern:
1.758 + return self._find_glob('', mozpack.path.split(pattern))
1.759 + elif os.path.isdir(os.path.join(self.base, pattern)):
1.760 + return self._find_dir(pattern)
1.761 + else:
1.762 + return self._find_file(pattern)
1.763 +
1.764 + def _find_dir(self, path):
1.765 + '''
1.766 + Actual implementation of FileFinder.find() when the given pattern
1.767 + corresponds to an existing directory under the base directory.
1.768 + Ignores file names starting with a '.' under the given path. If the
1.769 + path itself has leafs starting with a '.', they are not ignored.
1.770 + '''
1.771 + for p in self.ignore:
1.772 + if mozpack.path.match(path, p):
1.773 + return
1.774 +
1.775 + # The sorted makes the output idempotent. Otherwise, we are
1.776 + # likely dependent on filesystem implementation details, such as
1.777 + # inode ordering.
1.778 + for p in sorted(os.listdir(os.path.join(self.base, path))):
1.779 + if p.startswith('.'):
1.780 + continue
1.781 + for p_, f in self._find(mozpack.path.join(path, p)):
1.782 + yield p_, f
1.783 +
1.784 + def _find_file(self, path):
1.785 + '''
1.786 + Actual implementation of FileFinder.find() when the given pattern
1.787 + corresponds to an existing file under the base directory.
1.788 + '''
1.789 + srcpath = os.path.join(self.base, path)
1.790 + if not os.path.exists(srcpath):
1.791 + return
1.792 +
1.793 + for p in self.ignore:
1.794 + if mozpack.path.match(path, p):
1.795 + return
1.796 +
1.797 + if self.find_executables and is_executable(srcpath):
1.798 + yield path, ExecutableFile(srcpath)
1.799 + else:
1.800 + yield path, File(srcpath)
1.801 +
1.802 + def _find_glob(self, base, pattern):
1.803 + '''
1.804 + Actual implementation of FileFinder.find() when the given pattern
1.805 + contains globbing patterns ('*' or '**'). This is meant to be an
1.806 + equivalent of:
1.807 + for p, f in self:
1.808 + if mozpack.path.match(p, pattern):
1.809 + yield p, f
1.810 + but avoids scanning the entire tree.
1.811 + '''
1.812 + if not pattern:
1.813 + for p, f in self._find(base):
1.814 + yield p, f
1.815 + elif pattern[0] == '**':
1.816 + for p, f in self._find(base):
1.817 + if mozpack.path.match(p, mozpack.path.join(*pattern)):
1.818 + yield p, f
1.819 + elif '*' in pattern[0]:
1.820 + if not os.path.exists(os.path.join(self.base, base)):
1.821 + return
1.822 +
1.823 + for p in self.ignore:
1.824 + if mozpack.path.match(base, p):
1.825 + return
1.826 +
1.827 + # See above comment w.r.t. sorted() and idempotent behavior.
1.828 + for p in sorted(os.listdir(os.path.join(self.base, base))):
1.829 + if p.startswith('.') and not pattern[0].startswith('.'):
1.830 + continue
1.831 + if mozpack.path.match(p, pattern[0]):
1.832 + for p_, f in self._find_glob(mozpack.path.join(base, p),
1.833 + pattern[1:]):
1.834 + yield p_, f
1.835 + else:
1.836 + for p, f in self._find_glob(mozpack.path.join(base, pattern[0]),
1.837 + pattern[1:]):
1.838 + yield p, f
1.839 +
1.840 +
1.841 +class JarFinder(BaseFinder):
1.842 + '''
1.843 + Helper to get appropriate DeflatedFile instances from a JarReader.
1.844 + '''
1.845 + def __init__(self, base, reader, **kargs):
1.846 + '''
1.847 + Create a JarFinder for files in the given JarReader. The base argument
1.848 + is used as an indication of the Jar file location.
1.849 + '''
1.850 + assert isinstance(reader, JarReader)
1.851 + BaseFinder.__init__(self, base, **kargs)
1.852 + self._files = OrderedDict((f.filename, f) for f in reader)
1.853 +
1.854 + def _find(self, pattern):
1.855 + '''
1.856 + Actual implementation of JarFinder.find(), dispatching to specialized
1.857 + member functions depending on what kind of pattern was given.
1.858 + '''
1.859 + if '*' in pattern:
1.860 + for p in self._files:
1.861 + if mozpack.path.match(p, pattern):
1.862 + yield p, DeflatedFile(self._files[p])
1.863 + elif pattern == '':
1.864 + for p in self._files:
1.865 + yield p, DeflatedFile(self._files[p])
1.866 + elif pattern in self._files:
1.867 + yield pattern, DeflatedFile(self._files[pattern])
1.868 + else:
1.869 + for p in self._files:
1.870 + if mozpack.path.basedir(p, [pattern]) == pattern:
1.871 + yield p, DeflatedFile(self._files[p])
