michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: import errno
michael@0: import os
michael@0: import platform
michael@0: import shutil
michael@0: import stat
michael@0: import subprocess
michael@0: import uuid
michael@0: import mozbuild.makeutil as makeutil
michael@0: from mozbuild.preprocessor import Preprocessor
michael@0: from mozbuild.util import FileAvoidWrite
michael@0: from mozpack.executables import (
michael@0:     is_executable,
michael@0:     may_strip,
michael@0:     strip,
michael@0:     may_elfhack,
michael@0:     elfhack,
michael@0: )
michael@0: from mozpack.chrome.manifest import ManifestEntry
michael@0: from io import BytesIO
michael@0: from mozpack.errors import (
michael@0:     ErrorMessage,
michael@0:     errors,
michael@0: )
michael@0: from mozpack.mozjar import JarReader
michael@0: import mozpack.path
michael@0: from collections import OrderedDict
michael@0: from jsmin import JavascriptMinify
michael@0: from tempfile import (
michael@0:     mkstemp,
michael@0:     NamedTemporaryFile,
michael@0: )
michael@0: 
michael@0: 
michael@0: class Dest(object):
michael@0:     '''
michael@0:     Helper interface for BaseFile.copy. The interface works as follows:
michael@0:     - read() and write() can be used to sequentially read/write from the
michael@0:       underlying file.
michael@0:     - a call to read() after a write() will re-open the underlying file and
michael@0:       read from it.
michael@0:     - a call to write() after a read() will re-open the underlying file,
michael@0:       emptying it, and write to it.
michael@0:     '''
michael@0:     def __init__(self, path):
michael@0:         self.path = path
michael@0:         self.mode = None
michael@0: 
michael@0:     @property
michael@0:     def name(self):
michael@0:         return self.path
michael@0: 
michael@0:     def read(self, length=-1):
michael@0:         if self.mode != 'r':
michael@0:             self.file = open(self.path, 'rb')
michael@0:             self.mode = 'r'
michael@0:         return self.file.read(length)
michael@0: 
michael@0:     def write(self, data):
michael@0:         if self.mode != 'w':
michael@0:             self.file = open(self.path, 'wb')
michael@0:             self.mode = 'w'
michael@0:         return self.file.write(data)
michael@0: 
michael@0:     def exists(self):
michael@0:         return os.path.exists(self.path)
michael@0: 
michael@0:     def close(self):
michael@0:         if self.mode:
michael@0:             self.mode = None
michael@0:             self.file.close()
michael@0: 
michael@0: 
michael@0: class BaseFile(object):
michael@0:     '''
michael@0:     Base interface and helper for file copying. Derived class may implement
michael@0:     their own copy function, or rely on BaseFile.copy using the open() member
michael@0:     function and/or the path property.
michael@0:     '''
michael@0:     @staticmethod
michael@0:     def is_older(first, second):
michael@0:         '''
michael@0:         Compares the modification time of two files, and returns whether the
michael@0:         ``first`` file is older than the ``second`` file.
michael@0:         '''
michael@0:         # os.path.getmtime returns a result in seconds with precision up to
michael@0:         # the microsecond. But microsecond is too precise because
michael@0:         # shutil.copystat only copies milliseconds, and seconds is not
michael@0:         # enough precision.
michael@0:         return int(os.path.getmtime(first) * 1000) \
michael@0:                 <= int(os.path.getmtime(second) * 1000)
michael@0: 
michael@0:     @staticmethod
michael@0:     def any_newer(dest, inputs):
michael@0:         '''
michael@0:         Compares the modification time of ``dest`` to multiple input files, and
michael@0:         returns whether any of the ``inputs`` is newer (has a later mtime) than
michael@0:         ``dest``.
michael@0:         '''
michael@0:         # os.path.getmtime returns a result in seconds with precision up to
michael@0:         # the microsecond. But microsecond is too precise because
michael@0:         # shutil.copystat only copies milliseconds, and seconds is not
michael@0:         # enough precision.
michael@0:         dest_mtime = int(os.path.getmtime(dest) * 1000)
michael@0:         for input in inputs:
michael@0:             if dest_mtime < int(os.path.getmtime(input) * 1000):
michael@0:                 return True
michael@0:         return False
michael@0: 
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         '''
michael@0:         Copy the BaseFile content to the destination given as a string or a
michael@0:         Dest instance. Avoids replacing existing files if the BaseFile content
michael@0:         matches that of the destination, or in case of plain files, if the
michael@0:         destination is newer than the original file. This latter behaviour is
michael@0:         disabled when skip_if_older is False.
michael@0:         Returns whether a copy was actually performed (True) or not (False).
michael@0:         '''
michael@0:         if isinstance(dest, basestring):
michael@0:             dest = Dest(dest)
michael@0:         else:
michael@0:             assert isinstance(dest, Dest)
michael@0: 
michael@0:         can_skip_content_check = False
michael@0:         if not dest.exists():
michael@0:             can_skip_content_check = True
michael@0:         elif getattr(self, 'path', None) and getattr(dest, 'path', None):
michael@0:             if skip_if_older and BaseFile.is_older(self.path, dest.path):
michael@0:                 return False
michael@0:             elif os.path.getsize(self.path) != os.path.getsize(dest.path):
michael@0:                 can_skip_content_check = True
michael@0: 
michael@0:         if can_skip_content_check:
michael@0:             if getattr(self, 'path', None) and getattr(dest, 'path', None):
michael@0:                 shutil.copy2(self.path, dest.path)
michael@0:             else:
michael@0:                 # Ensure the file is always created
michael@0:                 if not dest.exists():
michael@0:                     dest.write('')
michael@0:                 shutil.copyfileobj(self.open(), dest)
michael@0:             return True
michael@0: 
michael@0:         src = self.open()
michael@0:         copy_content = ''
michael@0:         while True:
michael@0:             dest_content = dest.read(32768)
michael@0:             src_content = src.read(32768)
michael@0:             copy_content += src_content
michael@0:             if len(dest_content) == len(src_content) == 0:
michael@0:                 break
michael@0:             # If the read content differs between origin and destination,
michael@0:             # write what was read up to now, and copy the remainder.
michael@0:             if dest_content != src_content:
michael@0:                 dest.write(copy_content)
michael@0:                 shutil.copyfileobj(src, dest)
michael@0:                 break
michael@0:         if hasattr(self, 'path') and hasattr(dest, 'path'):
michael@0:             shutil.copystat(self.path, dest.path)
michael@0:         return True
michael@0: 
michael@0:     def open(self):
michael@0:         '''
michael@0:         Return a file-like object allowing to read() the content of the
michael@0:         associated file. This is meant to be overloaded in subclasses to return
michael@0:         a custom file-like object.
michael@0:         '''
michael@0:         assert self.path is not None
michael@0:         return open(self.path, 'rb')
michael@0: 
michael@0:     @property
michael@0:     def mode(self):
michael@0:         '''
michael@0:         Return the file's unix mode, or None if it has no meaning.
michael@0:         '''
michael@0:         return None
michael@0: 
michael@0: 
michael@0: class File(BaseFile):
michael@0:     '''
michael@0:     File class for plain files.
michael@0:     '''
michael@0:     def __init__(self, path):
michael@0:         self.path = path
michael@0: 
michael@0:     @property
michael@0:     def mode(self):
michael@0:         '''
michael@0:         Return the file's unix mode, as returned by os.stat().st_mode.
michael@0:         '''
michael@0:         if platform.system() == 'Windows':
michael@0:             return None
michael@0:         assert self.path is not None
michael@0:         return os.stat(self.path).st_mode
michael@0: 
michael@0: class ExecutableFile(File):
michael@0:     '''
michael@0:     File class for executable and library files on OS/2, OS/X and ELF systems.
michael@0:     (see mozpack.executables.is_executable documentation).
michael@0:     '''
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         real_dest = dest
michael@0:         if not isinstance(dest, basestring):
michael@0:             fd, dest = mkstemp()
michael@0:             os.close(fd)
michael@0:             os.remove(dest)
michael@0:         assert isinstance(dest, basestring)
michael@0:         # If File.copy didn't actually copy because dest is newer, check the
michael@0:         # file sizes. If dest is smaller, it means it is already stripped and
michael@0:         # elfhacked, so we can skip.
michael@0:         if not File.copy(self, dest, skip_if_older) and \
michael@0:                 os.path.getsize(self.path) > os.path.getsize(dest):
michael@0:             return False
michael@0:         try:
michael@0:             if may_strip(dest):
michael@0:                 strip(dest)
michael@0:             if may_elfhack(dest):
michael@0:                 elfhack(dest)
michael@0:         except ErrorMessage:
michael@0:             os.remove(dest)
michael@0:             raise
michael@0: 
michael@0:         if real_dest != dest:
michael@0:             f = File(dest)
michael@0:             ret = f.copy(real_dest, skip_if_older)
michael@0:             os.remove(dest)
michael@0:             return ret
michael@0:         return True
michael@0: 
michael@0: 
michael@0: class AbsoluteSymlinkFile(File):
michael@0:     '''File class that is copied by symlinking (if available).
michael@0: 
michael@0:     This class only works if the target path is absolute.
michael@0:     '''
michael@0: 
michael@0:     def __init__(self, path):
michael@0:         if not os.path.isabs(path):
michael@0:             raise ValueError('Symlink target not absolute: %s' % path)
michael@0: 
michael@0:         File.__init__(self, path)
michael@0: 
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         assert isinstance(dest, basestring)
michael@0: 
michael@0:         # The logic in this function is complicated by the fact that symlinks
michael@0:         # aren't universally supported. So, where symlinks aren't supported, we
michael@0:         # fall back to file copying. Keep in mind that symlink support is
michael@0:         # per-filesystem, not per-OS.
michael@0: 
michael@0:         # Handle the simple case where symlinks are definitely not supported by
michael@0:         # falling back to file copy.
michael@0:         if not hasattr(os, 'symlink'):
michael@0:             return File.copy(self, dest, skip_if_older=skip_if_older)
michael@0: 
michael@0:         # Always verify the symlink target path exists.
michael@0:         if not os.path.exists(self.path):
michael@0:             raise ErrorMessage('Symlink target path does not exist: %s' % self.path)
michael@0: 
michael@0:         st = None
michael@0: 
michael@0:         try:
michael@0:             st = os.lstat(dest)
michael@0:         except OSError as ose:
michael@0:             if ose.errno != errno.ENOENT:
michael@0:                 raise
michael@0: 
michael@0:         # If the dest is a symlink pointing to us, we have nothing to do.
michael@0:         # If it's the wrong symlink, the filesystem must support symlinks,
michael@0:         # so we replace with a proper symlink.
michael@0:         if st and stat.S_ISLNK(st.st_mode):
michael@0:             link = os.readlink(dest)
michael@0:             if link == self.path:
michael@0:                 return False
michael@0: 
michael@0:             os.remove(dest)
michael@0:             os.symlink(self.path, dest)
michael@0:             return True
michael@0: 
michael@0:         # If the destination doesn't exist, we try to create a symlink. If that
michael@0:         # fails, we fall back to copy code.
michael@0:         if not st:
michael@0:             try:
michael@0:                 os.symlink(self.path, dest)
michael@0:                 return True
michael@0:             except OSError:
michael@0:                 return File.copy(self, dest, skip_if_older=skip_if_older)
michael@0: 
michael@0:         # Now the complicated part. If the destination exists, we could be
michael@0:         # replacing a file with a symlink. Or, the filesystem may not support
michael@0:         # symlinks. We want to minimize I/O overhead for performance reasons,
michael@0:         # so we keep the existing destination file around as long as possible.
michael@0:         # A lot of the system calls would be eliminated if we cached whether
michael@0:         # symlinks are supported. However, even if we performed a single
michael@0:         # up-front test of whether the root of the destination directory
michael@0:         # supports symlinks, there's no guarantee that all operations for that
michael@0:         # dest (or source) would be on the same filesystem and would support
michael@0:         # symlinks.
michael@0:         #
michael@0:         # Our strategy is to attempt to create a new symlink with a random
michael@0:         # name. If that fails, we fall back to copy mode. If that works, we
michael@0:         # remove the old destination and move the newly-created symlink into
michael@0:         # its place.
michael@0: 
michael@0:         temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4()))
michael@0:         try:
michael@0:             os.symlink(self.path, temp_dest)
michael@0:         # TODO Figure out exactly how symlink creation fails and only trap
michael@0:         # that.
michael@0:         except EnvironmentError:
michael@0:             return File.copy(self, dest, skip_if_older=skip_if_older)
michael@0: 
michael@0:         # If removing the original file fails, don't forget to clean up the
michael@0:         # temporary symlink.
michael@0:         try:
michael@0:             os.remove(dest)
michael@0:         except EnvironmentError:
michael@0:             os.remove(temp_dest)
michael@0:             raise
michael@0: 
michael@0:         os.rename(temp_dest, dest)
michael@0:         return True
michael@0: 
michael@0: 
michael@0: class ExistingFile(BaseFile):
michael@0:     '''
michael@0:     File class that represents a file that may exist but whose content comes
michael@0:     from elsewhere.
michael@0: 
michael@0:     This purpose of this class is to account for files that are installed via
michael@0:     external means. It is typically only used in manifests or in registries to
michael@0:     account for files.
michael@0: 
michael@0:     When asked to copy, this class does nothing because nothing is known about
michael@0:     the source file/data.
michael@0: 
michael@0:     Instances of this class come in two flavors: required and optional. If an
michael@0:     existing file is required, it must exist during copy() or an error is
michael@0:     raised.
michael@0:     '''
michael@0:     def __init__(self, required):
michael@0:         self.required = required
michael@0: 
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         if isinstance(dest, basestring):
michael@0:             dest = Dest(dest)
michael@0:         else:
michael@0:             assert isinstance(dest, Dest)
michael@0: 
michael@0:         if not self.required:
michael@0:             return
michael@0: 
michael@0:         if not dest.exists():
michael@0:             errors.fatal("Required existing file doesn't exist: %s" %
michael@0:                 dest.path)
michael@0: 
michael@0: 
michael@0: class PreprocessedFile(BaseFile):
michael@0:     '''
michael@0:     File class for a file that is preprocessed. PreprocessedFile.copy() runs
michael@0:     the preprocessor on the file to create the output.
michael@0:     '''
michael@0:     def __init__(self, path, depfile_path, marker, defines, extra_depends=None):
michael@0:         self.path = path
michael@0:         self.depfile = depfile_path
michael@0:         self.marker = marker
michael@0:         self.defines = defines
michael@0:         self.extra_depends = list(extra_depends or [])
michael@0: 
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         '''
michael@0:         Invokes the preprocessor to create the destination file.
michael@0:         '''
michael@0:         if isinstance(dest, basestring):
michael@0:             dest = Dest(dest)
michael@0:         else:
michael@0:             assert isinstance(dest, Dest)
michael@0: 
michael@0:         # We have to account for the case where the destination exists and is a
michael@0:         # symlink to something. Since we know the preprocessor is certainly not
michael@0:         # going to create a symlink, we can just remove the existing one. If the
michael@0:         # destination is not a symlink, we leave it alone, since we're going to
michael@0:         # overwrite its contents anyway.
michael@0:         # If symlinks aren't supported at all, we can skip this step.
michael@0:         if hasattr(os, 'symlink'):
michael@0:             if os.path.islink(dest.path):
michael@0:                 os.remove(dest.path)
michael@0: 
michael@0:         pp_deps = set(self.extra_depends)
michael@0: 
michael@0:         # If a dependency file was specified, and it exists, add any
michael@0:         # dependencies from that file to our list.
michael@0:         if self.depfile and os.path.exists(self.depfile):
michael@0:             target = mozpack.path.normpath(dest.name)
michael@0:             with open(self.depfile, 'rb') as fileobj:
michael@0:                 for rule in makeutil.read_dep_makefile(fileobj):
michael@0:                     if target in rule.targets():
michael@0:                         pp_deps.update(rule.dependencies())
michael@0: 
michael@0:         skip = False
michael@0:         if dest.exists() and skip_if_older:
michael@0:             # If a dependency file was specified, and it doesn't exist,
michael@0:             # assume that the preprocessor needs to be rerun. That will
michael@0:             # regenerate the dependency file.
michael@0:             if self.depfile and not os.path.exists(self.depfile):
michael@0:                 skip = False
michael@0:             else:
michael@0:                 skip = not BaseFile.any_newer(dest.path, pp_deps)
michael@0: 
michael@0:         if skip:
michael@0:             return False
michael@0: 
michael@0:         deps_out = None
michael@0:         if self.depfile:
michael@0:             deps_out = FileAvoidWrite(self.depfile)
michael@0:         pp = Preprocessor(defines=self.defines, marker=self.marker)
michael@0: 
michael@0:         with open(self.path, 'rU') as input:
michael@0:             pp.processFile(input=input, output=dest, depfile=deps_out)
michael@0: 
michael@0:         dest.close()
michael@0:         if self.depfile:
michael@0:             deps_out.close()
michael@0: 
michael@0:         return True
michael@0: 
michael@0: 
michael@0: class GeneratedFile(BaseFile):
michael@0:     '''
michael@0:     File class for content with no previous existence on the filesystem.
michael@0:     '''
michael@0:     def __init__(self, content):
michael@0:         self.content = content
michael@0: 
michael@0:     def open(self):
michael@0:         return BytesIO(self.content)
michael@0: 
michael@0: 
michael@0: class DeflatedFile(BaseFile):
michael@0:     '''
michael@0:     File class for members of a jar archive. DeflatedFile.copy() effectively
michael@0:     extracts the file from the jar archive.
michael@0:     '''
michael@0:     def __init__(self, file):
michael@0:         from mozpack.mozjar import JarFileReader
michael@0:         assert isinstance(file, JarFileReader)
michael@0:         self.file = file
michael@0: 
michael@0:     def open(self):
michael@0:         self.file.seek(0)
michael@0:         return self.file
michael@0: 
michael@0: 
michael@0: class XPTFile(GeneratedFile):
michael@0:     '''
michael@0:     File class for a linked XPT file. It takes several XPT files as input
michael@0:     (using the add() and remove() member functions), and links them at copy()
michael@0:     time.
michael@0:     '''
michael@0:     def __init__(self):
michael@0:         self._files = set()
michael@0: 
michael@0:     def add(self, xpt):
michael@0:         '''
michael@0:         Add the given XPT file (as a BaseFile instance) to the list of XPTs
michael@0:         to link.
michael@0:         '''
michael@0:         assert isinstance(xpt, BaseFile)
michael@0:         self._files.add(xpt)
michael@0: 
michael@0:     def remove(self, xpt):
michael@0:         '''
michael@0:         Remove the given XPT file (as a BaseFile instance) from the list of
michael@0:         XPTs to link.
michael@0:         '''
michael@0:         assert isinstance(xpt, BaseFile)
michael@0:         self._files.remove(xpt)
michael@0: 
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         '''
michael@0:         Link the registered XPTs and place the resulting linked XPT at the
michael@0:         destination given as a string or a Dest instance. Avoids an expensive
michael@0:         XPT linking if the interfaces in an existing destination match those of
michael@0:         the individual XPTs to link.
michael@0:         skip_if_older is ignored.
michael@0:         '''
michael@0:         if isinstance(dest, basestring):
michael@0:             dest = Dest(dest)
michael@0:         assert isinstance(dest, Dest)
michael@0: 
michael@0:         from xpt import xpt_link, Typelib, Interface
michael@0:         all_typelibs = [Typelib.read(f.open()) for f in self._files]
michael@0:         if dest.exists():
michael@0:             # Typelib.read() needs to seek(), so use a BytesIO for dest
michael@0:             # content.
michael@0:             dest_interfaces = \
michael@0:                 dict((i.name, i)
michael@0:                      for i in Typelib.read(BytesIO(dest.read())).interfaces
michael@0:                      if i.iid != Interface.UNRESOLVED_IID)
michael@0:             identical = True
michael@0:             for f in self._files:
michael@0:                 typelib = Typelib.read(f.open())
michael@0:                 for i in typelib.interfaces:
michael@0:                     if i.iid != Interface.UNRESOLVED_IID and \
michael@0:                             not (i.name in dest_interfaces and
michael@0:                                  i == dest_interfaces[i.name]):
michael@0:                         identical = False
michael@0:                         break
michael@0:             if identical:
michael@0:                 return False
michael@0:         s = BytesIO()
michael@0:         xpt_link(all_typelibs).write(s)
michael@0:         dest.write(s.getvalue())
michael@0:         return True
michael@0: 
michael@0:     def open(self):
michael@0:         raise RuntimeError("Unsupported")
michael@0: 
michael@0:     def isempty(self):
michael@0:         '''
michael@0:         Return whether there are XPT files to link.
michael@0:         '''
michael@0:         return len(self._files) == 0
michael@0: 
michael@0: 
michael@0: class ManifestFile(BaseFile):
michael@0:     '''
michael@0:     File class for a manifest file. It takes individual manifest entries (using
michael@0:     the add() and remove() member functions), and adjusts them to be relative
michael@0:     to the base path for the manifest, given at creation.
michael@0:     Example:
michael@0:         There is a manifest entry "content webapprt webapprt/content/" relative
michael@0:         to "webapprt/chrome". When packaging, the entry will be stored in
michael@0:         jar:webapprt/omni.ja!/chrome/chrome.manifest, which means the entry
michael@0:         will have to be relative to "chrome" instead of "webapprt/chrome". This
michael@0:         doesn't really matter when serializing the entry, since this base path
michael@0:         is not written out, but it matters when moving the entry at the same
michael@0:         time, e.g. to jar:webapprt/omni.ja!/chrome.manifest, which we don't do
michael@0:         currently but could in the future.
michael@0:     '''
michael@0:     def __init__(self, base, entries=None):
michael@0:         self._entries = entries if entries else []
michael@0:         self._base = base
michael@0: 
michael@0:     def add(self, entry):
michael@0:         '''
michael@0:         Add the given entry to the manifest. Entries are rebased at open() time
michael@0:         instead of add() time so that they can be more easily remove()d.
michael@0:         '''
michael@0:         assert isinstance(entry, ManifestEntry)
michael@0:         self._entries.append(entry)
michael@0: 
michael@0:     def remove(self, entry):
michael@0:         '''
michael@0:         Remove the given entry from the manifest.
michael@0:         '''
michael@0:         assert isinstance(entry, ManifestEntry)
michael@0:         self._entries.remove(entry)
michael@0: 
michael@0:     def open(self):
michael@0:         '''
michael@0:         Return a file-like object allowing to read() the serialized content of
michael@0:         the manifest.
michael@0:         '''
michael@0:         return BytesIO(''.join('%s\n' % e.rebase(self._base)
michael@0:                                for e in self._entries))
michael@0: 
michael@0:     def __iter__(self):
michael@0:         '''
michael@0:         Iterate over entries in the manifest file.
michael@0:         '''
michael@0:         return iter(self._entries)
michael@0: 
michael@0:     def isempty(self):
michael@0:         '''
michael@0:         Return whether there are manifest entries to write
michael@0:         '''
michael@0:         return len(self._entries) == 0
michael@0: 
michael@0: 
michael@0: class MinifiedProperties(BaseFile):
michael@0:     '''
michael@0:     File class for minified properties. This wraps around a BaseFile instance,
michael@0:     and removes lines starting with a # from its content.
michael@0:     '''
michael@0:     def __init__(self, file):
michael@0:         assert isinstance(file, BaseFile)
michael@0:         self._file = file
michael@0: 
michael@0:     def open(self):
michael@0:         '''
michael@0:         Return a file-like object allowing to read() the minified content of
michael@0:         the properties file.
michael@0:         '''
michael@0:         return BytesIO(''.join(l for l in self._file.open().readlines()
michael@0:                                if not l.startswith('#')))
michael@0: 
michael@0: 
michael@0: class MinifiedJavaScript(BaseFile):
michael@0:     '''
michael@0:     File class for minifying JavaScript files.
michael@0:     '''
michael@0:     def __init__(self, file, verify_command=None):
michael@0:         assert isinstance(file, BaseFile)
michael@0:         self._file = file
michael@0:         self._verify_command = verify_command
michael@0: 
michael@0:     def open(self):
michael@0:         output = BytesIO()
michael@0:         minify = JavascriptMinify(self._file.open(), output)
michael@0:         minify.minify()
michael@0:         output.seek(0)
michael@0: 
michael@0:         if not self._verify_command:
michael@0:             return output
michael@0: 
michael@0:         input_source = self._file.open().read()
michael@0:         output_source = output.getvalue()
michael@0: 
michael@0:         with NamedTemporaryFile() as fh1, NamedTemporaryFile() as fh2:
michael@0:             fh1.write(input_source)
michael@0:             fh2.write(output_source)
michael@0:             fh1.flush()
michael@0:             fh2.flush()
michael@0: 
michael@0:             try:
michael@0:                 args = list(self._verify_command)
michael@0:                 args.extend([fh1.name, fh2.name])
michael@0:                 subprocess.check_output(args, stderr=subprocess.STDOUT)
michael@0:             except subprocess.CalledProcessError as e:
michael@0:                 errors.warn('JS minification verification failed for %s:' %
michael@0:                     (getattr(self._file, 'path', '<unknown>')))
michael@0:                 # Prefix each line with "Warning:" so mozharness doesn't
michael@0:                 # think these error messages are real errors.
michael@0:                 for line in e.output.splitlines():
michael@0:                     errors.warn(line)
michael@0: 
michael@0:                 return self._file.open()
michael@0: 
michael@0:         return output
michael@0: 
michael@0: 
michael@0: class BaseFinder(object):
michael@0:     def __init__(self, base, minify=False, minify_js=False,
michael@0:         minify_js_verify_command=None):
michael@0:         '''
michael@0:         Initializes the instance with a reference base directory.
michael@0: 
michael@0:         The optional minify argument specifies whether minification of code
michael@0:         should occur. minify_js is an additional option to control minification
michael@0:         of JavaScript. It requires minify to be True.
michael@0: 
michael@0:         minify_js_verify_command can be used to optionally verify the results
michael@0:         of JavaScript minification. If defined, it is expected to be an iterable
michael@0:         that will constitute the first arguments to a called process which will
michael@0:         receive the filenames of the original and minified JavaScript files.
michael@0:         The invoked process can then verify the results. If minification is
michael@0:         rejected, the process exits with a non-0 exit code and the original
michael@0:         JavaScript source is used. An example value for this argument is
michael@0:         ('/path/to/js', '/path/to/verify/script.js').
michael@0:         '''
michael@0:         if minify_js and not minify:
michael@0:             raise ValueError('minify_js requires minify.')
michael@0: 
michael@0:         self.base = base
michael@0:         self._minify = minify
michael@0:         self._minify_js = minify_js
michael@0:         self._minify_js_verify_command = minify_js_verify_command
michael@0: 
michael@0:     def find(self, pattern):
michael@0:         '''
michael@0:         Yield path, BaseFile_instance pairs for all files under the base
michael@0:         directory and its subdirectories that match the given pattern. See the
michael@0:         mozpack.path.match documentation for a description of the handled
michael@0:         patterns.
michael@0:         '''
michael@0:         while pattern.startswith('/'):
michael@0:             pattern = pattern[1:]
michael@0:         for p, f in self._find(pattern):
michael@0:             yield p, self._minify_file(p, f)
michael@0: 
michael@0:     def __iter__(self):
michael@0:         '''
michael@0:         Iterates over all files under the base directory (excluding files
michael@0:         starting with a '.' and files at any level under a directory starting
michael@0:         with a '.').
michael@0:             for path, file in finder:
michael@0:                 ...
michael@0:         '''
michael@0:         return self.find('')
michael@0: 
michael@0:     def __contains__(self, pattern):
michael@0:         raise RuntimeError("'in' operator forbidden for %s. Use contains()." %
michael@0:                            self.__class__.__name__)
michael@0: 
michael@0:     def contains(self, pattern):
michael@0:         '''
michael@0:         Return whether some files under the base directory match the given
michael@0:         pattern. See the mozpack.path.match documentation for a description of
michael@0:         the handled patterns.
michael@0:         '''
michael@0:         return any(self.find(pattern))
michael@0: 
michael@0:     def _minify_file(self, path, file):
michael@0:         '''
michael@0:         Return an appropriate MinifiedSomething wrapper for the given BaseFile
michael@0:         instance (file), according to the file type (determined by the given
michael@0:         path), if the FileFinder was created with minification enabled.
michael@0:         Otherwise, just return the given BaseFile instance.
michael@0:         '''
michael@0:         if not self._minify or isinstance(file, ExecutableFile):
michael@0:             return file
michael@0: 
michael@0:         if path.endswith('.properties'):
michael@0:             return MinifiedProperties(file)
michael@0: 
michael@0:         if self._minify_js and path.endswith(('.js', '.jsm')):
michael@0:             return MinifiedJavaScript(file, self._minify_js_verify_command)
michael@0: 
michael@0:         return file
michael@0: 
michael@0: 
michael@0: class FileFinder(BaseFinder):
michael@0:     '''
michael@0:     Helper to get appropriate BaseFile instances from the file system.
michael@0:     '''
michael@0:     def __init__(self, base, find_executables=True, ignore=(), **kargs):
michael@0:         '''
michael@0:         Create a FileFinder for files under the given base directory.
michael@0: 
michael@0:         The find_executables argument determines whether the finder needs to
michael@0:         try to guess whether files are executables. Disabling this guessing
michael@0:         when not necessary can speed up the finder significantly.
michael@0: 
michael@0:         ``ignore`` accepts an iterable of patterns to ignore. Entries are
michael@0:         strings that match paths relative to ``base`` using
michael@0:         ``mozpack.path.match()``. This means if an entry corresponds
michael@0:         to a directory, all files under that directory will be ignored. If
michael@0:         an entry corresponds to a file, that particular file will be ignored.
michael@0:         '''
michael@0:         BaseFinder.__init__(self, base, **kargs)
michael@0:         self.find_executables = find_executables
michael@0:         self.ignore = ignore
michael@0: 
michael@0:     def _find(self, pattern):
michael@0:         '''
michael@0:         Actual implementation of FileFinder.find(), dispatching to specialized
michael@0:         member functions depending on what kind of pattern was given.
michael@0:         Note all files with a name starting with a '.' are ignored when
michael@0:         scanning directories, but are not ignored when explicitely requested.
michael@0:         '''
michael@0:         if '*' in pattern:
michael@0:             return self._find_glob('', mozpack.path.split(pattern))
michael@0:         elif os.path.isdir(os.path.join(self.base, pattern)):
michael@0:             return self._find_dir(pattern)
michael@0:         else:
michael@0:             return self._find_file(pattern)
michael@0: 
michael@0:     def _find_dir(self, path):
michael@0:         '''
michael@0:         Actual implementation of FileFinder.find() when the given pattern
michael@0:         corresponds to an existing directory under the base directory.
michael@0:         Ignores file names starting with a '.' under the given path. If the
michael@0:         path itself has leafs starting with a '.', they are not ignored.
michael@0:         '''
michael@0:         for p in self.ignore:
michael@0:             if mozpack.path.match(path, p):
michael@0:                 return
michael@0: 
michael@0:         # The sorted makes the output idempotent. Otherwise, we are
michael@0:         # likely dependent on filesystem implementation details, such as
michael@0:         # inode ordering.
michael@0:         for p in sorted(os.listdir(os.path.join(self.base, path))):
michael@0:             if p.startswith('.'):
michael@0:                 continue
michael@0:             for p_, f in self._find(mozpack.path.join(path, p)):
michael@0:                 yield p_, f
michael@0: 
michael@0:     def _find_file(self, path):
michael@0:         '''
michael@0:         Actual implementation of FileFinder.find() when the given pattern
michael@0:         corresponds to an existing file under the base directory.
michael@0:         '''
michael@0:         srcpath = os.path.join(self.base, path)
michael@0:         if not os.path.exists(srcpath):
michael@0:             return
michael@0: 
michael@0:         for p in self.ignore:
michael@0:             if mozpack.path.match(path, p):
michael@0:                 return
michael@0: 
michael@0:         if self.find_executables and is_executable(srcpath):
michael@0:             yield path, ExecutableFile(srcpath)
michael@0:         else:
michael@0:             yield path, File(srcpath)
michael@0: 
michael@0:     def _find_glob(self, base, pattern):
michael@0:         '''
michael@0:         Actual implementation of FileFinder.find() when the given pattern
michael@0:         contains globbing patterns ('*' or '**'). This is meant to be an
michael@0:         equivalent of:
michael@0:             for p, f in self:
michael@0:                 if mozpack.path.match(p, pattern):
michael@0:                     yield p, f
michael@0:         but avoids scanning the entire tree.
michael@0:         '''
michael@0:         if not pattern:
michael@0:             for p, f in self._find(base):
michael@0:                 yield p, f
michael@0:         elif pattern[0] == '**':
michael@0:             for p, f in self._find(base):
michael@0:                 if mozpack.path.match(p, mozpack.path.join(*pattern)):
michael@0:                     yield p, f
michael@0:         elif '*' in pattern[0]:
michael@0:             if not os.path.exists(os.path.join(self.base, base)):
michael@0:                 return
michael@0: 
michael@0:             for p in self.ignore:
michael@0:                 if mozpack.path.match(base, p):
michael@0:                     return
michael@0: 
michael@0:             # See above comment w.r.t. sorted() and idempotent behavior.
michael@0:             for p in sorted(os.listdir(os.path.join(self.base, base))):
michael@0:                 if p.startswith('.') and not pattern[0].startswith('.'):
michael@0:                     continue
michael@0:                 if mozpack.path.match(p, pattern[0]):
michael@0:                     for p_, f in self._find_glob(mozpack.path.join(base, p),
michael@0:                                                  pattern[1:]):
michael@0:                         yield p_, f
michael@0:         else:
michael@0:             for p, f in self._find_glob(mozpack.path.join(base, pattern[0]),
michael@0:                                         pattern[1:]):
michael@0:                 yield p, f
michael@0: 
michael@0: 
michael@0: class JarFinder(BaseFinder):
michael@0:     '''
michael@0:     Helper to get appropriate DeflatedFile instances from a JarReader.
michael@0:     '''
michael@0:     def __init__(self, base, reader, **kargs):
michael@0:         '''
michael@0:         Create a JarFinder for files in the given JarReader. The base argument
michael@0:         is used as an indication of the Jar file location.
michael@0:         '''
michael@0:         assert isinstance(reader, JarReader)
michael@0:         BaseFinder.__init__(self, base, **kargs)
michael@0:         self._files = OrderedDict((f.filename, f) for f in reader)
michael@0: 
michael@0:     def _find(self, pattern):
michael@0:         '''
michael@0:         Actual implementation of JarFinder.find(), dispatching to specialized
michael@0:         member functions depending on what kind of pattern was given.
michael@0:         '''
michael@0:         if '*' in pattern:
michael@0:             for p in self._files:
michael@0:                 if mozpack.path.match(p, pattern):
michael@0:                     yield p, DeflatedFile(self._files[p])
michael@0:         elif pattern == '':
michael@0:             for p in self._files:
michael@0:                 yield p, DeflatedFile(self._files[p])
michael@0:         elif pattern in self._files:
michael@0:             yield pattern, DeflatedFile(self._files[pattern])
michael@0:         else:
michael@0:             for p in self._files:
michael@0:                 if mozpack.path.basedir(p, [pattern]) == pattern:
michael@0:                     yield p, DeflatedFile(self._files[p])