python/mozbuild/mozpack/files.py

Fri, 16 Jan 2015 18:13:44 +0100

author
Michael Schloh von Bennewitz <michael@schloh.com>
date
Fri, 16 Jan 2015 18:13:44 +0100
branch
TOR_BUG_9701
changeset 14
925c144e1f1f
permissions
-rw-r--r--

Integrate suggestion from review to improve consistency with existing code.

     1 # This Source Code Form is subject to the terms of the Mozilla Public
     2 # License, v. 2.0. If a copy of the MPL was not distributed with this
     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
     5 import errno
     6 import os
     7 import platform
     8 import shutil
     9 import stat
    10 import subprocess
    11 import uuid
    12 import mozbuild.makeutil as makeutil
    13 from mozbuild.preprocessor import Preprocessor
    14 from mozbuild.util import FileAvoidWrite
    15 from mozpack.executables import (
    16     is_executable,
    17     may_strip,
    18     strip,
    19     may_elfhack,
    20     elfhack,
    21 )
    22 from mozpack.chrome.manifest import ManifestEntry
    23 from io import BytesIO
    24 from mozpack.errors import (
    25     ErrorMessage,
    26     errors,
    27 )
    28 from mozpack.mozjar import JarReader
    29 import mozpack.path
    30 from collections import OrderedDict
    31 from jsmin import JavascriptMinify
    32 from tempfile import (
    33     mkstemp,
    34     NamedTemporaryFile,
    35 )
    38 class Dest(object):
    39     '''
    40     Helper interface for BaseFile.copy. The interface works as follows:
    41     - read() and write() can be used to sequentially read/write from the
    42       underlying file.
    43     - a call to read() after a write() will re-open the underlying file and
    44       read from it.
    45     - a call to write() after a read() will re-open the underlying file,
    46       emptying it, and write to it.
    47     '''
    48     def __init__(self, path):
    49         self.path = path
    50         self.mode = None
    52     @property
    53     def name(self):
    54         return self.path
    56     def read(self, length=-1):
    57         if self.mode != 'r':
    58             self.file = open(self.path, 'rb')
    59             self.mode = 'r'
    60         return self.file.read(length)
    62     def write(self, data):
    63         if self.mode != 'w':
    64             self.file = open(self.path, 'wb')
    65             self.mode = 'w'
    66         return self.file.write(data)
    68     def exists(self):
    69         return os.path.exists(self.path)
    71     def close(self):
    72         if self.mode:
    73             self.mode = None
    74             self.file.close()
    77 class BaseFile(object):
    78     '''
    79     Base interface and helper for file copying. Derived class may implement
    80     their own copy function, or rely on BaseFile.copy using the open() member
    81     function and/or the path property.
    82     '''
    83     @staticmethod
    84     def is_older(first, second):
    85         '''
    86         Compares the modification time of two files, and returns whether the
    87         ``first`` file is older than the ``second`` file.
    88         '''
    89         # os.path.getmtime returns a result in seconds with precision up to
    90         # the microsecond. But microsecond is too precise because
    91         # shutil.copystat only copies milliseconds, and seconds is not
    92         # enough precision.
    93         return int(os.path.getmtime(first) * 1000) \
    94                 <= int(os.path.getmtime(second) * 1000)
    96     @staticmethod
    97     def any_newer(dest, inputs):
    98         '''
    99         Compares the modification time of ``dest`` to multiple input files, and
   100         returns whether any of the ``inputs`` is newer (has a later mtime) than
   101         ``dest``.
   102         '''
   103         # os.path.getmtime returns a result in seconds with precision up to
   104         # the microsecond. But microsecond is too precise because
   105         # shutil.copystat only copies milliseconds, and seconds is not
   106         # enough precision.
   107         dest_mtime = int(os.path.getmtime(dest) * 1000)
   108         for input in inputs:
   109             if dest_mtime < int(os.path.getmtime(input) * 1000):
   110                 return True
   111         return False
   113     def copy(self, dest, skip_if_older=True):
   114         '''
   115         Copy the BaseFile content to the destination given as a string or a
   116         Dest instance. Avoids replacing existing files if the BaseFile content
   117         matches that of the destination, or in case of plain files, if the
   118         destination is newer than the original file. This latter behaviour is
   119         disabled when skip_if_older is False.
   120         Returns whether a copy was actually performed (True) or not (False).
   121         '''
   122         if isinstance(dest, basestring):
   123             dest = Dest(dest)
   124         else:
   125             assert isinstance(dest, Dest)
   127         can_skip_content_check = False
   128         if not dest.exists():
   129             can_skip_content_check = True
   130         elif getattr(self, 'path', None) and getattr(dest, 'path', None):
   131             if skip_if_older and BaseFile.is_older(self.path, dest.path):
   132                 return False
   133             elif os.path.getsize(self.path) != os.path.getsize(dest.path):
   134                 can_skip_content_check = True
   136         if can_skip_content_check:
   137             if getattr(self, 'path', None) and getattr(dest, 'path', None):
   138                 shutil.copy2(self.path, dest.path)
   139             else:
   140                 # Ensure the file is always created
   141                 if not dest.exists():
   142                     dest.write('')
   143                 shutil.copyfileobj(self.open(), dest)
   144             return True
   146         src = self.open()
   147         copy_content = ''
   148         while True:
   149             dest_content = dest.read(32768)
   150             src_content = src.read(32768)
   151             copy_content += src_content
   152             if len(dest_content) == len(src_content) == 0:
   153                 break
   154             # If the read content differs between origin and destination,
   155             # write what was read up to now, and copy the remainder.
   156             if dest_content != src_content:
   157                 dest.write(copy_content)
   158                 shutil.copyfileobj(src, dest)
   159                 break
   160         if hasattr(self, 'path') and hasattr(dest, 'path'):
   161             shutil.copystat(self.path, dest.path)
   162         return True
   164     def open(self):
   165         '''
   166         Return a file-like object allowing to read() the content of the
   167         associated file. This is meant to be overloaded in subclasses to return
   168         a custom file-like object.
   169         '''
   170         assert self.path is not None
   171         return open(self.path, 'rb')
   173     @property
   174     def mode(self):
   175         '''
   176         Return the file's unix mode, or None if it has no meaning.
   177         '''
   178         return None
   181 class File(BaseFile):
   182     '''
   183     File class for plain files.
   184     '''
   185     def __init__(self, path):
   186         self.path = path
   188     @property
   189     def mode(self):
   190         '''
   191         Return the file's unix mode, as returned by os.stat().st_mode.
   192         '''
   193         if platform.system() == 'Windows':
   194             return None
   195         assert self.path is not None
   196         return os.stat(self.path).st_mode
   198 class ExecutableFile(File):
   199     '''
   200     File class for executable and library files on OS/2, OS/X and ELF systems.
   201     (see mozpack.executables.is_executable documentation).
   202     '''
   203     def copy(self, dest, skip_if_older=True):
   204         real_dest = dest
   205         if not isinstance(dest, basestring):
   206             fd, dest = mkstemp()
   207             os.close(fd)
   208             os.remove(dest)
   209         assert isinstance(dest, basestring)
   210         # If File.copy didn't actually copy because dest is newer, check the
   211         # file sizes. If dest is smaller, it means it is already stripped and
   212         # elfhacked, so we can skip.
   213         if not File.copy(self, dest, skip_if_older) and \
   214                 os.path.getsize(self.path) > os.path.getsize(dest):
   215             return False
   216         try:
   217             if may_strip(dest):
   218                 strip(dest)
   219             if may_elfhack(dest):
   220                 elfhack(dest)
   221         except ErrorMessage:
   222             os.remove(dest)
   223             raise
   225         if real_dest != dest:
   226             f = File(dest)
   227             ret = f.copy(real_dest, skip_if_older)
   228             os.remove(dest)
   229             return ret
   230         return True
   233 class AbsoluteSymlinkFile(File):
   234     '''File class that is copied by symlinking (if available).
   236     This class only works if the target path is absolute.
   237     '''
   239     def __init__(self, path):
   240         if not os.path.isabs(path):
   241             raise ValueError('Symlink target not absolute: %s' % path)
   243         File.__init__(self, path)
   245     def copy(self, dest, skip_if_older=True):
   246         assert isinstance(dest, basestring)
   248         # The logic in this function is complicated by the fact that symlinks
   249         # aren't universally supported. So, where symlinks aren't supported, we
   250         # fall back to file copying. Keep in mind that symlink support is
   251         # per-filesystem, not per-OS.
   253         # Handle the simple case where symlinks are definitely not supported by
   254         # falling back to file copy.
   255         if not hasattr(os, 'symlink'):
   256             return File.copy(self, dest, skip_if_older=skip_if_older)
   258         # Always verify the symlink target path exists.
   259         if not os.path.exists(self.path):
   260             raise ErrorMessage('Symlink target path does not exist: %s' % self.path)
   262         st = None
   264         try:
   265             st = os.lstat(dest)
   266         except OSError as ose:
   267             if ose.errno != errno.ENOENT:
   268                 raise
   270         # If the dest is a symlink pointing to us, we have nothing to do.
   271         # If it's the wrong symlink, the filesystem must support symlinks,
   272         # so we replace with a proper symlink.
   273         if st and stat.S_ISLNK(st.st_mode):
   274             link = os.readlink(dest)
   275             if link == self.path:
   276                 return False
   278             os.remove(dest)
   279             os.symlink(self.path, dest)
   280             return True
   282         # If the destination doesn't exist, we try to create a symlink. If that
   283         # fails, we fall back to copy code.
   284         if not st:
   285             try:
   286                 os.symlink(self.path, dest)
   287                 return True
   288             except OSError:
   289                 return File.copy(self, dest, skip_if_older=skip_if_older)
   291         # Now the complicated part. If the destination exists, we could be
   292         # replacing a file with a symlink. Or, the filesystem may not support
   293         # symlinks. We want to minimize I/O overhead for performance reasons,
   294         # so we keep the existing destination file around as long as possible.
   295         # A lot of the system calls would be eliminated if we cached whether
   296         # symlinks are supported. However, even if we performed a single
   297         # up-front test of whether the root of the destination directory
   298         # supports symlinks, there's no guarantee that all operations for that
   299         # dest (or source) would be on the same filesystem and would support
   300         # symlinks.
   301         #
   302         # Our strategy is to attempt to create a new symlink with a random
   303         # name. If that fails, we fall back to copy mode. If that works, we
   304         # remove the old destination and move the newly-created symlink into
   305         # its place.
   307         temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4()))
   308         try:
   309             os.symlink(self.path, temp_dest)
   310         # TODO Figure out exactly how symlink creation fails and only trap
   311         # that.
   312         except EnvironmentError:
   313             return File.copy(self, dest, skip_if_older=skip_if_older)
   315         # If removing the original file fails, don't forget to clean up the
   316         # temporary symlink.
   317         try:
   318             os.remove(dest)
   319         except EnvironmentError:
   320             os.remove(temp_dest)
   321             raise
   323         os.rename(temp_dest, dest)
   324         return True
   327 class ExistingFile(BaseFile):
   328     '''
   329     File class that represents a file that may exist but whose content comes
   330     from elsewhere.
   332     This purpose of this class is to account for files that are installed via
   333     external means. It is typically only used in manifests or in registries to
   334     account for files.
   336     When asked to copy, this class does nothing because nothing is known about
   337     the source file/data.
   339     Instances of this class come in two flavors: required and optional. If an
   340     existing file is required, it must exist during copy() or an error is
   341     raised.
   342     '''
   343     def __init__(self, required):
   344         self.required = required
   346     def copy(self, dest, skip_if_older=True):
   347         if isinstance(dest, basestring):
   348             dest = Dest(dest)
   349         else:
   350             assert isinstance(dest, Dest)
   352         if not self.required:
   353             return
   355         if not dest.exists():
   356             errors.fatal("Required existing file doesn't exist: %s" %
   357                 dest.path)
   360 class PreprocessedFile(BaseFile):
   361     '''
   362     File class for a file that is preprocessed. PreprocessedFile.copy() runs
   363     the preprocessor on the file to create the output.
   364     '''
   365     def __init__(self, path, depfile_path, marker, defines, extra_depends=None):
   366         self.path = path
   367         self.depfile = depfile_path
   368         self.marker = marker
   369         self.defines = defines
   370         self.extra_depends = list(extra_depends or [])
   372     def copy(self, dest, skip_if_older=True):
   373         '''
   374         Invokes the preprocessor to create the destination file.
   375         '''
   376         if isinstance(dest, basestring):
   377             dest = Dest(dest)
   378         else:
   379             assert isinstance(dest, Dest)
   381         # We have to account for the case where the destination exists and is a
   382         # symlink to something. Since we know the preprocessor is certainly not
   383         # going to create a symlink, we can just remove the existing one. If the
   384         # destination is not a symlink, we leave it alone, since we're going to
   385         # overwrite its contents anyway.
   386         # If symlinks aren't supported at all, we can skip this step.
   387         if hasattr(os, 'symlink'):
   388             if os.path.islink(dest.path):
   389                 os.remove(dest.path)
   391         pp_deps = set(self.extra_depends)
   393         # If a dependency file was specified, and it exists, add any
   394         # dependencies from that file to our list.
   395         if self.depfile and os.path.exists(self.depfile):
   396             target = mozpack.path.normpath(dest.name)
   397             with open(self.depfile, 'rb') as fileobj:
   398                 for rule in makeutil.read_dep_makefile(fileobj):
   399                     if target in rule.targets():
   400                         pp_deps.update(rule.dependencies())
   402         skip = False
   403         if dest.exists() and skip_if_older:
   404             # If a dependency file was specified, and it doesn't exist,
   405             # assume that the preprocessor needs to be rerun. That will
   406             # regenerate the dependency file.
   407             if self.depfile and not os.path.exists(self.depfile):
   408                 skip = False
   409             else:
   410                 skip = not BaseFile.any_newer(dest.path, pp_deps)
   412         if skip:
   413             return False
   415         deps_out = None
   416         if self.depfile:
   417             deps_out = FileAvoidWrite(self.depfile)
   418         pp = Preprocessor(defines=self.defines, marker=self.marker)
   420         with open(self.path, 'rU') as input:
   421             pp.processFile(input=input, output=dest, depfile=deps_out)
   423         dest.close()
   424         if self.depfile:
   425             deps_out.close()
   427         return True
   430 class GeneratedFile(BaseFile):
   431     '''
   432     File class for content with no previous existence on the filesystem.
   433     '''
   434     def __init__(self, content):
   435         self.content = content
   437     def open(self):
   438         return BytesIO(self.content)
   441 class DeflatedFile(BaseFile):
   442     '''
   443     File class for members of a jar archive. DeflatedFile.copy() effectively
   444     extracts the file from the jar archive.
   445     '''
   446     def __init__(self, file):
   447         from mozpack.mozjar import JarFileReader
   448         assert isinstance(file, JarFileReader)
   449         self.file = file
   451     def open(self):
   452         self.file.seek(0)
   453         return self.file
   456 class XPTFile(GeneratedFile):
   457     '''
   458     File class for a linked XPT file. It takes several XPT files as input
   459     (using the add() and remove() member functions), and links them at copy()
   460     time.
   461     '''
   462     def __init__(self):
   463         self._files = set()
   465     def add(self, xpt):
   466         '''
   467         Add the given XPT file (as a BaseFile instance) to the list of XPTs
   468         to link.
   469         '''
   470         assert isinstance(xpt, BaseFile)
   471         self._files.add(xpt)
   473     def remove(self, xpt):
   474         '''
   475         Remove the given XPT file (as a BaseFile instance) from the list of
   476         XPTs to link.
   477         '''
   478         assert isinstance(xpt, BaseFile)
   479         self._files.remove(xpt)
   481     def copy(self, dest, skip_if_older=True):
   482         '''
   483         Link the registered XPTs and place the resulting linked XPT at the
   484         destination given as a string or a Dest instance. Avoids an expensive
   485         XPT linking if the interfaces in an existing destination match those of
   486         the individual XPTs to link.
   487         skip_if_older is ignored.
   488         '''
   489         if isinstance(dest, basestring):
   490             dest = Dest(dest)
   491         assert isinstance(dest, Dest)
   493         from xpt import xpt_link, Typelib, Interface
   494         all_typelibs = [Typelib.read(f.open()) for f in self._files]
   495         if dest.exists():
   496             # Typelib.read() needs to seek(), so use a BytesIO for dest
   497             # content.
   498             dest_interfaces = \
   499                 dict((i.name, i)
   500                      for i in Typelib.read(BytesIO(dest.read())).interfaces
   501                      if i.iid != Interface.UNRESOLVED_IID)
   502             identical = True
   503             for f in self._files:
   504                 typelib = Typelib.read(f.open())
   505                 for i in typelib.interfaces:
   506                     if i.iid != Interface.UNRESOLVED_IID and \
   507                             not (i.name in dest_interfaces and
   508                                  i == dest_interfaces[i.name]):
   509                         identical = False
   510                         break
   511             if identical:
   512                 return False
   513         s = BytesIO()
   514         xpt_link(all_typelibs).write(s)
   515         dest.write(s.getvalue())
   516         return True
   518     def open(self):
   519         raise RuntimeError("Unsupported")
   521     def isempty(self):
   522         '''
   523         Return whether there are XPT files to link.
   524         '''
   525         return len(self._files) == 0
   528 class ManifestFile(BaseFile):
   529     '''
   530     File class for a manifest file. It takes individual manifest entries (using
   531     the add() and remove() member functions), and adjusts them to be relative
   532     to the base path for the manifest, given at creation.
   533     Example:
   534         There is a manifest entry "content webapprt webapprt/content/" relative
   535         to "webapprt/chrome". When packaging, the entry will be stored in
   536         jar:webapprt/omni.ja!/chrome/chrome.manifest, which means the entry
   537         will have to be relative to "chrome" instead of "webapprt/chrome". This
   538         doesn't really matter when serializing the entry, since this base path
   539         is not written out, but it matters when moving the entry at the same
   540         time, e.g. to jar:webapprt/omni.ja!/chrome.manifest, which we don't do
   541         currently but could in the future.
   542     '''
   543     def __init__(self, base, entries=None):
   544         self._entries = entries if entries else []
   545         self._base = base
   547     def add(self, entry):
   548         '''
   549         Add the given entry to the manifest. Entries are rebased at open() time
   550         instead of add() time so that they can be more easily remove()d.
   551         '''
   552         assert isinstance(entry, ManifestEntry)
   553         self._entries.append(entry)
   555     def remove(self, entry):
   556         '''
   557         Remove the given entry from the manifest.
   558         '''
   559         assert isinstance(entry, ManifestEntry)
   560         self._entries.remove(entry)
   562     def open(self):
   563         '''
   564         Return a file-like object allowing to read() the serialized content of
   565         the manifest.
   566         '''
   567         return BytesIO(''.join('%s\n' % e.rebase(self._base)
   568                                for e in self._entries))
   570     def __iter__(self):
   571         '''
   572         Iterate over entries in the manifest file.
   573         '''
   574         return iter(self._entries)
   576     def isempty(self):
   577         '''
   578         Return whether there are manifest entries to write
   579         '''
   580         return len(self._entries) == 0
   583 class MinifiedProperties(BaseFile):
   584     '''
   585     File class for minified properties. This wraps around a BaseFile instance,
   586     and removes lines starting with a # from its content.
   587     '''
   588     def __init__(self, file):
   589         assert isinstance(file, BaseFile)
   590         self._file = file
   592     def open(self):
   593         '''
   594         Return a file-like object allowing to read() the minified content of
   595         the properties file.
   596         '''
   597         return BytesIO(''.join(l for l in self._file.open().readlines()
   598                                if not l.startswith('#')))
   601 class MinifiedJavaScript(BaseFile):
   602     '''
   603     File class for minifying JavaScript files.
   604     '''
   605     def __init__(self, file, verify_command=None):
   606         assert isinstance(file, BaseFile)
   607         self._file = file
   608         self._verify_command = verify_command
   610     def open(self):
   611         output = BytesIO()
   612         minify = JavascriptMinify(self._file.open(), output)
   613         minify.minify()
   614         output.seek(0)
   616         if not self._verify_command:
   617             return output
   619         input_source = self._file.open().read()
   620         output_source = output.getvalue()
   622         with NamedTemporaryFile() as fh1, NamedTemporaryFile() as fh2:
   623             fh1.write(input_source)
   624             fh2.write(output_source)
   625             fh1.flush()
   626             fh2.flush()
   628             try:
   629                 args = list(self._verify_command)
   630                 args.extend([fh1.name, fh2.name])
   631                 subprocess.check_output(args, stderr=subprocess.STDOUT)
   632             except subprocess.CalledProcessError as e:
   633                 errors.warn('JS minification verification failed for %s:' %
   634                     (getattr(self._file, 'path', '<unknown>')))
   635                 # Prefix each line with "Warning:" so mozharness doesn't
   636                 # think these error messages are real errors.
   637                 for line in e.output.splitlines():
   638                     errors.warn(line)
   640                 return self._file.open()
   642         return output
   645 class BaseFinder(object):
   646     def __init__(self, base, minify=False, minify_js=False,
   647         minify_js_verify_command=None):
   648         '''
   649         Initializes the instance with a reference base directory.
   651         The optional minify argument specifies whether minification of code
   652         should occur. minify_js is an additional option to control minification
   653         of JavaScript. It requires minify to be True.
   655         minify_js_verify_command can be used to optionally verify the results
   656         of JavaScript minification. If defined, it is expected to be an iterable
   657         that will constitute the first arguments to a called process which will
   658         receive the filenames of the original and minified JavaScript files.
   659         The invoked process can then verify the results. If minification is
   660         rejected, the process exits with a non-0 exit code and the original
   661         JavaScript source is used. An example value for this argument is
   662         ('/path/to/js', '/path/to/verify/script.js').
   663         '''
   664         if minify_js and not minify:
   665             raise ValueError('minify_js requires minify.')
   667         self.base = base
   668         self._minify = minify
   669         self._minify_js = minify_js
   670         self._minify_js_verify_command = minify_js_verify_command
   672     def find(self, pattern):
   673         '''
   674         Yield path, BaseFile_instance pairs for all files under the base
   675         directory and its subdirectories that match the given pattern. See the
   676         mozpack.path.match documentation for a description of the handled
   677         patterns.
   678         '''
   679         while pattern.startswith('/'):
   680             pattern = pattern[1:]
   681         for p, f in self._find(pattern):
   682             yield p, self._minify_file(p, f)
   684     def __iter__(self):
   685         '''
   686         Iterates over all files under the base directory (excluding files
   687         starting with a '.' and files at any level under a directory starting
   688         with a '.').
   689             for path, file in finder:
   690                 ...
   691         '''
   692         return self.find('')
   694     def __contains__(self, pattern):
   695         raise RuntimeError("'in' operator forbidden for %s. Use contains()." %
   696                            self.__class__.__name__)
   698     def contains(self, pattern):
   699         '''
   700         Return whether some files under the base directory match the given
   701         pattern. See the mozpack.path.match documentation for a description of
   702         the handled patterns.
   703         '''
   704         return any(self.find(pattern))
   706     def _minify_file(self, path, file):
   707         '''
   708         Return an appropriate MinifiedSomething wrapper for the given BaseFile
   709         instance (file), according to the file type (determined by the given
   710         path), if the FileFinder was created with minification enabled.
   711         Otherwise, just return the given BaseFile instance.
   712         '''
   713         if not self._minify or isinstance(file, ExecutableFile):
   714             return file
   716         if path.endswith('.properties'):
   717             return MinifiedProperties(file)
   719         if self._minify_js and path.endswith(('.js', '.jsm')):
   720             return MinifiedJavaScript(file, self._minify_js_verify_command)
   722         return file
   725 class FileFinder(BaseFinder):
   726     '''
   727     Helper to get appropriate BaseFile instances from the file system.
   728     '''
   729     def __init__(self, base, find_executables=True, ignore=(), **kargs):
   730         '''
   731         Create a FileFinder for files under the given base directory.
   733         The find_executables argument determines whether the finder needs to
   734         try to guess whether files are executables. Disabling this guessing
   735         when not necessary can speed up the finder significantly.
   737         ``ignore`` accepts an iterable of patterns to ignore. Entries are
   738         strings that match paths relative to ``base`` using
   739         ``mozpack.path.match()``. This means if an entry corresponds
   740         to a directory, all files under that directory will be ignored. If
   741         an entry corresponds to a file, that particular file will be ignored.
   742         '''
   743         BaseFinder.__init__(self, base, **kargs)
   744         self.find_executables = find_executables
   745         self.ignore = ignore
   747     def _find(self, pattern):
   748         '''
   749         Actual implementation of FileFinder.find(), dispatching to specialized
   750         member functions depending on what kind of pattern was given.
   751         Note all files with a name starting with a '.' are ignored when
   752         scanning directories, but are not ignored when explicitely requested.
   753         '''
   754         if '*' in pattern:
   755             return self._find_glob('', mozpack.path.split(pattern))
   756         elif os.path.isdir(os.path.join(self.base, pattern)):
   757             return self._find_dir(pattern)
   758         else:
   759             return self._find_file(pattern)
   761     def _find_dir(self, path):
   762         '''
   763         Actual implementation of FileFinder.find() when the given pattern
   764         corresponds to an existing directory under the base directory.
   765         Ignores file names starting with a '.' under the given path. If the
   766         path itself has leafs starting with a '.', they are not ignored.
   767         '''
   768         for p in self.ignore:
   769             if mozpack.path.match(path, p):
   770                 return
   772         # The sorted makes the output idempotent. Otherwise, we are
   773         # likely dependent on filesystem implementation details, such as
   774         # inode ordering.
   775         for p in sorted(os.listdir(os.path.join(self.base, path))):
   776             if p.startswith('.'):
   777                 continue
   778             for p_, f in self._find(mozpack.path.join(path, p)):
   779                 yield p_, f
   781     def _find_file(self, path):
   782         '''
   783         Actual implementation of FileFinder.find() when the given pattern
   784         corresponds to an existing file under the base directory.
   785         '''
   786         srcpath = os.path.join(self.base, path)
   787         if not os.path.exists(srcpath):
   788             return
   790         for p in self.ignore:
   791             if mozpack.path.match(path, p):
   792                 return
   794         if self.find_executables and is_executable(srcpath):
   795             yield path, ExecutableFile(srcpath)
   796         else:
   797             yield path, File(srcpath)
   799     def _find_glob(self, base, pattern):
   800         '''
   801         Actual implementation of FileFinder.find() when the given pattern
   802         contains globbing patterns ('*' or '**'). This is meant to be an
   803         equivalent of:
   804             for p, f in self:
   805                 if mozpack.path.match(p, pattern):
   806                     yield p, f
   807         but avoids scanning the entire tree.
   808         '''
   809         if not pattern:
   810             for p, f in self._find(base):
   811                 yield p, f
   812         elif pattern[0] == '**':
   813             for p, f in self._find(base):
   814                 if mozpack.path.match(p, mozpack.path.join(*pattern)):
   815                     yield p, f
   816         elif '*' in pattern[0]:
   817             if not os.path.exists(os.path.join(self.base, base)):
   818                 return
   820             for p in self.ignore:
   821                 if mozpack.path.match(base, p):
   822                     return
   824             # See above comment w.r.t. sorted() and idempotent behavior.
   825             for p in sorted(os.listdir(os.path.join(self.base, base))):
   826                 if p.startswith('.') and not pattern[0].startswith('.'):
   827                     continue
   828                 if mozpack.path.match(p, pattern[0]):
   829                     for p_, f in self._find_glob(mozpack.path.join(base, p),
   830                                                  pattern[1:]):
   831                         yield p_, f
   832         else:
   833             for p, f in self._find_glob(mozpack.path.join(base, pattern[0]),
   834                                         pattern[1:]):
   835                 yield p, f
   838 class JarFinder(BaseFinder):
   839     '''
   840     Helper to get appropriate DeflatedFile instances from a JarReader.
   841     '''
   842     def __init__(self, base, reader, **kargs):
   843         '''
   844         Create a JarFinder for files in the given JarReader. The base argument
   845         is used as an indication of the Jar file location.
   846         '''
   847         assert isinstance(reader, JarReader)
   848         BaseFinder.__init__(self, base, **kargs)
   849         self._files = OrderedDict((f.filename, f) for f in reader)
   851     def _find(self, pattern):
   852         '''
   853         Actual implementation of JarFinder.find(), dispatching to specialized
   854         member functions depending on what kind of pattern was given.
   855         '''
   856         if '*' in pattern:
   857             for p in self._files:
   858                 if mozpack.path.match(p, pattern):
   859                     yield p, DeflatedFile(self._files[p])
   860         elif pattern == '':
   861             for p in self._files:
   862                 yield p, DeflatedFile(self._files[p])
   863         elif pattern in self._files:
   864             yield pattern, DeflatedFile(self._files[pattern])
   865         else:
   866             for p in self._files:
   867                 if mozpack.path.basedir(p, [pattern]) == pattern:
   868                     yield p, DeflatedFile(self._files[p])

mercurial