michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: from mozpack.files import (
michael@0:     BaseFinder,
michael@0:     JarFinder,
michael@0:     ExecutableFile,
michael@0:     BaseFile,
michael@0:     GeneratedFile,
michael@0: )
michael@0: from mozpack.executables import (
michael@0:     MACHO_SIGNATURES,
michael@0: )
michael@0: from mozpack.mozjar import JarReader
michael@0: from mozpack.errors import errors
michael@0: from tempfile import mkstemp
michael@0: import mozpack.path
michael@0: import struct
michael@0: import os
michael@0: import subprocess
michael@0: from collections import OrderedDict
michael@0: 
michael@0: 
michael@0: def may_unify_binary(file):
michael@0:     '''
michael@0:     Return whether the given BaseFile instance is an ExecutableFile that
michael@0:     may be unified. Only non-fat Mach-O binaries are to be unified.
michael@0:     '''
michael@0:     if isinstance(file, ExecutableFile):
michael@0:         signature = file.open().read(4)
michael@0:         if len(signature) < 4:
michael@0:             return False
michael@0:         signature = struct.unpack('>L', signature)[0]
michael@0:         if signature in MACHO_SIGNATURES:
michael@0:             return True
michael@0:     return False
michael@0: 
michael@0: 
michael@0: class UnifiedExecutableFile(BaseFile):
michael@0:     '''
michael@0:     File class for executable and library files that to be unified with 'lipo'.
michael@0:     '''
michael@0:     def __init__(self, executable1, executable2):
michael@0:         '''
michael@0:         Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to
michael@0:         be unified. They are expected to be non-fat Mach-O executables.
michael@0:         '''
michael@0:         assert isinstance(executable1, ExecutableFile)
michael@0:         assert isinstance(executable2, ExecutableFile)
michael@0:         self._executables = (executable1, executable2)
michael@0: 
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         '''
michael@0:         Create a fat executable from the two Mach-O executable given when
michael@0:         creating the instance.
michael@0:         skip_if_older is ignored.
michael@0:         '''
michael@0:         assert isinstance(dest, basestring)
michael@0:         tmpfiles = []
michael@0:         try:
michael@0:             for e in self._executables:
michael@0:                 fd, f = mkstemp()
michael@0:                 os.close(fd)
michael@0:                 tmpfiles.append(f)
michael@0:                 e.copy(f, skip_if_older=False)
michael@0:             subprocess.call(['lipo', '-create'] + tmpfiles + ['-output', dest])
michael@0:         finally:
michael@0:             for f in tmpfiles:
michael@0:                 os.unlink(f)
michael@0: 
michael@0: 
michael@0: class UnifiedFinder(BaseFinder):
michael@0:     '''
michael@0:     Helper to get unified BaseFile instances from two distinct trees on the
michael@0:     file system.
michael@0:     '''
michael@0:     def __init__(self, finder1, finder2, sorted=[], **kargs):
michael@0:         '''
michael@0:         Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder
michael@0:         instances from which files are picked. UnifiedFinder.find() will act as
michael@0:         FileFinder.find() but will error out when matches can only be found in
michael@0:         one of the two trees and not the other. It will also error out if
michael@0:         matches can be found on both ends but their contents are not identical.
michael@0: 
michael@0:         The sorted argument gives a list of mozpack.path.match patterns. File
michael@0:         paths matching one of these patterns will have their contents compared
michael@0:         with their lines sorted.
michael@0:         '''
michael@0:         assert isinstance(finder1, BaseFinder)
michael@0:         assert isinstance(finder2, BaseFinder)
michael@0:         self._finder1 = finder1
michael@0:         self._finder2 = finder2
michael@0:         self._sorted = sorted
michael@0:         BaseFinder.__init__(self, finder1.base, **kargs)
michael@0: 
michael@0:     def _find(self, path):
michael@0:         '''
michael@0:         UnifiedFinder.find() implementation.
michael@0:         '''
michael@0:         files1 = OrderedDict()
michael@0:         for p, f in self._finder1.find(path):
michael@0:             files1[p] = f
michael@0:         files2 = set()
michael@0:         for p, f in self._finder2.find(path):
michael@0:             files2.add(p)
michael@0:             if p in files1:
michael@0:                 if may_unify_binary(files1[p]) and \
michael@0:                         may_unify_binary(f):
michael@0:                     yield p, UnifiedExecutableFile(files1[p], f)
michael@0:                 else:
michael@0:                     err = errors.count
michael@0:                     unified = self.unify_file(p, files1[p], f)
michael@0:                     if unified:
michael@0:                         yield p, unified
michael@0:                     elif err == errors.count:
michael@0:                         self._report_difference(p, files1[p], f)
michael@0:             else:
michael@0:                 errors.error('File missing in %s: %s' %
michael@0:                              (self._finder1.base, p))
michael@0:         for p in [p for p in files1 if not p in files2]:
michael@0:             errors.error('File missing in %s: %s' % (self._finder2.base, p))
michael@0: 
michael@0:     def _report_difference(self, path, file1, file2):
michael@0:         '''
michael@0:         Report differences between files in both trees.
michael@0:         '''
michael@0:         errors.error("Can't unify %s: file differs between %s and %s" %
michael@0:                      (path, self._finder1.base, self._finder2.base))
michael@0:         if not isinstance(file1, ExecutableFile) and \
michael@0:                 not isinstance(file2, ExecutableFile):
michael@0:             from difflib import unified_diff
michael@0:             for line in unified_diff(file1.open().readlines(),
michael@0:                                      file2.open().readlines(),
michael@0:                                      os.path.join(self._finder1.base, path),
michael@0:                                      os.path.join(self._finder2.base, path)):
michael@0:                 errors.out.write(line)
michael@0: 
michael@0:     def unify_file(self, path, file1, file2):
michael@0:         '''
michael@0:         Given two BaseFiles and the path they were found at, check whether
michael@0:         their content match and return the first BaseFile if they do.
michael@0:         '''
michael@0:         content1 = file1.open().readlines()
michael@0:         content2 = file2.open().readlines()
michael@0:         if content1 == content2:
michael@0:             return file1
michael@0:         for pattern in self._sorted:
michael@0:             if mozpack.path.match(path, pattern):
michael@0:                 if sorted(content1) == sorted(content2):
michael@0:                     return file1
michael@0:                 break
michael@0:         return None
michael@0: 
michael@0: 
michael@0: class UnifiedBuildFinder(UnifiedFinder):
michael@0:     '''
michael@0:     Specialized UnifiedFinder for Mozilla applications packaging. It allows
michael@0:     "*.manifest" files to differ in their order, and unifies "buildconfig.html"
michael@0:     files by merging their content.
michael@0:     '''
michael@0:     def __init__(self, finder1, finder2, **kargs):
michael@0:         UnifiedFinder.__init__(self, finder1, finder2,
michael@0:                                sorted=['**/*.manifest'], **kargs)
michael@0: 
michael@0:     def unify_file(self, path, file1, file2):
michael@0:         '''
michael@0:         Unify buildconfig.html contents, or defer to UnifiedFinder.unify_file.
michael@0:         '''
michael@0:         if mozpack.path.basename(path) == 'buildconfig.html':
michael@0:             content1 = file1.open().readlines()
michael@0:             content2 = file2.open().readlines()
michael@0:             # Copy everything from the first file up to the end of its <body>,
michael@0:             # insert a <hr> between the two files and copy the second file's
michael@0:             # content beginning after its leading <h1>.
michael@0:             return GeneratedFile(''.join(
michael@0:                 content1[:content1.index('</body>\n')] +
michael@0:                 ['<hr> </hr>\n'] +
michael@0:                 content2[content2.index('<h1>about:buildconfig</h1>\n') + 1:]
michael@0:             ))
michael@0:         if path.endswith('.xpi'):
michael@0:             finder1 = JarFinder(os.path.join(self._finder1.base, path),
michael@0:                                 JarReader(fileobj=file1.open()))
michael@0:             finder2 = JarFinder(os.path.join(self._finder2.base, path),
michael@0:                                 JarReader(fileobj=file2.open()))
michael@0:             unifier = UnifiedFinder(finder1, finder2, sorted=self._sorted)
michael@0:             err = errors.count
michael@0:             all(unifier.find(''))
michael@0:             if err == errors.count:
michael@0:                 return file1
michael@0:             return None
michael@0:         return UnifiedFinder.unify_file(self, path, file1, file2)