1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python/mozbuild/mozpack/unify.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,192 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +from mozpack.files import (
1.9 + BaseFinder,
1.10 + JarFinder,
1.11 + ExecutableFile,
1.12 + BaseFile,
1.13 + GeneratedFile,
1.14 +)
1.15 +from mozpack.executables import (
1.16 + MACHO_SIGNATURES,
1.17 +)
1.18 +from mozpack.mozjar import JarReader
1.19 +from mozpack.errors import errors
1.20 +from tempfile import mkstemp
1.21 +import mozpack.path
1.22 +import struct
1.23 +import os
1.24 +import subprocess
1.25 +from collections import OrderedDict
1.26 +
1.27 +
1.28 +def may_unify_binary(file):
1.29 + '''
1.30 + Return whether the given BaseFile instance is an ExecutableFile that
1.31 + may be unified. Only non-fat Mach-O binaries are to be unified.
1.32 + '''
1.33 + if isinstance(file, ExecutableFile):
1.34 + signature = file.open().read(4)
1.35 + if len(signature) < 4:
1.36 + return False
1.37 + signature = struct.unpack('>L', signature)[0]
1.38 + if signature in MACHO_SIGNATURES:
1.39 + return True
1.40 + return False
1.41 +
1.42 +
1.43 +class UnifiedExecutableFile(BaseFile):
1.44 + '''
1.45 + File class for executable and library files that to be unified with 'lipo'.
1.46 + '''
1.47 + def __init__(self, executable1, executable2):
1.48 + '''
1.49 + Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to
1.50 + be unified. They are expected to be non-fat Mach-O executables.
1.51 + '''
1.52 + assert isinstance(executable1, ExecutableFile)
1.53 + assert isinstance(executable2, ExecutableFile)
1.54 + self._executables = (executable1, executable2)
1.55 +
1.56 + def copy(self, dest, skip_if_older=True):
1.57 + '''
1.58 + Create a fat executable from the two Mach-O executable given when
1.59 + creating the instance.
1.60 + skip_if_older is ignored.
1.61 + '''
1.62 + assert isinstance(dest, basestring)
1.63 + tmpfiles = []
1.64 + try:
1.65 + for e in self._executables:
1.66 + fd, f = mkstemp()
1.67 + os.close(fd)
1.68 + tmpfiles.append(f)
1.69 + e.copy(f, skip_if_older=False)
1.70 + subprocess.call(['lipo', '-create'] + tmpfiles + ['-output', dest])
1.71 + finally:
1.72 + for f in tmpfiles:
1.73 + os.unlink(f)
1.74 +
1.75 +
1.76 +class UnifiedFinder(BaseFinder):
1.77 + '''
1.78 + Helper to get unified BaseFile instances from two distinct trees on the
1.79 + file system.
1.80 + '''
1.81 + def __init__(self, finder1, finder2, sorted=[], **kargs):
1.82 + '''
1.83 + Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder
1.84 + instances from which files are picked. UnifiedFinder.find() will act as
1.85 + FileFinder.find() but will error out when matches can only be found in
1.86 + one of the two trees and not the other. It will also error out if
1.87 + matches can be found on both ends but their contents are not identical.
1.88 +
1.89 + The sorted argument gives a list of mozpack.path.match patterns. File
1.90 + paths matching one of these patterns will have their contents compared
1.91 + with their lines sorted.
1.92 + '''
1.93 + assert isinstance(finder1, BaseFinder)
1.94 + assert isinstance(finder2, BaseFinder)
1.95 + self._finder1 = finder1
1.96 + self._finder2 = finder2
1.97 + self._sorted = sorted
1.98 + BaseFinder.__init__(self, finder1.base, **kargs)
1.99 +
1.100 + def _find(self, path):
1.101 + '''
1.102 + UnifiedFinder.find() implementation.
1.103 + '''
1.104 + files1 = OrderedDict()
1.105 + for p, f in self._finder1.find(path):
1.106 + files1[p] = f
1.107 + files2 = set()
1.108 + for p, f in self._finder2.find(path):
1.109 + files2.add(p)
1.110 + if p in files1:
1.111 + if may_unify_binary(files1[p]) and \
1.112 + may_unify_binary(f):
1.113 + yield p, UnifiedExecutableFile(files1[p], f)
1.114 + else:
1.115 + err = errors.count
1.116 + unified = self.unify_file(p, files1[p], f)
1.117 + if unified:
1.118 + yield p, unified
1.119 + elif err == errors.count:
1.120 + self._report_difference(p, files1[p], f)
1.121 + else:
1.122 + errors.error('File missing in %s: %s' %
1.123 + (self._finder1.base, p))
1.124 + for p in [p for p in files1 if not p in files2]:
1.125 + errors.error('File missing in %s: %s' % (self._finder2.base, p))
1.126 +
1.127 + def _report_difference(self, path, file1, file2):
1.128 + '''
1.129 + Report differences between files in both trees.
1.130 + '''
1.131 + errors.error("Can't unify %s: file differs between %s and %s" %
1.132 + (path, self._finder1.base, self._finder2.base))
1.133 + if not isinstance(file1, ExecutableFile) and \
1.134 + not isinstance(file2, ExecutableFile):
1.135 + from difflib import unified_diff
1.136 + for line in unified_diff(file1.open().readlines(),
1.137 + file2.open().readlines(),
1.138 + os.path.join(self._finder1.base, path),
1.139 + os.path.join(self._finder2.base, path)):
1.140 + errors.out.write(line)
1.141 +
1.142 + def unify_file(self, path, file1, file2):
1.143 + '''
1.144 + Given two BaseFiles and the path they were found at, check whether
1.145 + their content match and return the first BaseFile if they do.
1.146 + '''
1.147 + content1 = file1.open().readlines()
1.148 + content2 = file2.open().readlines()
1.149 + if content1 == content2:
1.150 + return file1
1.151 + for pattern in self._sorted:
1.152 + if mozpack.path.match(path, pattern):
1.153 + if sorted(content1) == sorted(content2):
1.154 + return file1
1.155 + break
1.156 + return None
1.157 +
1.158 +
1.159 +class UnifiedBuildFinder(UnifiedFinder):
1.160 + '''
1.161 + Specialized UnifiedFinder for Mozilla applications packaging. It allows
1.162 + "*.manifest" files to differ in their order, and unifies "buildconfig.html"
1.163 + files by merging their content.
1.164 + '''
1.165 + def __init__(self, finder1, finder2, **kargs):
1.166 + UnifiedFinder.__init__(self, finder1, finder2,
1.167 + sorted=['**/*.manifest'], **kargs)
1.168 +
1.169 + def unify_file(self, path, file1, file2):
1.170 + '''
1.171 + Unify buildconfig.html contents, or defer to UnifiedFinder.unify_file.
1.172 + '''
1.173 + if mozpack.path.basename(path) == 'buildconfig.html':
1.174 + content1 = file1.open().readlines()
1.175 + content2 = file2.open().readlines()
1.176 + # Copy everything from the first file up to the end of its <body>,
1.177 + # insert a <hr> between the two files and copy the second file's
1.178 + # content beginning after its leading <h1>.
1.179 + return GeneratedFile(''.join(
1.180 + content1[:content1.index('</body>\n')] +
1.181 + ['<hr> </hr>\n'] +
1.182 + content2[content2.index('<h1>about:buildconfig</h1>\n') + 1:]
1.183 + ))
1.184 + if path.endswith('.xpi'):
1.185 + finder1 = JarFinder(os.path.join(self._finder1.base, path),
1.186 + JarReader(fileobj=file1.open()))
1.187 + finder2 = JarFinder(os.path.join(self._finder2.base, path),
1.188 + JarReader(fileobj=file2.open()))
1.189 + unifier = UnifiedFinder(finder1, finder2, sorted=self._sorted)
1.190 + err = errors.count
1.191 + all(unifier.find(''))
1.192 + if err == errors.count:
1.193 + return file1
1.194 + return None
1.195 + return UnifiedFinder.unify_file(self, path, file1, file2)
