python/mozbuild/mozpack/unify.py@925c144e1f1f (annotated)
python/mozbuild/mozpack/unify.py
Fri, 16 Jan 2015 18:13:44 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Fri, 16 Jan 2015 18:13:44 +0100
- branch
- TOR_BUG_9701
- changeset 14
- 925c144e1f1f
- permissions
- -rw-r--r--
Integrate suggestion from review to improve consistency with existing code.
| michael@0 | 1 | # This Source Code Form is subject to the terms of the Mozilla Public |
| michael@0 | 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this |
| michael@0 | 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| michael@0 | 4 | |
| michael@0 | 5 | from mozpack.files import ( |
| michael@0 | 6 | BaseFinder, |
| michael@0 | 7 | JarFinder, |
| michael@0 | 8 | ExecutableFile, |
| michael@0 | 9 | BaseFile, |
| michael@0 | 10 | GeneratedFile, |
| michael@0 | 11 | ) |
| michael@0 | 12 | from mozpack.executables import ( |
| michael@0 | 13 | MACHO_SIGNATURES, |
| michael@0 | 14 | ) |
| michael@0 | 15 | from mozpack.mozjar import JarReader |
| michael@0 | 16 | from mozpack.errors import errors |
| michael@0 | 17 | from tempfile import mkstemp |
| michael@0 | 18 | import mozpack.path |
| michael@0 | 19 | import struct |
| michael@0 | 20 | import os |
| michael@0 | 21 | import subprocess |
| michael@0 | 22 | from collections import OrderedDict |
| michael@0 | 23 | |
| michael@0 | 24 | |
| michael@0 | 25 | def may_unify_binary(file): |
| michael@0 | 26 | ''' |
| michael@0 | 27 | Return whether the given BaseFile instance is an ExecutableFile that |
| michael@0 | 28 | may be unified. Only non-fat Mach-O binaries are to be unified. |
| michael@0 | 29 | ''' |
| michael@0 | 30 | if isinstance(file, ExecutableFile): |
| michael@0 | 31 | signature = file.open().read(4) |
| michael@0 | 32 | if len(signature) < 4: |
| michael@0 | 33 | return False |
| michael@0 | 34 | signature = struct.unpack('>L', signature)[0] |
| michael@0 | 35 | if signature in MACHO_SIGNATURES: |
| michael@0 | 36 | return True |
| michael@0 | 37 | return False |
| michael@0 | 38 | |
| michael@0 | 39 | |
| michael@0 | 40 | class UnifiedExecutableFile(BaseFile): |
| michael@0 | 41 | ''' |
| michael@0 | 42 | File class for executable and library files that to be unified with 'lipo'. |
| michael@0 | 43 | ''' |
| michael@0 | 44 | def __init__(self, executable1, executable2): |
| michael@0 | 45 | ''' |
| michael@0 | 46 | Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to |
| michael@0 | 47 | be unified. They are expected to be non-fat Mach-O executables. |
| michael@0 | 48 | ''' |
| michael@0 | 49 | assert isinstance(executable1, ExecutableFile) |
| michael@0 | 50 | assert isinstance(executable2, ExecutableFile) |
| michael@0 | 51 | self._executables = (executable1, executable2) |
| michael@0 | 52 | |
| michael@0 | 53 | def copy(self, dest, skip_if_older=True): |
| michael@0 | 54 | ''' |
| michael@0 | 55 | Create a fat executable from the two Mach-O executable given when |
| michael@0 | 56 | creating the instance. |
| michael@0 | 57 | skip_if_older is ignored. |
| michael@0 | 58 | ''' |
| michael@0 | 59 | assert isinstance(dest, basestring) |
| michael@0 | 60 | tmpfiles = [] |
| michael@0 | 61 | try: |
| michael@0 | 62 | for e in self._executables: |
| michael@0 | 63 | fd, f = mkstemp() |
| michael@0 | 64 | os.close(fd) |
| michael@0 | 65 | tmpfiles.append(f) |
| michael@0 | 66 | e.copy(f, skip_if_older=False) |
| michael@0 | 67 | subprocess.call(['lipo', '-create'] + tmpfiles + ['-output', dest]) |
| michael@0 | 68 | finally: |
| michael@0 | 69 | for f in tmpfiles: |
| michael@0 | 70 | os.unlink(f) |
| michael@0 | 71 | |
| michael@0 | 72 | |
| michael@0 | 73 | class UnifiedFinder(BaseFinder): |
| michael@0 | 74 | ''' |
| michael@0 | 75 | Helper to get unified BaseFile instances from two distinct trees on the |
| michael@0 | 76 | file system. |
| michael@0 | 77 | ''' |
| michael@0 | 78 | def __init__(self, finder1, finder2, sorted=[], **kargs): |
| michael@0 | 79 | ''' |
| michael@0 | 80 | Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder |
| michael@0 | 81 | instances from which files are picked. UnifiedFinder.find() will act as |
| michael@0 | 82 | FileFinder.find() but will error out when matches can only be found in |
| michael@0 | 83 | one of the two trees and not the other. It will also error out if |
| michael@0 | 84 | matches can be found on both ends but their contents are not identical. |
| michael@0 | 85 | |
| michael@0 | 86 | The sorted argument gives a list of mozpack.path.match patterns. File |
| michael@0 | 87 | paths matching one of these patterns will have their contents compared |
| michael@0 | 88 | with their lines sorted. |
| michael@0 | 89 | ''' |
| michael@0 | 90 | assert isinstance(finder1, BaseFinder) |
| michael@0 | 91 | assert isinstance(finder2, BaseFinder) |
| michael@0 | 92 | self._finder1 = finder1 |
| michael@0 | 93 | self._finder2 = finder2 |
| michael@0 | 94 | self._sorted = sorted |
| michael@0 | 95 | BaseFinder.__init__(self, finder1.base, **kargs) |
| michael@0 | 96 | |
| michael@0 | 97 | def _find(self, path): |
| michael@0 | 98 | ''' |
| michael@0 | 99 | UnifiedFinder.find() implementation. |
| michael@0 | 100 | ''' |
| michael@0 | 101 | files1 = OrderedDict() |
| michael@0 | 102 | for p, f in self._finder1.find(path): |
| michael@0 | 103 | files1[p] = f |
| michael@0 | 104 | files2 = set() |
| michael@0 | 105 | for p, f in self._finder2.find(path): |
| michael@0 | 106 | files2.add(p) |
| michael@0 | 107 | if p in files1: |
| michael@0 | 108 | if may_unify_binary(files1[p]) and \ |
| michael@0 | 109 | may_unify_binary(f): |
| michael@0 | 110 | yield p, UnifiedExecutableFile(files1[p], f) |
| michael@0 | 111 | else: |
| michael@0 | 112 | err = errors.count |
| michael@0 | 113 | unified = self.unify_file(p, files1[p], f) |
| michael@0 | 114 | if unified: |
| michael@0 | 115 | yield p, unified |
| michael@0 | 116 | elif err == errors.count: |
| michael@0 | 117 | self._report_difference(p, files1[p], f) |
| michael@0 | 118 | else: |
| michael@0 | 119 | errors.error('File missing in %s: %s' % |
| michael@0 | 120 | (self._finder1.base, p)) |
| michael@0 | 121 | for p in [p for p in files1 if not p in files2]: |
| michael@0 | 122 | errors.error('File missing in %s: %s' % (self._finder2.base, p)) |
| michael@0 | 123 | |
| michael@0 | 124 | def _report_difference(self, path, file1, file2): |
| michael@0 | 125 | ''' |
| michael@0 | 126 | Report differences between files in both trees. |
| michael@0 | 127 | ''' |
| michael@0 | 128 | errors.error("Can't unify %s: file differs between %s and %s" % |
| michael@0 | 129 | (path, self._finder1.base, self._finder2.base)) |
| michael@0 | 130 | if not isinstance(file1, ExecutableFile) and \ |
| michael@0 | 131 | not isinstance(file2, ExecutableFile): |
| michael@0 | 132 | from difflib import unified_diff |
| michael@0 | 133 | for line in unified_diff(file1.open().readlines(), |
| michael@0 | 134 | file2.open().readlines(), |
| michael@0 | 135 | os.path.join(self._finder1.base, path), |
| michael@0 | 136 | os.path.join(self._finder2.base, path)): |
| michael@0 | 137 | errors.out.write(line) |
| michael@0 | 138 | |
| michael@0 | 139 | def unify_file(self, path, file1, file2): |
| michael@0 | 140 | ''' |
| michael@0 | 141 | Given two BaseFiles and the path they were found at, check whether |
| michael@0 | 142 | their content match and return the first BaseFile if they do. |
| michael@0 | 143 | ''' |
| michael@0 | 144 | content1 = file1.open().readlines() |
| michael@0 | 145 | content2 = file2.open().readlines() |
| michael@0 | 146 | if content1 == content2: |
| michael@0 | 147 | return file1 |
| michael@0 | 148 | for pattern in self._sorted: |
| michael@0 | 149 | if mozpack.path.match(path, pattern): |
| michael@0 | 150 | if sorted(content1) == sorted(content2): |
| michael@0 | 151 | return file1 |
| michael@0 | 152 | break |
| michael@0 | 153 | return None |
| michael@0 | 154 | |
| michael@0 | 155 | |
| michael@0 | 156 | class UnifiedBuildFinder(UnifiedFinder): |
| michael@0 | 157 | ''' |
| michael@0 | 158 | Specialized UnifiedFinder for Mozilla applications packaging. It allows |
| michael@0 | 159 | "*.manifest" files to differ in their order, and unifies "buildconfig.html" |
| michael@0 | 160 | files by merging their content. |
| michael@0 | 161 | ''' |
| michael@0 | 162 | def __init__(self, finder1, finder2, **kargs): |
| michael@0 | 163 | UnifiedFinder.__init__(self, finder1, finder2, |
| michael@0 | 164 | sorted=['**/*.manifest'], **kargs) |
| michael@0 | 165 | |
| michael@0 | 166 | def unify_file(self, path, file1, file2): |
| michael@0 | 167 | ''' |
| michael@0 | 168 | Unify buildconfig.html contents, or defer to UnifiedFinder.unify_file. |
| michael@0 | 169 | ''' |
| michael@0 | 170 | if mozpack.path.basename(path) == 'buildconfig.html': |
| michael@0 | 171 | content1 = file1.open().readlines() |
| michael@0 | 172 | content2 = file2.open().readlines() |
| michael@0 | 173 | # Copy everything from the first file up to the end of its <body>, |
| michael@0 | 174 | # insert a <hr> between the two files and copy the second file's |
| michael@0 | 175 | # content beginning after its leading <h1>. |
| michael@0 | 176 | return GeneratedFile(''.join( |
| michael@0 | 177 | content1[:content1.index('</body>\n')] + |
| michael@0 | 178 | ['<hr> </hr>\n'] + |
| michael@0 | 179 | content2[content2.index('<h1>about:buildconfig</h1>\n') + 1:] |
| michael@0 | 180 | )) |
| michael@0 | 181 | if path.endswith('.xpi'): |
| michael@0 | 182 | finder1 = JarFinder(os.path.join(self._finder1.base, path), |
| michael@0 | 183 | JarReader(fileobj=file1.open())) |
| michael@0 | 184 | finder2 = JarFinder(os.path.join(self._finder2.base, path), |
| michael@0 | 185 | JarReader(fileobj=file2.open())) |
| michael@0 | 186 | unifier = UnifiedFinder(finder1, finder2, sorted=self._sorted) |
| michael@0 | 187 | err = errors.count |
| michael@0 | 188 | all(unifier.find('')) |
| michael@0 | 189 | if err == errors.count: |
| michael@0 | 190 | return file1 |
| michael@0 | 191 | return None |
| michael@0 | 192 | return UnifiedFinder.unify_file(self, path, file1, file2) |
