The Tor Browser / file comparison
comparison: config/expandlibs_exec.py
config/expandlibs_exec.py
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
equal
deleted
inserted
replaced
| -1:000000000000 | 0:3ddb0841fa6e |
|---|---|
| 1 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 2 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 4 | |
| 5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to | |
| 6 a given command line, and executes that command line with the expanded | |
| 7 arguments. | |
| 8 | |
| 9 With the --extract argument (useful for e.g. $(AR)), it extracts object files | |
| 10 from static libraries (or use those listed in library descriptors directly). | |
| 11 | |
| 12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object | |
| 13 files with a list file. This can be used to avoid limitations in the length | |
| 14 of a command line. The kind of list file format used depends on the | |
| 15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list) | |
| 16 or 'linkerscript' for GNU ld linker scripts. | |
| 17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details. | |
| 18 | |
| 19 With the --symbol-order argument, followed by a file name, it will add the | |
| 20 relevant linker options to change the order in which the linker puts the | |
| 21 symbols appear in the resulting binary. Only works for ELF targets. | |
| 22 ''' | |
| 23 from __future__ import with_statement | |
| 24 import sys | |
| 25 import os | |
| 26 from expandlibs import ( | |
| 27 ExpandArgs, | |
| 28 relativize, | |
| 29 isDynamicLib, | |
| 30 isObject, | |
| 31 ensureParentDir, | |
| 32 ExpandLibsDeps, | |
| 33 ) | |
| 34 import expandlibs_config as conf | |
| 35 from optparse import OptionParser | |
| 36 import subprocess | |
| 37 import tempfile | |
| 38 import shutil | |
| 39 import subprocess | |
| 40 import re | |
| 41 from mozbuild.makeutil import Makefile | |
| 42 | |
| 43 # The are the insert points for a GNU ld linker script, assuming a more | |
| 44 # or less "standard" default linker script. This is not a dict because | |
| 45 # order is important. | |
| 46 SECTION_INSERT_BEFORE = [ | |
| 47 ('.text', '.fini'), | |
| 48 ('.rodata', '.rodata1'), | |
| 49 ('.data.rel.ro', '.dynamic'), | |
| 50 ('.data', '.data1'), | |
| 51 ] | |
| 52 | |
| 53 class ExpandArgsMore(ExpandArgs): | |
| 54 ''' Meant to be used as 'with ExpandArgsMore(args) as ...: ''' | |
| 55 def __enter__(self): | |
| 56 self.tmp = [] | |
| 57 return self | |
| 58 | |
| 59 def __exit__(self, type, value, tb): | |
| 60 '''Automatically remove temporary files''' | |
| 61 for tmp in self.tmp: | |
| 62 if os.path.isdir(tmp): | |
| 63 shutil.rmtree(tmp, True) | |
| 64 else: | |
| 65 os.remove(tmp) | |
| 66 | |
| 67 def extract(self): | |
| 68 self[0:] = self._extract(self) | |
| 69 | |
| 70 def _extract(self, args): | |
| 71 '''When a static library name is found, either extract its contents | |
| 72 in a temporary directory or use the information found in the | |
| 73 corresponding lib descriptor. | |
| 74 ''' | |
| 75 ar_extract = conf.AR_EXTRACT.split() | |
| 76 newlist = [] | |
| 77 for arg in args: | |
| 78 if os.path.splitext(arg)[1] == conf.LIB_SUFFIX: | |
| 79 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX): | |
| 80 newlist += self._extract(self._expand_desc(arg)) | |
| 81 continue | |
| 82 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'): | |
| 83 tmp = tempfile.mkdtemp(dir=os.curdir) | |
| 84 self.tmp.append(tmp) | |
| 85 if conf.AR == 'lib': | |
| 86 out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg]) | |
| 87 files = out.splitlines() | |
| 88 # If lib -list returns a list full of dlls, it's an | |
| 89 # import lib. | |
| 90 if all(isDynamicLib(f) for f in files): | |
| 91 newlist += [arg] | |
| 92 continue | |
| 93 for f in files: | |
| 94 subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp) | |
| 95 else: | |
| 96 subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp) | |
| 97 objs = [] | |
| 98 for root, dirs, files in os.walk(tmp): | |
| 99 objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)] | |
| 100 newlist += sorted(objs) | |
| 101 continue | |
| 102 newlist += [arg] | |
| 103 return newlist | |
| 104 | |
| 105 def makelist(self): | |
| 106 '''Replaces object file names with a temporary list file, using a | |
| 107 list format depending on the EXPAND_LIBS_LIST_STYLE variable | |
| 108 ''' | |
| 109 objs = [o for o in self if isObject(o)] | |
| 110 if not len(objs): return | |
| 111 fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir) | |
| 112 if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript": | |
| 113 content = ['INPUT("%s")\n' % obj for obj in objs] | |
| 114 ref = tmp | |
| 115 elif conf.EXPAND_LIBS_LIST_STYLE == "filelist": | |
| 116 content = ["%s\n" % obj for obj in objs] | |
| 117 ref = "-Wl,-filelist," + tmp | |
| 118 elif conf.EXPAND_LIBS_LIST_STYLE == "list": | |
| 119 content = ["%s\n" % obj for obj in objs] | |
| 120 ref = "@" + tmp | |
| 121 else: | |
| 122 os.close(fd) | |
| 123 os.remove(tmp) | |
| 124 return | |
| 125 self.tmp.append(tmp) | |
| 126 f = os.fdopen(fd, "w") | |
| 127 f.writelines(content) | |
| 128 f.close() | |
| 129 idx = self.index(objs[0]) | |
| 130 newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs] | |
| 131 self[0:] = newlist | |
| 132 | |
| 133 def _getFoldedSections(self): | |
| 134 '''Returns a dict about folded sections. | |
| 135 When section A and B are folded into section C, the dict contains: | |
| 136 { 'A': 'C', | |
| 137 'B': 'C', | |
| 138 'C': ['A', 'B'] }''' | |
| 139 if not conf.LD_PRINT_ICF_SECTIONS: | |
| 140 return {} | |
| 141 | |
| 142 proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE) | |
| 143 (stdout, stderr) = proc.communicate() | |
| 144 result = {} | |
| 145 # gold's --print-icf-sections output looks like the following: | |
| 146 # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o' | |
| 147 # In terms of words, chances are this will change in the future, | |
| 148 # especially considering "into" is misplaced. Splitting on quotes | |
| 149 # seems safer. | |
| 150 for l in stderr.split('\n'): | |
| 151 quoted = l.split("'") | |
| 152 if len(quoted) > 5 and quoted[1] != quoted[5]: | |
| 153 result[quoted[1]] = [quoted[5]] | |
| 154 if quoted[5] in result: | |
| 155 result[quoted[5]].append(quoted[1]) | |
| 156 else: | |
| 157 result[quoted[5]] = [quoted[1]] | |
| 158 return result | |
| 159 | |
| 160 def _getOrderedSections(self, ordered_symbols): | |
| 161 '''Given an ordered list of symbols, returns the corresponding list | |
| 162 of sections following the order.''' | |
| 163 if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']: | |
| 164 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) | |
| 165 finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX]) | |
| 166 folded = self._getFoldedSections() | |
| 167 sections = set() | |
| 168 ordered_sections = [] | |
| 169 for symbol in ordered_symbols: | |
| 170 symbol_sections = finder.getSections(symbol) | |
| 171 all_symbol_sections = [] | |
| 172 for section in symbol_sections: | |
| 173 if section in folded: | |
| 174 if isinstance(folded[section], str): | |
| 175 section = folded[section] | |
| 176 all_symbol_sections.append(section) | |
| 177 all_symbol_sections.extend(folded[section]) | |
| 178 else: | |
| 179 all_symbol_sections.append(section) | |
| 180 for section in all_symbol_sections: | |
| 181 if not section in sections: | |
| 182 ordered_sections.append(section) | |
| 183 sections.add(section) | |
| 184 return ordered_sections | |
| 185 | |
| 186 def orderSymbols(self, order): | |
| 187 '''Given a file containing a list of symbols, adds the appropriate | |
| 188 argument to make the linker put the symbols in that order.''' | |
| 189 with open(order) as file: | |
| 190 sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()]) | |
| 191 split_sections = {} | |
| 192 linked_sections = [s[0] for s in SECTION_INSERT_BEFORE] | |
| 193 for s in sections: | |
| 194 for linked_section in linked_sections: | |
| 195 if s.startswith(linked_section): | |
| 196 if linked_section in split_sections: | |
| 197 split_sections[linked_section].append(s) | |
| 198 else: | |
| 199 split_sections[linked_section] = [s] | |
| 200 break | |
| 201 content = [] | |
| 202 # Order is important | |
| 203 linked_sections = [s for s in linked_sections if s in split_sections] | |
| 204 | |
| 205 if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file': | |
| 206 option = '-Wl,--section-ordering-file,%s' | |
| 207 content = sections | |
| 208 for linked_section in linked_sections: | |
| 209 content.extend(split_sections[linked_section]) | |
| 210 content.append('%s.*' % linked_section) | |
| 211 content.append(linked_section) | |
| 212 | |
| 213 elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript': | |
| 214 option = '-Wl,-T,%s' | |
| 215 section_insert_before = dict(SECTION_INSERT_BEFORE) | |
| 216 for linked_section in linked_sections: | |
| 217 content.append('SECTIONS {') | |
| 218 content.append(' %s : {' % linked_section) | |
| 219 content.extend(' *(%s)' % s for s in split_sections[linked_section]) | |
| 220 content.append(' }') | |
| 221 content.append('}') | |
| 222 content.append('INSERT BEFORE %s' % section_insert_before[linked_section]) | |
| 223 else: | |
| 224 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) | |
| 225 | |
| 226 fd, tmp = tempfile.mkstemp(dir=os.curdir) | |
| 227 f = os.fdopen(fd, "w") | |
| 228 f.write('\n'.join(content)+'\n') | |
| 229 f.close() | |
| 230 self.tmp.append(tmp) | |
| 231 self.append(option % tmp) | |
| 232 | |
| 233 class SectionFinder(object): | |
| 234 '''Instances of this class allow to map symbol names to sections in | |
| 235 object files.''' | |
| 236 | |
| 237 def __init__(self, objs): | |
| 238 '''Creates an instance, given a list of object files.''' | |
| 239 if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']: | |
| 240 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE) | |
| 241 self.mapping = {} | |
| 242 for obj in objs: | |
| 243 if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX: | |
| 244 raise Exception('%s is not an object nor a static library' % obj) | |
| 245 for symbol, section in SectionFinder._getSymbols(obj): | |
| 246 sym = SectionFinder._normalize(symbol) | |
| 247 if sym in self.mapping: | |
| 248 if not section in self.mapping[sym]: | |
| 249 self.mapping[sym].append(section) | |
| 250 else: | |
| 251 self.mapping[sym] = [section] | |
| 252 | |
| 253 def getSections(self, symbol): | |
| 254 '''Given a symbol, returns a list of sections containing it or the | |
| 255 corresponding thunks. When the given symbol is a thunk, returns the | |
| 256 list of sections containing its corresponding normal symbol and the | |
| 257 other thunks for that symbol.''' | |
| 258 sym = SectionFinder._normalize(symbol) | |
| 259 if sym in self.mapping: | |
| 260 return self.mapping[sym] | |
| 261 return [] | |
| 262 | |
| 263 @staticmethod | |
| 264 def _normalize(symbol): | |
| 265 '''For normal symbols, return the given symbol. For thunks, return | |
| 266 the corresponding normal symbol.''' | |
| 267 if re.match('^_ZThn[0-9]+_', symbol): | |
| 268 return re.sub('^_ZThn[0-9]+_', '_Z', symbol) | |
| 269 return symbol | |
| 270 | |
| 271 @staticmethod | |
| 272 def _getSymbols(obj): | |
| 273 '''Returns a list of (symbol, section) contained in the given object | |
| 274 file.''' | |
| 275 proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE) | |
| 276 (stdout, stderr) = proc.communicate() | |
| 277 syms = [] | |
| 278 for line in stdout.splitlines(): | |
| 279 # Each line has the following format: | |
| 280 # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol> | |
| 281 tmp = line.split(' ',1) | |
| 282 # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"] | |
| 283 # We only need to consider cases where "<section>\t<length> <symbol>" is present, | |
| 284 # and where the [FfO] flag is either F (function) or O (object). | |
| 285 if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']: | |
| 286 tmp = tmp[1][8:].split() | |
| 287 # That gives us ["<section>","<length>", "<symbol>"] | |
| 288 syms.append((tmp[-1], tmp[0])) | |
| 289 return syms | |
| 290 | |
| 291 def print_command(out, args): | |
| 292 print >>out, "Executing: " + " ".join(args) | |
| 293 for tmp in [f for f in args.tmp if os.path.isfile(f)]: | |
| 294 print >>out, tmp + ":" | |
| 295 with open(tmp) as file: | |
| 296 print >>out, "".join([" " + l for l in file.readlines()]) | |
| 297 out.flush() | |
| 298 | |
| 299 def main(): | |
| 300 parser = OptionParser() | |
| 301 parser.add_option("--depend", dest="depend", metavar="FILE", | |
| 302 help="generate dependencies for the given execution and store it in the given file") | |
| 303 parser.add_option("--target", dest="target", metavar="FILE", | |
| 304 help="designate the target for dependencies") | |
| 305 parser.add_option("--extract", action="store_true", dest="extract", | |
| 306 help="when a library has no descriptor file, extract it first, when possible") | |
| 307 parser.add_option("--uselist", action="store_true", dest="uselist", | |
| 308 help="use a list file for objects when executing a command") | |
| 309 parser.add_option("--verbose", action="store_true", dest="verbose", | |
| 310 help="display executed command and temporary files content") | |
| 311 parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE", | |
| 312 help="use the given list of symbols to order symbols in the resulting binary when using with a linker") | |
| 313 | |
| 314 (options, args) = parser.parse_args() | |
| 315 | |
| 316 if not options.target: | |
| 317 options.depend = False | |
| 318 if options.depend: | |
| 319 deps = ExpandLibsDeps(args) | |
| 320 # Filter out common command wrappers | |
| 321 while os.path.basename(deps[0]) in ['ccache', 'distcc']: | |
| 322 deps.pop(0) | |
| 323 # Remove command | |
| 324 deps.pop(0) | |
| 325 with ExpandArgsMore(args) as args: | |
| 326 if options.extract: | |
| 327 args.extract() | |
| 328 if options.symbol_order: | |
| 329 args.orderSymbols(options.symbol_order) | |
| 330 if options.uselist: | |
| 331 args.makelist() | |
| 332 | |
| 333 if options.verbose: | |
| 334 print_command(sys.stderr, args) | |
| 335 try: | |
| 336 proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) | |
| 337 except Exception, e: | |
| 338 print >>sys.stderr, 'error: Launching', args, ':', e | |
| 339 raise e | |
| 340 (stdout, stderr) = proc.communicate() | |
| 341 if proc.returncode and not options.verbose: | |
| 342 print_command(sys.stderr, args) | |
| 343 sys.stderr.write(stdout) | |
| 344 sys.stderr.flush() | |
| 345 if proc.returncode: | |
| 346 exit(proc.returncode) | |
| 347 if not options.depend: | |
| 348 return | |
| 349 ensureParentDir(options.depend) | |
| 350 mk = Makefile() | |
| 351 deps = [dep for dep in deps if os.path.isfile(dep) and dep != options.target | |
| 352 and os.path.abspath(dep) != os.path.abspath(options.depend)] | |
| 353 no_dynamic_lib = [dep for dep in deps if not isDynamicLib(dep)] | |
| 354 mk.create_rule([options.target]).add_dependencies(no_dynamic_lib) | |
| 355 if len(deps) != len(no_dynamic_lib): | |
| 356 mk.create_rule(['%s_order_only' % options.target]).add_dependencies(dep for dep in deps if isDynamicLib(dep)) | |
| 357 | |
| 358 with open(options.depend, 'w') as depfile: | |
| 359 mk.dump(depfile, removal_guard=True) | |
| 360 | |
| 361 if __name__ == '__main__': | |
| 362 main() |
