1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/config/expandlibs_exec.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,362 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +'''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
1.9 +a given command line, and executes that command line with the expanded
1.10 +arguments.
1.11 +
1.12 +With the --extract argument (useful for e.g. $(AR)), it extracts object files
1.13 +from static libraries (or use those listed in library descriptors directly).
1.14 +
1.15 +With the --uselist argument (useful for e.g. $(CC)), it replaces all object
1.16 +files with a list file. This can be used to avoid limitations in the length
1.17 +of a command line. The kind of list file format used depends on the
1.18 +EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
1.19 +or 'linkerscript' for GNU ld linker scripts.
1.20 +See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
1.21 +
1.22 +With the --symbol-order argument, followed by a file name, it will add the
1.23 +relevant linker options to change the order in which the linker puts the
1.24 +symbols appear in the resulting binary. Only works for ELF targets.
1.25 +'''
1.26 +from __future__ import with_statement
1.27 +import sys
1.28 +import os
1.29 +from expandlibs import (
1.30 + ExpandArgs,
1.31 + relativize,
1.32 + isDynamicLib,
1.33 + isObject,
1.34 + ensureParentDir,
1.35 + ExpandLibsDeps,
1.36 +)
1.37 +import expandlibs_config as conf
1.38 +from optparse import OptionParser
1.39 +import subprocess
1.40 +import tempfile
1.41 +import shutil
1.42 +import subprocess
1.43 +import re
1.44 +from mozbuild.makeutil import Makefile
1.45 +
1.46 +# The are the insert points for a GNU ld linker script, assuming a more
1.47 +# or less "standard" default linker script. This is not a dict because
1.48 +# order is important.
1.49 +SECTION_INSERT_BEFORE = [
1.50 + ('.text', '.fini'),
1.51 + ('.rodata', '.rodata1'),
1.52 + ('.data.rel.ro', '.dynamic'),
1.53 + ('.data', '.data1'),
1.54 +]
1.55 +
1.56 +class ExpandArgsMore(ExpandArgs):
1.57 + ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
1.58 + def __enter__(self):
1.59 + self.tmp = []
1.60 + return self
1.61 +
1.62 + def __exit__(self, type, value, tb):
1.63 + '''Automatically remove temporary files'''
1.64 + for tmp in self.tmp:
1.65 + if os.path.isdir(tmp):
1.66 + shutil.rmtree(tmp, True)
1.67 + else:
1.68 + os.remove(tmp)
1.69 +
1.70 + def extract(self):
1.71 + self[0:] = self._extract(self)
1.72 +
1.73 + def _extract(self, args):
1.74 + '''When a static library name is found, either extract its contents
1.75 + in a temporary directory or use the information found in the
1.76 + corresponding lib descriptor.
1.77 + '''
1.78 + ar_extract = conf.AR_EXTRACT.split()
1.79 + newlist = []
1.80 + for arg in args:
1.81 + if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
1.82 + if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
1.83 + newlist += self._extract(self._expand_desc(arg))
1.84 + continue
1.85 + elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
1.86 + tmp = tempfile.mkdtemp(dir=os.curdir)
1.87 + self.tmp.append(tmp)
1.88 + if conf.AR == 'lib':
1.89 + out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
1.90 + files = out.splitlines()
1.91 + # If lib -list returns a list full of dlls, it's an
1.92 + # import lib.
1.93 + if all(isDynamicLib(f) for f in files):
1.94 + newlist += [arg]
1.95 + continue
1.96 + for f in files:
1.97 + subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
1.98 + else:
1.99 + subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
1.100 + objs = []
1.101 + for root, dirs, files in os.walk(tmp):
1.102 + objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]
1.103 + newlist += sorted(objs)
1.104 + continue
1.105 + newlist += [arg]
1.106 + return newlist
1.107 +
1.108 + def makelist(self):
1.109 + '''Replaces object file names with a temporary list file, using a
1.110 + list format depending on the EXPAND_LIBS_LIST_STYLE variable
1.111 + '''
1.112 + objs = [o for o in self if isObject(o)]
1.113 + if not len(objs): return
1.114 + fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
1.115 + if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
1.116 + content = ['INPUT("%s")\n' % obj for obj in objs]
1.117 + ref = tmp
1.118 + elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
1.119 + content = ["%s\n" % obj for obj in objs]
1.120 + ref = "-Wl,-filelist," + tmp
1.121 + elif conf.EXPAND_LIBS_LIST_STYLE == "list":
1.122 + content = ["%s\n" % obj for obj in objs]
1.123 + ref = "@" + tmp
1.124 + else:
1.125 + os.close(fd)
1.126 + os.remove(tmp)
1.127 + return
1.128 + self.tmp.append(tmp)
1.129 + f = os.fdopen(fd, "w")
1.130 + f.writelines(content)
1.131 + f.close()
1.132 + idx = self.index(objs[0])
1.133 + newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
1.134 + self[0:] = newlist
1.135 +
1.136 + def _getFoldedSections(self):
1.137 + '''Returns a dict about folded sections.
1.138 + When section A and B are folded into section C, the dict contains:
1.139 + { 'A': 'C',
1.140 + 'B': 'C',
1.141 + 'C': ['A', 'B'] }'''
1.142 + if not conf.LD_PRINT_ICF_SECTIONS:
1.143 + return {}
1.144 +
1.145 + proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
1.146 + (stdout, stderr) = proc.communicate()
1.147 + result = {}
1.148 + # gold's --print-icf-sections output looks like the following:
1.149 + # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
1.150 + # In terms of words, chances are this will change in the future,
1.151 + # especially considering "into" is misplaced. Splitting on quotes
1.152 + # seems safer.
1.153 + for l in stderr.split('\n'):
1.154 + quoted = l.split("'")
1.155 + if len(quoted) > 5 and quoted[1] != quoted[5]:
1.156 + result[quoted[1]] = [quoted[5]]
1.157 + if quoted[5] in result:
1.158 + result[quoted[5]].append(quoted[1])
1.159 + else:
1.160 + result[quoted[5]] = [quoted[1]]
1.161 + return result
1.162 +
1.163 + def _getOrderedSections(self, ordered_symbols):
1.164 + '''Given an ordered list of symbols, returns the corresponding list
1.165 + of sections following the order.'''
1.166 + if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
1.167 + raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
1.168 + finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
1.169 + folded = self._getFoldedSections()
1.170 + sections = set()
1.171 + ordered_sections = []
1.172 + for symbol in ordered_symbols:
1.173 + symbol_sections = finder.getSections(symbol)
1.174 + all_symbol_sections = []
1.175 + for section in symbol_sections:
1.176 + if section in folded:
1.177 + if isinstance(folded[section], str):
1.178 + section = folded[section]
1.179 + all_symbol_sections.append(section)
1.180 + all_symbol_sections.extend(folded[section])
1.181 + else:
1.182 + all_symbol_sections.append(section)
1.183 + for section in all_symbol_sections:
1.184 + if not section in sections:
1.185 + ordered_sections.append(section)
1.186 + sections.add(section)
1.187 + return ordered_sections
1.188 +
1.189 + def orderSymbols(self, order):
1.190 + '''Given a file containing a list of symbols, adds the appropriate
1.191 + argument to make the linker put the symbols in that order.'''
1.192 + with open(order) as file:
1.193 + sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
1.194 + split_sections = {}
1.195 + linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
1.196 + for s in sections:
1.197 + for linked_section in linked_sections:
1.198 + if s.startswith(linked_section):
1.199 + if linked_section in split_sections:
1.200 + split_sections[linked_section].append(s)
1.201 + else:
1.202 + split_sections[linked_section] = [s]
1.203 + break
1.204 + content = []
1.205 + # Order is important
1.206 + linked_sections = [s for s in linked_sections if s in split_sections]
1.207 +
1.208 + if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
1.209 + option = '-Wl,--section-ordering-file,%s'
1.210 + content = sections
1.211 + for linked_section in linked_sections:
1.212 + content.extend(split_sections[linked_section])
1.213 + content.append('%s.*' % linked_section)
1.214 + content.append(linked_section)
1.215 +
1.216 + elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
1.217 + option = '-Wl,-T,%s'
1.218 + section_insert_before = dict(SECTION_INSERT_BEFORE)
1.219 + for linked_section in linked_sections:
1.220 + content.append('SECTIONS {')
1.221 + content.append(' %s : {' % linked_section)
1.222 + content.extend(' *(%s)' % s for s in split_sections[linked_section])
1.223 + content.append(' }')
1.224 + content.append('}')
1.225 + content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
1.226 + else:
1.227 + raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
1.228 +
1.229 + fd, tmp = tempfile.mkstemp(dir=os.curdir)
1.230 + f = os.fdopen(fd, "w")
1.231 + f.write('\n'.join(content)+'\n')
1.232 + f.close()
1.233 + self.tmp.append(tmp)
1.234 + self.append(option % tmp)
1.235 +
1.236 +class SectionFinder(object):
1.237 + '''Instances of this class allow to map symbol names to sections in
1.238 + object files.'''
1.239 +
1.240 + def __init__(self, objs):
1.241 + '''Creates an instance, given a list of object files.'''
1.242 + if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
1.243 + raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
1.244 + self.mapping = {}
1.245 + for obj in objs:
1.246 + if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
1.247 + raise Exception('%s is not an object nor a static library' % obj)
1.248 + for symbol, section in SectionFinder._getSymbols(obj):
1.249 + sym = SectionFinder._normalize(symbol)
1.250 + if sym in self.mapping:
1.251 + if not section in self.mapping[sym]:
1.252 + self.mapping[sym].append(section)
1.253 + else:
1.254 + self.mapping[sym] = [section]
1.255 +
1.256 + def getSections(self, symbol):
1.257 + '''Given a symbol, returns a list of sections containing it or the
1.258 + corresponding thunks. When the given symbol is a thunk, returns the
1.259 + list of sections containing its corresponding normal symbol and the
1.260 + other thunks for that symbol.'''
1.261 + sym = SectionFinder._normalize(symbol)
1.262 + if sym in self.mapping:
1.263 + return self.mapping[sym]
1.264 + return []
1.265 +
1.266 + @staticmethod
1.267 + def _normalize(symbol):
1.268 + '''For normal symbols, return the given symbol. For thunks, return
1.269 + the corresponding normal symbol.'''
1.270 + if re.match('^_ZThn[0-9]+_', symbol):
1.271 + return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
1.272 + return symbol
1.273 +
1.274 + @staticmethod
1.275 + def _getSymbols(obj):
1.276 + '''Returns a list of (symbol, section) contained in the given object
1.277 + file.'''
1.278 + proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
1.279 + (stdout, stderr) = proc.communicate()
1.280 + syms = []
1.281 + for line in stdout.splitlines():
1.282 + # Each line has the following format:
1.283 + # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
1.284 + tmp = line.split(' ',1)
1.285 + # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
1.286 + # We only need to consider cases where "<section>\t<length> <symbol>" is present,
1.287 + # and where the [FfO] flag is either F (function) or O (object).
1.288 + if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
1.289 + tmp = tmp[1][8:].split()
1.290 + # That gives us ["<section>","<length>", "<symbol>"]
1.291 + syms.append((tmp[-1], tmp[0]))
1.292 + return syms
1.293 +
1.294 +def print_command(out, args):
1.295 + print >>out, "Executing: " + " ".join(args)
1.296 + for tmp in [f for f in args.tmp if os.path.isfile(f)]:
1.297 + print >>out, tmp + ":"
1.298 + with open(tmp) as file:
1.299 + print >>out, "".join([" " + l for l in file.readlines()])
1.300 + out.flush()
1.301 +
1.302 +def main():
1.303 + parser = OptionParser()
1.304 + parser.add_option("--depend", dest="depend", metavar="FILE",
1.305 + help="generate dependencies for the given execution and store it in the given file")
1.306 + parser.add_option("--target", dest="target", metavar="FILE",
1.307 + help="designate the target for dependencies")
1.308 + parser.add_option("--extract", action="store_true", dest="extract",
1.309 + help="when a library has no descriptor file, extract it first, when possible")
1.310 + parser.add_option("--uselist", action="store_true", dest="uselist",
1.311 + help="use a list file for objects when executing a command")
1.312 + parser.add_option("--verbose", action="store_true", dest="verbose",
1.313 + help="display executed command and temporary files content")
1.314 + parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
1.315 + help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
1.316 +
1.317 + (options, args) = parser.parse_args()
1.318 +
1.319 + if not options.target:
1.320 + options.depend = False
1.321 + if options.depend:
1.322 + deps = ExpandLibsDeps(args)
1.323 + # Filter out common command wrappers
1.324 + while os.path.basename(deps[0]) in ['ccache', 'distcc']:
1.325 + deps.pop(0)
1.326 + # Remove command
1.327 + deps.pop(0)
1.328 + with ExpandArgsMore(args) as args:
1.329 + if options.extract:
1.330 + args.extract()
1.331 + if options.symbol_order:
1.332 + args.orderSymbols(options.symbol_order)
1.333 + if options.uselist:
1.334 + args.makelist()
1.335 +
1.336 + if options.verbose:
1.337 + print_command(sys.stderr, args)
1.338 + try:
1.339 + proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
1.340 + except Exception, e:
1.341 + print >>sys.stderr, 'error: Launching', args, ':', e
1.342 + raise e
1.343 + (stdout, stderr) = proc.communicate()
1.344 + if proc.returncode and not options.verbose:
1.345 + print_command(sys.stderr, args)
1.346 + sys.stderr.write(stdout)
1.347 + sys.stderr.flush()
1.348 + if proc.returncode:
1.349 + exit(proc.returncode)
1.350 + if not options.depend:
1.351 + return
1.352 + ensureParentDir(options.depend)
1.353 + mk = Makefile()
1.354 + deps = [dep for dep in deps if os.path.isfile(dep) and dep != options.target
1.355 + and os.path.abspath(dep) != os.path.abspath(options.depend)]
1.356 + no_dynamic_lib = [dep for dep in deps if not isDynamicLib(dep)]
1.357 + mk.create_rule([options.target]).add_dependencies(no_dynamic_lib)
1.358 + if len(deps) != len(no_dynamic_lib):
1.359 + mk.create_rule(['%s_order_only' % options.target]).add_dependencies(dep for dep in deps if isDynamicLib(dep))
1.360 +
1.361 + with open(options.depend, 'w') as depfile:
1.362 + mk.dump(depfile, removal_guard=True)
1.363 +
1.364 +if __name__ == '__main__':
1.365 + main()
