michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
michael@0: a given command line, and executes that command line with the expanded
michael@0: arguments.
michael@0: 
michael@0: With the --extract argument (useful for e.g. $(AR)), it extracts object files
michael@0: from static libraries (or use those listed in library descriptors directly).
michael@0: 
michael@0: With the --uselist argument (useful for e.g. $(CC)), it replaces all object
michael@0: files with a list file. This can be used to avoid limitations in the length
michael@0: of a command line. The kind of list file format used depends on the
michael@0: EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
michael@0: or 'linkerscript' for GNU ld linker scripts.
michael@0: See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
michael@0: 
michael@0: With the --symbol-order argument, followed by a file name, it will add the
michael@0: relevant linker options to change the order in which the linker puts the
michael@0: symbols appear in the resulting binary. Only works for ELF targets.
michael@0: '''
michael@0: from __future__ import with_statement
michael@0: import sys
michael@0: import os
michael@0: from expandlibs import (
michael@0:     ExpandArgs,
michael@0:     relativize,
michael@0:     isDynamicLib,
michael@0:     isObject,
michael@0:     ensureParentDir,
michael@0:     ExpandLibsDeps,
michael@0: )
michael@0: import expandlibs_config as conf
michael@0: from optparse import OptionParser
michael@0: import subprocess
michael@0: import tempfile
michael@0: import shutil
michael@0: import subprocess
michael@0: import re
michael@0: from mozbuild.makeutil import Makefile
michael@0: 
michael@0: # The are the insert points for a GNU ld linker script, assuming a more
michael@0: # or less "standard" default linker script. This is not a dict because
michael@0: # order is important.
michael@0: SECTION_INSERT_BEFORE = [
michael@0:   ('.text', '.fini'),
michael@0:   ('.rodata', '.rodata1'),
michael@0:   ('.data.rel.ro', '.dynamic'),
michael@0:   ('.data', '.data1'),
michael@0: ]
michael@0: 
michael@0: class ExpandArgsMore(ExpandArgs):
michael@0:     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
michael@0:     def __enter__(self):
michael@0:         self.tmp = []
michael@0:         return self
michael@0:         
michael@0:     def __exit__(self, type, value, tb):
michael@0:         '''Automatically remove temporary files'''
michael@0:         for tmp in self.tmp:
michael@0:             if os.path.isdir(tmp):
michael@0:                 shutil.rmtree(tmp, True)
michael@0:             else:
michael@0:                 os.remove(tmp)
michael@0: 
michael@0:     def extract(self):
michael@0:         self[0:] = self._extract(self)
michael@0: 
michael@0:     def _extract(self, args):
michael@0:         '''When a static library name is found, either extract its contents
michael@0:         in a temporary directory or use the information found in the
michael@0:         corresponding lib descriptor.
michael@0:         '''
michael@0:         ar_extract = conf.AR_EXTRACT.split()
michael@0:         newlist = []
michael@0:         for arg in args:
michael@0:             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
michael@0:                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
michael@0:                     newlist += self._extract(self._expand_desc(arg))
michael@0:                     continue
michael@0:                 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
michael@0:                     tmp = tempfile.mkdtemp(dir=os.curdir)
michael@0:                     self.tmp.append(tmp)
michael@0:                     if conf.AR == 'lib':
michael@0:                         out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
michael@0:                         files = out.splitlines()
michael@0:                         # If lib -list returns a list full of dlls, it's an
michael@0:                         # import lib.
michael@0:                         if all(isDynamicLib(f) for f in files):
michael@0:                             newlist += [arg]
michael@0:                             continue
michael@0:                         for f in files:
michael@0:                             subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
michael@0:                     else:
michael@0:                         subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
michael@0:                     objs = []
michael@0:                     for root, dirs, files in os.walk(tmp):
michael@0:                         objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]
michael@0:                     newlist += sorted(objs)
michael@0:                     continue
michael@0:             newlist += [arg]
michael@0:         return newlist
michael@0: 
michael@0:     def makelist(self):
michael@0:         '''Replaces object file names with a temporary list file, using a
michael@0:         list format depending on the EXPAND_LIBS_LIST_STYLE variable
michael@0:         '''
michael@0:         objs = [o for o in self if isObject(o)]
michael@0:         if not len(objs): return
michael@0:         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
michael@0:         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
michael@0:             content = ['INPUT("%s")\n' % obj for obj in objs]
michael@0:             ref = tmp
michael@0:         elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
michael@0:             content = ["%s\n" % obj for obj in objs]
michael@0:             ref = "-Wl,-filelist," + tmp
michael@0:         elif conf.EXPAND_LIBS_LIST_STYLE == "list":
michael@0:             content = ["%s\n" % obj for obj in objs]
michael@0:             ref = "@" + tmp
michael@0:         else:
michael@0:             os.close(fd)
michael@0:             os.remove(tmp)
michael@0:             return
michael@0:         self.tmp.append(tmp)
michael@0:         f = os.fdopen(fd, "w")
michael@0:         f.writelines(content)
michael@0:         f.close()
michael@0:         idx = self.index(objs[0])
michael@0:         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
michael@0:         self[0:] = newlist
michael@0: 
michael@0:     def _getFoldedSections(self):
michael@0:         '''Returns a dict about folded sections.
michael@0:         When section A and B are folded into section C, the dict contains:
michael@0:         { 'A': 'C',
michael@0:           'B': 'C',
michael@0:           'C': ['A', 'B'] }'''
michael@0:         if not conf.LD_PRINT_ICF_SECTIONS:
michael@0:             return {}
michael@0: 
michael@0:         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
michael@0:         (stdout, stderr) = proc.communicate()
michael@0:         result = {}
michael@0:         # gold's --print-icf-sections output looks like the following:
michael@0:         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
michael@0:         # In terms of words, chances are this will change in the future,
michael@0:         # especially considering "into" is misplaced. Splitting on quotes
michael@0:         # seems safer.
michael@0:         for l in stderr.split('\n'):
michael@0:             quoted = l.split("'")
michael@0:             if len(quoted) > 5 and quoted[1] != quoted[5]:
michael@0:                 result[quoted[1]] = [quoted[5]]
michael@0:                 if quoted[5] in result:
michael@0:                     result[quoted[5]].append(quoted[1])
michael@0:                 else:
michael@0:                     result[quoted[5]] = [quoted[1]]
michael@0:         return result
michael@0: 
michael@0:     def _getOrderedSections(self, ordered_symbols):
michael@0:         '''Given an ordered list of symbols, returns the corresponding list
michael@0:         of sections following the order.'''
michael@0:         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
michael@0:             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
michael@0:         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
michael@0:         folded = self._getFoldedSections()
michael@0:         sections = set()
michael@0:         ordered_sections = []
michael@0:         for symbol in ordered_symbols:
michael@0:             symbol_sections = finder.getSections(symbol)
michael@0:             all_symbol_sections = []
michael@0:             for section in symbol_sections:
michael@0:                 if section in folded:
michael@0:                     if isinstance(folded[section], str):
michael@0:                         section = folded[section]
michael@0:                     all_symbol_sections.append(section)
michael@0:                     all_symbol_sections.extend(folded[section])
michael@0:                 else:
michael@0:                     all_symbol_sections.append(section)
michael@0:             for section in all_symbol_sections:
michael@0:                 if not section in sections:
michael@0:                     ordered_sections.append(section)
michael@0:                     sections.add(section)
michael@0:         return ordered_sections
michael@0: 
michael@0:     def orderSymbols(self, order):
michael@0:         '''Given a file containing a list of symbols, adds the appropriate
michael@0:         argument to make the linker put the symbols in that order.'''
michael@0:         with open(order) as file:
michael@0:             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
michael@0:         split_sections = {}
michael@0:         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
michael@0:         for s in sections:
michael@0:             for linked_section in linked_sections:
michael@0:                 if s.startswith(linked_section):
michael@0:                     if linked_section in split_sections:
michael@0:                         split_sections[linked_section].append(s)
michael@0:                     else:
michael@0:                         split_sections[linked_section] = [s]
michael@0:                     break
michael@0:         content = []
michael@0:         # Order is important
michael@0:         linked_sections = [s for s in linked_sections if s in split_sections]
michael@0: 
michael@0:         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
michael@0:             option = '-Wl,--section-ordering-file,%s'
michael@0:             content = sections
michael@0:             for linked_section in linked_sections:
michael@0:                 content.extend(split_sections[linked_section])
michael@0:                 content.append('%s.*' % linked_section)
michael@0:                 content.append(linked_section)
michael@0: 
michael@0:         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
michael@0:             option = '-Wl,-T,%s'
michael@0:             section_insert_before = dict(SECTION_INSERT_BEFORE)
michael@0:             for linked_section in linked_sections:
michael@0:                 content.append('SECTIONS {')
michael@0:                 content.append('  %s : {' % linked_section)
michael@0:                 content.extend('    *(%s)' % s for s in split_sections[linked_section])
michael@0:                 content.append('  }')
michael@0:                 content.append('}')
michael@0:                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
michael@0:         else:
michael@0:             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
michael@0: 
michael@0:         fd, tmp = tempfile.mkstemp(dir=os.curdir)
michael@0:         f = os.fdopen(fd, "w")
michael@0:         f.write('\n'.join(content)+'\n')
michael@0:         f.close()
michael@0:         self.tmp.append(tmp)
michael@0:         self.append(option % tmp)
michael@0: 
michael@0: class SectionFinder(object):
michael@0:     '''Instances of this class allow to map symbol names to sections in
michael@0:     object files.'''
michael@0: 
michael@0:     def __init__(self, objs):
michael@0:         '''Creates an instance, given a list of object files.'''
michael@0:         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
michael@0:             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
michael@0:         self.mapping = {}
michael@0:         for obj in objs:
michael@0:             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
michael@0:                 raise Exception('%s is not an object nor a static library' % obj)
michael@0:             for symbol, section in SectionFinder._getSymbols(obj):
michael@0:                 sym = SectionFinder._normalize(symbol)
michael@0:                 if sym in self.mapping:
michael@0:                     if not section in self.mapping[sym]:
michael@0:                         self.mapping[sym].append(section)
michael@0:                 else:
michael@0:                     self.mapping[sym] = [section]
michael@0: 
michael@0:     def getSections(self, symbol):
michael@0:         '''Given a symbol, returns a list of sections containing it or the
michael@0:         corresponding thunks. When the given symbol is a thunk, returns the
michael@0:         list of sections containing its corresponding normal symbol and the
michael@0:         other thunks for that symbol.'''
michael@0:         sym = SectionFinder._normalize(symbol)
michael@0:         if sym in self.mapping:
michael@0:             return self.mapping[sym]
michael@0:         return []
michael@0: 
michael@0:     @staticmethod
michael@0:     def _normalize(symbol):
michael@0:         '''For normal symbols, return the given symbol. For thunks, return
michael@0:         the corresponding normal symbol.'''
michael@0:         if re.match('^_ZThn[0-9]+_', symbol):
michael@0:             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
michael@0:         return symbol
michael@0: 
michael@0:     @staticmethod
michael@0:     def _getSymbols(obj):
michael@0:         '''Returns a list of (symbol, section) contained in the given object
michael@0:         file.'''
michael@0:         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
michael@0:         (stdout, stderr) = proc.communicate()
michael@0:         syms = []
michael@0:         for line in stdout.splitlines():
michael@0:             # Each line has the following format:
michael@0:             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
michael@0:             tmp = line.split(' ',1)
michael@0:             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
michael@0:             # We only need to consider cases where "<section>\t<length> <symbol>" is present,
michael@0:             # and where the [FfO] flag is either F (function) or O (object).
michael@0:             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
michael@0:                 tmp = tmp[1][8:].split()
michael@0:                 # That gives us ["<section>","<length>", "<symbol>"]
michael@0:                 syms.append((tmp[-1], tmp[0]))
michael@0:         return syms
michael@0: 
michael@0: def print_command(out, args):
michael@0:     print >>out, "Executing: " + " ".join(args)
michael@0:     for tmp in [f for f in args.tmp if os.path.isfile(f)]:
michael@0:         print >>out, tmp + ":"
michael@0:         with open(tmp) as file:
michael@0:             print >>out, "".join(["    " + l for l in file.readlines()])
michael@0:     out.flush()
michael@0: 
michael@0: def main():
michael@0:     parser = OptionParser()
michael@0:     parser.add_option("--depend", dest="depend", metavar="FILE",
michael@0:         help="generate dependencies for the given execution and store it in the given file")
michael@0:     parser.add_option("--target", dest="target", metavar="FILE",
michael@0:         help="designate the target for dependencies")
michael@0:     parser.add_option("--extract", action="store_true", dest="extract",
michael@0:         help="when a library has no descriptor file, extract it first, when possible")
michael@0:     parser.add_option("--uselist", action="store_true", dest="uselist",
michael@0:         help="use a list file for objects when executing a command")
michael@0:     parser.add_option("--verbose", action="store_true", dest="verbose",
michael@0:         help="display executed command and temporary files content")
michael@0:     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
michael@0:         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
michael@0: 
michael@0:     (options, args) = parser.parse_args()
michael@0: 
michael@0:     if not options.target:
michael@0:         options.depend = False
michael@0:     if options.depend:
michael@0:         deps = ExpandLibsDeps(args)
michael@0:         # Filter out common command wrappers
michael@0:         while os.path.basename(deps[0]) in ['ccache', 'distcc']:
michael@0:             deps.pop(0)
michael@0:         # Remove command
michael@0:         deps.pop(0)
michael@0:     with ExpandArgsMore(args) as args:
michael@0:         if options.extract:
michael@0:             args.extract()
michael@0:         if options.symbol_order:
michael@0:             args.orderSymbols(options.symbol_order)
michael@0:         if options.uselist:
michael@0:             args.makelist()
michael@0: 
michael@0:         if options.verbose:
michael@0:             print_command(sys.stderr, args)
michael@0:         try:
michael@0:             proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
michael@0:         except Exception, e:
michael@0:             print >>sys.stderr, 'error: Launching', args, ':', e
michael@0:             raise e
michael@0:         (stdout, stderr) = proc.communicate()
michael@0:         if proc.returncode and not options.verbose:
michael@0:             print_command(sys.stderr, args)
michael@0:         sys.stderr.write(stdout)
michael@0:         sys.stderr.flush()
michael@0:         if proc.returncode:
michael@0:             exit(proc.returncode)
michael@0:     if not options.depend:
michael@0:         return
michael@0:     ensureParentDir(options.depend)
michael@0:     mk = Makefile()
michael@0:     deps = [dep for dep in deps if os.path.isfile(dep) and dep != options.target
michael@0:             and os.path.abspath(dep) != os.path.abspath(options.depend)]
michael@0:     no_dynamic_lib = [dep for dep in deps if not isDynamicLib(dep)]
michael@0:     mk.create_rule([options.target]).add_dependencies(no_dynamic_lib)
michael@0:     if len(deps) != len(no_dynamic_lib):
michael@0:         mk.create_rule(['%s_order_only' % options.target]).add_dependencies(dep for dep in deps if isDynamicLib(dep))
michael@0: 
michael@0:     with open(options.depend, 'w') as depfile:
michael@0:         mk.dump(depfile, removal_guard=True)
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     main()