The Tor Browser: config/expandlibs

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to

     6 a given command line, and executes that command line with the expanded

     7 arguments.

     9 With the --extract argument (useful for e.g. $(AR)), it extracts object files

    10 from static libraries (or use those listed in library descriptors directly).

    12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object

    13 files with a list file. This can be used to avoid limitations in the length

    14 of a command line. The kind of list file format used depends on the

    15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)

    16 or 'linkerscript' for GNU ld linker scripts.

    17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.

    19 With the --symbol-order argument, followed by a file name, it will add the

    20 relevant linker options to change the order in which the linker puts the

    21 symbols appear in the resulting binary. Only works for ELF targets.

    22 '''

    23 from __future__ import with_statement

    24 import sys

    25 import os

    26 from expandlibs import (

    27     ExpandArgs,

    28     relativize,

    29     isDynamicLib,

    30     isObject,

    31     ensureParentDir,

    32     ExpandLibsDeps,

    33 )

    34 import expandlibs_config as conf

    35 from optparse import OptionParser

    36 import subprocess

    37 import tempfile

    38 import shutil

    39 import subprocess

    40 import re

    41 from mozbuild.makeutil import Makefile

    43 # The are the insert points for a GNU ld linker script, assuming a more

    44 # or less "standard" default linker script. This is not a dict because

    45 # order is important.

    46 SECTION_INSERT_BEFORE = [

    47   ('.text', '.fini'),

    48   ('.rodata', '.rodata1'),

    49   ('.data.rel.ro', '.dynamic'),

    50   ('.data', '.data1'),

    51 ]

    53 class ExpandArgsMore(ExpandArgs):

    54     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''

    55     def __enter__(self):

    56         self.tmp = []

    57         return self

    59     def __exit__(self, type, value, tb):

    60         '''Automatically remove temporary files'''

    61         for tmp in self.tmp:

    62             if os.path.isdir(tmp):

    63                 shutil.rmtree(tmp, True)

    64             else:

    65                 os.remove(tmp)

    67     def extract(self):

    68         self[0:] = self._extract(self)

    70     def _extract(self, args):

    71         '''When a static library name is found, either extract its contents

    72         in a temporary directory or use the information found in the

    73         corresponding lib descriptor.

    74         '''

    75         ar_extract = conf.AR_EXTRACT.split()

    76         newlist = []

    77         for arg in args:

    78             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:

    79                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):

    80                     newlist += self._extract(self._expand_desc(arg))

    81                     continue

    82                 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):

    83                     tmp = tempfile.mkdtemp(dir=os.curdir)

    84                     self.tmp.append(tmp)

    85                     if conf.AR == 'lib':

    86                         out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])

    87                         files = out.splitlines()

    88                         # If lib -list returns a list full of dlls, it's an

    89                         # import lib.

    90                         if all(isDynamicLib(f) for f in files):

    91                             newlist += [arg]

    92                             continue

    93                         for f in files:

    94                             subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)

    95                     else:

    96                         subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)

    97                     objs = []

    98                     for root, dirs, files in os.walk(tmp):

    99                         objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]

   100                     newlist += sorted(objs)

   101                     continue

   102             newlist += [arg]

   103         return newlist

   105     def makelist(self):

   106         '''Replaces object file names with a temporary list file, using a

   107         list format depending on the EXPAND_LIBS_LIST_STYLE variable

   108         '''

   109         objs = [o for o in self if isObject(o)]

   110         if not len(objs): return

   111         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)

   112         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":

   113             content = ['INPUT("%s")\n' % obj for obj in objs]

   114             ref = tmp

   115         elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":

   116             content = ["%s\n" % obj for obj in objs]

   117             ref = "-Wl,-filelist," + tmp

   118         elif conf.EXPAND_LIBS_LIST_STYLE == "list":

   119             content = ["%s\n" % obj for obj in objs]

   120             ref = "@" + tmp

   121         else:

   122             os.close(fd)

   123             os.remove(tmp)

   124             return

   125         self.tmp.append(tmp)

   126         f = os.fdopen(fd, "w")

   127         f.writelines(content)

   128         f.close()

   129         idx = self.index(objs[0])

   130         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]

   131         self[0:] = newlist

   133     def _getFoldedSections(self):

   134         '''Returns a dict about folded sections.

   135         When section A and B are folded into section C, the dict contains:

   136         { 'A': 'C',

   137           'B': 'C',

   138           'C': ['A', 'B'] }'''

   139         if not conf.LD_PRINT_ICF_SECTIONS:

   140             return {}

   142         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)

   143         (stdout, stderr) = proc.communicate()

   144         result = {}

   145         # gold's --print-icf-sections output looks like the following:

   146         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'

   147         # In terms of words, chances are this will change in the future,

   148         # especially considering "into" is misplaced. Splitting on quotes

   149         # seems safer.

   150         for l in stderr.split('\n'):

   151             quoted = l.split("'")

   152             if len(quoted) > 5 and quoted[1] != quoted[5]:

   153                 result[quoted[1]] = [quoted[5]]

   154                 if quoted[5] in result:

   155                     result[quoted[5]].append(quoted[1])

   156                 else:

   157                     result[quoted[5]] = [quoted[1]]

   158         return result

   160     def _getOrderedSections(self, ordered_symbols):

   161         '''Given an ordered list of symbols, returns the corresponding list

   162         of sections following the order.'''

   163         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:

   164             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)

   165         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])

   166         folded = self._getFoldedSections()

   167         sections = set()

   168         ordered_sections = []

   169         for symbol in ordered_symbols:

   170             symbol_sections = finder.getSections(symbol)

   171             all_symbol_sections = []

   172             for section in symbol_sections:

   173                 if section in folded:

   174                     if isinstance(folded[section], str):

   175                         section = folded[section]

   176                     all_symbol_sections.append(section)

   177                     all_symbol_sections.extend(folded[section])

   178                 else:

   179                     all_symbol_sections.append(section)

   180             for section in all_symbol_sections:

   181                 if not section in sections:

   182                     ordered_sections.append(section)

   183                     sections.add(section)

   184         return ordered_sections

   186     def orderSymbols(self, order):

   187         '''Given a file containing a list of symbols, adds the appropriate

   188         argument to make the linker put the symbols in that order.'''

   189         with open(order) as file:

   190             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])

   191         split_sections = {}

   192         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]

   193         for s in sections:

   194             for linked_section in linked_sections:

   195                 if s.startswith(linked_section):

   196                     if linked_section in split_sections:

   197                         split_sections[linked_section].append(s)

   198                     else:

   199                         split_sections[linked_section] = [s]

   200                     break

   201         content = []

   202         # Order is important

   203         linked_sections = [s for s in linked_sections if s in split_sections]

   205         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':

   206             option = '-Wl,--section-ordering-file,%s'

   207             content = sections

   208             for linked_section in linked_sections:

   209                 content.extend(split_sections[linked_section])

   210                 content.append('%s.*' % linked_section)

   211                 content.append(linked_section)

   213         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':

   214             option = '-Wl,-T,%s'

   215             section_insert_before = dict(SECTION_INSERT_BEFORE)

   216             for linked_section in linked_sections:

   217                 content.append('SECTIONS {')

   218                 content.append('  %s : {' % linked_section)

   219                 content.extend('    *(%s)' % s for s in split_sections[linked_section])

   220                 content.append('  }')

   221                 content.append('}')

   222                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])

   223         else:

   224             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)

   226         fd, tmp = tempfile.mkstemp(dir=os.curdir)

   227         f = os.fdopen(fd, "w")

   228         f.write('\n'.join(content)+'\n')

   229         f.close()

   230         self.tmp.append(tmp)

   231         self.append(option % tmp)

   233 class SectionFinder(object):

   234     '''Instances of this class allow to map symbol names to sections in

   235     object files.'''

   237     def __init__(self, objs):

   238         '''Creates an instance, given a list of object files.'''

   239         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:

   240             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)

   241         self.mapping = {}

   242         for obj in objs:

   243             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:

   244                 raise Exception('%s is not an object nor a static library' % obj)

   245             for symbol, section in SectionFinder._getSymbols(obj):

   246                 sym = SectionFinder._normalize(symbol)

   247                 if sym in self.mapping:

   248                     if not section in self.mapping[sym]:

   249                         self.mapping[sym].append(section)

   250                 else:

   251                     self.mapping[sym] = [section]

   253     def getSections(self, symbol):

   254         '''Given a symbol, returns a list of sections containing it or the

   255         corresponding thunks. When the given symbol is a thunk, returns the

   256         list of sections containing its corresponding normal symbol and the

   257         other thunks for that symbol.'''

   258         sym = SectionFinder._normalize(symbol)

   259         if sym in self.mapping:

   260             return self.mapping[sym]

   261         return []

   263     @staticmethod

   264     def _normalize(symbol):

   265         '''For normal symbols, return the given symbol. For thunks, return

   266         the corresponding normal symbol.'''

   267         if re.match('^_ZThn[0-9]+_', symbol):

   268             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)

   269         return symbol

   271     @staticmethod

   272     def _getSymbols(obj):

   273         '''Returns a list of (symbol, section) contained in the given object

   274         file.'''

   275         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)

   276         (stdout, stderr) = proc.communicate()

   277         syms = []

   278         for line in stdout.splitlines():

   279             # Each line has the following format:

   280             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>

   281             tmp = line.split(' ',1)

   282             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]

   283             # We only need to consider cases where "<section>\t<length> <symbol>" is present,

   284             # and where the [FfO] flag is either F (function) or O (object).

   285             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:

   286                 tmp = tmp[1][8:].split()

   287                 # That gives us ["<section>","<length>", "<symbol>"]

   288                 syms.append((tmp[-1], tmp[0]))

   289         return syms

   291 def print_command(out, args):

   292     print >>out, "Executing: " + " ".join(args)

   293     for tmp in [f for f in args.tmp if os.path.isfile(f)]:

   294         print >>out, tmp + ":"

   295         with open(tmp) as file:

   296             print >>out, "".join(["    " + l for l in file.readlines()])

   297     out.flush()

   299 def main():

   300     parser = OptionParser()

   301     parser.add_option("--depend", dest="depend", metavar="FILE",

   302         help="generate dependencies for the given execution and store it in the given file")

   303     parser.add_option("--target", dest="target", metavar="FILE",

   304         help="designate the target for dependencies")

   305     parser.add_option("--extract", action="store_true", dest="extract",

   306         help="when a library has no descriptor file, extract it first, when possible")

   307     parser.add_option("--uselist", action="store_true", dest="uselist",

   308         help="use a list file for objects when executing a command")

   309     parser.add_option("--verbose", action="store_true", dest="verbose",

   310         help="display executed command and temporary files content")

   311     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",

   312         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")

   314     (options, args) = parser.parse_args()

   316     if not options.target:

   317         options.depend = False

   318     if options.depend:

   319         deps = ExpandLibsDeps(args)

   320         # Filter out common command wrappers

   321         while os.path.basename(deps[0]) in ['ccache', 'distcc']:

   322             deps.pop(0)

   323         # Remove command

   324         deps.pop(0)

   325     with ExpandArgsMore(args) as args:

   326         if options.extract:

   327             args.extract()

   328         if options.symbol_order:

   329             args.orderSymbols(options.symbol_order)

   330         if options.uselist:

   331             args.makelist()

   333         if options.verbose:

   334             print_command(sys.stderr, args)

   335         try:

   336             proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)

   337         except Exception, e:

   338             print >>sys.stderr, 'error: Launching', args, ':', e

   339             raise e

   340         (stdout, stderr) = proc.communicate()

   341         if proc.returncode and not options.verbose:

   342             print_command(sys.stderr, args)

   343         sys.stderr.write(stdout)

   344         sys.stderr.flush()

   345         if proc.returncode:

   346             exit(proc.returncode)

   347     if not options.depend:

   348         return

   349     ensureParentDir(options.depend)

   350     mk = Makefile()

   351     deps = [dep for dep in deps if os.path.isfile(dep) and dep != options.target

   352             and os.path.abspath(dep) != os.path.abspath(options.depend)]

   353     no_dynamic_lib = [dep for dep in deps if not isDynamicLib(dep)]

   354     mk.create_rule([options.target]).add_dependencies(no_dynamic_lib)

   355     if len(deps) != len(no_dynamic_lib):

   356         mk.create_rule(['%s_order_only' % options.target]).add_dependencies(dep for dep in deps if isDynamicLib(dep))

   358     with open(options.depend, 'w') as depfile:

   359         mk.dump(depfile, removal_guard=True)

   361 if __name__ == '__main__':

   362     main()

The Tor Browser / file revision

config/expandlibs_exec.py@97036ab72558

config/expandlibs_exec.py