1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/devtools/rootAnalysis/analyze.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,279 @@
1.4 +#!/usr/bin/python
1.5 +
1.6 +#
1.7 +# This Source Code Form is subject to the terms of the Mozilla Public
1.8 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.9 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.10 +
1.11 +"""
1.12 +Runs the static rooting analysis
1.13 +"""
1.14 +
1.15 +from subprocess import Popen
1.16 +import subprocess
1.17 +import os
1.18 +import argparse
1.19 +import sys
1.20 +import re
1.21 +
1.22 +def env(config):
1.23 + e = dict(os.environ)
1.24 + e['PATH'] = '%s:%s' % (e['PATH'], config['sixgill_bin'])
1.25 + e['XDB'] = '%(sixgill_bin)s/xdb.so' % config
1.26 + e['SOURCE'] = config['source']
1.27 + e['ANALYZED_OBJDIR'] = config['objdir']
1.28 + return e
1.29 +
1.30 +def fill(command, config):
1.31 + try:
1.32 + return tuple(s % config for s in command)
1.33 + except:
1.34 + print("Substitution failed:")
1.35 + problems = []
1.36 + for fragment in command:
1.37 + try:
1.38 + fragment % config
1.39 + except:
1.40 + problems.append(fragment)
1.41 + raise Exception("\n".join(["Substitution failed:"] + [ " %s" % s for s in problems ]))
1.42 +
1.43 +def print_command(command, outfile=None, env=None):
1.44 + output = ' '.join(command)
1.45 + if outfile:
1.46 + output += ' > ' + outfile
1.47 + if env:
1.48 + changed = {}
1.49 + e = os.environ
1.50 + for key,value in env.items():
1.51 + if (key not in e) or (e[key] != value):
1.52 + changed[key] = value
1.53 + if changed:
1.54 + outputs = []
1.55 + for key, value in changed.items():
1.56 + if key in e and e[key] in value:
1.57 + start = value.index(e[key])
1.58 + end = start + len(e[key])
1.59 + outputs.append('%s="%s${%s}%s"' % (key,
1.60 + value[:start],
1.61 + key,
1.62 + value[end:]))
1.63 + else:
1.64 + outputs.append("%s='%s'" % (key, value))
1.65 + output = ' '.join(outputs) + " " + output
1.66 +
1.67 + print output
1.68 +
1.69 +def generate_hazards(config, outfilename):
1.70 + jobs = []
1.71 + for i in range(config['jobs']):
1.72 + command = fill(('%(js)s',
1.73 + '%(analysis_scriptdir)s/analyzeRoots.js',
1.74 + '%(gcFunctions_list)s',
1.75 + '%(gcEdges)s',
1.76 + '%(suppressedFunctions_list)s',
1.77 + '%(gcTypes)s',
1.78 + str(i+1), '%(jobs)s',
1.79 + 'tmp.%s' % (i+1,)),
1.80 + config)
1.81 + outfile = 'rootingHazards.%s' % (i+1,)
1.82 + output = open(outfile, 'w')
1.83 + print_command(command, outfile=outfile, env=env(config))
1.84 + jobs.append((command, Popen(command, stdout=output, env=env(config))))
1.85 +
1.86 + final_status = 0
1.87 + while jobs:
1.88 + pid, status = os.wait()
1.89 + jobs = [ job for job in jobs if job[1].pid != pid ]
1.90 + final_status = final_status or status
1.91 +
1.92 + if final_status:
1.93 + raise subprocess.CalledProcessError(final_status, 'analyzeRoots.js')
1.94 +
1.95 + with open(outfilename, 'w') as output:
1.96 + command = ['cat'] + [ 'rootingHazards.%s' % (i+1,) for i in range(config['jobs']) ]
1.97 + print_command(command, outfile=outfilename)
1.98 + subprocess.call(command, stdout=output)
1.99 +
1.100 +JOBS = { 'dbs':
1.101 + (('%(ANALYSIS_SCRIPTDIR)s/run_complete',
1.102 + '--foreground',
1.103 + '--no-logs',
1.104 + '--build-root=%(objdir)s',
1.105 + '--wrap-dir=%(sixgill)s/scripts/wrap_gcc',
1.106 + '--work-dir=work',
1.107 + '-b', '%(sixgill_bin)s',
1.108 + '--buildcommand=%(buildcommand)s',
1.109 + '.'),
1.110 + ()),
1.111 +
1.112 + 'callgraph':
1.113 + (('%(js)s', '%(analysis_scriptdir)s/computeCallgraph.js'),
1.114 + 'callgraph.txt'),
1.115 +
1.116 + 'gcFunctions':
1.117 + (('%(js)s', '%(analysis_scriptdir)s/computeGCFunctions.js', '%(callgraph)s',
1.118 + '[gcFunctions]', '[gcFunctions_list]', '[gcEdges]', '[suppressedFunctions_list]'),
1.119 + ('gcFunctions.txt', 'gcFunctions.lst', 'gcEdges.txt', 'suppressedFunctions.lst')),
1.120 +
1.121 + 'gcTypes':
1.122 + (('%(js)s', '%(analysis_scriptdir)s/computeGCTypes.js',),
1.123 + 'gcTypes.txt'),
1.124 +
1.125 + 'allFunctions':
1.126 + (('%(sixgill_bin)s/xdbkeys', 'src_body.xdb',),
1.127 + 'allFunctions.txt'),
1.128 +
1.129 + 'hazards':
1.130 + (generate_hazards, 'rootingHazards.txt'),
1.131 +
1.132 + 'explain':
1.133 + (('python', '%(analysis_scriptdir)s/explain.py',
1.134 + '%(hazards)s', '%(gcFunctions)s',
1.135 + '[explained_hazards]', '[unnecessary]', '[refs]'),
1.136 + ('hazards.txt', 'unnecessary.txt', 'refs.txt'))
1.137 + }
1.138 +
1.139 +def out_indexes(command):
1.140 + for i in range(len(command)):
1.141 + m = re.match(r'^\[(.*)\]$', command[i])
1.142 + if m:
1.143 + yield (i, m.group(1))
1.144 +
1.145 +def run_job(name, config):
1.146 + cmdspec, outfiles = JOBS[name]
1.147 + print("Running " + name + " to generate " + str(outfiles))
1.148 + if hasattr(cmdspec, '__call__'):
1.149 + cmdspec(config, outfiles)
1.150 + else:
1.151 + temp_map = {}
1.152 + cmdspec = fill(cmdspec, config)
1.153 + if isinstance(outfiles, basestring):
1.154 + stdout_filename = '%s.tmp' % name
1.155 + temp_map[stdout_filename] = outfiles
1.156 + print_command(cmdspec, outfile=outfiles, env=env(config))
1.157 + else:
1.158 + stdout_filename = None
1.159 + pc = list(cmdspec)
1.160 + outfile = 0
1.161 + for (i, name) in out_indexes(cmdspec):
1.162 + pc[i] = outfiles[outfile]
1.163 + outfile += 1
1.164 + print_command(pc, env=env(config))
1.165 +
1.166 + command = list(cmdspec)
1.167 + outfile = 0
1.168 + for (i, name) in out_indexes(cmdspec):
1.169 + command[i] = '%s.tmp' % name
1.170 + temp_map[command[i]] = outfiles[outfile]
1.171 + outfile += 1
1.172 +
1.173 + sys.stdout.flush()
1.174 + if stdout_filename is None:
1.175 + subprocess.check_call(command, env=env(config))
1.176 + else:
1.177 + with open(stdout_filename, 'w') as output:
1.178 + subprocess.check_call(command, stdout=output, env=env(config))
1.179 + for (temp, final) in temp_map.items():
1.180 + try:
1.181 + os.rename(temp, final)
1.182 + except OSError:
1.183 + print("Error renaming %s -> %s" % (temp, final))
1.184 + raise
1.185 +
1.186 +config = { 'ANALYSIS_SCRIPTDIR': os.path.dirname(__file__) }
1.187 +
1.188 +defaults = [ '%s/defaults.py' % config['ANALYSIS_SCRIPTDIR'],
1.189 + '%s/defaults.py' % os.getcwd() ]
1.190 +
1.191 +for default in defaults:
1.192 + try:
1.193 + execfile(default, config)
1.194 + print("Loaded %s" % default)
1.195 + except:
1.196 + pass
1.197 +
1.198 +data = config.copy()
1.199 +
1.200 +parser = argparse.ArgumentParser(description='Statically analyze build tree for rooting hazards.')
1.201 +parser.add_argument('step', metavar='STEP', type=str, nargs='?',
1.202 + help='run starting from this step')
1.203 +parser.add_argument('--source', metavar='SOURCE', type=str, nargs='?',
1.204 + help='source code to analyze')
1.205 +parser.add_argument('--upto', metavar='UPTO', type=str, nargs='?',
1.206 + help='last step to execute')
1.207 +parser.add_argument('--jobs', '-j', default=None, metavar='JOBS', type=int,
1.208 + help='number of simultaneous analyzeRoots.js jobs')
1.209 +parser.add_argument('--list', const=True, nargs='?', type=bool,
1.210 + help='display available steps')
1.211 +parser.add_argument('--buildcommand', '--build', '-b', type=str, nargs='?',
1.212 + help='command to build the tree being analyzed')
1.213 +parser.add_argument('--tag', '-t', type=str, nargs='?',
1.214 + help='name of job, also sets build command to "build.<tag>"')
1.215 +parser.add_argument('--expect-file', type=str, nargs='?',
1.216 + help='deprecated option, temporarily still present for backwards compatibility')
1.217 +
1.218 +args = parser.parse_args()
1.219 +for k,v in vars(args).items():
1.220 + if v is not None:
1.221 + data[k] = v
1.222 +
1.223 +if args.tag and not args.buildcommand:
1.224 + args.buildcommand="build.%s" % args.tag
1.225 +
1.226 +if args.jobs is not None:
1.227 + data['jobs'] = args.jobs
1.228 +if not data.get('jobs'):
1.229 + data['jobs'] = subprocess.check_output(['nproc', '--ignore=1'])
1.230 +
1.231 +if args.buildcommand:
1.232 + data['buildcommand'] = args.buildcommand
1.233 +elif 'BUILD' in os.environ:
1.234 + data['buildcommand'] = os.environ['BUILD']
1.235 +else:
1.236 + data['buildcommand'] = 'make -j4 -s'
1.237 +
1.238 +if 'ANALYZED_OBJDIR' in os.environ:
1.239 + data['objdir'] = os.environ['ANALYZED_OBJDIR']
1.240 +
1.241 +if 'SOURCE' in os.environ:
1.242 + data['source'] = os.environ['SOURCE']
1.243 +if not data.get('source') and data.get('sixgill_bin'):
1.244 + path = subprocess.check_output(['sh', '-c', data['sixgill_bin'] + '/xdbkeys file_source.xdb | grep jsapi.cpp'])
1.245 + data['source'] = path.replace("/js/src/jsapi.cpp", "")
1.246 +
1.247 +steps = [ 'dbs',
1.248 + 'callgraph',
1.249 + 'gcTypes',
1.250 + 'gcFunctions',
1.251 + 'allFunctions',
1.252 + 'hazards',
1.253 + 'explain' ]
1.254 +
1.255 +if args.list:
1.256 + for step in steps:
1.257 + command, outfilename = JOBS[step]
1.258 + if outfilename:
1.259 + print("%s -> %s" % (step, outfilename))
1.260 + else:
1.261 + print(step)
1.262 + sys.exit(0)
1.263 +
1.264 +for step in steps:
1.265 + command, outfiles = JOBS[step]
1.266 + if isinstance(outfiles, basestring):
1.267 + data[step] = outfiles
1.268 + else:
1.269 + outfile = 0
1.270 + for (i, name) in out_indexes(command):
1.271 + data[name] = outfiles[outfile]
1.272 + outfile += 1
1.273 + assert len(outfiles) == outfile, 'step \'%s\': mismatched number of output files and params' % step
1.274 +
1.275 +if args.step:
1.276 + steps = steps[steps.index(args.step):]
1.277 +
1.278 +if args.upto:
1.279 + steps = steps[:steps.index(args.upto)+1]
1.280 +
1.281 +for step in steps:
1.282 + run_job(step, data)
