michael@0: #!/usr/bin/python
michael@0: 
michael@0: #
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: """
michael@0: Runs the static rooting analysis
michael@0: """
michael@0: 
michael@0: from subprocess import Popen
michael@0: import subprocess
michael@0: import os
michael@0: import argparse
michael@0: import sys
michael@0: import re
michael@0: 
michael@0: def env(config):
michael@0:     e = dict(os.environ)
michael@0:     e['PATH'] = '%s:%s' % (e['PATH'], config['sixgill_bin'])
michael@0:     e['XDB'] = '%(sixgill_bin)s/xdb.so' % config
michael@0:     e['SOURCE'] = config['source']
michael@0:     e['ANALYZED_OBJDIR'] = config['objdir']
michael@0:     return e
michael@0: 
michael@0: def fill(command, config):
michael@0:     try:
michael@0:         return tuple(s % config for s in command)
michael@0:     except:
michael@0:         print("Substitution failed:")
michael@0:         problems = []
michael@0:         for fragment in command:
michael@0:             try:
michael@0:                 fragment % config
michael@0:             except:
michael@0:                 problems.append(fragment)
michael@0:         raise Exception("\n".join(["Substitution failed:"] + [ "  %s" % s for s in problems ]))
michael@0: 
michael@0: def print_command(command, outfile=None, env=None):
michael@0:     output = ' '.join(command)
michael@0:     if outfile:
michael@0:         output += ' > ' + outfile
michael@0:     if env:
michael@0:         changed = {}
michael@0:         e = os.environ
michael@0:         for key,value in env.items():
michael@0:             if (key not in e) or (e[key] != value):
michael@0:                 changed[key] = value
michael@0:         if changed:
michael@0:             outputs = []
michael@0:             for key, value in changed.items():
michael@0:                 if key in e and e[key] in value:
michael@0:                     start = value.index(e[key])
michael@0:                     end = start + len(e[key])
michael@0:                     outputs.append('%s="%s${%s}%s"' % (key,
michael@0:                                                        value[:start],
michael@0:                                                        key,
michael@0:                                                        value[end:]))
michael@0:                 else:
michael@0:                     outputs.append("%s='%s'" % (key, value))
michael@0:             output = ' '.join(outputs) + " " + output
michael@0: 
michael@0:     print output
michael@0: 
michael@0: def generate_hazards(config, outfilename):
michael@0:     jobs = []
michael@0:     for i in range(config['jobs']):
michael@0:         command = fill(('%(js)s',
michael@0:                         '%(analysis_scriptdir)s/analyzeRoots.js',
michael@0:                         '%(gcFunctions_list)s',
michael@0:                         '%(gcEdges)s',
michael@0:                         '%(suppressedFunctions_list)s',
michael@0:                         '%(gcTypes)s',
michael@0:                         str(i+1), '%(jobs)s',
michael@0:                         'tmp.%s' % (i+1,)),
michael@0:                        config)
michael@0:         outfile = 'rootingHazards.%s' % (i+1,)
michael@0:         output = open(outfile, 'w')
michael@0:         print_command(command, outfile=outfile, env=env(config))
michael@0:         jobs.append((command, Popen(command, stdout=output, env=env(config))))
michael@0: 
michael@0:     final_status = 0
michael@0:     while jobs:
michael@0:         pid, status = os.wait()
michael@0:         jobs = [ job for job in jobs if job[1].pid != pid ]
michael@0:         final_status = final_status or status
michael@0: 
michael@0:     if final_status:
michael@0:         raise subprocess.CalledProcessError(final_status, 'analyzeRoots.js')
michael@0: 
michael@0:     with open(outfilename, 'w') as output:
michael@0:         command = ['cat'] + [ 'rootingHazards.%s' % (i+1,) for i in range(config['jobs']) ]
michael@0:         print_command(command, outfile=outfilename)
michael@0:         subprocess.call(command, stdout=output)
michael@0: 
michael@0: JOBS = { 'dbs':
michael@0:              (('%(ANALYSIS_SCRIPTDIR)s/run_complete',
michael@0:                '--foreground',
michael@0:                '--no-logs',
michael@0:                '--build-root=%(objdir)s',
michael@0:                '--wrap-dir=%(sixgill)s/scripts/wrap_gcc',
michael@0:                '--work-dir=work',
michael@0:                '-b', '%(sixgill_bin)s',
michael@0:                '--buildcommand=%(buildcommand)s',
michael@0:                '.'),
michael@0:               ()),
michael@0: 
michael@0:          'callgraph':
michael@0:              (('%(js)s', '%(analysis_scriptdir)s/computeCallgraph.js'),
michael@0:               'callgraph.txt'),
michael@0: 
michael@0:          'gcFunctions':
michael@0:              (('%(js)s', '%(analysis_scriptdir)s/computeGCFunctions.js', '%(callgraph)s',
michael@0:                '[gcFunctions]', '[gcFunctions_list]', '[gcEdges]', '[suppressedFunctions_list]'),
michael@0:               ('gcFunctions.txt', 'gcFunctions.lst', 'gcEdges.txt', 'suppressedFunctions.lst')),
michael@0: 
michael@0:          'gcTypes':
michael@0:              (('%(js)s', '%(analysis_scriptdir)s/computeGCTypes.js',),
michael@0:               'gcTypes.txt'),
michael@0: 
michael@0:          'allFunctions':
michael@0:              (('%(sixgill_bin)s/xdbkeys', 'src_body.xdb',),
michael@0:               'allFunctions.txt'),
michael@0: 
michael@0:          'hazards':
michael@0:              (generate_hazards, 'rootingHazards.txt'),
michael@0: 
michael@0:          'explain':
michael@0:              (('python', '%(analysis_scriptdir)s/explain.py',
michael@0:                '%(hazards)s', '%(gcFunctions)s',
michael@0:                '[explained_hazards]', '[unnecessary]', '[refs]'),
michael@0:               ('hazards.txt', 'unnecessary.txt', 'refs.txt'))
michael@0:          }
michael@0: 
michael@0: def out_indexes(command):
michael@0:     for i in range(len(command)):
michael@0:         m = re.match(r'^\[(.*)\]$', command[i])
michael@0:         if m:
michael@0:             yield (i, m.group(1))
michael@0: 
michael@0: def run_job(name, config):
michael@0:     cmdspec, outfiles = JOBS[name]
michael@0:     print("Running " + name + " to generate " + str(outfiles))
michael@0:     if hasattr(cmdspec, '__call__'):
michael@0:         cmdspec(config, outfiles)
michael@0:     else:
michael@0:         temp_map = {}
michael@0:         cmdspec = fill(cmdspec, config)
michael@0:         if isinstance(outfiles, basestring):
michael@0:             stdout_filename = '%s.tmp' % name
michael@0:             temp_map[stdout_filename] = outfiles
michael@0:             print_command(cmdspec, outfile=outfiles, env=env(config))
michael@0:         else:
michael@0:             stdout_filename = None
michael@0:             pc = list(cmdspec)
michael@0:             outfile = 0
michael@0:             for (i, name) in out_indexes(cmdspec):
michael@0:                 pc[i] = outfiles[outfile]
michael@0:                 outfile += 1
michael@0:             print_command(pc, env=env(config))
michael@0: 
michael@0:         command = list(cmdspec)
michael@0:         outfile = 0
michael@0:         for (i, name) in out_indexes(cmdspec):
michael@0:             command[i] = '%s.tmp' % name
michael@0:             temp_map[command[i]] = outfiles[outfile]
michael@0:             outfile += 1
michael@0: 
michael@0:         sys.stdout.flush()
michael@0:         if stdout_filename is None:
michael@0:             subprocess.check_call(command, env=env(config))
michael@0:         else:
michael@0:             with open(stdout_filename, 'w') as output:
michael@0:                 subprocess.check_call(command, stdout=output, env=env(config))
michael@0:         for (temp, final) in temp_map.items():
michael@0:             try:
michael@0:                 os.rename(temp, final)
michael@0:             except OSError:
michael@0:                 print("Error renaming %s -> %s" % (temp, final))
michael@0:                 raise
michael@0: 
michael@0: config = { 'ANALYSIS_SCRIPTDIR': os.path.dirname(__file__) }
michael@0: 
michael@0: defaults = [ '%s/defaults.py' % config['ANALYSIS_SCRIPTDIR'],
michael@0:              '%s/defaults.py' % os.getcwd() ]
michael@0: 
michael@0: for default in defaults:
michael@0:     try:
michael@0:         execfile(default, config)
michael@0:         print("Loaded %s" % default)
michael@0:     except:
michael@0:         pass
michael@0: 
michael@0: data = config.copy()
michael@0: 
michael@0: parser = argparse.ArgumentParser(description='Statically analyze build tree for rooting hazards.')
michael@0: parser.add_argument('step', metavar='STEP', type=str, nargs='?',
michael@0:                     help='run starting from this step')
michael@0: parser.add_argument('--source', metavar='SOURCE', type=str, nargs='?',
michael@0:                     help='source code to analyze')
michael@0: parser.add_argument('--upto', metavar='UPTO', type=str, nargs='?',
michael@0:                     help='last step to execute')
michael@0: parser.add_argument('--jobs', '-j', default=None, metavar='JOBS', type=int,
michael@0:                     help='number of simultaneous analyzeRoots.js jobs')
michael@0: parser.add_argument('--list', const=True, nargs='?', type=bool,
michael@0:                     help='display available steps')
michael@0: parser.add_argument('--buildcommand', '--build', '-b', type=str, nargs='?',
michael@0:                     help='command to build the tree being analyzed')
michael@0: parser.add_argument('--tag', '-t', type=str, nargs='?',
michael@0:                     help='name of job, also sets build command to "build.<tag>"')
michael@0: parser.add_argument('--expect-file', type=str, nargs='?',
michael@0:                     help='deprecated option, temporarily still present for backwards compatibility')
michael@0: 
michael@0: args = parser.parse_args()
michael@0: for k,v in vars(args).items():
michael@0:     if v is not None:
michael@0:         data[k] = v
michael@0: 
michael@0: if args.tag and not args.buildcommand:
michael@0:     args.buildcommand="build.%s" % args.tag
michael@0: 
michael@0: if args.jobs is not None:
michael@0:     data['jobs'] = args.jobs
michael@0: if not data.get('jobs'):
michael@0:     data['jobs'] = subprocess.check_output(['nproc', '--ignore=1'])
michael@0: 
michael@0: if args.buildcommand:
michael@0:     data['buildcommand'] = args.buildcommand
michael@0: elif 'BUILD' in os.environ:
michael@0:     data['buildcommand'] = os.environ['BUILD']
michael@0: else:
michael@0:     data['buildcommand'] = 'make -j4 -s'
michael@0: 
michael@0: if 'ANALYZED_OBJDIR' in os.environ:
michael@0:     data['objdir'] = os.environ['ANALYZED_OBJDIR']
michael@0: 
michael@0: if 'SOURCE' in os.environ:
michael@0:     data['source'] = os.environ['SOURCE']
michael@0: if not data.get('source') and data.get('sixgill_bin'):
michael@0:     path = subprocess.check_output(['sh', '-c', data['sixgill_bin'] + '/xdbkeys file_source.xdb | grep jsapi.cpp'])
michael@0:     data['source'] = path.replace("/js/src/jsapi.cpp", "")
michael@0: 
michael@0: steps = [ 'dbs',
michael@0:           'callgraph',
michael@0:           'gcTypes',
michael@0:           'gcFunctions',
michael@0:           'allFunctions',
michael@0:           'hazards',
michael@0:           'explain' ]
michael@0: 
michael@0: if args.list:
michael@0:     for step in steps:
michael@0:         command, outfilename = JOBS[step]
michael@0:         if outfilename:
michael@0:             print("%s -> %s" % (step, outfilename))
michael@0:         else:
michael@0:             print(step)
michael@0:     sys.exit(0)
michael@0: 
michael@0: for step in steps:
michael@0:     command, outfiles = JOBS[step]
michael@0:     if isinstance(outfiles, basestring):
michael@0:         data[step] = outfiles
michael@0:     else:
michael@0:         outfile = 0
michael@0:         for (i, name) in out_indexes(command):
michael@0:             data[name] = outfiles[outfile]
michael@0:             outfile += 1
michael@0:         assert len(outfiles) == outfile, 'step \'%s\': mismatched number of output files and params' % step
michael@0: 
michael@0: if args.step:
michael@0:     steps = steps[steps.index(args.step):]
michael@0: 
michael@0: if args.upto:
michael@0:     steps = steps[:steps.index(args.upto)+1]
michael@0: 
michael@0: for step in steps:
michael@0:     run_job(step, data)