1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/devtools/rootAnalysis/run_complete Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,367 @@
1.4 +#!/usr/bin/perl
1.5 +
1.6 +# Sixgill: Static assertion checker for C/C++ programs.
1.7 +# Copyright (C) 2009-2010 Stanford University
1.8 +# Author: Brian Hackett
1.9 +#
1.10 +# This program is free software: you can redistribute it and/or modify
1.11 +# it under the terms of the GNU General Public License as published by
1.12 +# the Free Software Foundation, either version 3 of the License, or
1.13 +# (at your option) any later version.
1.14 +#
1.15 +# This program is distributed in the hope that it will be useful,
1.16 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
1.17 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1.18 +# GNU General Public License for more details.
1.19 +#
1.20 +# You should have received a copy of the GNU General Public License
1.21 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
1.22 +
1.23 +# do a complete run of the system from raw source to reports. this requires
1.24 +# various run_monitor processes to be running in the background (maybe on other
1.25 +# machines) and watching a shared poll_file for jobs. if the output directory
1.26 +# for this script already exists then an incremental analysis will be performed
1.27 +# and the reports will only reflect the changes since the earlier run.
1.28 +
1.29 +use strict;
1.30 +use warnings;
1.31 +use IO::Handle;
1.32 +use File::Basename qw(dirname);
1.33 +use Getopt::Long;
1.34 +use Cwd;
1.35 +
1.36 +#################################
1.37 +# environment specific settings #
1.38 +#################################
1.39 +
1.40 +my $WORKDIR;
1.41 +my $SIXGILL_BIN;
1.42 +
1.43 +# poll file shared with the run_monitor script.
1.44 +my $poll_file;
1.45 +
1.46 +# root directory of the project.
1.47 +my $build_dir;
1.48 +
1.49 +# directory containing gcc wrapper scripts.
1.50 +my $wrap_dir;
1.51 +
1.52 +# optional file with annotations from the web interface.
1.53 +my $ann_file = "";
1.54 +
1.55 +# optional output directory to do a diff against.
1.56 +my $old_dir = "";
1.57 +
1.58 +# run in the foreground
1.59 +my $foreground;
1.60 +
1.61 +my $builder = "make -j4";
1.62 +
1.63 +my $suppress_logs;
1.64 +GetOptions("build-root|b=s" => \$build_dir,
1.65 + "poll-file=s" => \$poll_file,
1.66 + "no-logs!" => \$suppress_logs,
1.67 + "work-dir=s" => \$WORKDIR,
1.68 + "sixgill-binaries|binaries|b=s" => \$SIXGILL_BIN,
1.69 + "wrap-dir=s" => \$wrap_dir,
1.70 + "annotations-file|annotations|a=s" => \$ann_file,
1.71 + "old-dir|old=s" => \$old_dir,
1.72 + "foreground!" => \$foreground,
1.73 + "buildcommand=s" => \$builder,
1.74 + )
1.75 + or die;
1.76 +
1.77 +if (not -d $build_dir) {
1.78 + mkdir($build_dir);
1.79 +}
1.80 +if ($old_dir ne "" && not -d $old_dir) {
1.81 + die "Old directory '$old_dir' does not exist\n";
1.82 +}
1.83 +
1.84 +$WORKDIR ||= "sixgill-work";
1.85 +mkdir($WORKDIR, 0755) if ! -d $WORKDIR;
1.86 +$poll_file ||= "$WORKDIR/poll.file";
1.87 +$build_dir ||= "$WORKDIR/js-inbound-xgill";
1.88 +
1.89 +if (!defined $SIXGILL_BIN) {
1.90 + chomp(my $path = `which xmanager`);
1.91 + if ($path) {
1.92 + use File::Basename qw(dirname);
1.93 + $SIXGILL_BIN = dirname($path);
1.94 + } else {
1.95 + die "Cannot find sixgill binaries. Use the -b option.";
1.96 + }
1.97 +}
1.98 +
1.99 +$wrap_dir ||= "$WORKDIR/xgill-inbound/wrap_gcc";
1.100 +$wrap_dir = "$SIXGILL_BIN/../scripts/wrap_gcc" if not (-e "$wrap_dir/basecc");
1.101 +die "Bad wrapper directory: $wrap_dir" if not (-e "$wrap_dir/basecc");
1.102 +
1.103 +# code to clean the project from $build_dir.
1.104 +sub clean_project {
1.105 + system("make clean");
1.106 +}
1.107 +
1.108 +# code to build the project from $build_dir.
1.109 +sub build_project {
1.110 + return system($builder) >> 8;
1.111 +}
1.112 +
1.113 +our %kill_on_exit;
1.114 +END {
1.115 + for my $pid (keys %kill_on_exit) {
1.116 + kill($pid);
1.117 + }
1.118 +}
1.119 +
1.120 +# commands to start the various xgill binaries. timeouts can be specified
1.121 +# for the backend analyses here, and a memory limit can be specified for
1.122 +# xmanager if desired (and USE_COUNT_ALLOCATOR is defined in util/alloc.h).
1.123 +my $xmanager = "$SIXGILL_BIN/xmanager";
1.124 +my $xsource = "$SIXGILL_BIN/xsource";
1.125 +my $xmemlocal = "$SIXGILL_BIN/xmemlocal -timeout=20";
1.126 +my $xinfer = "$SIXGILL_BIN/xinfer -timeout=60";
1.127 +my $xcheck = "$SIXGILL_BIN/xcheck -timeout=30";
1.128 +
1.129 +# prefix directory to strip off source files.
1.130 +my $prefix_dir = $build_dir;
1.131 +
1.132 +##########################
1.133 +# general purpose script #
1.134 +##########################
1.135 +
1.136 +# Prevent ccache from being used. I don't think this does any good. The problem
1.137 +# I'm struggling with is that if autoconf.mk still has 'ccache gcc' in it, the
1.138 +# builds fail in a mysterious way.
1.139 +$ENV{CCACHE_COMPILERCHECK} = 'date +%s.%N';
1.140 +delete $ENV{CCACHE_PREFIX};
1.141 +
1.142 +my $usage = "USAGE: run_complete result-dir\n";
1.143 +my $result_dir = shift or die $usage;
1.144 +
1.145 +if (not $foreground) {
1.146 + my $pid = fork();
1.147 + if ($pid != 0) {
1.148 + print "Forked, exiting...\n";
1.149 + exit(0);
1.150 + }
1.151 +}
1.152 +
1.153 +# if the result directory does not already exist, mark for a clean build.
1.154 +my $do_clean = 0;
1.155 +if (not (-d $result_dir)) {
1.156 + $do_clean = 1;
1.157 + mkdir $result_dir;
1.158 +}
1.159 +
1.160 +if (!$suppress_logs) {
1.161 + my $log_file = "$result_dir/complete.log";
1.162 + open(OUT, ">>", $log_file) or die "append to $log_file: $!";
1.163 + OUT->autoflush(1); # don't buffer writes to the main log.
1.164 +
1.165 + # redirect stdout and stderr to the log.
1.166 + STDOUT->fdopen(\*OUT, "w");
1.167 + STDERR->fdopen(\*OUT, "w");
1.168 +}
1.169 +
1.170 +# pids to wait on before exiting. these are collating worker output.
1.171 +my @waitpids;
1.172 +
1.173 +chdir $result_dir;
1.174 +
1.175 +# to do a partial run, comment out the commands here you don't want to do.
1.176 +
1.177 +my $status = run_build();
1.178 +
1.179 +# end of run commands.
1.180 +
1.181 +for my $pid (@waitpids) {
1.182 + waitpid($pid, 0);
1.183 + $status ||= $? >> 8;
1.184 +}
1.185 +
1.186 +print "Exiting run_complete with status $status\n";
1.187 +exit $status;
1.188 +
1.189 +# get the IP address which a freshly created manager is listening on.
1.190 +sub get_manager_address
1.191 +{
1.192 + my $log_file = shift or die;
1.193 +
1.194 + # give the manager one second to start, any longer and something's broken.
1.195 + sleep(1);
1.196 +
1.197 + my $log_data = `cat $log_file`;
1.198 + my ($port) = $log_data =~ /Listening on ([\.\:0-9]*)/
1.199 + or die "no manager found";
1.200 + print OUT "Connecting to manager on port $port\n" unless $suppress_logs;
1.201 + print "Connecting to manager on port $port.\n";
1.202 + return $1;
1.203 +}
1.204 +
1.205 +sub run_build
1.206 +{
1.207 + print "build started: ";
1.208 + print scalar(localtime());
1.209 + print "\n";
1.210 +
1.211 + # fork off a process to run the build.
1.212 + defined(my $pid = fork) or die;
1.213 +
1.214 + # log file for the manager.
1.215 + my $manager_log_file = "$result_dir/build_manager.log";
1.216 +
1.217 + if (!$pid) {
1.218 + # this is the child process, fork another process to run a manager.
1.219 + defined(my $pid = fork) or die;
1.220 + exec("$xmanager -terminate-on-assert > $manager_log_file 2>&1") if (!$pid);
1.221 + $kill_on_exit{$pid} = 1;
1.222 +
1.223 + if (!$suppress_logs) {
1.224 + # open new streams to redirect stdout and stderr.
1.225 + open(LOGOUT, "> $result_dir/build.log");
1.226 + open(LOGERR, "> $result_dir/build_err.log");
1.227 + STDOUT->fdopen(\*LOGOUT, "w");
1.228 + STDERR->fdopen(\*LOGERR, "w");
1.229 + }
1.230 +
1.231 + my $address = get_manager_address($manager_log_file);
1.232 +
1.233 + # write the configuration file for the wrapper script.
1.234 + my $config_file = "$WORKDIR/xgill.config";
1.235 + open(CONFIG, ">", $config_file) or die "create $config_file: $!";
1.236 + print CONFIG "$prefix_dir\n";
1.237 + print CONFIG Cwd::abs_path("$result_dir/build_xgill.log")."\n";
1.238 + print CONFIG "$address\n";
1.239 + my @extra = ("-fplugin-arg-xgill-mangle=1");
1.240 + push(@extra, "-fplugin-arg-xgill-annfile=$ann_file")
1.241 + if ($ann_file ne "" && -e $ann_file);
1.242 + print CONFIG join(" ", @extra) . "\n";
1.243 + close(CONFIG);
1.244 +
1.245 + # Tell the wrapper where to find the config
1.246 + $ENV{"XGILL_CONFIG"} = Cwd::abs_path($config_file);
1.247 +
1.248 + # update the PATH so that the build will see the wrappers.
1.249 + $ENV{"PATH"} = "$wrap_dir:" . $ENV{"PATH"};
1.250 +
1.251 + # do the build, cleaning if necessary.
1.252 + chdir $build_dir;
1.253 + clean_project() if ($do_clean);
1.254 + my $exit_status = build_project();
1.255 +
1.256 + # signal the manager that it's over.
1.257 + system("$xsource -remote=$address -end-manager");
1.258 +
1.259 + # wait for the manager to clean up and terminate.
1.260 + print "Waiting for manager to finish (build status $exit_status)...\n";
1.261 + waitpid($pid, 0);
1.262 + my $manager_status = $?;
1.263 + delete $kill_on_exit{$pid};
1.264 +
1.265 + # build is finished, the complete run can resume.
1.266 + # return value only useful if --foreground
1.267 + print "Exiting with status " . ($manager_status || $exit_status) . "\n";
1.268 + exit($manager_status || $exit_status);
1.269 + }
1.270 +
1.271 + # this is the complete process, wait for the build to finish.
1.272 + waitpid($pid, 0);
1.273 + my $status = $? >> 8;
1.274 + print "build finished (status $status): ";
1.275 + print scalar(localtime());
1.276 + print "\n";
1.277 +
1.278 + return $status;
1.279 +}
1.280 +
1.281 +sub run_pass
1.282 +{
1.283 + my ($name, $command) = @_;
1.284 + my $log_file = "$result_dir/manager.$name.log";
1.285 +
1.286 + # extra commands to pass to the manager.
1.287 + my $manager_extra = "";
1.288 + $manager_extra .= "-modset-wait=10" if ($name eq "xmemlocal");
1.289 +
1.290 + # fork off a manager process for the analysis.
1.291 + defined(my $pid = fork) or die;
1.292 + exec("$xmanager $manager_extra > $log_file 2>&1") if (!$pid);
1.293 +
1.294 + my $address = get_manager_address($log_file);
1.295 +
1.296 + # write the poll file for this pass.
1.297 + if (! -d dirname($poll_file)) {
1.298 + system("mkdir", "-p", dirname($poll_file));
1.299 + }
1.300 + open(POLL, "> $poll_file");
1.301 + print POLL "$command\n";
1.302 + print POLL "$result_dir/$name\n";
1.303 + print POLL "$address\n";
1.304 + close(POLL);
1.305 +
1.306 + print "$name started: ";
1.307 + print scalar(localtime());
1.308 + print "\n";
1.309 +
1.310 + waitpid($pid, 0);
1.311 + unlink($poll_file);
1.312 +
1.313 + print "$name finished: ";
1.314 + print scalar(localtime());
1.315 + print "\n";
1.316 +
1.317 + # collate the worker's output into a single file. make this asynchronous
1.318 + # so we can wait a bit and make sure we get all worker output.
1.319 + defined($pid = fork) or die;
1.320 +
1.321 + if (!$pid) {
1.322 + sleep(20);
1.323 + exec("cat $name.*.log > $name.log");
1.324 + }
1.325 +
1.326 + push(@waitpids, $pid);
1.327 +}
1.328 +
1.329 +# the names of all directories containing reports to archive.
1.330 +my $indexes;
1.331 +
1.332 +sub run_index
1.333 +{
1.334 + my ($name, $kind) = @_;
1.335 +
1.336 + return if (not (-e "report_$kind.xdb"));
1.337 +
1.338 + print "$name started: ";
1.339 + print scalar(localtime());
1.340 + print "\n";
1.341 +
1.342 + # make an index for the report diff if applicable.
1.343 + if ($old_dir ne "") {
1.344 + system("make_index $kind $old_dir > $name.diff.log");
1.345 + system("mv $kind diff_$kind");
1.346 + $indexes .= " diff_$kind";
1.347 + }
1.348 +
1.349 + # make an index for the full set of reports.
1.350 + system("make_index $kind > $name.log");
1.351 + $indexes .= " $kind";
1.352 +
1.353 + print "$name finished: ";
1.354 + print scalar(localtime());
1.355 + print "\n";
1.356 +}
1.357 +
1.358 +sub archive_indexes
1.359 +{
1.360 + print "archive started: ";
1.361 + print scalar(localtime());
1.362 + print "\n";
1.363 +
1.364 + system("tar -czf reports.tgz $indexes");
1.365 + system("rm -rf $indexes");
1.366 +
1.367 + print "archive finished: ";
1.368 + print scalar(localtime());
1.369 + print "\n";
1.370 +}
