1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/mochitest/runtestsvmware.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,397 @@
1.4 +#
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +
1.9 +import sys
1.10 +import os
1.11 +import re
1.12 +import types
1.13 +from optparse import OptionValueError
1.14 +from subprocess import PIPE
1.15 +from time import sleep
1.16 +from tempfile import mkstemp
1.17 +
1.18 +sys.path.insert(0, os.path.abspath(os.path.realpath(
1.19 + os.path.dirname(sys.argv[0]))))
1.20 +
1.21 +from automation import Automation
1.22 +from runtests import Mochitest, MochitestOptions
1.23 +
1.24 +class VMwareOptions(MochitestOptions):
1.25 + def __init__(self, automation, mochitest, **kwargs):
1.26 + defaults = {}
1.27 + self._automation = automation or Automation()
1.28 + MochitestOptions.__init__(self, mochitest.SCRIPT_DIRECTORY)
1.29 +
1.30 + def checkPathCallback(option, opt_str, value, parser):
1.31 + path = mochitest.getFullPath(value)
1.32 + if not os.path.exists(path):
1.33 + raise OptionValueError("Path %s does not exist for %s option"
1.34 + % (path, opt_str))
1.35 + setattr(parser.values, option.dest, path)
1.36 +
1.37 + self.add_option("--with-vmware-vm",
1.38 + action = "callback", type = "string", dest = "vmx",
1.39 + callback = checkPathCallback,
1.40 + help = "launches the given VM and runs mochitests inside")
1.41 + defaults["vmx"] = None
1.42 +
1.43 + self.add_option("--with-vmrun-executable",
1.44 + action = "callback", type = "string", dest = "vmrun",
1.45 + callback = checkPathCallback,
1.46 + help = "specifies the vmrun.exe to use for VMware control")
1.47 + defaults["vmrun"] = None
1.48 +
1.49 + self.add_option("--shutdown-vm-when-done",
1.50 + action = "store_true", dest = "shutdownVM",
1.51 + help = "shuts down the VM when mochitests complete")
1.52 + defaults["shutdownVM"] = False
1.53 +
1.54 + self.add_option("--repeat-until-failure",
1.55 + action = "store_true", dest = "repeatUntilFailure",
1.56 + help = "Runs tests continuously until failure")
1.57 + defaults["repeatUntilFailure"] = False
1.58 +
1.59 + self.set_defaults(**defaults)
1.60 +
1.61 +class VMwareMochitest(Mochitest):
1.62 + _pathFixRegEx = re.compile(r'^[cC](\:[\\\/]+)')
1.63 +
1.64 + def convertHostPathsToGuestPaths(self, string):
1.65 + """ converts a path on the host machine to a path on the guest machine """
1.66 + # XXXbent Lame!
1.67 + return self._pathFixRegEx.sub(r'z\1', string)
1.68 +
1.69 + def prepareGuestArguments(self, parser, options):
1.70 + """ returns an array of command line arguments needed to replicate the
1.71 + current set of options in the guest """
1.72 + args = []
1.73 + for key in options.__dict__.keys():
1.74 + # Don't send these args to the vm test runner!
1.75 + if key == "vmrun" or key == "vmx" or key == "repeatUntilFailure":
1.76 + continue
1.77 +
1.78 + value = options.__dict__[key]
1.79 + valueType = type(value)
1.80 +
1.81 + # Find the option in the parser's list.
1.82 + option = None
1.83 + for index in range(len(parser.option_list)):
1.84 + if str(parser.option_list[index].dest) == key:
1.85 + option = parser.option_list[index]
1.86 + break
1.87 + if not option:
1.88 + continue
1.89 +
1.90 + # No need to pass args on the command line if they're just going to set
1.91 + # default values. The exception is list values... For some reason the
1.92 + # option parser modifies the defaults as well as the values when using the
1.93 + # "append" action.
1.94 + if value == parser.defaults[option.dest]:
1.95 + if valueType == types.StringType and \
1.96 + value == self.convertHostPathsToGuestPaths(value):
1.97 + continue
1.98 + if valueType != types.ListType:
1.99 + continue
1.100 +
1.101 + def getArgString(arg, option):
1.102 + if option.action == "store_true" or option.action == "store_false":
1.103 + return str(option)
1.104 + return "%s=%s" % (str(option),
1.105 + self.convertHostPathsToGuestPaths(str(arg)))
1.106 +
1.107 + if valueType == types.ListType:
1.108 + # Expand lists into separate args.
1.109 + for item in value:
1.110 + args.append(getArgString(item, option))
1.111 + else:
1.112 + args.append(getArgString(value, option))
1.113 +
1.114 + return tuple(args)
1.115 +
1.116 + def launchVM(self, options):
1.117 + """ launches the VM and enables shared folders """
1.118 + # Launch VM first.
1.119 + self.automation.log.info("INFO | runtests.py | Launching the VM.")
1.120 + (result, stdout) = self.runVMCommand(self.vmrunargs + ("start", self.vmx))
1.121 + if result:
1.122 + return result
1.123 +
1.124 + # Make sure that shared folders are enabled.
1.125 + self.automation.log.info("INFO | runtests.py | Enabling shared folders in "
1.126 + "the VM.")
1.127 + (result, stdout) = self.runVMCommand(self.vmrunargs + \
1.128 + ("enableSharedFolders", self.vmx))
1.129 + if result:
1.130 + return result
1.131 +
1.132 + def shutdownVM(self):
1.133 + """ shuts down the VM """
1.134 + self.automation.log.info("INFO | runtests.py | Shutting down the VM.")
1.135 + command = self.vmrunargs + ("runProgramInGuest", self.vmx,
1.136 + "c:\\windows\\system32\\shutdown.exe", "/s", "/t", "1")
1.137 + (result, stdout) = self.runVMCommand(command)
1.138 + return result
1.139 +
1.140 + def runVMCommand(self, command, expectedErrors=[], silent=False):
1.141 + """ runs a command in the VM using the vmrun.exe helper """
1.142 + commandString = ""
1.143 + for part in command:
1.144 + commandString += str(part) + " "
1.145 + if not silent:
1.146 + self.automation.log.info("INFO | runtests.py | Running command: %s"
1.147 + % commandString)
1.148 +
1.149 + commonErrors = ["Error: Invalid user name or password for the guest OS",
1.150 + "Unable to connect to host."]
1.151 + expectedErrors.extend(commonErrors)
1.152 +
1.153 + # VMware can't run commands until the VM has fully loaded so keep running
1.154 + # this command in a loop until it succeeds or we try 100 times.
1.155 + errorString = ""
1.156 + for i in range(100):
1.157 + process = Automation.Process(command, stdout=PIPE)
1.158 + result = process.wait()
1.159 + if result == 0:
1.160 + break
1.161 +
1.162 + for line in process.stdout.readlines():
1.163 + line = line.strip()
1.164 + if not line:
1.165 + continue
1.166 + errorString = line
1.167 + break
1.168 +
1.169 + expected = False
1.170 + for error in expectedErrors:
1.171 + if errorString.startswith(error):
1.172 + expected = True
1.173 +
1.174 + if not expected:
1.175 + self.automation.log.warning("WARNING | runtests.py | Command \"%s\" "
1.176 + "failed with result %d, : %s"
1.177 + % (commandString, result, errorString))
1.178 + break
1.179 +
1.180 + if not silent:
1.181 + self.automation.log.info("INFO | runtests.py | Running command again.")
1.182 +
1.183 + return (result, process.stdout.readlines())
1.184 +
1.185 + def monitorVMExecution(self, appname, logfilepath):
1.186 + """ monitors test execution in the VM. Waits for the test process to start,
1.187 + then watches the log file for test failures and checks the status of the
1.188 + process to catch crashes. Returns True if mochitests ran successfully.
1.189 + """
1.190 + success = True
1.191 +
1.192 + self.automation.log.info("INFO | runtests.py | Waiting for test process to "
1.193 + "start.")
1.194 +
1.195 + listProcessesCommand = self.vmrunargs + ("listProcessesInGuest", self.vmx)
1.196 + expectedErrors = [ "Error: The virtual machine is not powered on" ]
1.197 +
1.198 + running = False
1.199 + for i in range(100):
1.200 + (result, stdout) = self.runVMCommand(listProcessesCommand, expectedErrors,
1.201 + silent=True)
1.202 + if result:
1.203 + self.automation.log.warning("WARNING | runtests.py | Failed to get "
1.204 + "list of processes in VM!")
1.205 + return False
1.206 + for line in stdout:
1.207 + line = line.strip()
1.208 + if line.find(appname) != -1:
1.209 + running = True
1.210 + break
1.211 + if running:
1.212 + break
1.213 + sleep(1)
1.214 +
1.215 + self.automation.log.info("INFO | runtests.py | Found test process, "
1.216 + "monitoring log.")
1.217 +
1.218 + completed = False
1.219 + nextLine = 0
1.220 + while running:
1.221 + log = open(logfilepath, "rb")
1.222 + lines = log.readlines()
1.223 + if len(lines) > nextLine:
1.224 + linesToPrint = lines[nextLine:]
1.225 + for line in linesToPrint:
1.226 + line = line.strip()
1.227 + if line.find("INFO SimpleTest FINISHED") != -1:
1.228 + completed = True
1.229 + continue
1.230 + if line.find("ERROR TEST-UNEXPECTED-FAIL") != -1:
1.231 + self.automation.log.info("INFO | runtests.py | Detected test "
1.232 + "failure: \"%s\"" % line)
1.233 + success = False
1.234 + nextLine = len(lines)
1.235 + log.close()
1.236 +
1.237 + (result, stdout) = self.runVMCommand(listProcessesCommand, expectedErrors,
1.238 + silent=True)
1.239 + if result:
1.240 + self.automation.log.warning("WARNING | runtests.py | Failed to get "
1.241 + "list of processes in VM!")
1.242 + return False
1.243 +
1.244 + stillRunning = False
1.245 + for line in stdout:
1.246 + line = line.strip()
1.247 + if line.find(appname) != -1:
1.248 + stillRunning = True
1.249 + break
1.250 + if stillRunning:
1.251 + sleep(5)
1.252 + else:
1.253 + if not completed:
1.254 + self.automation.log.info("INFO | runtests.py | Test process exited "
1.255 + "without finishing tests, maybe crashed.")
1.256 + success = False
1.257 + running = stillRunning
1.258 +
1.259 + return success
1.260 +
1.261 + def getCurentSnapshotList(self):
1.262 + """ gets a list of snapshots from the VM """
1.263 + (result, stdout) = self.runVMCommand(self.vmrunargs + ("listSnapshots",
1.264 + self.vmx))
1.265 + snapshots = []
1.266 + if result != 0:
1.267 + self.automation.log.warning("WARNING | runtests.py | Failed to get list "
1.268 + "of snapshots in VM!")
1.269 + return snapshots
1.270 + for line in stdout:
1.271 + if line.startswith("Total snapshots:"):
1.272 + continue
1.273 + snapshots.append(line.strip())
1.274 + return snapshots
1.275 +
1.276 + def runTests(self, parser, options):
1.277 + """ runs mochitests in the VM """
1.278 + # Base args that must always be passed to vmrun.
1.279 + self.vmrunargs = (options.vmrun, "-T", "ws", "-gu", "Replay", "-gp",
1.280 + "mozilla")
1.281 + self.vmrun = options.vmrun
1.282 + self.vmx = options.vmx
1.283 +
1.284 + result = self.launchVM(options)
1.285 + if result:
1.286 + return result
1.287 +
1.288 + if options.vmwareRecording:
1.289 + snapshots = self.getCurentSnapshotList()
1.290 +
1.291 + def innerRun():
1.292 + """ subset of the function that must run every time if we're running until
1.293 + failure """
1.294 + # Make a new shared file for the log file.
1.295 + (logfile, logfilepath) = mkstemp(suffix=".log")
1.296 + os.close(logfile)
1.297 + # Get args to pass to VM process. Make sure we autorun and autoclose.
1.298 + options.autorun = True
1.299 + options.closeWhenDone = True
1.300 + options.logFile = logfilepath
1.301 + self.automation.log.info("INFO | runtests.py | Determining guest "
1.302 + "arguments.")
1.303 + runtestsArgs = self.prepareGuestArguments(parser, options)
1.304 + runtestsPath = self.convertHostPathsToGuestPaths(self.SCRIPT_DIRECTORY)
1.305 + runtestsPath = os.path.join(runtestsPath, "runtests.py")
1.306 + runtestsCommand = self.vmrunargs + ("runProgramInGuest", self.vmx,
1.307 + "-activeWindow", "-interactive", "-noWait",
1.308 + "c:\\mozilla-build\\python25\\python.exe",
1.309 + runtestsPath) + runtestsArgs
1.310 + expectedErrors = [ "Unable to connect to host.",
1.311 + "Error: The virtual machine is not powered on" ]
1.312 + self.automation.log.info("INFO | runtests.py | Launching guest test "
1.313 + "runner.")
1.314 + (result, stdout) = self.runVMCommand(runtestsCommand, expectedErrors)
1.315 + if result:
1.316 + return (result, False)
1.317 + self.automation.log.info("INFO | runtests.py | Waiting for guest test "
1.318 + "runner to complete.")
1.319 + mochitestsSucceeded = self.monitorVMExecution(
1.320 + os.path.basename(options.app), logfilepath)
1.321 + if mochitestsSucceeded:
1.322 + self.automation.log.info("INFO | runtests.py | Guest tests passed!")
1.323 + else:
1.324 + self.automation.log.info("INFO | runtests.py | Guest tests failed.")
1.325 + if mochitestsSucceeded and options.vmwareRecording:
1.326 + newSnapshots = self.getCurentSnapshotList()
1.327 + if len(newSnapshots) > len(snapshots):
1.328 + self.automation.log.info("INFO | runtests.py | Removing last "
1.329 + "recording.")
1.330 + (result, stdout) = self.runVMCommand(self.vmrunargs + \
1.331 + ("deleteSnapshot", self.vmx,
1.332 + newSnapshots[-1]))
1.333 + self.automation.log.info("INFO | runtests.py | Removing guest log file.")
1.334 + for i in range(30):
1.335 + try:
1.336 + os.remove(logfilepath)
1.337 + break
1.338 + except:
1.339 + sleep(1)
1.340 + self.automation.log.warning("WARNING | runtests.py | Couldn't remove "
1.341 + "guest log file, trying again.")
1.342 + return (result, mochitestsSucceeded)
1.343 +
1.344 + if options.repeatUntilFailure:
1.345 + succeeded = True
1.346 + result = 0
1.347 + count = 1
1.348 + while result == 0 and succeeded:
1.349 + self.automation.log.info("INFO | runtests.py | Beginning mochitest run "
1.350 + "(%d)." % count)
1.351 + count += 1
1.352 + (result, succeeded) = innerRun()
1.353 + else:
1.354 + self.automation.log.info("INFO | runtests.py | Beginning mochitest run.")
1.355 + (result, succeeded) = innerRun()
1.356 +
1.357 + if not succeeded and options.vmwareRecording:
1.358 + newSnapshots = self.getCurentSnapshotList()
1.359 + if len(newSnapshots) > len(snapshots):
1.360 + self.automation.log.info("INFO | runtests.py | Failed recording saved "
1.361 + "as '%s'." % newSnapshots[-1])
1.362 +
1.363 + if result:
1.364 + return result
1.365 +
1.366 + if options.shutdownVM:
1.367 + result = self.shutdownVM()
1.368 + if result:
1.369 + return result
1.370 +
1.371 + return 0
1.372 +
1.373 +def main():
1.374 + automation = Automation()
1.375 + mochitest = VMwareMochitest(automation)
1.376 +
1.377 + parser = VMwareOptions(automation, mochitest)
1.378 + options, args = parser.parse_args()
1.379 + options = parser.verifyOptions(options, mochitest)
1.380 + if (options == None):
1.381 + sys.exit(1)
1.382 +
1.383 + if options.vmx is None:
1.384 + parser.error("A virtual machine must be specified with " +
1.385 + "--with-vmware-vm")
1.386 +
1.387 + if options.vmrun is None:
1.388 + options.vmrun = os.path.join("c:\\", "Program Files", "VMware",
1.389 + "VMware VIX", "vmrun.exe")
1.390 + if not os.path.exists(options.vmrun):
1.391 + options.vmrun = os.path.join("c:\\", "Program Files (x86)", "VMware",
1.392 + "VMware VIX", "vmrun.exe")
1.393 + if not os.path.exists(options.vmrun):
1.394 + parser.error("Could not locate vmrun.exe, use --with-vmrun-executable" +
1.395 + " to identify its location")
1.396 +
1.397 + sys.exit(mochitest.runTests(parser, options))
1.398 +
1.399 +if __name__ == "__main__":
1.400 + main()
