layout/tools/reftest/remotereftest.py@deefc01c0e14
layout/tools/reftest/remotereftest.py
Thu, 15 Jan 2015 21:03:48 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Thu, 15 Jan 2015 21:03:48 +0100
- branch
- TOR_BUG_9701
- changeset 11
- deefc01c0e14
- permissions
- -rw-r--r--
Integrate friendly tips from Tor colleagues to make (or not) 4.5 alpha 3;
This includes removal of overloaded (but unused) methods, and addition of
a overlooked call to DataStruct::SetData(nsISupports, uint32_t, bool.)
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 import sys
6 import os
7 import time
8 import tempfile
9 import traceback
11 # We need to know our current directory so that we can serve our test files from it.
12 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
14 from runreftest import RefTest
15 from runreftest import ReftestOptions
16 from automation import Automation
17 import devicemanager
18 import droid
19 import moznetwork
20 from remoteautomation import RemoteAutomation, fennecLogcatFilters
22 class RemoteOptions(ReftestOptions):
23 def __init__(self, automation):
24 ReftestOptions.__init__(self, automation)
26 defaults = {}
27 defaults["logFile"] = "reftest.log"
28 # app, xrePath and utilityPath variables are set in main function
29 defaults["app"] = ""
30 defaults["xrePath"] = ""
31 defaults["utilityPath"] = ""
32 defaults["runTestsInParallel"] = False
34 self.add_option("--remote-app-path", action="store",
35 type = "string", dest = "remoteAppPath",
36 help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
37 defaults["remoteAppPath"] = None
39 self.add_option("--deviceIP", action="store",
40 type = "string", dest = "deviceIP",
41 help = "ip address of remote device to test")
42 defaults["deviceIP"] = None
44 self.add_option("--devicePort", action="store",
45 type = "string", dest = "devicePort",
46 help = "port of remote device to test")
47 defaults["devicePort"] = 20701
49 self.add_option("--remote-product-name", action="store",
50 type = "string", dest = "remoteProductName",
51 help = "Name of product to test - either fennec or firefox, defaults to fennec")
52 defaults["remoteProductName"] = "fennec"
54 self.add_option("--remote-webserver", action="store",
55 type = "string", dest = "remoteWebServer",
56 help = "IP Address of the webserver hosting the reftest content")
57 defaults["remoteWebServer"] = moznetwork.get_ip()
59 self.add_option("--http-port", action = "store",
60 type = "string", dest = "httpPort",
61 help = "port of the web server for http traffic")
62 defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
64 self.add_option("--ssl-port", action = "store",
65 type = "string", dest = "sslPort",
66 help = "Port for https traffic to the web server")
67 defaults["sslPort"] = automation.DEFAULT_SSL_PORT
69 self.add_option("--remote-logfile", action="store",
70 type = "string", dest = "remoteLogFile",
71 help = "Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
72 defaults["remoteLogFile"] = None
74 self.add_option("--enable-privilege", action="store_true", dest = "enablePrivilege",
75 help = "add webserver and port to the user.js file for remote script access and universalXPConnect")
76 defaults["enablePrivilege"] = False
78 self.add_option("--pidfile", action = "store",
79 type = "string", dest = "pidFile",
80 help = "name of the pidfile to generate")
81 defaults["pidFile"] = ""
83 self.add_option("--bootstrap", action="store_true", dest = "bootstrap",
84 help = "test with a bootstrap addon required for native Fennec")
85 defaults["bootstrap"] = False
87 self.add_option("--dm_trans", action="store",
88 type = "string", dest = "dm_trans",
89 help = "the transport to use to communicate with device: [adb|sut]; default=sut")
90 defaults["dm_trans"] = "sut"
92 self.add_option("--remoteTestRoot", action = "store",
93 type = "string", dest = "remoteTestRoot",
94 help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
95 defaults["remoteTestRoot"] = None
97 self.add_option("--httpd-path", action = "store",
98 type = "string", dest = "httpdPath",
99 help = "path to the httpd.js file")
100 defaults["httpdPath"] = None
102 defaults["localLogName"] = None
104 self.set_defaults(**defaults)
106 def verifyRemoteOptions(self, options):
107 if options.runTestsInParallel:
108 self.error("Cannot run parallel tests here")
110 # Ensure our defaults are set properly for everything we can infer
111 if not options.remoteTestRoot:
112 options.remoteTestRoot = self.automation._devicemanager.getDeviceRoot() + '/reftest'
113 options.remoteProfile = options.remoteTestRoot + "/profile"
115 # Verify that our remotewebserver is set properly
116 if (options.remoteWebServer == None or
117 options.remoteWebServer == '127.0.0.1'):
118 print "ERROR: Either you specified the loopback for the remote webserver or ",
119 print "your local IP cannot be detected. Please provide the local ip in --remote-webserver"
120 return None
122 # One of remoteAppPath (relative path to application) or the app (executable) must be
123 # set, but not both. If both are set, we destroy the user's selection for app
124 # so instead of silently destroying a user specificied setting, we error.
125 if (options.remoteAppPath and options.app):
126 print "ERROR: You cannot specify both the remoteAppPath and the app"
127 return None
128 elif (options.remoteAppPath):
129 options.app = options.remoteTestRoot + "/" + options.remoteAppPath
130 elif (options.app == None):
131 # Neither remoteAppPath nor app are set -- error
132 print "ERROR: You must specify either appPath or app"
133 return None
135 if (options.xrePath == None):
136 print "ERROR: You must specify the path to the controller xre directory"
137 return None
138 else:
139 # Ensure xrepath is a full path
140 options.xrePath = os.path.abspath(options.xrePath)
142 # Default to <deviceroot>/reftest/reftest.log
143 if (options.remoteLogFile == None):
144 options.remoteLogFile = 'reftest.log'
146 options.localLogName = options.remoteLogFile
147 options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
149 # Ensure that the options.logfile (which the base class uses) is set to
150 # the remote setting when running remote. Also, if the user set the
151 # log file name there, use that instead of reusing the remotelogfile as above.
152 if (options.logFile):
153 # If the user specified a local logfile name use that
154 options.localLogName = options.logFile
156 options.logFile = options.remoteLogFile
158 if (options.pidFile != ""):
159 f = open(options.pidFile, 'w')
160 f.write("%s" % os.getpid())
161 f.close()
163 # httpd-path is specified by standard makefile targets and may be specified
164 # on the command line to select a particular version of httpd.js. If not
165 # specified, try to select the one from hostutils.zip, as required in bug 882932.
166 if not options.httpdPath:
167 options.httpdPath = os.path.join(options.utilityPath, "components")
169 # TODO: Copied from main, but I think these are no longer used in a post xulrunner world
170 #options.xrePath = options.remoteTestRoot + self.automation._product + '/xulrunner'
171 #options.utilityPath = options.testRoot + self.automation._product + '/bin'
172 return options
174 class ReftestServer:
175 """ Web server used to serve Reftests, for closer fidelity to the real web.
176 It is virtually identical to the server used in mochitest and will only
177 be used for running reftests remotely.
178 Bug 581257 has been filed to refactor this wrapper around httpd.js into
179 it's own class and use it in both remote and non-remote testing. """
181 def __init__(self, automation, options, scriptDir):
182 self.automation = automation
183 self._utilityPath = options.utilityPath
184 self._xrePath = options.xrePath
185 self._profileDir = options.serverProfilePath
186 self.webServer = options.remoteWebServer
187 self.httpPort = options.httpPort
188 self.scriptDir = scriptDir
189 self.pidFile = options.pidFile
190 self._httpdPath = os.path.abspath(options.httpdPath)
191 self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort }
193 def start(self):
194 "Run the Refest server, returning the process ID of the server."
196 env = self.automation.environment(xrePath = self._xrePath)
197 env["XPCOM_DEBUG_BREAK"] = "warn"
198 if self.automation.IS_WIN32:
199 env["PATH"] = env["PATH"] + ";" + self._xrePath
201 args = ["-g", self._xrePath,
202 "-v", "170",
203 "-f", os.path.join(self._httpdPath, "httpd.js"),
204 "-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = '%(port)s'; const _SERVER_ADDR ='%(server)s';" %
205 {"profile" : self._profileDir.replace('\\', '\\\\'), "port" : self.httpPort, "server" : self.webServer },
206 "-f", os.path.join(self.scriptDir, "server.js")]
208 xpcshell = os.path.join(self._utilityPath,
209 "xpcshell" + self.automation.BIN_SUFFIX)
211 if not os.access(xpcshell, os.F_OK):
212 raise Exception('xpcshell not found at %s' % xpcshell)
213 if self.automation.elf_arm(xpcshell):
214 raise Exception('xpcshell at %s is an ARM binary; please use '
215 'the --utility-path argument to specify the path '
216 'to a desktop version.' % xpcshell)
218 self._process = self.automation.Process([xpcshell] + args, env = env)
219 pid = self._process.pid
220 if pid < 0:
221 print "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server."
222 return 2
223 self.automation.log.info("INFO | remotereftests.py | Server pid: %d", pid)
225 if (self.pidFile != ""):
226 f = open(self.pidFile + ".xpcshell.pid", 'w')
227 f.write("%s" % pid)
228 f.close()
230 def ensureReady(self, timeout):
231 assert timeout >= 0
233 aliveFile = os.path.join(self._profileDir, "server_alive.txt")
234 i = 0
235 while i < timeout:
236 if os.path.exists(aliveFile):
237 break
238 time.sleep(1)
239 i += 1
240 else:
241 print "TEST-UNEXPECTED-FAIL | remotereftests.py | Timed out while waiting for server startup."
242 self.stop()
243 return 1
245 def stop(self):
246 if hasattr(self, '_process'):
247 try:
248 c = urllib2.urlopen(self.shutdownURL)
249 c.read()
250 c.close()
252 rtncode = self._process.poll()
253 if (rtncode == None):
254 self._process.terminate()
255 except:
256 self._process.kill()
258 class RemoteReftest(RefTest):
259 remoteApp = ''
261 def __init__(self, automation, devicemanager, options, scriptDir):
262 RefTest.__init__(self, automation)
263 self._devicemanager = devicemanager
264 self.scriptDir = scriptDir
265 self.remoteApp = options.app
266 self.remoteProfile = options.remoteProfile
267 self.remoteTestRoot = options.remoteTestRoot
268 self.remoteLogFile = options.remoteLogFile
269 self.localLogName = options.localLogName
270 self.pidFile = options.pidFile
271 if self.automation.IS_DEBUG_BUILD:
272 self.SERVER_STARTUP_TIMEOUT = 180
273 else:
274 self.SERVER_STARTUP_TIMEOUT = 90
275 self.automation.deleteANRs()
277 def findPath(self, paths, filename = None):
278 for path in paths:
279 p = path
280 if filename:
281 p = os.path.join(p, filename)
282 if os.path.exists(self.getFullPath(p)):
283 return path
284 return None
286 def startWebServer(self, options):
287 """ Create the webserver on the host and start it up """
288 remoteXrePath = options.xrePath
289 remoteUtilityPath = options.utilityPath
290 localAutomation = Automation()
291 localAutomation.IS_WIN32 = False
292 localAutomation.IS_LINUX = False
293 localAutomation.IS_MAC = False
294 localAutomation.UNIXISH = False
295 hostos = sys.platform
296 if (hostos == 'mac' or hostos == 'darwin'):
297 localAutomation.IS_MAC = True
298 elif (hostos == 'linux' or hostos == 'linux2'):
299 localAutomation.IS_LINUX = True
300 localAutomation.UNIXISH = True
301 elif (hostos == 'win32' or hostos == 'win64'):
302 localAutomation.BIN_SUFFIX = ".exe"
303 localAutomation.IS_WIN32 = True
305 paths = [options.xrePath, localAutomation.DIST_BIN, self.automation._product, os.path.join('..', self.automation._product)]
306 options.xrePath = self.findPath(paths)
307 if options.xrePath == None:
308 print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
309 return 1
310 paths.append("bin")
311 paths.append(os.path.join("..", "bin"))
313 xpcshell = "xpcshell"
314 if (os.name == "nt"):
315 xpcshell += ".exe"
317 if (options.utilityPath):
318 paths.insert(0, options.utilityPath)
319 options.utilityPath = self.findPath(paths, xpcshell)
320 if options.utilityPath == None:
321 print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
322 return 1
324 options.serverProfilePath = tempfile.mkdtemp()
325 self.server = ReftestServer(localAutomation, options, self.scriptDir)
326 retVal = self.server.start()
327 if retVal:
328 return retVal
329 retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
330 if retVal:
331 return retVal
333 options.xrePath = remoteXrePath
334 options.utilityPath = remoteUtilityPath
335 return 0
337 def stopWebServer(self, options):
338 self.server.stop()
340 def createReftestProfile(self, options, reftestlist):
341 profile = RefTest.createReftestProfile(self, options, reftestlist, server=options.remoteWebServer)
342 profileDir = profile.profile
344 prefs = {}
345 prefs["browser.firstrun.show.localepicker"] = False
346 prefs["font.size.inflation.emPerLine"] = 0
347 prefs["font.size.inflation.minTwips"] = 0
348 prefs["reftest.remote"] = True
349 # Set a future policy version to avoid the telemetry prompt.
350 prefs["toolkit.telemetry.prompted"] = 999
351 prefs["toolkit.telemetry.notifiedOptOut"] = 999
352 prefs["reftest.uri"] = "%s" % reftestlist
353 prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
355 # Point the url-classifier to the local testing server for fast failures
356 prefs["browser.safebrowsing.gethashURL"] = "http://127.0.0.1:8888/safebrowsing-dummy/gethash"
357 prefs["browser.safebrowsing.updateURL"] = "http://127.0.0.1:8888/safebrowsing-dummy/update"
358 # Point update checks to the local testing server for fast failures
359 prefs["extensions.update.url"] = "http://127.0.0.1:8888/extensions-dummy/updateURL"
360 prefs["extensions.update.background.url"] = "http://127.0.0.1:8888/extensions-dummy/updateBackgroundURL"
361 prefs["extensions.blocklist.url"] = "http://127.0.0.1:8888/extensions-dummy/blocklistURL"
362 prefs["extensions.hotfix.url"] = "http://127.0.0.1:8888/extensions-dummy/hotfixURL"
363 # Turn off extension updates so they don't bother tests
364 prefs["extensions.update.enabled"] = False
365 # Make sure opening about:addons won't hit the network
366 prefs["extensions.webservice.discoverURL"] = "http://127.0.0.1:8888/extensions-dummy/discoveryURL"
367 # Make sure AddonRepository won't hit the network
368 prefs["extensions.getAddons.maxResults"] = 0
369 prefs["extensions.getAddons.get.url"] = "http://127.0.0.1:8888/extensions-dummy/repositoryGetURL"
370 prefs["extensions.getAddons.getWithPerformance.url"] = "http://127.0.0.1:8888/extensions-dummy/repositoryGetWithPerformanceURL"
371 prefs["extensions.getAddons.search.browseURL"] = "http://127.0.0.1:8888/extensions-dummy/repositoryBrowseURL"
372 prefs["extensions.getAddons.search.url"] = "http://127.0.0.1:8888/extensions-dummy/repositorySearchURL"
373 # Make sure that opening the plugins check page won't hit the network
374 prefs["plugins.update.url"] = "http://127.0.0.1:8888/plugins-dummy/updateCheckURL"
375 prefs["layout.css.devPixelsPerPx"] = "1.0"
377 # Disable skia-gl: see bug 907351
378 prefs["gfx.canvas.azure.accelerated"] = False
380 # Set the extra prefs.
381 profile.set_preferences(prefs)
383 try:
384 self._devicemanager.pushDir(profileDir, options.remoteProfile)
385 except devicemanager.DMError:
386 print "Automation Error: Failed to copy profiledir to device"
387 raise
389 return profile
391 def copyExtraFilesToProfile(self, options, profile):
392 profileDir = profile.profile
393 RefTest.copyExtraFilesToProfile(self, options, profile)
394 try:
395 self._devicemanager.pushDir(profileDir, options.remoteProfile)
396 except devicemanager.DMError:
397 print "Automation Error: Failed to copy extra files to device"
398 raise
400 def getManifestPath(self, path):
401 return path
403 def printDeviceInfo(self, printLogcat=False):
404 try:
405 if printLogcat:
406 logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
407 print ''.join(logcat)
408 print "Device info: %s" % self._devicemanager.getInfo()
409 print "Test root: %s" % self._devicemanager.getDeviceRoot()
410 except devicemanager.DMError:
411 print "WARNING: Error getting device information"
413 def cleanup(self, profileDir):
414 # Pull results back from device
415 if self.remoteLogFile and \
416 self._devicemanager.fileExists(self.remoteLogFile):
417 self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
418 else:
419 print "WARNING: Unable to retrieve log file (%s) from remote " \
420 "device" % self.remoteLogFile
421 self._devicemanager.removeDir(self.remoteProfile)
422 self._devicemanager.removeDir(self.remoteTestRoot)
423 RefTest.cleanup(self, profileDir)
424 if (self.pidFile != ""):
425 try:
426 os.remove(self.pidFile)
427 os.remove(self.pidFile + ".xpcshell.pid")
428 except:
429 print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
431 def main(args):
432 automation = RemoteAutomation(None)
433 parser = RemoteOptions(automation)
434 options, args = parser.parse_args()
436 if (options.deviceIP == None):
437 print "Error: you must provide a device IP to connect to via the --device option"
438 return 1
440 try:
441 if (options.dm_trans == "adb"):
442 if (options.deviceIP):
443 dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
444 else:
445 dm = droid.DroidADB(None, None, deviceRoot=options.remoteTestRoot)
446 else:
447 dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
448 except devicemanager.DMError:
449 print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state."
450 return 1
452 automation.setDeviceManager(dm)
454 if (options.remoteProductName != None):
455 automation.setProduct(options.remoteProductName)
457 # Set up the defaults and ensure options are set
458 options = parser.verifyRemoteOptions(options)
459 if (options == None):
460 print "ERROR: Invalid options specified, use --help for a list of valid options"
461 return 1
463 if not options.ignoreWindowSize:
464 parts = dm.getInfo('screen')['screen'][0].split()
465 width = int(parts[0].split(':')[1])
466 height = int(parts[1].split(':')[1])
467 if (width < 1050 or height < 1050):
468 print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height)
469 return 1
471 automation.setAppName(options.app)
472 automation.setRemoteProfile(options.remoteProfile)
473 automation.setRemoteLog(options.remoteLogFile)
474 reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
475 options = parser.verifyCommonOptions(options, reftest)
477 # Hack in a symbolic link for jsreftest
478 os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
480 # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
481 manifest = args[0]
482 if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])):
483 manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0]
484 elif os.path.exists(args[0]):
485 manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/')
486 manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath
487 else:
488 print "ERROR: Could not find test manifest '%s'" % manifest
489 return 1
491 # Start the webserver
492 retVal = reftest.startWebServer(options)
493 if retVal:
494 return retVal
496 procName = options.app.split('/')[-1]
497 if (dm.processExist(procName)):
498 dm.killProcess(procName)
500 reftest.printDeviceInfo()
502 #an example manifest name to use on the cli
503 # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
504 retVal = 0
505 try:
506 cmdlineArgs = ["-reftest", manifest]
507 if options.bootstrap:
508 cmdlineArgs = []
509 dm.recordLogcat()
510 retVal = reftest.runTests(manifest, options, cmdlineArgs)
511 except:
512 print "Automation Error: Exception caught while running tests"
513 traceback.print_exc()
514 retVal = 1
516 reftest.stopWebServer(options)
518 reftest.printDeviceInfo(printLogcat=True)
520 return retVal
522 if __name__ == "__main__":
523 sys.exit(main(sys.argv[1:]))
