The Tor Browser / file comparison

comparison: layout/tools/reftest/remotereftest.py

layout/tools/reftest/remotereftest.py

branch
TOR_BUG_3246
changeset 6
8bccb770b82d
equal deleted inserted replaced
-1:000000000000 0:8032336738d3
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import sys
6 import os
7 import time
8 import tempfile
9 import traceback
10
11 # We need to know our current directory so that we can serve our test files from it.
12 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
13
14 from runreftest import RefTest
15 from runreftest import ReftestOptions
16 from automation import Automation
17 import devicemanager
18 import droid
19 import moznetwork
20 from remoteautomation import RemoteAutomation, fennecLogcatFilters
21
22 class RemoteOptions(ReftestOptions):
23 def __init__(self, automation):
24 ReftestOptions.__init__(self, automation)
25
26 defaults = {}
27 defaults["logFile"] = "reftest.log"
28 # app, xrePath and utilityPath variables are set in main function
29 defaults["app"] = ""
30 defaults["xrePath"] = ""
31 defaults["utilityPath"] = ""
32 defaults["runTestsInParallel"] = False
33
34 self.add_option("--remote-app-path", action="store",
35 type = "string", dest = "remoteAppPath",
36 help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
37 defaults["remoteAppPath"] = None
38
39 self.add_option("--deviceIP", action="store",
40 type = "string", dest = "deviceIP",
41 help = "ip address of remote device to test")
42 defaults["deviceIP"] = None
43
44 self.add_option("--devicePort", action="store",
45 type = "string", dest = "devicePort",
46 help = "port of remote device to test")
47 defaults["devicePort"] = 20701
48
49 self.add_option("--remote-product-name", action="store",
50 type = "string", dest = "remoteProductName",
51 help = "Name of product to test - either fennec or firefox, defaults to fennec")
52 defaults["remoteProductName"] = "fennec"
53
54 self.add_option("--remote-webserver", action="store",
55 type = "string", dest = "remoteWebServer",
56 help = "IP Address of the webserver hosting the reftest content")
57 defaults["remoteWebServer"] = moznetwork.get_ip()
58
59 self.add_option("--http-port", action = "store",
60 type = "string", dest = "httpPort",
61 help = "port of the web server for http traffic")
62 defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
63
64 self.add_option("--ssl-port", action = "store",
65 type = "string", dest = "sslPort",
66 help = "Port for https traffic to the web server")
67 defaults["sslPort"] = automation.DEFAULT_SSL_PORT
68
69 self.add_option("--remote-logfile", action="store",
70 type = "string", dest = "remoteLogFile",
71 help = "Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
72 defaults["remoteLogFile"] = None
73
74 self.add_option("--enable-privilege", action="store_true", dest = "enablePrivilege",
75 help = "add webserver and port to the user.js file for remote script access and universalXPConnect")
76 defaults["enablePrivilege"] = False
77
78 self.add_option("--pidfile", action = "store",
79 type = "string", dest = "pidFile",
80 help = "name of the pidfile to generate")
81 defaults["pidFile"] = ""
82
83 self.add_option("--bootstrap", action="store_true", dest = "bootstrap",
84 help = "test with a bootstrap addon required for native Fennec")
85 defaults["bootstrap"] = False
86
87 self.add_option("--dm_trans", action="store",
88 type = "string", dest = "dm_trans",
89 help = "the transport to use to communicate with device: [adb|sut]; default=sut")
90 defaults["dm_trans"] = "sut"
91
92 self.add_option("--remoteTestRoot", action = "store",
93 type = "string", dest = "remoteTestRoot",
94 help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
95 defaults["remoteTestRoot"] = None
96
97 self.add_option("--httpd-path", action = "store",
98 type = "string", dest = "httpdPath",
99 help = "path to the httpd.js file")
100 defaults["httpdPath"] = None
101
102 defaults["localLogName"] = None
103
104 self.set_defaults(**defaults)
105
106 def verifyRemoteOptions(self, options):
107 if options.runTestsInParallel:
108 self.error("Cannot run parallel tests here")
109
110 # Ensure our defaults are set properly for everything we can infer
111 if not options.remoteTestRoot:
112 options.remoteTestRoot = self.automation._devicemanager.getDeviceRoot() + '/reftest'
113 options.remoteProfile = options.remoteTestRoot + "/profile"
114
115 # Verify that our remotewebserver is set properly
116 if (options.remoteWebServer == None or
117 options.remoteWebServer == '127.0.0.1'):
118 print "ERROR: Either you specified the loopback for the remote webserver or ",
119 print "your local IP cannot be detected. Please provide the local ip in --remote-webserver"
120 return None
121
122 # One of remoteAppPath (relative path to application) or the app (executable) must be
123 # set, but not both. If both are set, we destroy the user's selection for app
124 # so instead of silently destroying a user specificied setting, we error.
125 if (options.remoteAppPath and options.app):
126 print "ERROR: You cannot specify both the remoteAppPath and the app"
127 return None
128 elif (options.remoteAppPath):
129 options.app = options.remoteTestRoot + "/" + options.remoteAppPath
130 elif (options.app == None):
131 # Neither remoteAppPath nor app are set -- error
132 print "ERROR: You must specify either appPath or app"
133 return None
134
135 if (options.xrePath == None):
136 print "ERROR: You must specify the path to the controller xre directory"
137 return None
138 else:
139 # Ensure xrepath is a full path
140 options.xrePath = os.path.abspath(options.xrePath)
141
142 # Default to <deviceroot>/reftest/reftest.log
143 if (options.remoteLogFile == None):
144 options.remoteLogFile = 'reftest.log'
145
146 options.localLogName = options.remoteLogFile
147 options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
148
149 # Ensure that the options.logfile (which the base class uses) is set to
150 # the remote setting when running remote. Also, if the user set the
151 # log file name there, use that instead of reusing the remotelogfile as above.
152 if (options.logFile):
153 # If the user specified a local logfile name use that
154 options.localLogName = options.logFile
155
156 options.logFile = options.remoteLogFile
157
158 if (options.pidFile != ""):
159 f = open(options.pidFile, 'w')
160 f.write("%s" % os.getpid())
161 f.close()
162
163 # httpd-path is specified by standard makefile targets and may be specified
164 # on the command line to select a particular version of httpd.js. If not
165 # specified, try to select the one from hostutils.zip, as required in bug 882932.
166 if not options.httpdPath:
167 options.httpdPath = os.path.join(options.utilityPath, "components")
168
169 # TODO: Copied from main, but I think these are no longer used in a post xulrunner world
170 #options.xrePath = options.remoteTestRoot + self.automation._product + '/xulrunner'
171 #options.utilityPath = options.testRoot + self.automation._product + '/bin'
172 return options
173
174 class ReftestServer:
175 """ Web server used to serve Reftests, for closer fidelity to the real web.
176 It is virtually identical to the server used in mochitest and will only
177 be used for running reftests remotely.
178 Bug 581257 has been filed to refactor this wrapper around httpd.js into
179 it's own class and use it in both remote and non-remote testing. """
180
181 def __init__(self, automation, options, scriptDir):
182 self.automation = automation
183 self._utilityPath = options.utilityPath
184 self._xrePath = options.xrePath
185 self._profileDir = options.serverProfilePath
186 self.webServer = options.remoteWebServer
187 self.httpPort = options.httpPort
188 self.scriptDir = scriptDir
189 self.pidFile = options.pidFile
190 self._httpdPath = os.path.abspath(options.httpdPath)
191 self.shutdownURL = "http://%(server)s:%(port)s/server/shutdown" % { "server" : self.webServer, "port" : self.httpPort }
192
193 def start(self):
194 "Run the Refest server, returning the process ID of the server."
195
196 env = self.automation.environment(xrePath = self._xrePath)
197 env["XPCOM_DEBUG_BREAK"] = "warn"
198 if self.automation.IS_WIN32:
199 env["PATH"] = env["PATH"] + ";" + self._xrePath
200
201 args = ["-g", self._xrePath,
202 "-v", "170",
203 "-f", os.path.join(self._httpdPath, "httpd.js"),
204 "-e", "const _PROFILE_PATH = '%(profile)s';const _SERVER_PORT = '%(port)s'; const _SERVER_ADDR ='%(server)s';" %
205 {"profile" : self._profileDir.replace('\\', '\\\\'), "port" : self.httpPort, "server" : self.webServer },
206 "-f", os.path.join(self.scriptDir, "server.js")]
207
208 xpcshell = os.path.join(self._utilityPath,
209 "xpcshell" + self.automation.BIN_SUFFIX)
210
211 if not os.access(xpcshell, os.F_OK):
212 raise Exception('xpcshell not found at %s' % xpcshell)
213 if self.automation.elf_arm(xpcshell):
214 raise Exception('xpcshell at %s is an ARM binary; please use '
215 'the --utility-path argument to specify the path '
216 'to a desktop version.' % xpcshell)
217
218 self._process = self.automation.Process([xpcshell] + args, env = env)
219 pid = self._process.pid
220 if pid < 0:
221 print "TEST-UNEXPECTED-FAIL | remotereftests.py | Error starting server."
222 return 2
223 self.automation.log.info("INFO | remotereftests.py | Server pid: %d", pid)
224
225 if (self.pidFile != ""):
226 f = open(self.pidFile + ".xpcshell.pid", 'w')
227 f.write("%s" % pid)
228 f.close()
229
230 def ensureReady(self, timeout):
231 assert timeout >= 0
232
233 aliveFile = os.path.join(self._profileDir, "server_alive.txt")
234 i = 0
235 while i < timeout:
236 if os.path.exists(aliveFile):
237 break
238 time.sleep(1)
239 i += 1
240 else:
241 print "TEST-UNEXPECTED-FAIL | remotereftests.py | Timed out while waiting for server startup."
242 self.stop()
243 return 1
244
245 def stop(self):
246 if hasattr(self, '_process'):
247 try:
248 c = urllib2.urlopen(self.shutdownURL)
249 c.read()
250 c.close()
251
252 rtncode = self._process.poll()
253 if (rtncode == None):
254 self._process.terminate()
255 except:
256 self._process.kill()
257
258 class RemoteReftest(RefTest):
259 remoteApp = ''
260
261 def __init__(self, automation, devicemanager, options, scriptDir):
262 RefTest.__init__(self, automation)
263 self._devicemanager = devicemanager
264 self.scriptDir = scriptDir
265 self.remoteApp = options.app
266 self.remoteProfile = options.remoteProfile
267 self.remoteTestRoot = options.remoteTestRoot
268 self.remoteLogFile = options.remoteLogFile
269 self.localLogName = options.localLogName
270 self.pidFile = options.pidFile
271 if self.automation.IS_DEBUG_BUILD:
272 self.SERVER_STARTUP_TIMEOUT = 180
273 else:
274 self.SERVER_STARTUP_TIMEOUT = 90
275 self.automation.deleteANRs()
276
277 def findPath(self, paths, filename = None):
278 for path in paths:
279 p = path
280 if filename:
281 p = os.path.join(p, filename)
282 if os.path.exists(self.getFullPath(p)):
283 return path
284 return None
285
286 def startWebServer(self, options):
287 """ Create the webserver on the host and start it up """
288 remoteXrePath = options.xrePath
289 remoteUtilityPath = options.utilityPath
290 localAutomation = Automation()
291 localAutomation.IS_WIN32 = False
292 localAutomation.IS_LINUX = False
293 localAutomation.IS_MAC = False
294 localAutomation.UNIXISH = False
295 hostos = sys.platform
296 if (hostos == 'mac' or hostos == 'darwin'):
297 localAutomation.IS_MAC = True
298 elif (hostos == 'linux' or hostos == 'linux2'):
299 localAutomation.IS_LINUX = True
300 localAutomation.UNIXISH = True
301 elif (hostos == 'win32' or hostos == 'win64'):
302 localAutomation.BIN_SUFFIX = ".exe"
303 localAutomation.IS_WIN32 = True
304
305 paths = [options.xrePath, localAutomation.DIST_BIN, self.automation._product, os.path.join('..', self.automation._product)]
306 options.xrePath = self.findPath(paths)
307 if options.xrePath == None:
308 print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
309 return 1
310 paths.append("bin")
311 paths.append(os.path.join("..", "bin"))
312
313 xpcshell = "xpcshell"
314 if (os.name == "nt"):
315 xpcshell += ".exe"
316
317 if (options.utilityPath):
318 paths.insert(0, options.utilityPath)
319 options.utilityPath = self.findPath(paths, xpcshell)
320 if options.utilityPath == None:
321 print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
322 return 1
323
324 options.serverProfilePath = tempfile.mkdtemp()
325 self.server = ReftestServer(localAutomation, options, self.scriptDir)
326 retVal = self.server.start()
327 if retVal:
328 return retVal
329 retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
330 if retVal:
331 return retVal
332
333 options.xrePath = remoteXrePath
334 options.utilityPath = remoteUtilityPath
335 return 0
336
337 def stopWebServer(self, options):
338 self.server.stop()
339
340 def createReftestProfile(self, options, reftestlist):
341 profile = RefTest.createReftestProfile(self, options, reftestlist, server=options.remoteWebServer)
342 profileDir = profile.profile
343
344 prefs = {}
345 prefs["browser.firstrun.show.localepicker"] = False
346 prefs["font.size.inflation.emPerLine"] = 0
347 prefs["font.size.inflation.minTwips"] = 0
348 prefs["reftest.remote"] = True
349 # Set a future policy version to avoid the telemetry prompt.
350 prefs["toolkit.telemetry.prompted"] = 999
351 prefs["toolkit.telemetry.notifiedOptOut"] = 999
352 prefs["reftest.uri"] = "%s" % reftestlist
353 prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
354
355 # Point the url-classifier to the local testing server for fast failures
356 prefs["browser.safebrowsing.gethashURL"] = "http://127.0.0.1:8888/safebrowsing-dummy/gethash"
357 prefs["browser.safebrowsing.updateURL"] = "http://127.0.0.1:8888/safebrowsing-dummy/update"
358 # Point update checks to the local testing server for fast failures
359 prefs["extensions.update.url"] = "http://127.0.0.1:8888/extensions-dummy/updateURL"
360 prefs["extensions.update.background.url"] = "http://127.0.0.1:8888/extensions-dummy/updateBackgroundURL"
361 prefs["extensions.blocklist.url"] = "http://127.0.0.1:8888/extensions-dummy/blocklistURL"
362 prefs["extensions.hotfix.url"] = "http://127.0.0.1:8888/extensions-dummy/hotfixURL"
363 # Turn off extension updates so they don't bother tests
364 prefs["extensions.update.enabled"] = False
365 # Make sure opening about:addons won't hit the network
366 prefs["extensions.webservice.discoverURL"] = "http://127.0.0.1:8888/extensions-dummy/discoveryURL"
367 # Make sure AddonRepository won't hit the network
368 prefs["extensions.getAddons.maxResults"] = 0
369 prefs["extensions.getAddons.get.url"] = "http://127.0.0.1:8888/extensions-dummy/repositoryGetURL"
370 prefs["extensions.getAddons.getWithPerformance.url"] = "http://127.0.0.1:8888/extensions-dummy/repositoryGetWithPerformanceURL"
371 prefs["extensions.getAddons.search.browseURL"] = "http://127.0.0.1:8888/extensions-dummy/repositoryBrowseURL"
372 prefs["extensions.getAddons.search.url"] = "http://127.0.0.1:8888/extensions-dummy/repositorySearchURL"
373 # Make sure that opening the plugins check page won't hit the network
374 prefs["plugins.update.url"] = "http://127.0.0.1:8888/plugins-dummy/updateCheckURL"
375 prefs["layout.css.devPixelsPerPx"] = "1.0"
376
377 # Disable skia-gl: see bug 907351
378 prefs["gfx.canvas.azure.accelerated"] = False
379
380 # Set the extra prefs.
381 profile.set_preferences(prefs)
382
383 try:
384 self._devicemanager.pushDir(profileDir, options.remoteProfile)
385 except devicemanager.DMError:
386 print "Automation Error: Failed to copy profiledir to device"
387 raise
388
389 return profile
390
391 def copyExtraFilesToProfile(self, options, profile):
392 profileDir = profile.profile
393 RefTest.copyExtraFilesToProfile(self, options, profile)
394 try:
395 self._devicemanager.pushDir(profileDir, options.remoteProfile)
396 except devicemanager.DMError:
397 print "Automation Error: Failed to copy extra files to device"
398 raise
399
400 def getManifestPath(self, path):
401 return path
402
403 def printDeviceInfo(self, printLogcat=False):
404 try:
405 if printLogcat:
406 logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
407 print ''.join(logcat)
408 print "Device info: %s" % self._devicemanager.getInfo()
409 print "Test root: %s" % self._devicemanager.getDeviceRoot()
410 except devicemanager.DMError:
411 print "WARNING: Error getting device information"
412
413 def cleanup(self, profileDir):
414 # Pull results back from device
415 if self.remoteLogFile and \
416 self._devicemanager.fileExists(self.remoteLogFile):
417 self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
418 else:
419 print "WARNING: Unable to retrieve log file (%s) from remote " \
420 "device" % self.remoteLogFile
421 self._devicemanager.removeDir(self.remoteProfile)
422 self._devicemanager.removeDir(self.remoteTestRoot)
423 RefTest.cleanup(self, profileDir)
424 if (self.pidFile != ""):
425 try:
426 os.remove(self.pidFile)
427 os.remove(self.pidFile + ".xpcshell.pid")
428 except:
429 print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
430
431 def main(args):
432 automation = RemoteAutomation(None)
433 parser = RemoteOptions(automation)
434 options, args = parser.parse_args()
435
436 if (options.deviceIP == None):
437 print "Error: you must provide a device IP to connect to via the --device option"
438 return 1
439
440 try:
441 if (options.dm_trans == "adb"):
442 if (options.deviceIP):
443 dm = droid.DroidADB(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
444 else:
445 dm = droid.DroidADB(None, None, deviceRoot=options.remoteTestRoot)
446 else:
447 dm = droid.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)
448 except devicemanager.DMError:
449 print "Automation Error: exception while initializing devicemanager. Most likely the device is not in a testable state."
450 return 1
451
452 automation.setDeviceManager(dm)
453
454 if (options.remoteProductName != None):
455 automation.setProduct(options.remoteProductName)
456
457 # Set up the defaults and ensure options are set
458 options = parser.verifyRemoteOptions(options)
459 if (options == None):
460 print "ERROR: Invalid options specified, use --help for a list of valid options"
461 return 1
462
463 if not options.ignoreWindowSize:
464 parts = dm.getInfo('screen')['screen'][0].split()
465 width = int(parts[0].split(':')[1])
466 height = int(parts[1].split(':')[1])
467 if (width < 1050 or height < 1050):
468 print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height)
469 return 1
470
471 automation.setAppName(options.app)
472 automation.setRemoteProfile(options.remoteProfile)
473 automation.setRemoteLog(options.remoteLogFile)
474 reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
475 options = parser.verifyCommonOptions(options, reftest)
476
477 # Hack in a symbolic link for jsreftest
478 os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
479
480 # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
481 manifest = args[0]
482 if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])):
483 manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0]
484 elif os.path.exists(args[0]):
485 manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/')
486 manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath
487 else:
488 print "ERROR: Could not find test manifest '%s'" % manifest
489 return 1
490
491 # Start the webserver
492 retVal = reftest.startWebServer(options)
493 if retVal:
494 return retVal
495
496 procName = options.app.split('/')[-1]
497 if (dm.processExist(procName)):
498 dm.killProcess(procName)
499
500 reftest.printDeviceInfo()
501
502 #an example manifest name to use on the cli
503 # manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
504 retVal = 0
505 try:
506 cmdlineArgs = ["-reftest", manifest]
507 if options.bootstrap:
508 cmdlineArgs = []
509 dm.recordLogcat()
510 retVal = reftest.runTests(manifest, options, cmdlineArgs)
511 except:
512 print "Automation Error: Exception caught while running tests"
513 traceback.print_exc()
514 retVal = 1
515
516 reftest.stopWebServer(options)
517
518 reftest.printDeviceInfo(printLogcat=True)
519
520 return retVal
521
522 if __name__ == "__main__":
523 sys.exit(main(sys.argv[1:]))
524

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial