layout/tools/reftest/runreftestb2g.py@44a2da4a2ab2
layout/tools/reftest/runreftestb2g.py
Fri, 16 Jan 2015 04:50:19 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Fri, 16 Jan 2015 04:50:19 +0100
- branch
- TOR_BUG_9701
- changeset 13
- 44a2da4a2ab2
- permissions
- -rw-r--r--
Replace accessor implementation with direct member state manipulation, by
request https://trac.torproject.org/projects/tor/ticket/9701#comment:32
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 import ConfigParser
6 import os
7 import sys
8 import tempfile
9 import traceback
11 # We need to know our current directory so that we can serve our test files from it.
12 here = os.path.abspath(os.path.dirname(__file__))
14 from automation import Automation
15 from b2gautomation import B2GRemoteAutomation
16 from b2g_desktop import run_desktop_reftests
17 from runreftest import RefTest
18 from runreftest import ReftestOptions
19 from remotereftest import ReftestServer
21 from mozdevice import DeviceManagerADB, DMError
22 from marionette import Marionette
23 import moznetwork
25 class B2GOptions(ReftestOptions):
27 def __init__(self, automation=None, **kwargs):
28 defaults = {}
29 if not automation:
30 automation = B2GRemoteAutomation(None, "fennec", context_chrome=True)
32 ReftestOptions.__init__(self, automation)
34 self.add_option("--browser-arg", action="store",
35 type = "string", dest = "browser_arg",
36 help = "Optional command-line arg to pass to the browser")
37 defaults["browser_arg"] = None
39 self.add_option("--b2gpath", action="store",
40 type = "string", dest = "b2gPath",
41 help = "path to B2G repo or qemu dir")
42 defaults["b2gPath"] = None
44 self.add_option("--marionette", action="store",
45 type = "string", dest = "marionette",
46 help = "host:port to use when connecting to Marionette")
47 defaults["marionette"] = None
49 self.add_option("--emulator", action="store",
50 type="string", dest = "emulator",
51 help = "Architecture of emulator to use: x86 or arm")
52 defaults["emulator"] = None
53 self.add_option("--emulator-res", action="store",
54 type="string", dest = "emulator_res",
55 help = "Emulator resolution of the format '<width>x<height>'")
56 defaults["emulator_res"] = None
58 self.add_option("--no-window", action="store_true",
59 dest = "noWindow",
60 help = "Pass --no-window to the emulator")
61 defaults["noWindow"] = False
63 self.add_option("--adbpath", action="store",
64 type = "string", dest = "adbPath",
65 help = "path to adb")
66 defaults["adbPath"] = "adb"
68 self.add_option("--deviceIP", action="store",
69 type = "string", dest = "deviceIP",
70 help = "ip address of remote device to test")
71 defaults["deviceIP"] = None
73 self.add_option("--devicePort", action="store",
74 type = "string", dest = "devicePort",
75 help = "port of remote device to test")
76 defaults["devicePort"] = 20701
78 self.add_option("--remote-logfile", action="store",
79 type = "string", dest = "remoteLogFile",
80 help = "Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.")
81 defaults["remoteLogFile"] = None
83 self.add_option("--remote-webserver", action = "store",
84 type = "string", dest = "remoteWebServer",
85 help = "ip address where the remote web server is hosted at")
86 defaults["remoteWebServer"] = None
88 self.add_option("--http-port", action = "store",
89 type = "string", dest = "httpPort",
90 help = "ip address where the remote web server is hosted at")
91 defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
93 self.add_option("--ssl-port", action = "store",
94 type = "string", dest = "sslPort",
95 help = "ip address where the remote web server is hosted at")
96 defaults["sslPort"] = automation.DEFAULT_SSL_PORT
98 self.add_option("--pidfile", action = "store",
99 type = "string", dest = "pidFile",
100 help = "name of the pidfile to generate")
101 defaults["pidFile"] = ""
102 self.add_option("--gecko-path", action="store",
103 type="string", dest="geckoPath",
104 help="the path to a gecko distribution that should "
105 "be installed on the emulator prior to test")
106 defaults["geckoPath"] = None
107 self.add_option("--logcat-dir", action="store",
108 type="string", dest="logcat_dir",
109 help="directory to store logcat dump files")
110 defaults["logcat_dir"] = None
111 self.add_option('--busybox', action='store',
112 type='string', dest='busybox',
113 help="Path to busybox binary to install on device")
114 defaults['busybox'] = None
115 self.add_option("--httpd-path", action = "store",
116 type = "string", dest = "httpdPath",
117 help = "path to the httpd.js file")
118 defaults["httpdPath"] = None
119 self.add_option("--profile", action="store",
120 type="string", dest="profile",
121 help="for desktop testing, the path to the "
122 "gaia profile to use")
123 defaults["profile"] = None
124 self.add_option("--desktop", action="store_true",
125 dest="desktop",
126 help="Run the tests on a B2G desktop build")
127 defaults["desktop"] = False
128 defaults["remoteTestRoot"] = "/data/local/tests"
129 defaults["logFile"] = "reftest.log"
130 defaults["autorun"] = True
131 defaults["closeWhenDone"] = True
132 defaults["testPath"] = ""
133 defaults["runTestsInParallel"] = False
135 self.set_defaults(**defaults)
137 def verifyRemoteOptions(self, options):
138 if options.runTestsInParallel:
139 self.error("Cannot run parallel tests here")
141 if not options.remoteTestRoot:
142 options.remoteTestRoot = self.automation._devicemanager.getDeviceRoot() + "/reftest"
143 options.remoteProfile = options.remoteTestRoot + "/profile"
145 productRoot = options.remoteTestRoot + "/" + self.automation._product
146 if options.utilityPath == self.automation.DIST_BIN:
147 options.utilityPath = productRoot + "/bin"
149 if options.remoteWebServer == None:
150 if os.name != "nt":
151 options.remoteWebServer = moznetwork.get_ip()
152 else:
153 print "ERROR: you must specify a --remote-webserver=<ip address>\n"
154 return None
156 options.webServer = options.remoteWebServer
158 if options.geckoPath and not options.emulator:
159 self.error("You must specify --emulator if you specify --gecko-path")
161 if options.logcat_dir and not options.emulator:
162 self.error("You must specify --emulator if you specify --logcat-dir")
164 #if not options.emulator and not options.deviceIP:
165 # print "ERROR: you must provide a device IP"
166 # return None
168 if options.remoteLogFile == None:
169 options.remoteLogFile = "reftest.log"
171 options.localLogName = options.remoteLogFile
172 options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
174 # Ensure that the options.logfile (which the base class uses) is set to
175 # the remote setting when running remote. Also, if the user set the
176 # log file name there, use that instead of reusing the remotelogfile as above.
177 if (options.logFile):
178 # If the user specified a local logfile name use that
179 options.localLogName = options.logFile
180 options.logFile = options.remoteLogFile
182 # Only reset the xrePath if it wasn't provided
183 if options.xrePath == None:
184 options.xrePath = options.utilityPath
185 options.xrePath = os.path.abspath(options.xrePath)
187 if options.pidFile != "":
188 f = open(options.pidFile, 'w')
189 f.write("%s" % os.getpid())
190 f.close()
192 # httpd-path is specified by standard makefile targets and may be specified
193 # on the command line to select a particular version of httpd.js. If not
194 # specified, try to select the one from from the xre bundle, as required in bug 882932.
195 if not options.httpdPath:
196 options.httpdPath = os.path.join(options.xrePath, "components")
198 return options
201 class ProfileConfigParser(ConfigParser.RawConfigParser):
202 """Subclass of RawConfigParser that outputs .ini files in the exact
203 format expected for profiles.ini, which is slightly different
204 than the default format.
205 """
207 def optionxform(self, optionstr):
208 return optionstr
210 def write(self, fp):
211 if self._defaults:
212 fp.write("[%s]\n" % ConfigParser.DEFAULTSECT)
213 for (key, value) in self._defaults.items():
214 fp.write("%s=%s\n" % (key, str(value).replace('\n', '\n\t')))
215 fp.write("\n")
216 for section in self._sections:
217 fp.write("[%s]\n" % section)
218 for (key, value) in self._sections[section].items():
219 if key == "__name__":
220 continue
221 if (value is not None) or (self._optcre == self.OPTCRE):
222 key = "=".join((key, str(value).replace('\n', '\n\t')))
223 fp.write("%s\n" % (key))
224 fp.write("\n")
226 class B2GRemoteReftest(RefTest):
228 _devicemanager = None
229 localProfile = None
230 remoteApp = ''
231 profile = None
233 def __init__(self, automation, devicemanager, options, scriptDir):
234 RefTest.__init__(self, automation)
235 self._devicemanager = devicemanager
236 self.runSSLTunnel = False
237 self.remoteTestRoot = options.remoteTestRoot
238 self.remoteProfile = options.remoteProfile
239 self.automation.setRemoteProfile(self.remoteProfile)
240 self.localLogName = options.localLogName
241 self.remoteLogFile = options.remoteLogFile
242 self.bundlesDir = '/system/b2g/distribution/bundles'
243 self.userJS = '/data/local/user.js'
244 self.remoteMozillaPath = '/data/b2g/mozilla'
245 self.remoteProfilesIniPath = os.path.join(self.remoteMozillaPath, 'profiles.ini')
246 self.originalProfilesIni = None
247 self.scriptDir = scriptDir
248 self.SERVER_STARTUP_TIMEOUT = 90
249 if self.automation.IS_DEBUG_BUILD:
250 self.SERVER_STARTUP_TIMEOUT = 180
252 def cleanup(self, profileDir):
253 # Pull results back from device
254 if (self.remoteLogFile):
255 try:
256 self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
257 except:
258 print "ERROR: We were not able to retrieve the info from %s" % self.remoteLogFile
259 sys.exit(5)
261 # Delete any bundled extensions
262 if profileDir:
263 extensionDir = os.path.join(profileDir, 'extensions', 'staged')
264 for filename in os.listdir(extensionDir):
265 try:
266 self._devicemanager._checkCmd(['shell', 'rm', '-rf',
267 os.path.join(self.bundlesDir, filename)])
268 except DMError:
269 pass
271 # Restore the original profiles.ini.
272 if self.originalProfilesIni:
273 try:
274 if not self.automation._is_emulator:
275 self.restoreProfilesIni()
276 os.remove(self.originalProfilesIni)
277 except:
278 pass
280 if not self.automation._is_emulator:
281 self._devicemanager.removeFile(self.remoteLogFile)
282 self._devicemanager.removeDir(self.remoteProfile)
283 self._devicemanager.removeDir(self.remoteTestRoot)
285 # Restore the original user.js.
286 self._devicemanager._checkCmd(['shell', 'rm', '-f', self.userJS])
287 self._devicemanager._checkCmd(['shell', 'dd', 'if=%s.orig' % self.userJS, 'of=%s' % self.userJS])
289 # We've restored the original profile, so reboot the device so that
290 # it gets picked up.
291 self.automation.rebootDevice()
293 RefTest.cleanup(self, profileDir)
294 if getattr(self, 'pidFile', '') != '':
295 try:
296 os.remove(self.pidFile)
297 os.remove(self.pidFile + ".xpcshell.pid")
298 except:
299 print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
301 def findPath(self, paths, filename = None):
302 for path in paths:
303 p = path
304 if filename:
305 p = os.path.join(p, filename)
306 if os.path.exists(self.getFullPath(p)):
307 return path
308 return None
310 def startWebServer(self, options):
311 """ Create the webserver on the host and start it up """
312 remoteXrePath = options.xrePath
313 remoteProfilePath = self.remoteProfile
314 remoteUtilityPath = options.utilityPath
315 localAutomation = Automation()
316 localAutomation.IS_WIN32 = False
317 localAutomation.IS_LINUX = False
318 localAutomation.IS_MAC = False
319 localAutomation.UNIXISH = False
320 hostos = sys.platform
321 if hostos in ['mac', 'darwin']:
322 localAutomation.IS_MAC = True
323 elif hostos in ['linux', 'linux2']:
324 localAutomation.IS_LINUX = True
325 localAutomation.UNIXISH = True
326 elif hostos in ['win32', 'win64']:
327 localAutomation.BIN_SUFFIX = ".exe"
328 localAutomation.IS_WIN32 = True
330 paths = [options.xrePath,
331 localAutomation.DIST_BIN,
332 self.automation._product,
333 os.path.join('..', self.automation._product)]
334 options.xrePath = self.findPath(paths)
335 if options.xrePath == None:
336 print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
337 sys.exit(1)
338 paths.append("bin")
339 paths.append(os.path.join("..", "bin"))
341 xpcshell = "xpcshell"
342 if (os.name == "nt"):
343 xpcshell += ".exe"
345 if (options.utilityPath):
346 paths.insert(0, options.utilityPath)
347 options.utilityPath = self.findPath(paths, xpcshell)
348 if options.utilityPath == None:
349 print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
350 sys.exit(1)
352 xpcshell = os.path.join(options.utilityPath, xpcshell)
353 if self.automation.elf_arm(xpcshell):
354 raise Exception('xpcshell at %s is an ARM binary; please use '
355 'the --utility-path argument to specify the path '
356 'to a desktop version.' % xpcshell)
358 options.serverProfilePath = tempfile.mkdtemp()
359 self.server = ReftestServer(localAutomation, options, self.scriptDir)
360 retVal = self.server.start()
361 if retVal:
362 return retVal
364 if (options.pidFile != ""):
365 f = open(options.pidFile + ".xpcshell.pid", 'w')
366 f.write("%s" % self.server._process.pid)
367 f.close()
369 retVal = self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
370 if retVal:
371 return retVal
373 options.xrePath = remoteXrePath
374 options.utilityPath = remoteUtilityPath
375 options.profilePath = remoteProfilePath
376 return 0
378 def stopWebServer(self, options):
379 if hasattr(self, 'server'):
380 self.server.stop()
382 def restoreProfilesIni(self):
383 # restore profiles.ini on the device to its previous state
384 if not self.originalProfilesIni or not os.access(self.originalProfilesIni, os.F_OK):
385 raise DMError('Unable to install original profiles.ini; file not found: %s',
386 self.originalProfilesIni)
388 self._devicemanager.pushFile(self.originalProfilesIni, self.remoteProfilesIniPath)
390 def updateProfilesIni(self, profilePath):
391 # update profiles.ini on the device to point to the test profile
392 self.originalProfilesIni = tempfile.mktemp()
393 self._devicemanager.getFile(self.remoteProfilesIniPath, self.originalProfilesIni)
395 config = ProfileConfigParser()
396 config.read(self.originalProfilesIni)
397 for section in config.sections():
398 if 'Profile' in section:
399 config.set(section, 'IsRelative', 0)
400 config.set(section, 'Path', profilePath)
402 newProfilesIni = tempfile.mktemp()
403 with open(newProfilesIni, 'wb') as configfile:
404 config.write(configfile)
406 self._devicemanager.pushFile(newProfilesIni, self.remoteProfilesIniPath)
407 try:
408 os.remove(newProfilesIni)
409 except:
410 pass
413 def createReftestProfile(self, options, reftestlist):
414 profile = RefTest.createReftestProfile(self, options, reftestlist,
415 server=options.remoteWebServer,
416 special_powers=False)
417 profileDir = profile.profile
419 prefs = {}
420 # Turn off the locale picker screen
421 prefs["browser.firstrun.show.localepicker"] = False
422 prefs["browser.homescreenURL"] = "app://test-container.gaiamobile.org/index.html"
423 prefs["browser.manifestURL"] = "app://test-container.gaiamobile.org/manifest.webapp"
424 prefs["browser.tabs.remote"] = False
425 prefs["dom.ipc.tabs.disabled"] = False
426 prefs["dom.mozBrowserFramesEnabled"] = True
427 prefs["font.size.inflation.emPerLine"] = 0
428 prefs["font.size.inflation.minTwips"] = 0
429 prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
430 prefs["reftest.browser.iframe.enabled"] = False
431 prefs["reftest.remote"] = True
432 prefs["reftest.uri"] = "%s" % reftestlist
433 # Set a future policy version to avoid the telemetry prompt.
434 prefs["toolkit.telemetry.prompted"] = 999
435 prefs["toolkit.telemetry.notifiedOptOut"] = 999
437 # Set the extra prefs.
438 profile.set_preferences(prefs)
440 # Copy the profile to the device.
441 self._devicemanager.removeDir(self.remoteProfile)
442 try:
443 self._devicemanager.pushDir(profileDir, self.remoteProfile)
444 except DMError:
445 print "Automation Error: Unable to copy profile to device."
446 raise
448 # Copy the extensions to the B2G bundles dir.
449 extensionDir = os.path.join(profileDir, 'extensions', 'staged')
450 # need to write to read-only dir
451 self._devicemanager._checkCmd(['remount'])
452 for filename in os.listdir(extensionDir):
453 self._devicemanager._checkCmd(['shell', 'rm', '-rf',
454 os.path.join(self.bundlesDir, filename)])
455 try:
456 self._devicemanager.pushDir(extensionDir, self.bundlesDir)
457 except DMError:
458 print "Automation Error: Unable to copy extensions to device."
459 raise
461 # In B2G, user.js is always read from /data/local, not the profile
462 # directory. Backup the original user.js first so we can restore it.
463 self._devicemanager._checkCmd(['shell', 'rm', '-f', '%s.orig' % self.userJS])
464 self._devicemanager._checkCmd(['shell', 'dd', 'if=%s' % self.userJS, 'of=%s.orig' % self.userJS])
465 self._devicemanager.pushFile(os.path.join(profileDir, "user.js"), self.userJS)
467 self.updateProfilesIni(self.remoteProfile)
469 options.profilePath = self.remoteProfile
470 return profile
472 def copyExtraFilesToProfile(self, options, profile):
473 profileDir = profile.profile
474 RefTest.copyExtraFilesToProfile(self, options, profile)
475 try:
476 self._devicemanager.pushDir(profileDir, options.remoteProfile)
477 except DMError:
478 print "Automation Error: Failed to copy extra files to device"
479 raise
481 def getManifestPath(self, path):
482 return path
485 def run_remote_reftests(parser, options, args):
486 auto = B2GRemoteAutomation(None, "fennec", context_chrome=True)
488 # create our Marionette instance
489 kwargs = {}
490 if options.emulator:
491 kwargs['emulator'] = options.emulator
492 auto.setEmulator(True)
493 if options.noWindow:
494 kwargs['noWindow'] = True
495 if options.geckoPath:
496 kwargs['gecko_path'] = options.geckoPath
497 if options.logcat_dir:
498 kwargs['logcat_dir'] = options.logcat_dir
499 if options.busybox:
500 kwargs['busybox'] = options.busybox
501 if options.symbolsPath:
502 kwargs['symbols_path'] = options.symbolsPath
503 if options.emulator_res:
504 kwargs['emulator_res'] = options.emulator_res
505 if options.b2gPath:
506 kwargs['homedir'] = options.b2gPath
507 if options.marionette:
508 host,port = options.marionette.split(':')
509 kwargs['host'] = host
510 kwargs['port'] = int(port)
511 marionette = Marionette.getMarionetteOrExit(**kwargs)
512 auto.marionette = marionette
514 if options.emulator:
515 dm = marionette.emulator.dm
516 else:
517 # create the DeviceManager
518 kwargs = {'adbPath': options.adbPath,
519 'deviceRoot': options.remoteTestRoot}
520 if options.deviceIP:
521 kwargs.update({'host': options.deviceIP,
522 'port': options.devicePort})
523 dm = DeviagerADB(**kwargs)
524 auto.setDeviceManager(dm)
526 options = parser.verifyRemoteOptions(options)
528 if (options == None):
529 print "ERROR: Invalid options specified, use --help for a list of valid options"
530 sys.exit(1)
532 # TODO fix exception
533 if not options.ignoreWindowSize:
534 parts = dm.getInfo('screen')['screen'][0].split()
535 width = int(parts[0].split(':')[1])
536 height = int(parts[1].split(':')[1])
537 if (width < 1366 or height < 1050):
538 print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height)
539 return 1
541 auto.setProduct("b2g")
542 auto.test_script = os.path.join(here, 'b2g_start_script.js')
543 auto.test_script_args = [options.remoteWebServer, options.httpPort]
544 auto.logFinish = "REFTEST TEST-START | Shutdown"
546 reftest = B2GRemoteReftest(auto, dm, options, here)
547 options = parser.verifyCommonOptions(options, reftest)
549 logParent = os.path.dirname(options.remoteLogFile)
550 dm.mkDir(logParent);
551 auto.setRemoteLog(options.remoteLogFile)
552 auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
554 # Hack in a symbolic link for jsreftest
555 os.system("ln -s %s %s" % (os.path.join('..', 'jsreftest'), os.path.join(here, 'jsreftest')))
557 # Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
558 manifest = args[0]
559 if os.path.exists(os.path.join(here, args[0])):
560 manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, args[0])
561 elif os.path.exists(args[0]):
562 manifestPath = os.path.abspath(args[0]).split(here)[1].strip('/')
563 manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, manifestPath)
564 else:
565 print "ERROR: Could not find test manifest '%s'" % manifest
566 return 1
568 # Start the webserver
569 retVal = 1
570 try:
571 retVal = reftest.startWebServer(options)
572 if retVal:
573 return retVal
574 procName = options.app.split('/')[-1]
575 if (dm.processExist(procName)):
576 dm.killProcess(procName)
578 cmdlineArgs = ["-reftest", manifest]
579 if getattr(options, 'bootstrap', False):
580 cmdlineArgs = []
582 retVal = reftest.runTests(manifest, options, cmdlineArgs)
583 except:
584 print "Automation Error: Exception caught while running tests"
585 traceback.print_exc()
586 reftest.stopWebServer(options)
587 try:
588 reftest.cleanup(None)
589 except:
590 pass
591 return 1
593 reftest.stopWebServer(options)
594 return retVal
596 def main(args=sys.argv[1:]):
597 parser = B2GOptions()
598 options, args = parser.parse_args(args)
600 if options.desktop:
601 return run_desktop_reftests(parser, options, args)
602 return run_remote_reftests(parser, options, args)
605 if __name__ == "__main__":
606 sys.exit(main())
