michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0: # You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: import mozcrash
michael@0: import threading
michael@0: import os
michael@0: import Queue
michael@0: import re
michael@0: import shutil
michael@0: import tempfile
michael@0: import time
michael@0: import traceback
michael@0: 
michael@0: from automation import Automation
michael@0: from mozprocess import ProcessHandlerMixin
michael@0: 
michael@0: 
michael@0: class StdOutProc(ProcessHandlerMixin):
michael@0:     """Process handler for b2g which puts all output in a Queue.
michael@0:     """
michael@0: 
michael@0:     def __init__(self, cmd, queue, **kwargs):
michael@0:         self.queue = queue
michael@0:         kwargs.setdefault('processOutputLine', []).append(self.handle_output)
michael@0:         ProcessHandlerMixin.__init__(self, cmd, **kwargs)
michael@0: 
michael@0:     def handle_output(self, line):
michael@0:         self.queue.put_nowait(line)
michael@0: 
michael@0: 
michael@0: class B2GRemoteAutomation(Automation):
michael@0:     _devicemanager = None
michael@0: 
michael@0:     def __init__(self, deviceManager, appName='', remoteLog=None,
michael@0:                  marionette=None, context_chrome=True):
michael@0:         self._devicemanager = deviceManager
michael@0:         self._appName = appName
michael@0:         self._remoteProfile = None
michael@0:         self._remoteLog = remoteLog
michael@0:         self.marionette = marionette
michael@0:         self.context_chrome = context_chrome
michael@0:         self._is_emulator = False
michael@0:         self.test_script = None
michael@0:         self.test_script_args = None
michael@0: 
michael@0:         # Default our product to b2g
michael@0:         self._product = "b2g"
michael@0:         self.lastTestSeen = "b2gautomation.py"
michael@0:         # Default log finish to mochitest standard
michael@0:         self.logFinish = 'INFO SimpleTest FINISHED'
michael@0:         Automation.__init__(self)
michael@0: 
michael@0:     def setEmulator(self, is_emulator):
michael@0:         self._is_emulator = is_emulator
michael@0: 
michael@0:     def setDeviceManager(self, deviceManager):
michael@0:         self._devicemanager = deviceManager
michael@0: 
michael@0:     def setAppName(self, appName):
michael@0:         self._appName = appName
michael@0: 
michael@0:     def setRemoteProfile(self, remoteProfile):
michael@0:         self._remoteProfile = remoteProfile
michael@0: 
michael@0:     def setProduct(self, product):
michael@0:         self._product = product
michael@0: 
michael@0:     def setRemoteLog(self, logfile):
michael@0:         self._remoteLog = logfile
michael@0: 
michael@0:     def installExtension(self, extensionSource, profileDir, extensionID=None):
michael@0:         # Bug 827504 - installing special-powers extension separately causes problems in B2G
michael@0:         if extensionID != "special-powers@mozilla.org":
michael@0:             Automation.installExtension(self, extensionSource, profileDir, extensionID)
michael@0: 
michael@0:     # Set up what we need for the remote environment
michael@0:     def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False):
michael@0:         # Because we are running remote, we don't want to mimic the local env
michael@0:         # so no copying of os.environ
michael@0:         if env is None:
michael@0:             env = {}
michael@0: 
michael@0:         if crashreporter:
michael@0:             env['MOZ_CRASHREPORTER'] = '1'
michael@0:             env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
michael@0: 
michael@0:         # We always hide the results table in B2G; it's much slower if we don't.
michael@0:         env['MOZ_HIDE_RESULTS_TABLE'] = '1'
michael@0:         return env
michael@0: 
michael@0:     def waitForNet(self):
michael@0:         active = False
michael@0:         time_out = 0
michael@0:         while not active and time_out < 40:
michael@0:             data = self._devicemanager._runCmd(['shell', '/system/bin/netcfg']).stdout.readlines()
michael@0:             data.pop(0)
michael@0:             for line in data:
michael@0:                 if (re.search(r'UP\s+(?:[0-9]{1,3}\.){3}[0-9]{1,3}', line)):
michael@0:                     active = True
michael@0:                     break
michael@0:             time_out += 1
michael@0:             time.sleep(1)
michael@0:         return active
michael@0: 
michael@0:     def checkForCrashes(self, directory, symbolsPath):
michael@0:         crashed = False
michael@0:         remote_dump_dir = self._remoteProfile + '/minidumps'
michael@0:         print "checking for crashes in '%s'" % remote_dump_dir
michael@0:         if self._devicemanager.dirExists(remote_dump_dir):
michael@0:             local_dump_dir = tempfile.mkdtemp()
michael@0:             self._devicemanager.getDirectory(remote_dump_dir, local_dump_dir)
michael@0:             try:
michael@0:                 crashed = mozcrash.check_for_crashes(local_dump_dir, symbolsPath, test_name=self.lastTestSeen)
michael@0:             except:
michael@0:                 traceback.print_exc()
michael@0:             finally:
michael@0:                 shutil.rmtree(local_dump_dir)
michael@0:                 self._devicemanager.removeDir(remote_dump_dir)
michael@0:         return crashed
michael@0: 
michael@0:     def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
michael@0:         # if remote profile is specified, use that instead
michael@0:         if (self._remoteProfile):
michael@0:             profileDir = self._remoteProfile
michael@0: 
michael@0:         cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
michael@0: 
michael@0:         return app, args
michael@0: 
michael@0:     def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime,
michael@0:                       debuggerInfo, symbolsPath):
michael@0:         """ Wait for tests to finish (as evidenced by a signature string
michael@0:             in logcat), or for a given amount of time to elapse with no
michael@0:             output.
michael@0:         """
michael@0:         timeout = timeout or 120
michael@0:         while True:
michael@0:             currentlog = proc.getStdoutLines(timeout)
michael@0:             if currentlog:
michael@0:                 print currentlog
michael@0:                 # Match the test filepath from the last TEST-START line found in the new
michael@0:                 # log content. These lines are in the form:
michael@0:                 # ... INFO TEST-START | /filepath/we/wish/to/capture.html\n
michael@0:                 testStartFilenames = re.findall(r"TEST-START \| ([^\s]*)", currentlog)
michael@0:                 if testStartFilenames:
michael@0:                     self.lastTestSeen = testStartFilenames[-1]
michael@0:                 if hasattr(self, 'logFinish') and self.logFinish in currentlog:
michael@0:                     return 0
michael@0:             else:
michael@0:                 self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed "
michael@0:                               "out after %d seconds with no output",
michael@0:                               self.lastTestSeen, int(timeout))
michael@0:                 return 1
michael@0: 
michael@0:     def getDeviceStatus(self, serial=None):
michael@0:         # Get the current status of the device.  If we know the device
michael@0:         # serial number, we look for that, otherwise we use the (presumably
michael@0:         # only) device shown in 'adb devices'.
michael@0:         serial = serial or self._devicemanager._deviceSerial
michael@0:         status = 'unknown'
michael@0: 
michael@0:         for line in self._devicemanager._runCmd(['devices']).stdout.readlines():
michael@0:             result = re.match('(.*?)\t(.*)', line)
michael@0:             if result:
michael@0:                 thisSerial = result.group(1)
michael@0:                 if not serial or thisSerial == serial:
michael@0:                     serial = thisSerial
michael@0:                     status = result.group(2)
michael@0: 
michael@0:         return (serial, status)
michael@0: 
michael@0:     def restartB2G(self):
michael@0:         # TODO hangs in subprocess.Popen without this delay
michael@0:         time.sleep(5)
michael@0:         self._devicemanager._checkCmd(['shell', 'stop', 'b2g'])
michael@0:         # Wait for a bit to make sure B2G has completely shut down.
michael@0:         time.sleep(10)
michael@0:         self._devicemanager._checkCmd(['shell', 'start', 'b2g'])
michael@0:         if self._is_emulator:
michael@0:             self.marionette.emulator.wait_for_port()
michael@0: 
michael@0:     def rebootDevice(self):
michael@0:         # find device's current status and serial number
michael@0:         serial, status = self.getDeviceStatus()
michael@0: 
michael@0:         # reboot!
michael@0:         self._devicemanager._runCmd(['shell', '/system/bin/reboot'])
michael@0: 
michael@0:         # The above command can return while adb still thinks the device is
michael@0:         # connected, so wait a little bit for it to disconnect from adb.
michael@0:         time.sleep(10)
michael@0: 
michael@0:         # wait for device to come back to previous status
michael@0:         print 'waiting for device to come back online after reboot'
michael@0:         start = time.time()
michael@0:         rserial, rstatus = self.getDeviceStatus(serial)
michael@0:         while rstatus != 'device':
michael@0:             if time.time() - start > 120:
michael@0:                 # device hasn't come back online in 2 minutes, something's wrong
michael@0:                 raise Exception("Device %s (status: %s) not back online after reboot" % (serial, rstatus))
michael@0:             time.sleep(5)
michael@0:             rserial, rstatus = self.getDeviceStatus(serial)
michael@0:         print 'device:', serial, 'status:', rstatus
michael@0: 
michael@0:     def Process(self, cmd, stdout=None, stderr=None, env=None, cwd=None):
michael@0:         # On a desktop or fennec run, the Process method invokes a gecko
michael@0:         # process in which to the tests.  For B2G, we simply
michael@0:         # reboot the device (which was configured with a test profile
michael@0:         # already), wait for B2G to start up, and then navigate to the
michael@0:         # test url using Marionette.  There doesn't seem to be any way
michael@0:         # to pass env variables into the B2G process, but this doesn't
michael@0:         # seem to matter.
michael@0: 
michael@0:         # reboot device so it starts up with the mochitest profile
michael@0:         # XXX:  We could potentially use 'stop b2g' + 'start b2g' to achieve
michael@0:         # a similar effect; will see which is more stable while attempting
michael@0:         # to bring up the continuous integration.
michael@0:         if not self._is_emulator:
michael@0:             self.rebootDevice()
michael@0:             time.sleep(5)
michael@0:             #wait for wlan to come up
michael@0:             if not self.waitForNet():
michael@0:                 raise Exception("network did not come up, please configure the network" +
michael@0:                                 " prior to running before running the automation framework")
michael@0: 
michael@0:         # stop b2g
michael@0:         self._devicemanager._runCmd(['shell', 'stop', 'b2g'])
michael@0:         time.sleep(5)
michael@0: 
michael@0:         # relaunch b2g inside b2g instance
michael@0:         instance = self.B2GInstance(self._devicemanager, env=env)
michael@0: 
michael@0:         time.sleep(5)
michael@0: 
michael@0:         # Set up port forwarding again for Marionette, since any that
michael@0:         # existed previously got wiped out by the reboot.
michael@0:         if not self._is_emulator:
michael@0:             self._devicemanager._checkCmd(['forward',
michael@0:                                            'tcp:%s' % self.marionette.port,
michael@0:                                            'tcp:%s' % self.marionette.port])
michael@0: 
michael@0:         if self._is_emulator:
michael@0:             self.marionette.emulator.wait_for_port()
michael@0:         else:
michael@0:             time.sleep(5)
michael@0: 
michael@0:         # start a marionette session
michael@0:         session = self.marionette.start_session()
michael@0:         if 'b2g' not in session:
michael@0:             raise Exception("bad session value %s returned by start_session" % session)
michael@0: 
michael@0:         if self._is_emulator:
michael@0:             # Disable offline status management (bug 777145), otherwise the network
michael@0:             # will be 'offline' when the mochitests start.  Presumably, the network
michael@0:             # won't be offline on a real device, so we only do this for emulators.
michael@0:             self.marionette.set_context(self.marionette.CONTEXT_CHROME)
michael@0:             self.marionette.execute_script("""
michael@0:                 Components.utils.import("resource://gre/modules/Services.jsm");
michael@0:                 Services.io.manageOfflineStatus = false;
michael@0:                 Services.io.offline = false;
michael@0:                 """)
michael@0: 
michael@0:         if self.context_chrome:
michael@0:             self.marionette.set_context(self.marionette.CONTEXT_CHROME)
michael@0:         else:
michael@0:             self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
michael@0: 
michael@0:         # run the script that starts the tests
michael@0:         if self.test_script:
michael@0:             if os.path.isfile(self.test_script):
michael@0:                 script = open(self.test_script, 'r')
michael@0:                 self.marionette.execute_script(script.read(), script_args=self.test_script_args)
michael@0:                 script.close()
michael@0:             elif isinstance(self.test_script, basestring):
michael@0:                 self.marionette.execute_script(self.test_script, script_args=self.test_script_args)
michael@0:         else:
michael@0:             # assumes the tests are started on startup automatically
michael@0:             pass
michael@0: 
michael@0:         return instance
michael@0: 
michael@0:     # be careful here as this inner class doesn't have access to outer class members
michael@0:     class B2GInstance(object):
michael@0:         """Represents a B2G instance running on a device, and exposes
michael@0:            some process-like methods/properties that are expected by the
michael@0:            automation.
michael@0:         """
michael@0: 
michael@0:         def __init__(self, dm, env=None):
michael@0:             self.dm = dm
michael@0:             self.env = env or {}
michael@0:             self.stdout_proc = None
michael@0:             self.queue = Queue.Queue()
michael@0: 
michael@0:             # Launch b2g in a separate thread, and dump all output lines
michael@0:             # into a queue.  The lines in this queue are
michael@0:             # retrieved and returned by accessing the stdout property of
michael@0:             # this class.
michael@0:             cmd = [self.dm._adbPath]
michael@0:             if self.dm._deviceSerial:
michael@0:                 cmd.extend(['-s', self.dm._deviceSerial])
michael@0:             cmd.append('shell')
michael@0:             for k, v in self.env.iteritems():
michael@0:                 cmd.append("%s=%s" % (k, v))
michael@0:             cmd.append('/system/bin/b2g.sh')
michael@0:             proc = threading.Thread(target=self._save_stdout_proc, args=(cmd, self.queue))
michael@0:             proc.daemon = True
michael@0:             proc.start()
michael@0: 
michael@0:         def _save_stdout_proc(self, cmd, queue):
michael@0:             self.stdout_proc = StdOutProc(cmd, queue)
michael@0:             self.stdout_proc.run()
michael@0:             if hasattr(self.stdout_proc, 'processOutput'):
michael@0:                 self.stdout_proc.processOutput()
michael@0:             self.stdout_proc.wait()
michael@0:             self.stdout_proc = None
michael@0: 
michael@0:         @property
michael@0:         def pid(self):
michael@0:             # a dummy value to make the automation happy
michael@0:             return 0
michael@0: 
michael@0:         def getStdoutLines(self, timeout):
michael@0:             # Return any lines in the queue used by the
michael@0:             # b2g process handler.
michael@0:             lines = []
michael@0:             # get all of the lines that are currently available
michael@0:             while True:
michael@0:                 try:
michael@0:                     lines.append(self.queue.get_nowait())
michael@0:                 except Queue.Empty:
michael@0:                     break
michael@0: 
michael@0:             # wait 'timeout' for any additional lines
michael@0:             try:
michael@0:                 lines.append(self.queue.get(True, timeout))
michael@0:             except Queue.Empty:
michael@0:                 pass
michael@0:             return '\n'.join(lines)
michael@0: 
michael@0:         def wait(self, timeout=None):
michael@0:             # this should never happen
michael@0:             raise Exception("'wait' called on B2GInstance")
michael@0: 
michael@0:         def kill(self):
michael@0:             # this should never happen
michael@0:             raise Exception("'kill' called on B2GInstance")
michael@0: