michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: import os
michael@0: import sys
michael@0: import time
michael@0: import tempfile
michael@0: import atexit
michael@0: import shlex
michael@0: import subprocess
michael@0: import re
michael@0: import shutil
michael@0: 
michael@0: import mozrunner
michael@0: from cuddlefish.prefs import DEFAULT_COMMON_PREFS
michael@0: from cuddlefish.prefs import DEFAULT_FIREFOX_PREFS
michael@0: from cuddlefish.prefs import DEFAULT_THUNDERBIRD_PREFS
michael@0: from cuddlefish.prefs import DEFAULT_FENNEC_PREFS
michael@0: 
michael@0: # Used to remove noise from ADB output
michael@0: CLEANUP_ADB = re.compile(r'^(I|E)/(stdout|stderr|GeckoConsole)\s*\(\s*\d+\):\s*(.*)$')
michael@0: # Used to filter only messages send by `console` module
michael@0: FILTER_ONLY_CONSOLE_FROM_ADB = re.compile(r'^I/(stdout|stderr)\s*\(\s*\d+\):\s*((info|warning|error|debug): .*)$')
michael@0: 
michael@0: # Used to detect the currently running test
michael@0: PARSEABLE_TEST_NAME = re.compile(r'TEST-START \| ([^\n]+)\n')
michael@0: 
michael@0: # Maximum time we'll wait for tests to finish, in seconds.
michael@0: # The purpose of this timeout is to recover from infinite loops.  It should be
michael@0: # longer than the amount of time any test run takes, including those on slow
michael@0: # machines running slow (debug) versions of Firefox.
michael@0: RUN_TIMEOUT = 1.5 * 60 * 60 # 1.5 Hour
michael@0: 
michael@0: # Maximum time we'll wait for tests to emit output, in seconds.
michael@0: # The purpose of this timeout is to recover from hangs.  It should be longer
michael@0: # than the amount of time any test takes to report results.
michael@0: OUTPUT_TIMEOUT = 60 * 5 # five minutes
michael@0: 
michael@0: def follow_file(filename):
michael@0:     """
michael@0:     Generator that yields the latest unread content from the given
michael@0:     file, or None if no new content is available.
michael@0: 
michael@0:     For example:
michael@0: 
michael@0:       >>> f = open('temp.txt', 'w')
michael@0:       >>> f.write('hello')
michael@0:       >>> f.flush()
michael@0:       >>> tail = follow_file('temp.txt')
michael@0:       >>> tail.next()
michael@0:       'hello'
michael@0:       >>> tail.next() is None
michael@0:       True
michael@0:       >>> f.write('there')
michael@0:       >>> f.flush()
michael@0:       >>> tail.next()
michael@0:       'there'
michael@0:       >>> f.close()
michael@0:       >>> os.remove('temp.txt')
michael@0:     """
michael@0: 
michael@0:     last_pos = 0
michael@0:     last_size = 0
michael@0:     while True:
michael@0:         newstuff = None
michael@0:         if os.path.exists(filename):
michael@0:             size = os.stat(filename).st_size
michael@0:             if size > last_size:
michael@0:                 last_size = size
michael@0:                 f = open(filename, 'r')
michael@0:                 f.seek(last_pos)
michael@0:                 newstuff = f.read()
michael@0:                 last_pos = f.tell()
michael@0:                 f.close()
michael@0:         yield newstuff
michael@0: 
michael@0: # subprocess.check_output only appeared in python2.7, so this code is taken
michael@0: # from python source code for compatibility with py2.5/2.6
michael@0: class CalledProcessError(Exception):
michael@0:     def __init__(self, returncode, cmd, output=None):
michael@0:         self.returncode = returncode
michael@0:         self.cmd = cmd
michael@0:         self.output = output
michael@0:     def __str__(self):
michael@0:         return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
michael@0: 
michael@0: def check_output(*popenargs, **kwargs):
michael@0:     if 'stdout' in kwargs:
michael@0:         raise ValueError('stdout argument not allowed, it will be overridden.')
michael@0:     process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
michael@0:     output, unused_err = process.communicate()
michael@0:     retcode = process.poll()
michael@0:     if retcode:
michael@0:         cmd = kwargs.get("args")
michael@0:         if cmd is None:
michael@0:             cmd = popenargs[0]
michael@0:         raise CalledProcessError(retcode, cmd, output=output)
michael@0:     return output
michael@0: 
michael@0: 
michael@0: class FennecProfile(mozrunner.Profile):
michael@0:     preferences = {}
michael@0:     names = ['fennec']
michael@0: 
michael@0: class FennecRunner(mozrunner.Runner):
michael@0:     profile_class = FennecProfile
michael@0: 
michael@0:     names = ['fennec']
michael@0: 
michael@0:     __DARWIN_PATH = '/Applications/Fennec.app/Contents/MacOS/fennec'
michael@0: 
michael@0:     def __init__(self, binary=None, **kwargs):
michael@0:         if sys.platform == 'darwin' and binary and binary.endswith('.app'):
michael@0:             # Assume it's a Fennec app dir.
michael@0:             binary = os.path.join(binary, 'Contents/MacOS/fennec')
michael@0: 
michael@0:         self.__real_binary = binary
michael@0: 
michael@0:         mozrunner.Runner.__init__(self, **kwargs)
michael@0: 
michael@0:     def find_binary(self):
michael@0:         if not self.__real_binary:
michael@0:             if sys.platform == 'darwin':
michael@0:                 if os.path.exists(self.__DARWIN_PATH):
michael@0:                     return self.__DARWIN_PATH
michael@0:             self.__real_binary = mozrunner.Runner.find_binary(self)
michael@0:         return self.__real_binary
michael@0: 
michael@0: FENNEC_REMOTE_PATH = '/mnt/sdcard/jetpack-profile'
michael@0: 
michael@0: class RemoteFennecRunner(mozrunner.Runner):
michael@0:     profile_class = FennecProfile
michael@0: 
michael@0:     names = ['fennec']
michael@0: 
michael@0:     _INTENT_PREFIX = 'org.mozilla.'
michael@0: 
michael@0:     _adb_path = None
michael@0: 
michael@0:     def __init__(self, binary=None, **kwargs):
michael@0:         # Check that we have a binary set
michael@0:         if not binary:
michael@0:             raise ValueError("You have to define `--binary` option set to the "
michael@0:                             "path to your ADB executable.")
michael@0:         # Ensure that binary refer to a valid ADB executable
michael@0:         output = subprocess.Popen([binary], stdout=subprocess.PIPE,
michael@0:                                   stderr=subprocess.PIPE).communicate()
michael@0:         output = "".join(output)
michael@0:         if not ("Android Debug Bridge" in output):
michael@0:             raise ValueError("`--binary` option should be the path to your "
michael@0:                             "ADB executable.")
michael@0:         self.binary = binary
michael@0: 
michael@0:         mobile_app_name = kwargs['cmdargs'][0]
michael@0:         self.profile = kwargs['profile']
michael@0:         self._adb_path = binary
michael@0: 
michael@0:         # This pref has to be set to `false` otherwise, we do not receive
michael@0:         # output of adb commands!
michael@0:         subprocess.call([self._adb_path, "shell",
michael@0:                         "setprop log.redirect-stdio false"])
michael@0: 
michael@0:         # Android apps are launched by their "intent" name,
michael@0:         # Automatically detect already installed firefox by using `pm` program
michael@0:         # or use name given as cfx `--mobile-app` argument.
michael@0:         intents = self.getIntentNames()
michael@0:         if not intents:
michael@0:             raise ValueError("Unable to found any Firefox "
michael@0:                              "application on your device.")
michael@0:         elif mobile_app_name:
michael@0:             if not mobile_app_name in intents:
michael@0:                 raise ValueError("Unable to found Firefox application "
michael@0:                                  "with intent name '%s'\n"
michael@0:                                  "Available ones are: %s" %
michael@0:                                  (mobile_app_name, ", ".join(intents)))
michael@0:             self._intent_name = self._INTENT_PREFIX + mobile_app_name
michael@0:         else:
michael@0:             if "firefox" in intents:
michael@0:                 self._intent_name = self._INTENT_PREFIX + "firefox"
michael@0:             elif "firefox_beta" in intents:
michael@0:                 self._intent_name = self._INTENT_PREFIX + "firefox_beta"
michael@0:             elif "firefox_nightly" in intents:
michael@0:                 self._intent_name = self._INTENT_PREFIX + "firefox_nightly"
michael@0:             else:
michael@0:                 self._intent_name = self._INTENT_PREFIX + intents[0]
michael@0: 
michael@0:         print "Launching mobile application with intent name " + self._intent_name
michael@0: 
michael@0:         # First try to kill firefox if it is already running
michael@0:         pid = self.getProcessPID(self._intent_name)
michael@0:         if pid != None:
michael@0:             print "Killing running Firefox instance ..."
michael@0:             subprocess.call([self._adb_path, "shell",
michael@0:                              "am force-stop " + self._intent_name])
michael@0:             time.sleep(7)
michael@0:             # It appears recently that the PID still exists even after
michael@0:             # Fennec closes, so removing this error still allows the tests
michael@0:             # to pass as the new Fennec instance is able to start.
michael@0:             # Leaving error in but commented out for now.
michael@0:             #
michael@0:             #if self.getProcessPID(self._intent_name) != None:
michael@0:             #    raise Exception("Unable to automatically kill running Firefox" +
michael@0:             #                    " instance. Please close it manually before " +
michael@0:             #                    "executing cfx.")
michael@0: 
michael@0:         print "Pushing the addon to your device"
michael@0: 
michael@0:         # Create a clean empty profile on the sd card
michael@0:         subprocess.call([self._adb_path, "shell", "rm -r " + FENNEC_REMOTE_PATH])
michael@0:         subprocess.call([self._adb_path, "shell", "mkdir " + FENNEC_REMOTE_PATH])
michael@0: 
michael@0:         # Push the profile folder created by mozrunner to the device
michael@0:         # (we can't simply use `adb push` as it doesn't copy empty folders)
michael@0:         localDir = self.profile.profile
michael@0:         remoteDir = FENNEC_REMOTE_PATH
michael@0:         for root, dirs, files in os.walk(localDir, followlinks='true'):
michael@0:             relRoot = os.path.relpath(root, localDir)
michael@0:             # Note about os.path usage below:
michael@0:             # Local files may be using Windows `\` separators but
michael@0:             # remote are always `/`, so we need to convert local ones to `/`
michael@0:             for file in files:
michael@0:                 localFile = os.path.join(root, file)
michael@0:                 remoteFile = remoteDir.replace("/", os.sep)
michael@0:                 if relRoot != ".":
michael@0:                     remoteFile = os.path.join(remoteFile, relRoot)
michael@0:                 remoteFile = os.path.join(remoteFile, file)
michael@0:                 remoteFile = "/".join(remoteFile.split(os.sep))
michael@0:                 subprocess.Popen([self._adb_path, "push", localFile, remoteFile],
michael@0:                                  stderr=subprocess.PIPE).wait()
michael@0:             for dir in dirs:
michael@0:                 targetDir = remoteDir.replace("/", os.sep)
michael@0:                 if relRoot != ".":
michael@0:                     targetDir = os.path.join(targetDir, relRoot)
michael@0:                 targetDir = os.path.join(targetDir, dir)
michael@0:                 targetDir = "/".join(targetDir.split(os.sep))
michael@0:                 # `-p` option is not supported on all devices!
michael@0:                 subprocess.call([self._adb_path, "shell", "mkdir " + targetDir])
michael@0: 
michael@0:     @property
michael@0:     def command(self):
michael@0:         """Returns the command list to run."""
michael@0:         return [self._adb_path,
michael@0:             "shell",
michael@0:             "am start " +
michael@0:                 "-a android.activity.MAIN " +
michael@0:                 "-n " + self._intent_name + "/" + self._intent_name + ".App " +
michael@0:                 "--es args \"-profile " + FENNEC_REMOTE_PATH + "\""
michael@0:         ]
michael@0: 
michael@0:     def start(self):
michael@0:         subprocess.call(self.command)
michael@0: 
michael@0:     def getProcessPID(self, processName):
michael@0:         p = subprocess.Popen([self._adb_path, "shell", "ps"],
michael@0:                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
michael@0:         line = p.stdout.readline()
michael@0:         while line:
michael@0:             columns = line.split()
michael@0:             pid = columns[1]
michael@0:             name = columns[-1]
michael@0:             line = p.stdout.readline()
michael@0:             if processName in name:
michael@0:                 return pid
michael@0:         return None
michael@0: 
michael@0:     def getIntentNames(self):
michael@0:         p = subprocess.Popen([self._adb_path, "shell", "pm list packages"],
michael@0:                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
michael@0:         names = []
michael@0:         for line in p.stdout.readlines():
michael@0:             line = re.sub("(^package:)|\s", "", line)
michael@0:             if self._INTENT_PREFIX in line:
michael@0:                 names.append(line.replace(self._INTENT_PREFIX, ""))
michael@0:         return names
michael@0: 
michael@0: 
michael@0: class XulrunnerAppProfile(mozrunner.Profile):
michael@0:     preferences = {}
michael@0:     names = []
michael@0: 
michael@0: class XulrunnerAppRunner(mozrunner.Runner):
michael@0:     """
michael@0:     Runner for any XULRunner app. Can use a Firefox binary in XULRunner
michael@0:     mode to execute the app, or can use XULRunner itself. Expects the
michael@0:     app's application.ini to be passed in as one of the items in
michael@0:     'cmdargs' in the constructor.
michael@0: 
michael@0:     This class relies a lot on the particulars of mozrunner.Runner's
michael@0:     implementation, and does some unfortunate acrobatics to get around
michael@0:     some of the class' limitations/assumptions.
michael@0:     """
michael@0: 
michael@0:     profile_class = XulrunnerAppProfile
michael@0: 
michael@0:     # This is a default, and will be overridden in the instance if
michael@0:     # Firefox is used in XULRunner mode.
michael@0:     names = ['xulrunner']
michael@0: 
michael@0:     # Default location of XULRunner on OS X.
michael@0:     __DARWIN_PATH = "/Library/Frameworks/XUL.framework/xulrunner-bin"
michael@0:     __LINUX_PATH  = "/usr/bin/xulrunner"
michael@0: 
michael@0:     # What our application.ini's path looks like if it's part of
michael@0:     # an "installed" XULRunner app on OS X.
michael@0:     __DARWIN_APP_INI_SUFFIX = '.app/Contents/Resources/application.ini'
michael@0: 
michael@0:     def __init__(self, binary=None, **kwargs):
michael@0:         if sys.platform == 'darwin' and binary and binary.endswith('.app'):
michael@0:             # Assume it's a Firefox app dir.
michael@0:             binary = os.path.join(binary, 'Contents/MacOS/firefox-bin')
michael@0: 
michael@0:         self.__app_ini = None
michael@0:         self.__real_binary = binary
michael@0: 
michael@0:         mozrunner.Runner.__init__(self, **kwargs)
michael@0: 
michael@0:         # See if we're using a genuine xulrunner-bin from the XULRunner SDK,
michael@0:         # or if we're being asked to use Firefox in XULRunner mode.
michael@0:         self.__is_xulrunner_sdk = 'xulrunner' in self.binary
michael@0: 
michael@0:         if sys.platform == 'linux2' and not self.env.get('LD_LIBRARY_PATH'):
michael@0:             self.env['LD_LIBRARY_PATH'] = os.path.dirname(self.binary)
michael@0: 
michael@0:         newargs = []
michael@0:         for item in self.cmdargs:
michael@0:             if 'application.ini' in item:
michael@0:                 self.__app_ini = item
michael@0:             else:
michael@0:                 newargs.append(item)
michael@0:         self.cmdargs = newargs
michael@0: 
michael@0:         if not self.__app_ini:
michael@0:             raise ValueError('application.ini not found in cmdargs')
michael@0:         if not os.path.exists(self.__app_ini):
michael@0:             raise ValueError("file does not exist: '%s'" % self.__app_ini)
michael@0: 
michael@0:         if (sys.platform == 'darwin' and
michael@0:             self.binary == self.__DARWIN_PATH and
michael@0:             self.__app_ini.endswith(self.__DARWIN_APP_INI_SUFFIX)):
michael@0:             # If the application.ini is in an app bundle, then
michael@0:             # it could be inside an "installed" XULRunner app.
michael@0:             # If this is the case, use the app's actual
michael@0:             # binary instead of the XUL framework's, so we get
michael@0:             # a proper app icon, etc.
michael@0:             new_binary = '/'.join(self.__app_ini.split('/')[:-2] +
michael@0:                                   ['MacOS', 'xulrunner'])
michael@0:             if os.path.exists(new_binary):
michael@0:                 self.binary = new_binary
michael@0: 
michael@0:     @property
michael@0:     def command(self):
michael@0:         """Returns the command list to run."""
michael@0: 
michael@0:         if self.__is_xulrunner_sdk:
michael@0:             return [self.binary, self.__app_ini, '-profile',
michael@0:                     self.profile.profile]
michael@0:         else:
michael@0:             return [self.binary, '-app', self.__app_ini, '-profile',
michael@0:                     self.profile.profile]
michael@0: 
michael@0:     def __find_xulrunner_binary(self):
michael@0:         if sys.platform == 'darwin':
michael@0:             if os.path.exists(self.__DARWIN_PATH):
michael@0:                 return self.__DARWIN_PATH
michael@0:         if sys.platform == 'linux2':
michael@0:             if os.path.exists(self.__LINUX_PATH):
michael@0:                 return self.__LINUX_PATH
michael@0:         return None
michael@0: 
michael@0:     def find_binary(self):
michael@0:         # This gets called by the superclass constructor. It will
michael@0:         # always get called, even if a binary was passed into the
michael@0:         # constructor, because we want to have full control over
michael@0:         # what the exact setting of self.binary is.
michael@0: 
michael@0:         if not self.__real_binary:
michael@0:             self.__real_binary = self.__find_xulrunner_binary()
michael@0:             if not self.__real_binary:
michael@0:                 dummy_profile = {}
michael@0:                 runner = mozrunner.FirefoxRunner(profile=dummy_profile)
michael@0:                 self.__real_binary = runner.find_binary()
michael@0:                 self.names = runner.names
michael@0:         return self.__real_binary
michael@0: 
michael@0: def set_overloaded_modules(env_root, app_type, addon_id, preferences, overloads):
michael@0:     # win32 file scheme needs 3 slashes
michael@0:     desktop_file_scheme = "file://"
michael@0:     if not env_root.startswith("/"):
michael@0:       desktop_file_scheme = desktop_file_scheme + "/"
michael@0: 
michael@0:     pref_prefix = "extensions.modules." + addon_id + ".path"
michael@0: 
michael@0:     # Set preferences that will map require prefix to a given path
michael@0:     for name, path in overloads.items():
michael@0:         if len(name) == 0:
michael@0:             prefName = pref_prefix
michael@0:         else:
michael@0:             prefName = pref_prefix + "." + name
michael@0:         if app_type == "fennec-on-device":
michael@0:             # For testing on device, we have to copy overloaded files from fs
michael@0:             # to the device and use device path instead of local fs path.
michael@0:             # Actual copy of files if done after the call to Profile constructor
michael@0:             preferences[prefName] = "file://" + \
michael@0:                 FENNEC_REMOTE_PATH + "/overloads/" + name
michael@0:         else:
michael@0:             preferences[prefName] = desktop_file_scheme + \
michael@0:                 path.replace("\\", "/") + "/"
michael@0: 
michael@0: def run_app(harness_root_dir, manifest_rdf, harness_options,
michael@0:             app_type, binary=None, profiledir=None, verbose=False,
michael@0:             parseable=False, enforce_timeouts=False,
michael@0:             logfile=None, addons=None, args=None, extra_environment={},
michael@0:             norun=None,
michael@0:             used_files=None, enable_mobile=False,
michael@0:             mobile_app_name=None,
michael@0:             env_root=None,
michael@0:             is_running_tests=False,
michael@0:             overload_modules=False,
michael@0:             bundle_sdk=True,
michael@0:             pkgdir=""):
michael@0:     if binary:
michael@0:         binary = os.path.expanduser(binary)
michael@0: 
michael@0:     if addons is None:
michael@0:         addons = []
michael@0:     else:
michael@0:         addons = list(addons)
michael@0: 
michael@0:     cmdargs = []
michael@0:     preferences = dict(DEFAULT_COMMON_PREFS)
michael@0: 
michael@0:     # For now, only allow running on Mobile with --force-mobile argument
michael@0:     if app_type in ["fennec", "fennec-on-device"] and not enable_mobile:
michael@0:         print """
michael@0:   WARNING: Firefox Mobile support is still experimental.
michael@0:   If you would like to run an addon on this platform, use --force-mobile flag:
michael@0: 
michael@0:     cfx --force-mobile"""
michael@0:         return 0
michael@0: 
michael@0:     if app_type == "fennec-on-device":
michael@0:         profile_class = FennecProfile
michael@0:         preferences.update(DEFAULT_FENNEC_PREFS)
michael@0:         runner_class = RemoteFennecRunner
michael@0:         # We pass the intent name through command arguments
michael@0:         cmdargs.append(mobile_app_name)
michael@0:     elif enable_mobile or app_type == "fennec":
michael@0:         profile_class = FennecProfile
michael@0:         preferences.update(DEFAULT_FENNEC_PREFS)
michael@0:         runner_class = FennecRunner
michael@0:     elif app_type == "xulrunner":
michael@0:         profile_class = XulrunnerAppProfile
michael@0:         runner_class = XulrunnerAppRunner
michael@0:         cmdargs.append(os.path.join(harness_root_dir, 'application.ini'))
michael@0:     elif app_type == "firefox":
michael@0:         profile_class = mozrunner.FirefoxProfile
michael@0:         preferences.update(DEFAULT_FIREFOX_PREFS)
michael@0:         runner_class = mozrunner.FirefoxRunner
michael@0:     elif app_type == "thunderbird":
michael@0:         profile_class = mozrunner.ThunderbirdProfile
michael@0:         preferences.update(DEFAULT_THUNDERBIRD_PREFS)
michael@0:         runner_class = mozrunner.ThunderbirdRunner
michael@0:     else:
michael@0:         raise ValueError("Unknown app: %s" % app_type)
michael@0:     if sys.platform == 'darwin' and app_type != 'xulrunner':
michael@0:         cmdargs.append('-foreground')
michael@0: 
michael@0:     if args:
michael@0:         cmdargs.extend(shlex.split(args))
michael@0: 
michael@0:     # TODO: handle logs on remote device
michael@0:     if app_type != "fennec-on-device":
michael@0:         # tempfile.gettempdir() was constant, preventing two simultaneous "cfx
michael@0:         # run"/"cfx test" on the same host. On unix it points at /tmp (which is
michael@0:         # world-writeable), enabling a symlink attack (e.g. imagine some bad guy
michael@0:         # does 'ln -s ~/.ssh/id_rsa /tmp/harness_result'). NamedTemporaryFile
michael@0:         # gives us a unique filename that fixes both problems. We leave the
michael@0:         # (0-byte) file in place until the browser-side code starts writing to
michael@0:         # it, otherwise the symlink attack becomes possible again.
michael@0:         fileno,resultfile = tempfile.mkstemp(prefix="harness-result-")
michael@0:         os.close(fileno)
michael@0:         harness_options['resultFile'] = resultfile
michael@0: 
michael@0:     def maybe_remove_logfile():
michael@0:         if os.path.exists(logfile):
michael@0:             os.remove(logfile)
michael@0: 
michael@0:     logfile_tail = None
michael@0: 
michael@0:     # We always buffer output through a logfile for two reasons:
michael@0:     # 1. On Windows, it's the only way to print console output to stdout/err.
michael@0:     # 2. It enables us to keep track of the last time output was emitted,
michael@0:     #    so we can raise an exception if the test runner hangs.
michael@0:     if not logfile:
michael@0:         fileno,logfile = tempfile.mkstemp(prefix="harness-log-")
michael@0:         os.close(fileno)
michael@0:     logfile_tail = follow_file(logfile)
michael@0:     atexit.register(maybe_remove_logfile)
michael@0: 
michael@0:     logfile = os.path.abspath(os.path.expanduser(logfile))
michael@0:     maybe_remove_logfile()
michael@0: 
michael@0:     env = {}
michael@0:     env.update(os.environ)
michael@0:     env['MOZ_NO_REMOTE'] = '1'
michael@0:     env['XPCOM_DEBUG_BREAK'] = 'stack'
michael@0:     env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
michael@0:     env.update(extra_environment)
michael@0:     if norun:
michael@0:         cmdargs.append("-no-remote")
michael@0: 
michael@0:     # Create the addon XPI so mozrunner will copy it to the profile it creates.
michael@0:     # We delete it below after getting mozrunner to create the profile.
michael@0:     from cuddlefish.xpi import build_xpi
michael@0:     xpi_path = tempfile.mktemp(suffix='cfx-tmp.xpi')
michael@0:     build_xpi(template_root_dir=harness_root_dir,
michael@0:               manifest=manifest_rdf,
michael@0:               xpi_path=xpi_path,
michael@0:               harness_options=harness_options,
michael@0:               limit_to=used_files,
michael@0:               bundle_sdk=bundle_sdk,
michael@0:               pkgdir=pkgdir)
michael@0:     addons.append(xpi_path)
michael@0: 
michael@0:     starttime = last_output_time = time.time()
michael@0: 
michael@0:     # Redirect runner output to a file so we can catch output not generated
michael@0:     # by us.
michael@0:     # In theory, we could do this using simple redirection on all platforms
michael@0:     # other than Windows, but this way we only have a single codepath to
michael@0:     # maintain.
michael@0:     fileno,outfile = tempfile.mkstemp(prefix="harness-stdout-")
michael@0:     os.close(fileno)
michael@0:     outfile_tail = follow_file(outfile)
michael@0:     def maybe_remove_outfile():
michael@0:         if os.path.exists(outfile):
michael@0:             os.remove(outfile)
michael@0:     atexit.register(maybe_remove_outfile)
michael@0:     outf = open(outfile, "w")
michael@0:     popen_kwargs = { 'stdout': outf, 'stderr': outf}
michael@0: 
michael@0:     profile = None
michael@0: 
michael@0:     if app_type == "fennec-on-device":
michael@0:         # Install a special addon when we run firefox on mobile device
michael@0:         # in order to be able to kill it
michael@0:         mydir = os.path.dirname(os.path.abspath(__file__))
michael@0:         addon_dir = os.path.join(mydir, "mobile-utils")
michael@0:         addons.append(addon_dir)
michael@0: 
michael@0:     # Overload addon-specific commonjs modules path with lib/ folder
michael@0:     overloads = dict()
michael@0:     if overload_modules:
michael@0:         overloads[""] = os.path.join(env_root, "lib")
michael@0: 
michael@0:     # Overload tests/ mapping with test/ folder, only when running test
michael@0:     if is_running_tests:
michael@0:         overloads["tests"] = os.path.join(env_root, "test")
michael@0: 
michael@0:     set_overloaded_modules(env_root, app_type, harness_options["jetpackID"], \
michael@0:                            preferences, overloads)
michael@0: 
michael@0:     # the XPI file is copied into the profile here
michael@0:     profile = profile_class(addons=addons,
michael@0:                             profile=profiledir,
michael@0:                             preferences=preferences)
michael@0: 
michael@0:     # Delete the temporary xpi file
michael@0:     os.remove(xpi_path)
michael@0: 
michael@0:     # Copy overloaded files registered in set_overloaded_modules
michael@0:     # For testing on device, we have to copy overloaded files from fs
michael@0:     # to the device and use device path instead of local fs path.
michael@0:     # (has to be done after the call to profile_class() which eventualy creates
michael@0:     #  profile folder)
michael@0:     if app_type == "fennec-on-device":
michael@0:         profile_path = profile.profile
michael@0:         for name, path in overloads.items():
michael@0:             shutil.copytree(path, \
michael@0:                 os.path.join(profile_path, "overloads", name))
michael@0: 
michael@0:     runner = runner_class(profile=profile,
michael@0:                           binary=binary,
michael@0:                           env=env,
michael@0:                           cmdargs=cmdargs,
michael@0:                           kp_kwargs=popen_kwargs)
michael@0: 
michael@0:     sys.stdout.flush(); sys.stderr.flush()
michael@0: 
michael@0:     if app_type == "fennec-on-device":
michael@0:         if not enable_mobile:
michael@0:             print >>sys.stderr, """
michael@0:   WARNING: Firefox Mobile support is still experimental.
michael@0:   If you would like to run an addon on this platform, use --force-mobile flag:
michael@0: 
michael@0:     cfx --force-mobile"""
michael@0:             return 0
michael@0: 
michael@0:         # In case of mobile device, we need to get stdio from `adb logcat` cmd:
michael@0: 
michael@0:         # First flush logs in order to avoid catching previous ones
michael@0:         subprocess.call([binary, "logcat", "-c"])
michael@0: 
michael@0:         # Launch adb command
michael@0:         runner.start()
michael@0: 
michael@0:         # We can immediatly remove temporary profile folder
michael@0:         # as it has been uploaded to the device
michael@0:         profile.cleanup()
michael@0:         # We are not going to use the output log file
michael@0:         outf.close()
michael@0: 
michael@0:         # Then we simply display stdout of `adb logcat`
michael@0:         p = subprocess.Popen([binary, "logcat", "stderr:V stdout:V GeckoConsole:V *:S"], stdout=subprocess.PIPE)
michael@0:         while True:
michael@0:             line = p.stdout.readline()
michael@0:             if line == '':
michael@0:                 break
michael@0:             # mobile-utils addon contains an application quit event observer
michael@0:             # that will print this string:
michael@0:             if "APPLICATION-QUIT" in line:
michael@0:                 break
michael@0: 
michael@0:             if verbose:
michael@0:                 # if --verbose is given, we display everything:
michael@0:                 # All JS Console messages, stdout and stderr.
michael@0:                 m = CLEANUP_ADB.match(line)
michael@0:                 if not m:
michael@0:                     print line.rstrip()
michael@0:                     continue
michael@0:                 print m.group(3)
michael@0:             else:
michael@0:                 # Otherwise, display addons messages dispatched through
michael@0:                 # console.[info, log, debug, warning, error](msg)
michael@0:                 m = FILTER_ONLY_CONSOLE_FROM_ADB.match(line)
michael@0:                 if m:
michael@0:                     print m.group(2)
michael@0: 
michael@0:         print >>sys.stderr, "Program terminated successfully."
michael@0:         return 0
michael@0: 
michael@0: 
michael@0:     print >>sys.stderr, "Using binary at '%s'." % runner.binary
michael@0: 
michael@0:     # Ensure cfx is being used with Firefox 4.0+.
michael@0:     # TODO: instead of dying when Firefox is < 4, warn when Firefox is outside
michael@0:     # the minVersion/maxVersion boundaries.
michael@0:     version_output = check_output(runner.command + ["-v"])
michael@0:     # Note: this regex doesn't handle all valid versions in the Toolkit Version
michael@0:     # Format <https://developer.mozilla.org/en/Toolkit_version_format>, just the
michael@0:     # common subset that we expect Mozilla apps to use.
michael@0:     mo = re.search(r"Mozilla (Firefox|Iceweasel|Fennec)\b[^ ]* ((\d+)\.\S*)",
michael@0:                    version_output)
michael@0:     if not mo:
michael@0:         # cfx may be used with Thunderbird, SeaMonkey or an exotic Firefox
michael@0:         # version.
michael@0:         print """
michael@0:   WARNING: cannot determine Firefox version; please ensure you are running
michael@0:   a Mozilla application equivalent to Firefox 4.0 or greater.
michael@0:   """
michael@0:     elif mo.group(1) == "Fennec":
michael@0:         # For now, only allow running on Mobile with --force-mobile argument
michael@0:         if not enable_mobile:
michael@0:             print """
michael@0:   WARNING: Firefox Mobile support is still experimental.
michael@0:   If you would like to run an addon on this platform, use --force-mobile flag:
michael@0: 
michael@0:     cfx --force-mobile"""
michael@0:             return
michael@0:     else:
michael@0:         version = mo.group(3)
michael@0:         if int(version) < 4:
michael@0:             print """
michael@0:   cfx requires Firefox 4 or greater and is unable to find a compatible
michael@0:   binary. Please install a newer version of Firefox or provide the path to
michael@0:   your existing compatible version with the --binary flag:
michael@0: 
michael@0:     cfx --binary=PATH_TO_FIREFOX_BINARY"""
michael@0:             return
michael@0: 
michael@0:         # Set the appropriate extensions.checkCompatibility preference to false,
michael@0:         # so the tests run even if the SDK is not marked as compatible with the
michael@0:         # version of Firefox on which they are running, and we don't have to
michael@0:         # ensure we update the maxVersion before the version of Firefox changes
michael@0:         # every six weeks.
michael@0:         #
michael@0:         # The regex we use here is effectively the same as BRANCH_REGEX from
michael@0:         # /toolkit/mozapps/extensions/content/extensions.js, which toolkit apps
michael@0:         # use to determine whether or not to load an incompatible addon.
michael@0:         #
michael@0:         br = re.search(r"^([^\.]+\.[0-9]+[a-z]*).*", mo.group(2), re.I)
michael@0:         if br:
michael@0:             prefname = 'extensions.checkCompatibility.' + br.group(1)
michael@0:             profile.preferences[prefname] = False
michael@0:             # Calling profile.set_preferences here duplicates the list of prefs
michael@0:             # in prefs.js, since the profile calls self.set_preferences in its
michael@0:             # constructor, but that is ok, because it doesn't change the set of
michael@0:             # preferences that are ultimately registered in Firefox.
michael@0:             profile.set_preferences(profile.preferences)
michael@0: 
michael@0:     print >>sys.stderr, "Using profile at '%s'." % profile.profile
michael@0:     sys.stderr.flush()
michael@0: 
michael@0:     if norun:
michael@0:         print "To launch the application, enter the following command:"
michael@0:         print " ".join(runner.command) + " " + (" ".join(runner.cmdargs))
michael@0:         return 0
michael@0: 
michael@0:     runner.start()
michael@0: 
michael@0:     done = False
michael@0:     result = None
michael@0:     test_name = "unknown"
michael@0: 
michael@0:     def Timeout(message, test_name, parseable):
michael@0:         if parseable:
michael@0:             sys.stderr.write("TEST-UNEXPECTED-FAIL | %s | %s\n" % (test_name, message))
michael@0:             sys.stderr.flush()
michael@0:         return Exception(message)
michael@0: 
michael@0:     try:
michael@0:         while not done:
michael@0:             time.sleep(0.05)
michael@0:             for tail in (logfile_tail, outfile_tail):
michael@0:                 if tail:
michael@0:                     new_chars = tail.next()
michael@0:                     if new_chars:
michael@0:                         last_output_time = time.time()
michael@0:                         sys.stderr.write(new_chars)
michael@0:                         sys.stderr.flush()
michael@0:                         if is_running_tests and parseable:
michael@0:                             match = PARSEABLE_TEST_NAME.search(new_chars)
michael@0:                             if match:
michael@0:                                 test_name = match.group(1)
michael@0:             if os.path.exists(resultfile):
michael@0:                 result = open(resultfile).read()
michael@0:                 if result:
michael@0:                     if result in ['OK', 'FAIL']:
michael@0:                         done = True
michael@0:                     else:
michael@0:                         sys.stderr.write("Hrm, resultfile (%s) contained something weird (%d bytes)\n" % (resultfile, len(result)))
michael@0:                         sys.stderr.write("'"+result+"'\n")
michael@0:             if enforce_timeouts:
michael@0:                 if time.time() - last_output_time > OUTPUT_TIMEOUT:
michael@0:                     raise Timeout("Test output exceeded timeout (%ds)." %
michael@0:                                   OUTPUT_TIMEOUT, test_name, parseable)
michael@0:                 if time.time() - starttime > RUN_TIMEOUT:
michael@0:                     raise Timeout("Test run exceeded timeout (%ds)." %
michael@0:                                   RUN_TIMEOUT, test_name, parseable)
michael@0:     except:
michael@0:         runner.stop()
michael@0:         raise
michael@0:     else:
michael@0:         runner.wait(10)
michael@0:     finally:
michael@0:         outf.close()
michael@0:         if profile:
michael@0:             profile.cleanup()
michael@0: 
michael@0:     print >>sys.stderr, "Total time: %f seconds" % (time.time() - starttime)
michael@0: 
michael@0:     if result == 'OK':
michael@0:         print >>sys.stderr, "Program terminated successfully."
michael@0:         return 0
michael@0:     else:
michael@0:         print >>sys.stderr, "Program terminated unsuccessfully."
michael@0:         return -1