The Tor Browser: addon-sdk/source/python-lib/mozrunner/__init_

addon-sdk/source/python-lib/mozrunner/init.py@b8a032363ba2 (annotated)

addon-sdk/source/python-lib/mozrunner/init.py

Thu, 22 Jan 2015 13:21:57 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 22 Jan 2015 13:21:57 +0100
branch: TOR_BUG_9701
changeset 15: b8a032363ba2
permissions: -rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
 import sys
 import copy
 import tempfile
 import signal
 import commands
 import zipfile
 import optparse
 import killableprocess
 import subprocess
 import platform
 import shutil
 from StringIO import StringIO
 from xml.dom import minidom
 from distutils import dir_util
 from time import sleep
 # conditional (version-dependent) imports
 try:
     import simplejson
 except ImportError:
     import json as simplejson
 import logging
 logger = logging.getLogger(__name__)
 # Use dir_util for copy/rm operations because shutil is all kinds of broken
 copytree = dir_util.copy_tree
 rmtree = dir_util.remove_tree
 def findInPath(fileName, path=os.environ['PATH']):
     dirs = path.split(os.pathsep)
     for dir in dirs:
         if os.path.isfile(os.path.join(dir, fileName)):
             return os.path.join(dir, fileName)
         if os.name == 'nt' or sys.platform == 'cygwin':
             if os.path.isfile(os.path.join(dir, fileName + ".exe")):
                 return os.path.join(dir, fileName + ".exe")
     return None
 stdout = sys.stdout
 stderr = sys.stderr
 stdin = sys.stdin
 def run_command(cmd, env=None, **kwargs):
     """Run the given command in killable process."""
     killable_kwargs = {'stdout':stdout ,'stderr':stderr, 'stdin':stdin}
     killable_kwargs.update(kwargs)
     if sys.platform != "win32":
         return killableprocess.Popen(cmd, preexec_fn=lambda : os.setpgid(0, 0),
                                      env=env, **killable_kwargs)
     else:
         return killableprocess.Popen(cmd, env=env, **killable_kwargs)
 def getoutput(l):
     tmp = tempfile.mktemp()
     x = open(tmp, 'w')
     subprocess.call(l, stdout=x, stderr=x)
     x.close(); x = open(tmp, 'r')
     r = x.read() ; x.close()
     os.remove(tmp)
     return r
 def get_pids(name, minimun_pid=0):
     """Get all the pids matching name, exclude any pids below minimum_pid."""
     if os.name == 'nt' or sys.platform == 'cygwin':
         import wpk
         pids = wpk.get_pids(name)
     else:
         data = getoutput(['ps', 'ax']).splitlines()
         pids = [int(line.split()[0]) for line in data if line.find(name) is not -1]
     matching_pids = [m for m in pids if m > minimun_pid]
     return matching_pids
 def makedirs(name):
     head, tail = os.path.split(name)
     if not tail:
         head, tail = os.path.split(head)
     if head and tail and not os.path.exists(head):
         try:
             makedirs(head)
         except OSError, e:
             pass
         if tail == os.curdir:           # xxx/newdir/. exists if xxx/newdir exists
             return
     try:
         os.mkdir(name)
     except:
         pass
 # addon_details() copied from mozprofile
 def addon_details(install_rdf_fh):
     """
     returns a dictionary of details about the addon
     - addon_path : path to the addon directory
     Returns:
     {'id':      u'rainbow@colors.org', # id of the addon
      'version': u'1.4',                # version of the addon
      'name':    u'Rainbow',            # name of the addon
      'unpack': # whether to unpack the addon
     """
     details = {
         'id': None,
         'unpack': False,
         'name': None,
         'version': None
     }
     def get_namespace_id(doc, url):
         attributes = doc.documentElement.attributes
         namespace = ""
         for i in range(attributes.length):
             if attributes.item(i).value == url:
                 if ":" in attributes.item(i).name:
                     # If the namespace is not the default one remove 'xlmns:'
                     namespace = attributes.item(i).name.split(':')[1] + ":"
                     break
         return namespace
     def get_text(element):
         """Retrieve the text value of a given node"""
         rc = []
         for node in element.childNodes:
             if node.nodeType == node.TEXT_NODE:
                 rc.append(node.data)
         return ''.join(rc).strip()
     doc = minidom.parse(install_rdf_fh)
     # Get the namespaces abbreviations
     em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#")
     rdf = get_namespace_id(doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#")
     description = doc.getElementsByTagName(rdf + "Description").item(0)
     for node in description.childNodes:
         # Remove the namespace prefix from the tag for comparison
         entry = node.nodeName.replace(em, "")
         if entry in details.keys():
             details.update({ entry: get_text(node) })
     # turn unpack into a true/false value
     if isinstance(details['unpack'], basestring):
         details['unpack'] = details['unpack'].lower() == 'true'
     return details
 class Profile(object):
     """Handles all operations regarding profile. Created new profiles, installs extensions,
     sets preferences and handles cleanup."""
     def __init__(self, binary=None, profile=None, addons=None,
                  preferences=None):
         self.binary = binary
         self.create_new = not(bool(profile))
         if profile:
             self.profile = profile
         else:
             self.profile = self.create_new_profile(self.binary)
         self.addons_installed = []
         self.addons = addons or []
         ### set preferences from class preferences
         preferences = preferences or {}
         if hasattr(self.__class__, 'preferences'):
             self.preferences = self.__class__.preferences.copy()
         else:
             self.preferences = {}
         self.preferences.update(preferences)
         for addon in self.addons:
             self.install_addon(addon)
         self.set_preferences(self.preferences)
     def create_new_profile(self, binary):
         """Create a new clean profile in tmp which is a simple empty folder"""
         profile = tempfile.mkdtemp(suffix='.mozrunner')
         return profile
     def unpack_addon(self, xpi_zipfile, addon_path):
         for name in xpi_zipfile.namelist():
             if name.endswith('/'):
                 makedirs(os.path.join(addon_path, name))
             else:
                 if not os.path.isdir(os.path.dirname(os.path.join(addon_path, name))):
                     makedirs(os.path.dirname(os.path.join(addon_path, name)))
                 data = xpi_zipfile.read(name)
                 f = open(os.path.join(addon_path, name), 'wb')
                 f.write(data) ; f.close()
                 zi = xpi_zipfile.getinfo(name)
                 os.chmod(os.path.join(addon_path,name), (zi.external_attr>>16))
     def install_addon(self, path):
         """Installs the given addon or directory of addons in the profile."""
         extensions_path = os.path.join(self.profile, 'extensions')
         if not os.path.exists(extensions_path):
             os.makedirs(extensions_path)
         addons = [path]
         if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')):
             addons = [os.path.join(path, x) for x in os.listdir(path)]
         for addon in addons:
             if addon.endswith('.xpi'):
                 xpi_zipfile = zipfile.ZipFile(addon, "r")
                 details = addon_details(StringIO(xpi_zipfile.read('install.rdf')))
                 addon_path = os.path.join(extensions_path, details["id"])
                 if details.get("unpack", True):
                     self.unpack_addon(xpi_zipfile, addon_path)
                     self.addons_installed.append(addon_path)
                 else:
                     shutil.copy(addon, addon_path + '.xpi')
             else:
                 # it's already unpacked, but we need to extract the id so we
                 # can copy it
                 details = addon_details(open(os.path.join(addon, "install.rdf"), "rb"))
                 addon_path = os.path.join(extensions_path, details["id"])
                 shutil.copytree(addon, addon_path, symlinks=True)
     def set_preferences(self, preferences):
         """Adds preferences dict to profile preferences"""
         prefs_file = os.path.join(self.profile, 'user.js')
         # Ensure that the file exists first otherwise create an empty file
         if os.path.isfile(prefs_file):
             f = open(prefs_file, 'a+')
         else:
             f = open(prefs_file, 'w')
         f.write('\n#MozRunner Prefs Start\n')
         pref_lines = ['user_pref(%s, %s);' %
                       (simplejson.dumps(k), simplejson.dumps(v) ) for k, v in
                        preferences.items()]
         for line in pref_lines:
             f.write(line+'\n')
         f.write('#MozRunner Prefs End\n')
         f.flush() ; f.close()
     def pop_preferences(self):
         """
         pop the last set of preferences added
         returns True if popped
         """
         # our magic markers
         delimeters = ('#MozRunner Prefs Start', '#MozRunner Prefs End')
         lines = file(os.path.join(self.profile, 'user.js')).read().splitlines()
         def last_index(_list, value):
             """
             returns the last index of an item;
             this should actually be part of python code but it isn't
             """
             for index in reversed(range(len(_list))):
                 if _list[index] == value:
                     return index
         s = last_index(lines, delimeters[0])
         e = last_index(lines, delimeters[1])
         # ensure both markers are found
         if s is None:
             assert e is None, '%s found without %s' % (delimeters[1], delimeters[0])
             return False # no preferences found
         elif e is None:
             assert e is None, '%s found without %s' % (delimeters[0], delimeters[1])
         # ensure the markers are in the proper order
         assert e > s, '%s found at %s, while %s found at %s' (delimeter[1], e, delimeter[0], s)
         # write the prefs
         cleaned_prefs = '\n'.join(lines[:s] + lines[e+1:])
         f = file(os.path.join(self.profile, 'user.js'), 'w')
         f.write(cleaned_prefs)
         f.close()
         return True
     def clean_preferences(self):
         """Removed preferences added by mozrunner."""
         while True:
             if not self.pop_preferences():
                 break
     def clean_addons(self):
         """Cleans up addons in the profile."""
         for addon in self.addons_installed:
             if os.path.isdir(addon):
                 rmtree(addon)
     def cleanup(self):
         """Cleanup operations on the profile."""
         def oncleanup_error(function, path, excinfo):
             #TODO: How should we handle this?
             print "Error Cleaning up: " + str(excinfo[1])
         if self.create_new:
             shutil.rmtree(self.profile, False, oncleanup_error)
         else:
             self.clean_preferences()
             self.clean_addons()
 class FirefoxProfile(Profile):
     """Specialized Profile subclass for Firefox"""
     preferences = {# Don't automatically update the application
                    'app.update.enabled' : False,
                    # Don't restore the last open set of tabs if the browser has crashed
                    'browser.sessionstore.resume_from_crash': False,
                    # Don't check for the default web browser
                    'browser.shell.checkDefaultBrowser' : False,
                    # Don't warn on exit when multiple tabs are open
                    'browser.tabs.warnOnClose' : False,
                    # Don't warn when exiting the browser
                    'browser.warnOnQuit': False,
                    # Only install add-ons from the profile and the app folder
                    'extensions.enabledScopes' : 5,
                    # Don't automatically update add-ons
                    'extensions.update.enabled'    : False,
                    # Don't open a dialog to show available add-on updates
                    'extensions.update.notifyUser' : False,
                    }
     # The possible names of application bundles on Mac OS X, in order of
     # preference from most to least preferred.
     # Note: Nightly is obsolete, as it has been renamed to FirefoxNightly,
     # but it will still be present if users update an older nightly build
     # via the app update service.
     bundle_names = ['Firefox', 'FirefoxNightly', 'Nightly']
     # The possible names of binaries, in order of preference from most to least
     # preferred.
     @property
     def names(self):
         if sys.platform == 'darwin':
             return ['firefox', 'nightly', 'shiretoko']
         if (sys.platform == 'linux2') or (sys.platform in ('sunos5', 'solaris')):
             return ['firefox', 'mozilla-firefox', 'iceweasel']
         if os.name == 'nt' or sys.platform == 'cygwin':
             return ['firefox']
 class ThunderbirdProfile(Profile):
     preferences = {'extensions.update.enabled'    : False,
                    'extensions.update.notifyUser' : False,
                    'browser.shell.checkDefaultBrowser' : False,
                    'browser.tabs.warnOnClose' : False,
                    'browser.warnOnQuit': False,
                    'browser.sessionstore.resume_from_crash': False,
                    }
     # The possible names of application bundles on Mac OS X, in order of
     # preference from most to least preferred.
     bundle_names = ["Thunderbird", "Shredder"]
     # The possible names of binaries, in order of preference from most to least
     # preferred.
     names = ["thunderbird", "shredder"]
 class Runner(object):
     """Handles all running operations. Finds bins, runs and kills the process."""
     def __init__(self, binary=None, profile=None, cmdargs=[], env=None,
                  kp_kwargs={}):
         if binary is None:
             self.binary = self.find_binary()
         elif sys.platform == 'darwin' and binary.find('Contents/MacOS/') == -1:
             self.binary = os.path.join(binary, 'Contents/MacOS/%s-bin' % self.names[0])
         else:
             self.binary = binary
         if not os.path.exists(self.binary):
             raise Exception("Binary path does not exist "+self.binary)
         if sys.platform == 'linux2' and self.binary.endswith('-bin'):
             dirname = os.path.dirname(self.binary)
             if os.environ.get('LD_LIBRARY_PATH', None):
                 os.environ['LD_LIBRARY_PATH'] = '%s:%s' % (os.environ['LD_LIBRARY_PATH'], dirname)
             else:
                 os.environ['LD_LIBRARY_PATH'] = dirname
         # Disable the crash reporter by default
         os.environ['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
         self.profile = profile
         self.cmdargs = cmdargs
         if env is None:
             self.env = copy.copy(os.environ)
             self.env.update({'MOZ_NO_REMOTE':"1",})
         else:
             self.env = env
         self.kp_kwargs = kp_kwargs or {}
     def find_binary(self):
         """Finds the binary for self.names if one was not provided."""
         binary = None
         if sys.platform in ('linux2', 'sunos5', 'solaris') \
                 or sys.platform.startswith('freebsd'):
             for name in reversed(self.names):
                 binary = findInPath(name)
         elif os.name == 'nt' or sys.platform == 'cygwin':
             # find the default executable from the windows registry
             try:
                 import _winreg
             except ImportError:
                 pass
             else:
                 sam_flags = [0]
                 # KEY_WOW64_32KEY etc only appeared in 2.6+, but that's OK as
                 # only 2.6+ has functioning 64bit builds.
                 if hasattr(_winreg, "KEY_WOW64_32KEY"):
                     if "64 bit" in sys.version:
                         # a 64bit Python should also look in the 32bit registry
                         sam_flags.append(_winreg.KEY_WOW64_32KEY)
                     else:
                         # possibly a 32bit Python on 64bit Windows, so look in
                         # the 64bit registry incase there is a 64bit app.
                         sam_flags.append(_winreg.KEY_WOW64_64KEY)
                 for sam_flag in sam_flags:
                     try:
                         # assumes self.app_name is defined, as it should be for
                         # implementors
                         keyname = r"Software\Mozilla\Mozilla %s" % self.app_name
                         sam = _winreg.KEY_READ | sam_flag
                         app_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, keyname, 0, sam)
                         version, _type = _winreg.QueryValueEx(app_key, "CurrentVersion")
                         version_key = _winreg.OpenKey(app_key, version + r"\Main")
                         path, _ = _winreg.QueryValueEx(version_key, "PathToExe")
                         return path
                     except _winreg.error:
                         pass
             # search for the binary in the path
             for name in reversed(self.names):
                 binary = findInPath(name)
                 if sys.platform == 'cygwin':
                     program_files = os.environ['PROGRAMFILES']
                 else:
                     program_files = os.environ['ProgramFiles']
                 if binary is None:
                     for bin in [(program_files, 'Mozilla Firefox', 'firefox.exe'),
                                 (os.environ.get("ProgramFiles(x86)"),'Mozilla Firefox', 'firefox.exe'),
                                 (program_files, 'Nightly', 'firefox.exe'),
                                 (os.environ.get("ProgramFiles(x86)"),'Nightly', 'firefox.exe'),
                                 (program_files, 'Aurora', 'firefox.exe'),
                                 (os.environ.get("ProgramFiles(x86)"),'Aurora', 'firefox.exe')
                                 ]:
                         path = os.path.join(*bin)
                         if os.path.isfile(path):
                             binary = path
                             break
         elif sys.platform == 'darwin':
             for bundle_name in self.bundle_names:
                 # Look for the application bundle in the user's home directory
                 # or the system-wide /Applications directory.  If we don't find
                 # it in one of those locations, we move on to the next possible
                 # bundle name.
                 appdir = os.path.join("~/Applications/%s.app" % bundle_name)
                 if not os.path.isdir(appdir):
                     appdir = "/Applications/%s.app" % bundle_name
                 if not os.path.isdir(appdir):
                     continue
                 # Look for a binary with any of the possible binary names
                 # inside the application bundle.
                 for binname in self.names:
                     binpath = os.path.join(appdir,
                                            "Contents/MacOS/%s-bin" % binname)
                     if (os.path.isfile(binpath)):
                         binary = binpath
                         break
                 if binary:
                     break
         if binary is None:
             raise Exception('Mozrunner could not locate your binary, you will need to set it.')
         return binary
     @property
     def command(self):
         """Returns the command list to run."""
         cmd = [self.binary, '-profile', self.profile.profile]
         # On i386 OS X machines, i386+x86_64 universal binaries need to be told
         # to run as i386 binaries.  If we're not running a i386+x86_64 universal
         # binary, then this command modification is harmless.
         if sys.platform == 'darwin':
             if hasattr(platform, 'architecture') and platform.architecture()[0] == '32bit':
                 cmd = ['arch', '-i386'] + cmd
         return cmd
     def get_repositoryInfo(self):
         """Read repository information from application.ini and platform.ini."""
         import ConfigParser
         config = ConfigParser.RawConfigParser()
         dirname = os.path.dirname(self.binary)
         repository = { }
         for entry in [['application', 'App'], ['platform', 'Build']]:
             (file, section) = entry
             config.read(os.path.join(dirname, '%s.ini' % file))
             for entry in [['SourceRepository', 'repository'], ['SourceStamp', 'changeset']]:
                 (key, id) = entry
                 try:
                     repository['%s_%s' % (file, id)] = config.get(section, key);
                 except:
                     repository['%s_%s' % (file, id)] = None
         return repository
     def start(self):
         """Run self.command in the proper environment."""
         if self.profile is None:
             self.profile = self.profile_class()
         self.process_handler = run_command(self.command+self.cmdargs, self.env, **self.kp_kwargs)
     def wait(self, timeout=None):
         """Wait for the browser to exit."""
         self.process_handler.wait(timeout=timeout)
         if sys.platform != 'win32':
             for name in self.names:
                 for pid in get_pids(name, self.process_handler.pid):
                     self.process_handler.pid = pid
                     self.process_handler.wait(timeout=timeout)
     def kill(self, kill_signal=signal.SIGTERM):
         """Kill the browser"""
         if sys.platform != 'win32':
             self.process_handler.kill()
             for name in self.names:
                 for pid in get_pids(name, self.process_handler.pid):
                     self.process_handler.pid = pid
                     self.process_handler.kill()
         else:
             try:
                 self.process_handler.kill(group=True)
                 # On windows, it sometimes behooves one to wait for dust to settle
                 # after killing processes.  Let's try that.
                 # TODO: Bug 640047 is invesitgating the correct way to handle this case
                 self.process_handler.wait(timeout=10)
             except Exception, e:
                 logger.error('Cannot kill process, '+type(e).__name__+' '+e.message)
     def stop(self):
         self.kill()
 class FirefoxRunner(Runner):
     """Specialized Runner subclass for running Firefox."""
     app_name = 'Firefox'
     profile_class = FirefoxProfile
     # The possible names of application bundles on Mac OS X, in order of
     # preference from most to least preferred.
     # Note: Nightly is obsolete, as it has been renamed to FirefoxNightly,
     # but it will still be present if users update an older nightly build
     # only via the app update service.
     bundle_names = ['Firefox', 'FirefoxNightly', 'Nightly']
     @property
     def names(self):
         if sys.platform == 'darwin':
             return ['firefox', 'nightly', 'shiretoko']
         if sys.platform in ('linux2', 'sunos5', 'solaris') \
                 or sys.platform.startswith('freebsd'):
             return ['firefox', 'mozilla-firefox', 'iceweasel']
         if os.name == 'nt' or sys.platform == 'cygwin':
             return ['firefox']
 class ThunderbirdRunner(Runner):
     """Specialized Runner subclass for running Thunderbird"""
     app_name = 'Thunderbird'
     profile_class = ThunderbirdProfile
     # The possible names of application bundles on Mac OS X, in order of
     # preference from most to least preferred.
     bundle_names = ["Thunderbird", "Shredder"]
     # The possible names of binaries, in order of preference from most to least
     # preferred.
     names = ["thunderbird", "shredder"]
 class CLI(object):
     """Command line interface."""
     runner_class = FirefoxRunner
     profile_class = FirefoxProfile
     module = "mozrunner"
     parser_options = {("-b", "--binary",): dict(dest="binary", help="Binary path.",
                                                 metavar=None, default=None),
                       ('-p', "--profile",): dict(dest="profile", help="Profile path.",
                                                  metavar=None, default=None),
                       ('-a', "--addons",): dict(dest="addons",
                                                 help="Addons paths to install.",
                                                 metavar=None, default=None),
                       ("--info",): dict(dest="info", default=False,
                                         action="store_true",
                                         help="Print module information")
                      }
     def __init__(self):
         """ Setup command line parser and parse arguments """
         self.metadata = self.get_metadata_from_egg()
         self.parser = optparse.OptionParser(version="%prog " + self.metadata["Version"])
         for names, opts in self.parser_options.items():
             self.parser.add_option(*names, **opts)
         (self.options, self.args) = self.parser.parse_args()
         if self.options.info:
             self.print_metadata()
             sys.exit(0)
         # XXX should use action='append' instead of rolling our own
         try:
             self.addons = self.options.addons.split(',')
         except:
             self.addons = []
     def get_metadata_from_egg(self):
         import pkg_resources
         ret = {}
         dist = pkg_resources.get_distribution(self.module)
         if dist.has_metadata("PKG-INFO"):
             for line in dist.get_metadata_lines("PKG-INFO"):
                 key, value = line.split(':', 1)
                 ret[key] = value
         if dist.has_metadata("requires.txt"):
             ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")
         return ret
     def print_metadata(self, data=("Name", "Version", "Summary", "Home-page",
                                    "Author", "Author-email", "License", "Platform", "Dependencies")):
         for key in data:
             if key in self.metadata:
                 print key + ": " + self.metadata[key]
     def create_runner(self):
         """ Get the runner object """
         runner = self.get_runner(binary=self.options.binary)
         profile = self.get_profile(binary=runner.binary,
                                    profile=self.options.profile,
                                    addons=self.addons)
         runner.profile = profile
         return runner
     def get_runner(self, binary=None, profile=None):
         """Returns the runner instance for the given command line binary argument
         the profile instance returned from self.get_profile()."""
         return self.runner_class(binary, profile)
     def get_profile(self, binary=None, profile=None, addons=None, preferences=None):
         """Returns the profile instance for the given command line arguments."""
         addons = addons or []
         preferences = preferences or {}
         return self.profile_class(binary, profile, addons, preferences)
     def run(self):
         runner = self.create_runner()
         self.start(runner)
         runner.profile.cleanup()
     def start(self, runner):
         """Starts the runner and waits for Firefox to exitor Keyboard Interrupt.
         Shoule be overwritten to provide custom running of the runner instance."""
         runner.start()
         print 'Started:', ' '.join(runner.command)
         try:
             runner.wait()
         except KeyboardInterrupt:
             runner.stop()
 def cli():
     CLI().run()

The Tor Browser / annotate

addon-sdk/source/python-lib/mozrunner/__init__.py@b8a032363ba2 (annotated)

addon-sdk/source/python-lib/mozrunner/__init__.py

addon-sdk/source/python-lib/mozrunner/init.py@b8a032363ba2 (annotated)

addon-sdk/source/python-lib/mozrunner/init.py