michael@0: #!/usr/bin/env python
michael@0: 
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0: # You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: """
michael@0: Setup mozbase packages for development.
michael@0: 
michael@0: Packages may be specified as command line arguments.
michael@0: If no arguments are given, install all packages.
michael@0: 
michael@0: See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase
michael@0: """
michael@0: 
michael@0: import os
michael@0: import subprocess
michael@0: import sys
michael@0: from optparse import OptionParser
michael@0: from subprocess import PIPE
michael@0: try:
michael@0:     from subprocess import check_call as call
michael@0: except ImportError:
michael@0:     from subprocess import call
michael@0: 
michael@0: 
michael@0: # directory containing this file
michael@0: here = os.path.dirname(os.path.abspath(__file__))
michael@0: 
michael@0: # all python packages
michael@0: mozbase_packages = [i for i in os.listdir(here)
michael@0:                     if os.path.exists(os.path.join(here, i, 'setup.py'))]
michael@0: test_packages = [ "mock" # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests
michael@0:                   ]
michael@0: extra_packages = [ "sphinx" # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation
michael@0:                   ]
michael@0: 
michael@0: def cycle_check(order, dependencies):
michael@0:     """ensure no cyclic dependencies"""
michael@0:     order_dict = dict([(j, i) for i, j in enumerate(order)])
michael@0:     for package, deps in dependencies.items():
michael@0:         index = order_dict[package]
michael@0:         for d in deps:
michael@0:             assert index > order_dict[d], "Cyclic dependencies detected"
michael@0: 
michael@0: def info(directory):
michael@0:     "get the package setup.py information"
michael@0: 
michael@0:     assert os.path.exists(os.path.join(directory, 'setup.py'))
michael@0: 
michael@0:     # setup the egg info
michael@0:     try:
michael@0:         call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
michael@0:     except subprocess.CalledProcessError:
michael@0:         print "Error running setup.py in %s" % directory
michael@0:         raise
michael@0: 
michael@0:     # get the .egg-info directory
michael@0:     egg_info = [entry for entry in os.listdir(directory)
michael@0:                 if entry.endswith('.egg-info')]
michael@0:     assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info)
michael@0:     egg_info = os.path.join(directory, egg_info[0])
michael@0:     assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
michael@0: 
michael@0:     # read the package information
michael@0:     pkg_info = os.path.join(egg_info, 'PKG-INFO')
michael@0:     info_dict = {}
michael@0:     for line in file(pkg_info).readlines():
michael@0:         if not line or line[0].isspace():
michael@0:             continue # XXX neglects description
michael@0:         assert ':' in line
michael@0:         key, value = [i.strip() for i in line.split(':', 1)]
michael@0:         info_dict[key] = value
michael@0: 
michael@0:     return info_dict
michael@0: 
michael@0: def get_dependencies(directory):
michael@0:     "returns the package name and dependencies given a package directory"
michael@0: 
michael@0:     # get the package metadata
michael@0:     info_dict = info(directory)
michael@0: 
michael@0:     # get the .egg-info directory
michael@0:     egg_info = [entry for entry in os.listdir(directory)
michael@0:                 if entry.endswith('.egg-info')][0]
michael@0: 
michael@0:     # read the dependencies
michael@0:     requires = os.path.join(directory, egg_info, 'requires.txt')
michael@0:     if os.path.exists(requires):
michael@0:         dependencies = [line.strip()
michael@0:                         for line in file(requires).readlines()
michael@0:                         if line.strip()]
michael@0:     else:
michael@0:         dependencies = []
michael@0: 
michael@0:     # return the information
michael@0:     return info_dict['Name'], dependencies
michael@0: 
michael@0: def dependency_info(dep):
michael@0:     "return dictionary of dependency information from a dependency string"
michael@0:     retval = dict(Name=None, Type=None, Version=None)
michael@0:     for joiner in ('==', '<=', '>='):
michael@0:         if joiner in dep:
michael@0:             retval['Type'] = joiner
michael@0:             name, version = [i.strip() for i in dep.split(joiner, 1)]
michael@0:             retval['Name'] = name
michael@0:             retval['Version'] = version
michael@0:             break
michael@0:     else:
michael@0:         retval['Name'] = dep.strip()
michael@0:     return retval
michael@0: 
michael@0: def unroll_dependencies(dependencies):
michael@0:     """
michael@0:     unroll a set of dependencies to a flat list
michael@0: 
michael@0:     dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
michael@0:                     'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
michael@0:                     'packageC': set(['packageE']),
michael@0:                     'packageE': set(['packageF', 'packageG']),
michael@0:                     'packageF': set(['packageG']),
michael@0:                     'packageX': set(['packageA', 'packageG'])}
michael@0:     """
michael@0: 
michael@0:     order = []
michael@0: 
michael@0:     # flatten all
michael@0:     packages = set(dependencies.keys())
michael@0:     for deps in dependencies.values():
michael@0:         packages.update(deps)
michael@0: 
michael@0:     while len(order) != len(packages):
michael@0: 
michael@0:         for package in packages.difference(order):
michael@0:             if set(dependencies.get(package, set())).issubset(order):
michael@0:                 order.append(package)
michael@0:                 break
michael@0:         else:
michael@0:             raise AssertionError("Cyclic dependencies detected")
michael@0: 
michael@0:     cycle_check(order, dependencies) # sanity check
michael@0: 
michael@0:     return order
michael@0: 
michael@0: 
michael@0: def main(args=sys.argv[1:]):
michael@0: 
michael@0:     # parse command line options
michael@0:     usage = '%prog [options] [package] [package] [...]'
michael@0:     parser = OptionParser(usage=usage, description=__doc__)
michael@0:     parser.add_option('-d', '--dependencies', dest='list_dependencies',
michael@0:                       action='store_true', default=False,
michael@0:                       help="list dependencies for the packages")
michael@0:     parser.add_option('--list', action='store_true', default=False,
michael@0:                       help="list what will be installed")
michael@0:     parser.add_option('--extra', '--install-extra-packages', action='store_true', default=False,
michael@0:                       help="installs extra supporting packages as well as core mozbase ones")
michael@0:     options, packages = parser.parse_args(args)
michael@0: 
michael@0:     if not packages:
michael@0:         # install all packages
michael@0:         packages = sorted(mozbase_packages)
michael@0: 
michael@0:     # ensure specified packages are in the list
michael@0:     assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
michael@0: 
michael@0:     if options.list_dependencies:
michael@0:         # list the package dependencies
michael@0:         for package in packages:
michael@0:             print '%s: %s' % get_dependencies(os.path.join(here, package))
michael@0:         parser.exit()
michael@0: 
michael@0:     # gather dependencies
michael@0:     # TODO: version conflict checking
michael@0:     deps = {}
michael@0:     alldeps = {}
michael@0:     mapping = {} # mapping from subdir name to package name
michael@0:     # core dependencies
michael@0:     for package in packages:
michael@0:         key, value = get_dependencies(os.path.join(here, package))
michael@0:         deps[key] = [dependency_info(dep)['Name'] for dep in value]
michael@0:         mapping[package] = key
michael@0: 
michael@0:         # keep track of all dependencies for non-mozbase packages
michael@0:         for dep in value:
michael@0:             alldeps[dependency_info(dep)['Name']] = ''.join(dep.split())
michael@0: 
michael@0:     # indirect dependencies
michael@0:     flag = True
michael@0:     while flag:
michael@0:         flag = False
michael@0:         for value in deps.values():
michael@0:             for dep in value:
michael@0:                 if dep in mozbase_packages and dep not in deps:
michael@0:                     key, value = get_dependencies(os.path.join(here, dep))
michael@0:                     deps[key] = [sanitize_dependency(dep) for dep in value]
michael@0: 
michael@0:                     for dep in value:
michael@0:                         alldeps[sanitize_dependency(dep)] = ''.join(dep.split())
michael@0:                     mapping[package] = key
michael@0:                     flag = True
michael@0:                     break
michael@0:             if flag:
michael@0:                 break
michael@0: 
michael@0:     # get the remaining names for the mapping
michael@0:     for package in mozbase_packages:
michael@0:         if package in mapping:
michael@0:             continue
michael@0:         key, value = get_dependencies(os.path.join(here, package))
michael@0:         mapping[package] = key
michael@0: 
michael@0:     # unroll dependencies
michael@0:     unrolled = unroll_dependencies(deps)
michael@0: 
michael@0:     # make a reverse mapping: package name -> subdirectory
michael@0:     reverse_mapping = dict([(j,i) for i, j in mapping.items()])
michael@0: 
michael@0:     # we only care about dependencies in mozbase
michael@0:     unrolled = [package for package in unrolled if package in reverse_mapping]
michael@0: 
michael@0:     if options.list:
michael@0:         # list what will be installed
michael@0:         for package in unrolled:
michael@0:             print package
michael@0:         parser.exit()
michael@0: 
michael@0:     # set up the packages for development
michael@0:     for package in unrolled:
michael@0:         call([sys.executable, 'setup.py', 'develop', '--no-deps'],
michael@0:              cwd=os.path.join(here, reverse_mapping[package]))
michael@0: 
michael@0:     # add the directory of sys.executable to path to aid the correct
michael@0:     # `easy_install` getting called
michael@0:     # https://bugzilla.mozilla.org/show_bug.cgi?id=893878
michael@0:     os.environ['PATH'] = '%s%s%s' % (os.path.dirname(os.path.abspath(sys.executable)),
michael@0:                                      os.path.pathsep,
michael@0:                                      os.environ.get('PATH', '').strip(os.path.pathsep))
michael@0: 
michael@0:     # install non-mozbase dependencies
michael@0:     # these need to be installed separately and the --no-deps flag
michael@0:     # subsequently used due to a bug in setuptools; see
michael@0:     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
michael@0:     pypi_deps = dict([(i, j) for i,j in alldeps.items()
michael@0:                       if i not in unrolled])
michael@0:     for package, version in pypi_deps.items():
michael@0:         # easy_install should be available since we rely on setuptools
michael@0:         call(['easy_install', version])
michael@0: 
michael@0:     # install packages required for unit testing
michael@0:     for package in test_packages:
michael@0:         call(['easy_install', package])
michael@0: 
michael@0:     # install extra non-mozbase packages if desired
michael@0:     if options.extra:
michael@0:         for package in extra_packages:
michael@0:             call(['easy_install', package])
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     main()