1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/mozbase/setup_development.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,260 @@
1.4 +#!/usr/bin/env python
1.5 +
1.6 +# This Source Code Form is subject to the terms of the Mozilla Public
1.7 +# License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.8 +# You can obtain one at http://mozilla.org/MPL/2.0/.
1.9 +
1.10 +"""
1.11 +Setup mozbase packages for development.
1.12 +
1.13 +Packages may be specified as command line arguments.
1.14 +If no arguments are given, install all packages.
1.15 +
1.16 +See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase
1.17 +"""
1.18 +
1.19 +import os
1.20 +import subprocess
1.21 +import sys
1.22 +from optparse import OptionParser
1.23 +from subprocess import PIPE
1.24 +try:
1.25 + from subprocess import check_call as call
1.26 +except ImportError:
1.27 + from subprocess import call
1.28 +
1.29 +
1.30 +# directory containing this file
1.31 +here = os.path.dirname(os.path.abspath(__file__))
1.32 +
1.33 +# all python packages
1.34 +mozbase_packages = [i for i in os.listdir(here)
1.35 + if os.path.exists(os.path.join(here, i, 'setup.py'))]
1.36 +test_packages = [ "mock" # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests
1.37 + ]
1.38 +extra_packages = [ "sphinx" # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation
1.39 + ]
1.40 +
1.41 +def cycle_check(order, dependencies):
1.42 + """ensure no cyclic dependencies"""
1.43 + order_dict = dict([(j, i) for i, j in enumerate(order)])
1.44 + for package, deps in dependencies.items():
1.45 + index = order_dict[package]
1.46 + for d in deps:
1.47 + assert index > order_dict[d], "Cyclic dependencies detected"
1.48 +
1.49 +def info(directory):
1.50 + "get the package setup.py information"
1.51 +
1.52 + assert os.path.exists(os.path.join(directory, 'setup.py'))
1.53 +
1.54 + # setup the egg info
1.55 + try:
1.56 + call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
1.57 + except subprocess.CalledProcessError:
1.58 + print "Error running setup.py in %s" % directory
1.59 + raise
1.60 +
1.61 + # get the .egg-info directory
1.62 + egg_info = [entry for entry in os.listdir(directory)
1.63 + if entry.endswith('.egg-info')]
1.64 + assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info)
1.65 + egg_info = os.path.join(directory, egg_info[0])
1.66 + assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
1.67 +
1.68 + # read the package information
1.69 + pkg_info = os.path.join(egg_info, 'PKG-INFO')
1.70 + info_dict = {}
1.71 + for line in file(pkg_info).readlines():
1.72 + if not line or line[0].isspace():
1.73 + continue # XXX neglects description
1.74 + assert ':' in line
1.75 + key, value = [i.strip() for i in line.split(':', 1)]
1.76 + info_dict[key] = value
1.77 +
1.78 + return info_dict
1.79 +
1.80 +def get_dependencies(directory):
1.81 + "returns the package name and dependencies given a package directory"
1.82 +
1.83 + # get the package metadata
1.84 + info_dict = info(directory)
1.85 +
1.86 + # get the .egg-info directory
1.87 + egg_info = [entry for entry in os.listdir(directory)
1.88 + if entry.endswith('.egg-info')][0]
1.89 +
1.90 + # read the dependencies
1.91 + requires = os.path.join(directory, egg_info, 'requires.txt')
1.92 + if os.path.exists(requires):
1.93 + dependencies = [line.strip()
1.94 + for line in file(requires).readlines()
1.95 + if line.strip()]
1.96 + else:
1.97 + dependencies = []
1.98 +
1.99 + # return the information
1.100 + return info_dict['Name'], dependencies
1.101 +
1.102 +def dependency_info(dep):
1.103 + "return dictionary of dependency information from a dependency string"
1.104 + retval = dict(Name=None, Type=None, Version=None)
1.105 + for joiner in ('==', '<=', '>='):
1.106 + if joiner in dep:
1.107 + retval['Type'] = joiner
1.108 + name, version = [i.strip() for i in dep.split(joiner, 1)]
1.109 + retval['Name'] = name
1.110 + retval['Version'] = version
1.111 + break
1.112 + else:
1.113 + retval['Name'] = dep.strip()
1.114 + return retval
1.115 +
1.116 +def unroll_dependencies(dependencies):
1.117 + """
1.118 + unroll a set of dependencies to a flat list
1.119 +
1.120 + dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']),
1.121 + 'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']),
1.122 + 'packageC': set(['packageE']),
1.123 + 'packageE': set(['packageF', 'packageG']),
1.124 + 'packageF': set(['packageG']),
1.125 + 'packageX': set(['packageA', 'packageG'])}
1.126 + """
1.127 +
1.128 + order = []
1.129 +
1.130 + # flatten all
1.131 + packages = set(dependencies.keys())
1.132 + for deps in dependencies.values():
1.133 + packages.update(deps)
1.134 +
1.135 + while len(order) != len(packages):
1.136 +
1.137 + for package in packages.difference(order):
1.138 + if set(dependencies.get(package, set())).issubset(order):
1.139 + order.append(package)
1.140 + break
1.141 + else:
1.142 + raise AssertionError("Cyclic dependencies detected")
1.143 +
1.144 + cycle_check(order, dependencies) # sanity check
1.145 +
1.146 + return order
1.147 +
1.148 +
1.149 +def main(args=sys.argv[1:]):
1.150 +
1.151 + # parse command line options
1.152 + usage = '%prog [options] [package] [package] [...]'
1.153 + parser = OptionParser(usage=usage, description=__doc__)
1.154 + parser.add_option('-d', '--dependencies', dest='list_dependencies',
1.155 + action='store_true', default=False,
1.156 + help="list dependencies for the packages")
1.157 + parser.add_option('--list', action='store_true', default=False,
1.158 + help="list what will be installed")
1.159 + parser.add_option('--extra', '--install-extra-packages', action='store_true', default=False,
1.160 + help="installs extra supporting packages as well as core mozbase ones")
1.161 + options, packages = parser.parse_args(args)
1.162 +
1.163 + if not packages:
1.164 + # install all packages
1.165 + packages = sorted(mozbase_packages)
1.166 +
1.167 + # ensure specified packages are in the list
1.168 + assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
1.169 +
1.170 + if options.list_dependencies:
1.171 + # list the package dependencies
1.172 + for package in packages:
1.173 + print '%s: %s' % get_dependencies(os.path.join(here, package))
1.174 + parser.exit()
1.175 +
1.176 + # gather dependencies
1.177 + # TODO: version conflict checking
1.178 + deps = {}
1.179 + alldeps = {}
1.180 + mapping = {} # mapping from subdir name to package name
1.181 + # core dependencies
1.182 + for package in packages:
1.183 + key, value = get_dependencies(os.path.join(here, package))
1.184 + deps[key] = [dependency_info(dep)['Name'] for dep in value]
1.185 + mapping[package] = key
1.186 +
1.187 + # keep track of all dependencies for non-mozbase packages
1.188 + for dep in value:
1.189 + alldeps[dependency_info(dep)['Name']] = ''.join(dep.split())
1.190 +
1.191 + # indirect dependencies
1.192 + flag = True
1.193 + while flag:
1.194 + flag = False
1.195 + for value in deps.values():
1.196 + for dep in value:
1.197 + if dep in mozbase_packages and dep not in deps:
1.198 + key, value = get_dependencies(os.path.join(here, dep))
1.199 + deps[key] = [sanitize_dependency(dep) for dep in value]
1.200 +
1.201 + for dep in value:
1.202 + alldeps[sanitize_dependency(dep)] = ''.join(dep.split())
1.203 + mapping[package] = key
1.204 + flag = True
1.205 + break
1.206 + if flag:
1.207 + break
1.208 +
1.209 + # get the remaining names for the mapping
1.210 + for package in mozbase_packages:
1.211 + if package in mapping:
1.212 + continue
1.213 + key, value = get_dependencies(os.path.join(here, package))
1.214 + mapping[package] = key
1.215 +
1.216 + # unroll dependencies
1.217 + unrolled = unroll_dependencies(deps)
1.218 +
1.219 + # make a reverse mapping: package name -> subdirectory
1.220 + reverse_mapping = dict([(j,i) for i, j in mapping.items()])
1.221 +
1.222 + # we only care about dependencies in mozbase
1.223 + unrolled = [package for package in unrolled if package in reverse_mapping]
1.224 +
1.225 + if options.list:
1.226 + # list what will be installed
1.227 + for package in unrolled:
1.228 + print package
1.229 + parser.exit()
1.230 +
1.231 + # set up the packages for development
1.232 + for package in unrolled:
1.233 + call([sys.executable, 'setup.py', 'develop', '--no-deps'],
1.234 + cwd=os.path.join(here, reverse_mapping[package]))
1.235 +
1.236 + # add the directory of sys.executable to path to aid the correct
1.237 + # `easy_install` getting called
1.238 + # https://bugzilla.mozilla.org/show_bug.cgi?id=893878
1.239 + os.environ['PATH'] = '%s%s%s' % (os.path.dirname(os.path.abspath(sys.executable)),
1.240 + os.path.pathsep,
1.241 + os.environ.get('PATH', '').strip(os.path.pathsep))
1.242 +
1.243 + # install non-mozbase dependencies
1.244 + # these need to be installed separately and the --no-deps flag
1.245 + # subsequently used due to a bug in setuptools; see
1.246 + # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
1.247 + pypi_deps = dict([(i, j) for i,j in alldeps.items()
1.248 + if i not in unrolled])
1.249 + for package, version in pypi_deps.items():
1.250 + # easy_install should be available since we rely on setuptools
1.251 + call(['easy_install', version])
1.252 +
1.253 + # install packages required for unit testing
1.254 + for package in test_packages:
1.255 + call(['easy_install', package])
1.256 +
1.257 + # install extra non-mozbase packages if desired
1.258 + if options.extra:
1.259 + for package in extra_packages:
1.260 + call(['easy_install', package])
1.261 +
1.262 +if __name__ == '__main__':
1.263 + main()
