1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/addon-sdk/source/python-lib/cuddlefish/packaging.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,471 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +import os
1.9 +import sys
1.10 +import re
1.11 +import copy
1.12 +
1.13 +import simplejson as json
1.14 +from cuddlefish.bunch import Bunch
1.15 +
1.16 +MANIFEST_NAME = 'package.json'
1.17 +DEFAULT_LOADER = 'addon-sdk'
1.18 +
1.19 +# Is different from root_dir when running tests
1.20 +env_root = os.environ.get('CUDDLEFISH_ROOT')
1.21 +
1.22 +DEFAULT_PROGRAM_MODULE = 'main'
1.23 +
1.24 +DEFAULT_ICON = 'icon.png'
1.25 +DEFAULT_ICON64 = 'icon64.png'
1.26 +
1.27 +METADATA_PROPS = ['name', 'description', 'keywords', 'author', 'version',
1.28 + 'translators', 'contributors', 'license', 'homepage', 'icon',
1.29 + 'icon64', 'main', 'directories', 'permissions']
1.30 +
1.31 +RESOURCE_HOSTNAME_RE = re.compile(r'^[a-z0-9_\-]+$')
1.32 +
1.33 +class Error(Exception):
1.34 + pass
1.35 +
1.36 +class MalformedPackageError(Error):
1.37 + pass
1.38 +
1.39 +class MalformedJsonFileError(Error):
1.40 + pass
1.41 +
1.42 +class DuplicatePackageError(Error):
1.43 + pass
1.44 +
1.45 +class PackageNotFoundError(Error):
1.46 + def __init__(self, missing_package, reason):
1.47 + self.missing_package = missing_package
1.48 + self.reason = reason
1.49 + def __str__(self):
1.50 + return "%s (%s)" % (self.missing_package, self.reason)
1.51 +
1.52 +class BadChromeMarkerError(Error):
1.53 + pass
1.54 +
1.55 +def validate_resource_hostname(name):
1.56 + """
1.57 + Validates the given hostname for a resource: URI.
1.58 +
1.59 + For more information, see:
1.60 +
1.61 + https://bugzilla.mozilla.org/show_bug.cgi?id=566812#c13
1.62 +
1.63 + Examples:
1.64 +
1.65 + >>> validate_resource_hostname('blarg')
1.66 +
1.67 + >>> validate_resource_hostname('bl arg')
1.68 + Traceback (most recent call last):
1.69 + ...
1.70 + ValueError: Error: the name of your package contains an invalid character.
1.71 + Package names can contain only lower-case letters, numbers, underscores, and dashes.
1.72 + Current package name: bl arg
1.73 +
1.74 + >>> validate_resource_hostname('BLARG')
1.75 + Traceback (most recent call last):
1.76 + ...
1.77 + ValueError: Error: the name of your package contains upper-case letters.
1.78 + Package names can contain only lower-case letters, numbers, underscores, and dashes.
1.79 + Current package name: BLARG
1.80 +
1.81 + >>> validate_resource_hostname('foo@bar')
1.82 + Traceback (most recent call last):
1.83 + ...
1.84 + ValueError: Error: the name of your package contains an invalid character.
1.85 + Package names can contain only lower-case letters, numbers, underscores, and dashes.
1.86 + Current package name: foo@bar
1.87 + """
1.88 +
1.89 + # See https://bugzilla.mozilla.org/show_bug.cgi?id=568131 for details.
1.90 + if not name.islower():
1.91 + raise ValueError("""Error: the name of your package contains upper-case letters.
1.92 +Package names can contain only lower-case letters, numbers, underscores, and dashes.
1.93 +Current package name: %s""" % name)
1.94 +
1.95 + if not RESOURCE_HOSTNAME_RE.match(name):
1.96 + raise ValueError("""Error: the name of your package contains an invalid character.
1.97 +Package names can contain only lower-case letters, numbers, underscores, and dashes.
1.98 +Current package name: %s""" % name)
1.99 +
1.100 +def find_packages_with_module(pkg_cfg, name):
1.101 + # TODO: Make this support more than just top-level modules.
1.102 + filename = "%s.js" % name
1.103 + packages = []
1.104 + for cfg in pkg_cfg.packages.itervalues():
1.105 + if 'lib' in cfg:
1.106 + matches = [dirname for dirname in resolve_dirs(cfg, cfg.lib)
1.107 + if os.path.exists(os.path.join(dirname, filename))]
1.108 + if matches:
1.109 + packages.append(cfg.name)
1.110 + return packages
1.111 +
1.112 +def resolve_dirs(pkg_cfg, dirnames):
1.113 + for dirname in dirnames:
1.114 + yield resolve_dir(pkg_cfg, dirname)
1.115 +
1.116 +def resolve_dir(pkg_cfg, dirname):
1.117 + return os.path.join(pkg_cfg.root_dir, dirname)
1.118 +
1.119 +def validate_permissions(perms):
1.120 + if (perms.get('cross-domain-content') and
1.121 + not isinstance(perms.get('cross-domain-content'), list)):
1.122 + raise ValueError("Error: `cross-domain-content` permissions in \
1.123 + package.json file must be an array of strings:\n %s" % perms)
1.124 +
1.125 +def get_metadata(pkg_cfg, deps):
1.126 + metadata = Bunch()
1.127 + for pkg_name in deps:
1.128 + cfg = pkg_cfg.packages[pkg_name]
1.129 + metadata[pkg_name] = Bunch()
1.130 + for prop in METADATA_PROPS:
1.131 + if cfg.get(prop):
1.132 + if prop == 'permissions':
1.133 + validate_permissions(cfg[prop])
1.134 + metadata[pkg_name][prop] = cfg[prop]
1.135 + return metadata
1.136 +
1.137 +def set_section_dir(base_json, name, base_path, dirnames, allow_root=False):
1.138 + resolved = compute_section_dir(base_json, base_path, dirnames, allow_root)
1.139 + if resolved:
1.140 + base_json[name] = os.path.abspath(resolved)
1.141 +
1.142 +def compute_section_dir(base_json, base_path, dirnames, allow_root):
1.143 + # PACKAGE_JSON.lib is highest priority
1.144 + # then PACKAGE_JSON.directories.lib
1.145 + # then lib/ (if it exists)
1.146 + # then . (but only if allow_root=True)
1.147 + for dirname in dirnames:
1.148 + if base_json.get(dirname):
1.149 + return os.path.join(base_path, base_json[dirname])
1.150 + if "directories" in base_json:
1.151 + for dirname in dirnames:
1.152 + if dirname in base_json.directories:
1.153 + return os.path.join(base_path, base_json.directories[dirname])
1.154 + for dirname in dirnames:
1.155 + if os.path.isdir(os.path.join(base_path, dirname)):
1.156 + return os.path.join(base_path, dirname)
1.157 + if allow_root:
1.158 + return os.path.abspath(base_path)
1.159 + return None
1.160 +
1.161 +def normalize_string_or_array(base_json, key):
1.162 + if base_json.get(key):
1.163 + if isinstance(base_json[key], basestring):
1.164 + base_json[key] = [base_json[key]]
1.165 +
1.166 +def load_json_file(path):
1.167 + data = open(path, 'r').read()
1.168 + try:
1.169 + return Bunch(json.loads(data))
1.170 + except ValueError, e:
1.171 + raise MalformedJsonFileError('%s when reading "%s"' % (str(e),
1.172 + path))
1.173 +
1.174 +def get_config_in_dir(path):
1.175 + package_json = os.path.join(path, MANIFEST_NAME)
1.176 + if not (os.path.exists(package_json) and
1.177 + os.path.isfile(package_json)):
1.178 + raise MalformedPackageError('%s not found in "%s"' % (MANIFEST_NAME,
1.179 + path))
1.180 + base_json = load_json_file(package_json)
1.181 +
1.182 + if 'name' not in base_json:
1.183 + base_json.name = os.path.basename(path)
1.184 +
1.185 + # later processing steps will expect to see the following keys in the
1.186 + # base_json that we return:
1.187 + #
1.188 + # name: name of the package
1.189 + # lib: list of directories with .js files
1.190 + # test: list of directories with test-*.js files
1.191 + # doc: list of directories with documentation .md files
1.192 + # data: list of directories with bundled arbitrary data files
1.193 + # packages: ?
1.194 +
1.195 + if (not base_json.get('tests') and
1.196 + os.path.isdir(os.path.join(path, 'test'))):
1.197 + base_json['tests'] = 'test'
1.198 +
1.199 + set_section_dir(base_json, 'lib', path, ['lib'], True)
1.200 + set_section_dir(base_json, 'tests', path, ['test', 'tests'], False)
1.201 + set_section_dir(base_json, 'doc', path, ['doc', 'docs'])
1.202 + set_section_dir(base_json, 'data', path, ['data'])
1.203 + set_section_dir(base_json, 'packages', path, ['packages'])
1.204 + set_section_dir(base_json, 'locale', path, ['locale'])
1.205 +
1.206 + if (not base_json.get('icon') and
1.207 + os.path.isfile(os.path.join(path, DEFAULT_ICON))):
1.208 + base_json['icon'] = DEFAULT_ICON
1.209 +
1.210 + if (not base_json.get('icon64') and
1.211 + os.path.isfile(os.path.join(path, DEFAULT_ICON64))):
1.212 + base_json['icon64'] = DEFAULT_ICON64
1.213 +
1.214 + for key in ['lib', 'tests', 'dependencies', 'packages']:
1.215 + # TODO: lib/tests can be an array?? consider interaction with
1.216 + # compute_section_dir above
1.217 + normalize_string_or_array(base_json, key)
1.218 +
1.219 + if 'main' not in base_json and 'lib' in base_json:
1.220 + for dirname in base_json['lib']:
1.221 + program = os.path.join(path, dirname,
1.222 + '%s.js' % DEFAULT_PROGRAM_MODULE)
1.223 + if os.path.exists(program):
1.224 + base_json['main'] = DEFAULT_PROGRAM_MODULE
1.225 + break
1.226 +
1.227 + base_json.root_dir = path
1.228 +
1.229 + if "dependencies" in base_json:
1.230 + deps = base_json["dependencies"]
1.231 + deps = [x for x in deps if x not in ["addon-kit", "api-utils"]]
1.232 + deps.append("addon-sdk")
1.233 + base_json["dependencies"] = deps
1.234 +
1.235 + return base_json
1.236 +
1.237 +def _is_same_file(a, b):
1.238 + if hasattr(os.path, 'samefile'):
1.239 + return os.path.samefile(a, b)
1.240 + return a == b
1.241 +
1.242 +def build_config(root_dir, target_cfg, packagepath=[]):
1.243 + dirs_to_scan = [env_root] # root is addon-sdk dir, diff from root_dir in tests
1.244 +
1.245 + def add_packages_from_config(pkgconfig):
1.246 + if 'packages' in pkgconfig:
1.247 + for package_dir in resolve_dirs(pkgconfig, pkgconfig.packages):
1.248 + dirs_to_scan.append(package_dir)
1.249 +
1.250 + add_packages_from_config(target_cfg)
1.251 +
1.252 + packages_dir = os.path.join(root_dir, 'packages')
1.253 + if os.path.exists(packages_dir) and os.path.isdir(packages_dir):
1.254 + dirs_to_scan.append(packages_dir)
1.255 + dirs_to_scan.extend(packagepath)
1.256 +
1.257 + packages = Bunch({target_cfg.name: target_cfg})
1.258 +
1.259 + while dirs_to_scan:
1.260 + packages_dir = dirs_to_scan.pop()
1.261 + if os.path.exists(os.path.join(packages_dir, "package.json")):
1.262 + package_paths = [packages_dir]
1.263 + else:
1.264 + package_paths = [os.path.join(packages_dir, dirname)
1.265 + for dirname in os.listdir(packages_dir)
1.266 + if not dirname.startswith('.')]
1.267 + package_paths = [dirname for dirname in package_paths
1.268 + if os.path.isdir(dirname)]
1.269 +
1.270 + for path in package_paths:
1.271 + pkgconfig = get_config_in_dir(path)
1.272 + if pkgconfig.name in packages:
1.273 + otherpkg = packages[pkgconfig.name]
1.274 + if not _is_same_file(otherpkg.root_dir, path):
1.275 + raise DuplicatePackageError(path, otherpkg.root_dir)
1.276 + else:
1.277 + packages[pkgconfig.name] = pkgconfig
1.278 + add_packages_from_config(pkgconfig)
1.279 +
1.280 + return Bunch(packages=packages)
1.281 +
1.282 +def get_deps_for_targets(pkg_cfg, targets):
1.283 + visited = []
1.284 + deps_left = [[dep, None] for dep in list(targets)]
1.285 +
1.286 + while deps_left:
1.287 + [dep, required_by] = deps_left.pop()
1.288 + if dep not in visited:
1.289 + visited.append(dep)
1.290 + if dep not in pkg_cfg.packages:
1.291 + required_reason = ("required by '%s'" % (required_by)) \
1.292 + if required_by is not None \
1.293 + else "specified as target"
1.294 + raise PackageNotFoundError(dep, required_reason)
1.295 + dep_cfg = pkg_cfg.packages[dep]
1.296 + deps_left.extend([[i, dep] for i in dep_cfg.get('dependencies', [])])
1.297 + deps_left.extend([[i, dep] for i in dep_cfg.get('extra_dependencies', [])])
1.298 +
1.299 + return visited
1.300 +
1.301 +def generate_build_for_target(pkg_cfg, target, deps,
1.302 + include_tests=True,
1.303 + include_dep_tests=False,
1.304 + is_running_tests=False,
1.305 + default_loader=DEFAULT_LOADER):
1.306 +
1.307 + build = Bunch(# Contains section directories for all packages:
1.308 + packages=Bunch(),
1.309 + locale=Bunch()
1.310 + )
1.311 +
1.312 + def add_section_to_build(cfg, section, is_code=False,
1.313 + is_data=False):
1.314 + if section in cfg:
1.315 + dirnames = cfg[section]
1.316 + if isinstance(dirnames, basestring):
1.317 + # This is just for internal consistency within this
1.318 + # function, it has nothing to do w/ a non-canonical
1.319 + # configuration dict.
1.320 + dirnames = [dirnames]
1.321 + for dirname in resolve_dirs(cfg, dirnames):
1.322 + # ensure that package name is valid
1.323 + try:
1.324 + validate_resource_hostname(cfg.name)
1.325 + except ValueError, err:
1.326 + print err
1.327 + sys.exit(1)
1.328 + # ensure that this package has an entry
1.329 + if not cfg.name in build.packages:
1.330 + build.packages[cfg.name] = Bunch()
1.331 + # detect duplicated sections
1.332 + if section in build.packages[cfg.name]:
1.333 + raise KeyError("package's section already defined",
1.334 + cfg.name, section)
1.335 + # Register this section (lib, data, tests)
1.336 + build.packages[cfg.name][section] = dirname
1.337 +
1.338 + def add_locale_to_build(cfg):
1.339 + # Bug 730776: Ignore locales for addon-kit, that are only for unit tests
1.340 + if not is_running_tests and cfg.name == "addon-sdk":
1.341 + return
1.342 +
1.343 + path = resolve_dir(cfg, cfg['locale'])
1.344 + files = os.listdir(path)
1.345 + for filename in files:
1.346 + fullpath = os.path.join(path, filename)
1.347 + if os.path.isfile(fullpath) and filename.endswith('.properties'):
1.348 + language = filename[:-len('.properties')]
1.349 +
1.350 + from property_parser import parse_file, MalformedLocaleFileError
1.351 + try:
1.352 + content = parse_file(fullpath)
1.353 + except MalformedLocaleFileError, msg:
1.354 + print msg[0]
1.355 + sys.exit(1)
1.356 +
1.357 + # Merge current locales into global locale hashtable.
1.358 + # Locale files only contains one big JSON object
1.359 + # that act as an hastable of:
1.360 + # "keys to translate" => "translated keys"
1.361 + if language in build.locale:
1.362 + merge = (build.locale[language].items() +
1.363 + content.items())
1.364 + build.locale[language] = Bunch(merge)
1.365 + else:
1.366 + build.locale[language] = content
1.367 +
1.368 + def add_dep_to_build(dep):
1.369 + dep_cfg = pkg_cfg.packages[dep]
1.370 + add_section_to_build(dep_cfg, "lib", is_code=True)
1.371 + add_section_to_build(dep_cfg, "data", is_data=True)
1.372 + if include_tests and include_dep_tests:
1.373 + add_section_to_build(dep_cfg, "tests", is_code=True)
1.374 + if 'locale' in dep_cfg:
1.375 + add_locale_to_build(dep_cfg)
1.376 + if ("loader" in dep_cfg) and ("loader" not in build):
1.377 + build.loader = "%s/%s" % (dep,
1.378 + dep_cfg.loader)
1.379 +
1.380 + target_cfg = pkg_cfg.packages[target]
1.381 +
1.382 + if include_tests and not include_dep_tests:
1.383 + add_section_to_build(target_cfg, "tests", is_code=True)
1.384 +
1.385 + for dep in deps:
1.386 + add_dep_to_build(dep)
1.387 +
1.388 + if 'loader' not in build:
1.389 + add_dep_to_build(DEFAULT_LOADER)
1.390 +
1.391 + if 'icon' in target_cfg:
1.392 + build['icon'] = os.path.join(target_cfg.root_dir, target_cfg.icon)
1.393 + del target_cfg['icon']
1.394 +
1.395 + if 'icon64' in target_cfg:
1.396 + build['icon64'] = os.path.join(target_cfg.root_dir, target_cfg.icon64)
1.397 + del target_cfg['icon64']
1.398 +
1.399 + if ('preferences' in target_cfg):
1.400 + build['preferences'] = target_cfg.preferences
1.401 +
1.402 + if 'id' in target_cfg:
1.403 + # NOTE: logic duplicated from buildJID()
1.404 + jid = target_cfg['id']
1.405 + if not ('@' in jid or jid.startswith('{')):
1.406 + jid += '@jetpack'
1.407 + build['preferencesBranch'] = jid
1.408 +
1.409 + if 'preferences-branch' in target_cfg:
1.410 + # check it's a non-empty, valid branch name
1.411 + preferencesBranch = target_cfg['preferences-branch']
1.412 + if re.match('^[\w{@}-]+$', preferencesBranch):
1.413 + build['preferencesBranch'] = preferencesBranch
1.414 + elif not is_running_tests:
1.415 + print >>sys.stderr, "IGNORING preferences-branch (not a valid branch name)"
1.416 +
1.417 + return build
1.418 +
1.419 +def _get_files_in_dir(path):
1.420 + data = {}
1.421 + files = os.listdir(path)
1.422 + for filename in files:
1.423 + fullpath = os.path.join(path, filename)
1.424 + if os.path.isdir(fullpath):
1.425 + data[filename] = _get_files_in_dir(fullpath)
1.426 + else:
1.427 + try:
1.428 + info = os.stat(fullpath)
1.429 + data[filename] = ("file", dict(size=info.st_size))
1.430 + except OSError:
1.431 + pass
1.432 + return ("directory", data)
1.433 +
1.434 +def build_pkg_index(pkg_cfg):
1.435 + pkg_cfg = copy.deepcopy(pkg_cfg)
1.436 + for pkg in pkg_cfg.packages:
1.437 + root_dir = pkg_cfg.packages[pkg].root_dir
1.438 + files = _get_files_in_dir(root_dir)
1.439 + pkg_cfg.packages[pkg].files = files
1.440 + try:
1.441 + readme = open(root_dir + '/README.md').read()
1.442 + pkg_cfg.packages[pkg].readme = readme
1.443 + except IOError:
1.444 + pass
1.445 + del pkg_cfg.packages[pkg].root_dir
1.446 + return pkg_cfg.packages
1.447 +
1.448 +def build_pkg_cfg(root):
1.449 + pkg_cfg = build_config(root, Bunch(name='dummy'))
1.450 + del pkg_cfg.packages['dummy']
1.451 + return pkg_cfg
1.452 +
1.453 +def call_plugins(pkg_cfg, deps):
1.454 + for dep in deps:
1.455 + dep_cfg = pkg_cfg.packages[dep]
1.456 + dirnames = dep_cfg.get('python-lib', [])
1.457 + for dirname in resolve_dirs(dep_cfg, dirnames):
1.458 + sys.path.append(dirname)
1.459 + module_names = dep_cfg.get('python-plugins', [])
1.460 + for module_name in module_names:
1.461 + module = __import__(module_name)
1.462 + module.init(root_dir=dep_cfg.root_dir)
1.463 +
1.464 +def call_cmdline_tool(env_root, pkg_name):
1.465 + pkg_cfg = build_config(env_root, Bunch(name='dummy'))
1.466 + if pkg_name not in pkg_cfg.packages:
1.467 + print "This tool requires the '%s' package." % pkg_name
1.468 + sys.exit(1)
1.469 + cfg = pkg_cfg.packages[pkg_name]
1.470 + for dirname in resolve_dirs(cfg, cfg['python-lib']):
1.471 + sys.path.append(dirname)
1.472 + module_name = cfg.get('python-cmdline-tool')
1.473 + module = __import__(module_name)
1.474 + module.run()
