michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: import os
michael@0: import sys
michael@0: import re
michael@0: import copy
michael@0: 
michael@0: import simplejson as json
michael@0: from cuddlefish.bunch import Bunch
michael@0: 
michael@0: MANIFEST_NAME = 'package.json'
michael@0: DEFAULT_LOADER = 'addon-sdk'
michael@0: 
michael@0: # Is different from root_dir when running tests
michael@0: env_root = os.environ.get('CUDDLEFISH_ROOT')
michael@0: 
michael@0: DEFAULT_PROGRAM_MODULE = 'main'
michael@0: 
michael@0: DEFAULT_ICON = 'icon.png'
michael@0: DEFAULT_ICON64 = 'icon64.png'
michael@0: 
michael@0: METADATA_PROPS = ['name', 'description', 'keywords', 'author', 'version',
michael@0:                   'translators', 'contributors', 'license', 'homepage', 'icon',
michael@0:                   'icon64', 'main', 'directories', 'permissions']
michael@0: 
michael@0: RESOURCE_HOSTNAME_RE = re.compile(r'^[a-z0-9_\-]+$')
michael@0: 
michael@0: class Error(Exception):
michael@0:     pass
michael@0: 
michael@0: class MalformedPackageError(Error):
michael@0:     pass
michael@0: 
michael@0: class MalformedJsonFileError(Error):
michael@0:     pass
michael@0: 
michael@0: class DuplicatePackageError(Error):
michael@0:     pass
michael@0: 
michael@0: class PackageNotFoundError(Error):
michael@0:     def __init__(self, missing_package, reason):
michael@0:         self.missing_package = missing_package
michael@0:         self.reason = reason
michael@0:     def __str__(self):
michael@0:         return "%s (%s)" % (self.missing_package, self.reason)
michael@0: 
michael@0: class BadChromeMarkerError(Error):
michael@0:     pass
michael@0: 
michael@0: def validate_resource_hostname(name):
michael@0:     """
michael@0:     Validates the given hostname for a resource: URI.
michael@0: 
michael@0:     For more information, see:
michael@0: 
michael@0:       https://bugzilla.mozilla.org/show_bug.cgi?id=566812#c13
michael@0: 
michael@0:     Examples:
michael@0: 
michael@0:       >>> validate_resource_hostname('blarg')
michael@0: 
michael@0:       >>> validate_resource_hostname('bl arg')
michael@0:       Traceback (most recent call last):
michael@0:       ...
michael@0:       ValueError: Error: the name of your package contains an invalid character.
michael@0:       Package names can contain only lower-case letters, numbers, underscores, and dashes.
michael@0:       Current package name: bl arg
michael@0: 
michael@0:       >>> validate_resource_hostname('BLARG')
michael@0:       Traceback (most recent call last):
michael@0:       ...
michael@0:       ValueError: Error: the name of your package contains upper-case letters.
michael@0:       Package names can contain only lower-case letters, numbers, underscores, and dashes.
michael@0:       Current package name: BLARG
michael@0: 
michael@0:       >>> validate_resource_hostname('foo@bar')
michael@0:       Traceback (most recent call last):
michael@0:       ...
michael@0:       ValueError: Error: the name of your package contains an invalid character.
michael@0:       Package names can contain only lower-case letters, numbers, underscores, and dashes.
michael@0:       Current package name: foo@bar
michael@0:     """
michael@0: 
michael@0:     # See https://bugzilla.mozilla.org/show_bug.cgi?id=568131 for details.
michael@0:     if not name.islower():
michael@0:         raise ValueError("""Error: the name of your package contains upper-case letters.
michael@0: Package names can contain only lower-case letters, numbers, underscores, and dashes.
michael@0: Current package name: %s""" % name)
michael@0: 
michael@0:     if not RESOURCE_HOSTNAME_RE.match(name):
michael@0:         raise ValueError("""Error: the name of your package contains an invalid character.
michael@0: Package names can contain only lower-case letters, numbers, underscores, and dashes.
michael@0: Current package name: %s""" % name)
michael@0: 
michael@0: def find_packages_with_module(pkg_cfg, name):
michael@0:     # TODO: Make this support more than just top-level modules.
michael@0:     filename = "%s.js" % name
michael@0:     packages = []
michael@0:     for cfg in pkg_cfg.packages.itervalues():
michael@0:         if 'lib' in cfg:
michael@0:             matches = [dirname for dirname in resolve_dirs(cfg, cfg.lib)
michael@0:                        if os.path.exists(os.path.join(dirname, filename))]
michael@0:             if matches:
michael@0:                 packages.append(cfg.name)
michael@0:     return packages
michael@0: 
michael@0: def resolve_dirs(pkg_cfg, dirnames):
michael@0:     for dirname in dirnames:
michael@0:         yield resolve_dir(pkg_cfg, dirname)
michael@0: 
michael@0: def resolve_dir(pkg_cfg, dirname):
michael@0:     return os.path.join(pkg_cfg.root_dir, dirname)
michael@0: 
michael@0: def validate_permissions(perms):
michael@0:     if (perms.get('cross-domain-content') and
michael@0:         not isinstance(perms.get('cross-domain-content'), list)):
michael@0:         raise ValueError("Error: `cross-domain-content` permissions in \
michael@0:  package.json file must be an array of strings:\n  %s" % perms)
michael@0: 
michael@0: def get_metadata(pkg_cfg, deps):
michael@0:     metadata = Bunch()
michael@0:     for pkg_name in deps:
michael@0:         cfg = pkg_cfg.packages[pkg_name]
michael@0:         metadata[pkg_name] = Bunch()
michael@0:         for prop in METADATA_PROPS:
michael@0:             if cfg.get(prop):
michael@0:                 if prop == 'permissions':
michael@0:                     validate_permissions(cfg[prop])
michael@0:                 metadata[pkg_name][prop] = cfg[prop]
michael@0:     return metadata
michael@0: 
michael@0: def set_section_dir(base_json, name, base_path, dirnames, allow_root=False):
michael@0:     resolved = compute_section_dir(base_json, base_path, dirnames, allow_root)
michael@0:     if resolved:
michael@0:         base_json[name] = os.path.abspath(resolved)
michael@0: 
michael@0: def compute_section_dir(base_json, base_path, dirnames, allow_root):
michael@0:     # PACKAGE_JSON.lib is highest priority
michael@0:     # then PACKAGE_JSON.directories.lib
michael@0:     # then lib/ (if it exists)
michael@0:     # then . (but only if allow_root=True)
michael@0:     for dirname in dirnames:
michael@0:         if base_json.get(dirname):
michael@0:             return os.path.join(base_path, base_json[dirname])
michael@0:     if "directories" in base_json:
michael@0:         for dirname in dirnames:
michael@0:             if dirname in base_json.directories:
michael@0:                 return os.path.join(base_path, base_json.directories[dirname])
michael@0:     for dirname in dirnames:
michael@0:         if os.path.isdir(os.path.join(base_path, dirname)):
michael@0:             return os.path.join(base_path, dirname)
michael@0:     if allow_root:
michael@0:         return os.path.abspath(base_path)
michael@0:     return None
michael@0: 
michael@0: def normalize_string_or_array(base_json, key):
michael@0:     if base_json.get(key):
michael@0:         if isinstance(base_json[key], basestring):
michael@0:             base_json[key] = [base_json[key]]
michael@0: 
michael@0: def load_json_file(path):
michael@0:     data = open(path, 'r').read()
michael@0:     try:
michael@0:         return Bunch(json.loads(data))
michael@0:     except ValueError, e:
michael@0:         raise MalformedJsonFileError('%s when reading "%s"' % (str(e),
michael@0:                                                                path))
michael@0: 
michael@0: def get_config_in_dir(path):
michael@0:     package_json = os.path.join(path, MANIFEST_NAME)
michael@0:     if not (os.path.exists(package_json) and
michael@0:             os.path.isfile(package_json)):
michael@0:         raise MalformedPackageError('%s not found in "%s"' % (MANIFEST_NAME,
michael@0:                                                               path))
michael@0:     base_json = load_json_file(package_json)
michael@0: 
michael@0:     if 'name' not in base_json:
michael@0:         base_json.name = os.path.basename(path)
michael@0: 
michael@0:     # later processing steps will expect to see the following keys in the
michael@0:     # base_json that we return:
michael@0:     #
michael@0:     #  name: name of the package
michael@0:     #  lib: list of directories with .js files
michael@0:     #  test: list of directories with test-*.js files
michael@0:     #  doc: list of directories with documentation .md files
michael@0:     #  data: list of directories with bundled arbitrary data files
michael@0:     #  packages: ?
michael@0: 
michael@0:     if (not base_json.get('tests') and
michael@0:         os.path.isdir(os.path.join(path, 'test'))):
michael@0:         base_json['tests'] = 'test'
michael@0: 
michael@0:     set_section_dir(base_json, 'lib', path, ['lib'], True)
michael@0:     set_section_dir(base_json, 'tests', path, ['test', 'tests'], False)
michael@0:     set_section_dir(base_json, 'doc', path, ['doc', 'docs'])
michael@0:     set_section_dir(base_json, 'data', path, ['data'])
michael@0:     set_section_dir(base_json, 'packages', path, ['packages'])
michael@0:     set_section_dir(base_json, 'locale', path, ['locale'])
michael@0: 
michael@0:     if (not base_json.get('icon') and
michael@0:         os.path.isfile(os.path.join(path, DEFAULT_ICON))):
michael@0:         base_json['icon'] = DEFAULT_ICON
michael@0: 
michael@0:     if (not base_json.get('icon64') and
michael@0:         os.path.isfile(os.path.join(path, DEFAULT_ICON64))):
michael@0:         base_json['icon64'] = DEFAULT_ICON64
michael@0: 
michael@0:     for key in ['lib', 'tests', 'dependencies', 'packages']:
michael@0:         # TODO: lib/tests can be an array?? consider interaction with
michael@0:         # compute_section_dir above
michael@0:         normalize_string_or_array(base_json, key)
michael@0: 
michael@0:     if 'main' not in base_json and 'lib' in base_json:
michael@0:         for dirname in base_json['lib']:
michael@0:             program = os.path.join(path, dirname,
michael@0:                                    '%s.js' % DEFAULT_PROGRAM_MODULE)
michael@0:             if os.path.exists(program):
michael@0:                 base_json['main'] = DEFAULT_PROGRAM_MODULE
michael@0:                 break
michael@0: 
michael@0:     base_json.root_dir = path
michael@0: 
michael@0:     if "dependencies" in base_json:
michael@0:       deps = base_json["dependencies"]
michael@0:       deps = [x for x in deps if x not in ["addon-kit", "api-utils"]]
michael@0:       deps.append("addon-sdk")
michael@0:       base_json["dependencies"] = deps
michael@0: 
michael@0:     return base_json
michael@0: 
michael@0: def _is_same_file(a, b):
michael@0:     if hasattr(os.path, 'samefile'):
michael@0:         return os.path.samefile(a, b)
michael@0:     return a == b
michael@0: 
michael@0: def build_config(root_dir, target_cfg, packagepath=[]):
michael@0:     dirs_to_scan = [env_root] # root is addon-sdk dir, diff from root_dir in tests
michael@0: 
michael@0:     def add_packages_from_config(pkgconfig):
michael@0:         if 'packages' in pkgconfig:
michael@0:             for package_dir in resolve_dirs(pkgconfig, pkgconfig.packages):
michael@0:                 dirs_to_scan.append(package_dir)
michael@0: 
michael@0:     add_packages_from_config(target_cfg)
michael@0: 
michael@0:     packages_dir = os.path.join(root_dir, 'packages')
michael@0:     if os.path.exists(packages_dir) and os.path.isdir(packages_dir):
michael@0:         dirs_to_scan.append(packages_dir)
michael@0:     dirs_to_scan.extend(packagepath)
michael@0: 
michael@0:     packages = Bunch({target_cfg.name: target_cfg})
michael@0: 
michael@0:     while dirs_to_scan:
michael@0:         packages_dir = dirs_to_scan.pop()
michael@0:         if os.path.exists(os.path.join(packages_dir, "package.json")):
michael@0:             package_paths = [packages_dir]
michael@0:         else:
michael@0:             package_paths = [os.path.join(packages_dir, dirname)
michael@0:                              for dirname in os.listdir(packages_dir)
michael@0:                              if not dirname.startswith('.')]
michael@0:             package_paths = [dirname for dirname in package_paths
michael@0:                              if os.path.isdir(dirname)]
michael@0: 
michael@0:         for path in package_paths:
michael@0:             pkgconfig = get_config_in_dir(path)
michael@0:             if pkgconfig.name in packages:
michael@0:                 otherpkg = packages[pkgconfig.name]
michael@0:                 if not _is_same_file(otherpkg.root_dir, path):
michael@0:                     raise DuplicatePackageError(path, otherpkg.root_dir)
michael@0:             else:
michael@0:                 packages[pkgconfig.name] = pkgconfig
michael@0:                 add_packages_from_config(pkgconfig)
michael@0: 
michael@0:     return Bunch(packages=packages)
michael@0: 
michael@0: def get_deps_for_targets(pkg_cfg, targets):
michael@0:     visited = []
michael@0:     deps_left = [[dep, None] for dep in list(targets)]
michael@0: 
michael@0:     while deps_left:
michael@0:         [dep, required_by] = deps_left.pop()
michael@0:         if dep not in visited:
michael@0:             visited.append(dep)
michael@0:             if dep not in pkg_cfg.packages:
michael@0:                 required_reason = ("required by '%s'" % (required_by)) \
michael@0:                                     if required_by is not None \
michael@0:                                     else "specified as target"
michael@0:                 raise PackageNotFoundError(dep, required_reason)
michael@0:             dep_cfg = pkg_cfg.packages[dep]
michael@0:             deps_left.extend([[i, dep] for i in dep_cfg.get('dependencies', [])])
michael@0:             deps_left.extend([[i, dep] for i in dep_cfg.get('extra_dependencies', [])])
michael@0: 
michael@0:     return visited
michael@0: 
michael@0: def generate_build_for_target(pkg_cfg, target, deps,
michael@0:                               include_tests=True,
michael@0:                               include_dep_tests=False,
michael@0:                               is_running_tests=False,
michael@0:                               default_loader=DEFAULT_LOADER):
michael@0: 
michael@0:     build = Bunch(# Contains section directories for all packages:
michael@0:                   packages=Bunch(),
michael@0:                   locale=Bunch()
michael@0:                   )
michael@0: 
michael@0:     def add_section_to_build(cfg, section, is_code=False,
michael@0:                              is_data=False):
michael@0:         if section in cfg:
michael@0:             dirnames = cfg[section]
michael@0:             if isinstance(dirnames, basestring):
michael@0:                 # This is just for internal consistency within this
michael@0:                 # function, it has nothing to do w/ a non-canonical
michael@0:                 # configuration dict.
michael@0:                 dirnames = [dirnames]
michael@0:             for dirname in resolve_dirs(cfg, dirnames):
michael@0:                 # ensure that package name is valid
michael@0:                 try:
michael@0:                     validate_resource_hostname(cfg.name)
michael@0:                 except ValueError, err:
michael@0:                     print err
michael@0:                     sys.exit(1)
michael@0:                 # ensure that this package has an entry
michael@0:                 if not cfg.name in build.packages:
michael@0:                     build.packages[cfg.name] = Bunch()
michael@0:                 # detect duplicated sections
michael@0:                 if section in build.packages[cfg.name]:
michael@0:                     raise KeyError("package's section already defined",
michael@0:                                    cfg.name, section)
michael@0:                 # Register this section (lib, data, tests)
michael@0:                 build.packages[cfg.name][section] = dirname
michael@0: 
michael@0:     def add_locale_to_build(cfg):
michael@0:         # Bug 730776: Ignore locales for addon-kit, that are only for unit tests
michael@0:         if not is_running_tests and cfg.name == "addon-sdk":
michael@0:             return
michael@0: 
michael@0:         path = resolve_dir(cfg, cfg['locale'])
michael@0:         files = os.listdir(path)
michael@0:         for filename in files:
michael@0:             fullpath = os.path.join(path, filename)
michael@0:             if os.path.isfile(fullpath) and filename.endswith('.properties'):
michael@0:                 language = filename[:-len('.properties')]
michael@0: 
michael@0:                 from property_parser import parse_file, MalformedLocaleFileError
michael@0:                 try:
michael@0:                     content = parse_file(fullpath)
michael@0:                 except MalformedLocaleFileError, msg:
michael@0:                     print msg[0]
michael@0:                     sys.exit(1)
michael@0: 
michael@0:                 # Merge current locales into global locale hashtable.
michael@0:                 # Locale files only contains one big JSON object
michael@0:                 # that act as an hastable of:
michael@0:                 # "keys to translate" => "translated keys"
michael@0:                 if language in build.locale:
michael@0:                     merge = (build.locale[language].items() +
michael@0:                              content.items())
michael@0:                     build.locale[language] = Bunch(merge)
michael@0:                 else:
michael@0:                     build.locale[language] = content
michael@0: 
michael@0:     def add_dep_to_build(dep):
michael@0:         dep_cfg = pkg_cfg.packages[dep]
michael@0:         add_section_to_build(dep_cfg, "lib", is_code=True)
michael@0:         add_section_to_build(dep_cfg, "data", is_data=True)
michael@0:         if include_tests and include_dep_tests:
michael@0:             add_section_to_build(dep_cfg, "tests", is_code=True)
michael@0:         if 'locale' in dep_cfg:
michael@0:             add_locale_to_build(dep_cfg)
michael@0:         if ("loader" in dep_cfg) and ("loader" not in build):
michael@0:             build.loader = "%s/%s" % (dep,
michael@0:                                                  dep_cfg.loader)
michael@0: 
michael@0:     target_cfg = pkg_cfg.packages[target]
michael@0: 
michael@0:     if include_tests and not include_dep_tests:
michael@0:         add_section_to_build(target_cfg, "tests", is_code=True)
michael@0: 
michael@0:     for dep in deps:
michael@0:         add_dep_to_build(dep)
michael@0: 
michael@0:     if 'loader' not in build:
michael@0:         add_dep_to_build(DEFAULT_LOADER)
michael@0: 
michael@0:     if 'icon' in target_cfg:
michael@0:         build['icon'] = os.path.join(target_cfg.root_dir, target_cfg.icon)
michael@0:         del target_cfg['icon']
michael@0: 
michael@0:     if 'icon64' in target_cfg:
michael@0:         build['icon64'] = os.path.join(target_cfg.root_dir, target_cfg.icon64)
michael@0:         del target_cfg['icon64']
michael@0: 
michael@0:     if ('preferences' in target_cfg):
michael@0:         build['preferences'] = target_cfg.preferences
michael@0: 
michael@0:     if 'id' in target_cfg:
michael@0:         # NOTE: logic duplicated from buildJID()
michael@0:         jid = target_cfg['id']
michael@0:         if not ('@' in jid or jid.startswith('{')):
michael@0:             jid += '@jetpack'
michael@0:         build['preferencesBranch'] = jid
michael@0: 
michael@0:     if 'preferences-branch' in target_cfg:
michael@0:         # check it's a non-empty, valid branch name
michael@0:         preferencesBranch = target_cfg['preferences-branch']
michael@0:         if re.match('^[\w{@}-]+$', preferencesBranch):
michael@0:             build['preferencesBranch'] = preferencesBranch
michael@0:         elif not is_running_tests:
michael@0:             print >>sys.stderr, "IGNORING preferences-branch (not a valid branch name)"
michael@0: 
michael@0:     return build
michael@0: 
michael@0: def _get_files_in_dir(path):
michael@0:     data = {}
michael@0:     files = os.listdir(path)
michael@0:     for filename in files:
michael@0:         fullpath = os.path.join(path, filename)
michael@0:         if os.path.isdir(fullpath):
michael@0:             data[filename] = _get_files_in_dir(fullpath)
michael@0:         else:
michael@0:             try:
michael@0:                 info = os.stat(fullpath)
michael@0:                 data[filename] = ("file", dict(size=info.st_size))
michael@0:             except OSError:
michael@0:                 pass
michael@0:     return ("directory", data)
michael@0: 
michael@0: def build_pkg_index(pkg_cfg):
michael@0:     pkg_cfg = copy.deepcopy(pkg_cfg)
michael@0:     for pkg in pkg_cfg.packages:
michael@0:         root_dir = pkg_cfg.packages[pkg].root_dir
michael@0:         files = _get_files_in_dir(root_dir)
michael@0:         pkg_cfg.packages[pkg].files = files
michael@0:         try:
michael@0:             readme = open(root_dir + '/README.md').read()
michael@0:             pkg_cfg.packages[pkg].readme = readme
michael@0:         except IOError:
michael@0:             pass
michael@0:         del pkg_cfg.packages[pkg].root_dir
michael@0:     return pkg_cfg.packages
michael@0: 
michael@0: def build_pkg_cfg(root):
michael@0:     pkg_cfg = build_config(root, Bunch(name='dummy'))
michael@0:     del pkg_cfg.packages['dummy']
michael@0:     return pkg_cfg
michael@0: 
michael@0: def call_plugins(pkg_cfg, deps):
michael@0:     for dep in deps:
michael@0:         dep_cfg = pkg_cfg.packages[dep]
michael@0:         dirnames = dep_cfg.get('python-lib', [])
michael@0:         for dirname in resolve_dirs(dep_cfg, dirnames):
michael@0:             sys.path.append(dirname)
michael@0:         module_names = dep_cfg.get('python-plugins', [])
michael@0:         for module_name in module_names:
michael@0:             module = __import__(module_name)
michael@0:             module.init(root_dir=dep_cfg.root_dir)
michael@0: 
michael@0: def call_cmdline_tool(env_root, pkg_name):
michael@0:     pkg_cfg = build_config(env_root, Bunch(name='dummy'))
michael@0:     if pkg_name not in pkg_cfg.packages:
michael@0:         print "This tool requires the '%s' package." % pkg_name
michael@0:         sys.exit(1)
michael@0:     cfg = pkg_cfg.packages[pkg_name]
michael@0:     for dirname in resolve_dirs(cfg, cfg['python-lib']):
michael@0:         sys.path.append(dirname)
michael@0:     module_name = cfg.get('python-cmdline-tool')
michael@0:     module = __import__(module_name)
michael@0:     module.run()