The Tor Browser: addon-sdk/source/python-lib/cuddlefish/packaging.py@ac0c01689b40

Implement a real Private Browsing Mode condition by changing the API/ABI;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 import os

     6 import sys

     7 import re

     8 import copy

    10 import simplejson as json

    11 from cuddlefish.bunch import Bunch

    13 MANIFEST_NAME = 'package.json'

    14 DEFAULT_LOADER = 'addon-sdk'

    16 # Is different from root_dir when running tests

    17 env_root = os.environ.get('CUDDLEFISH_ROOT')

    19 DEFAULT_PROGRAM_MODULE = 'main'

    21 DEFAULT_ICON = 'icon.png'

    22 DEFAULT_ICON64 = 'icon64.png'

    24 METADATA_PROPS = ['name', 'description', 'keywords', 'author', 'version',

    25                   'translators', 'contributors', 'license', 'homepage', 'icon',

    26                   'icon64', 'main', 'directories', 'permissions']

    28 RESOURCE_HOSTNAME_RE = re.compile(r'^[a-z0-9_\-]+$')

    30 class Error(Exception):

    31     pass

    33 class MalformedPackageError(Error):

    34     pass

    36 class MalformedJsonFileError(Error):

    37     pass

    39 class DuplicatePackageError(Error):

    40     pass

    42 class PackageNotFoundError(Error):

    43     def __init__(self, missing_package, reason):

    44         self.missing_package = missing_package

    45         self.reason = reason

    46     def __str__(self):

    47         return "%s (%s)" % (self.missing_package, self.reason)

    49 class BadChromeMarkerError(Error):

    50     pass

    52 def validate_resource_hostname(name):

    53     """

    54     Validates the given hostname for a resource: URI.

    56     For more information, see:

    58       https://bugzilla.mozilla.org/show_bug.cgi?id=566812#c13

    60     Examples:

    62       >>> validate_resource_hostname('blarg')

    64       >>> validate_resource_hostname('bl arg')

    65       Traceback (most recent call last):

    66       ...

    67       ValueError: Error: the name of your package contains an invalid character.

    68       Package names can contain only lower-case letters, numbers, underscores, and dashes.

    69       Current package name: bl arg

    71       >>> validate_resource_hostname('BLARG')

    72       Traceback (most recent call last):

    73       ...

    74       ValueError: Error: the name of your package contains upper-case letters.

    75       Package names can contain only lower-case letters, numbers, underscores, and dashes.

    76       Current package name: BLARG

    78       >>> validate_resource_hostname('foo@bar')

    79       Traceback (most recent call last):

    80       ...

    81       ValueError: Error: the name of your package contains an invalid character.

    82       Package names can contain only lower-case letters, numbers, underscores, and dashes.

    83       Current package name: foo@bar

    84     """

    86     # See https://bugzilla.mozilla.org/show_bug.cgi?id=568131 for details.

    87     if not name.islower():

    88         raise ValueError("""Error: the name of your package contains upper-case letters.

    89 Package names can contain only lower-case letters, numbers, underscores, and dashes.

    90 Current package name: %s""" % name)

    92     if not RESOURCE_HOSTNAME_RE.match(name):

    93         raise ValueError("""Error: the name of your package contains an invalid character.

    94 Package names can contain only lower-case letters, numbers, underscores, and dashes.

    95 Current package name: %s""" % name)

    97 def find_packages_with_module(pkg_cfg, name):

    98     # TODO: Make this support more than just top-level modules.

    99     filename = "%s.js" % name

   100     packages = []

   101     for cfg in pkg_cfg.packages.itervalues():

   102         if 'lib' in cfg:

   103             matches = [dirname for dirname in resolve_dirs(cfg, cfg.lib)

   104                        if os.path.exists(os.path.join(dirname, filename))]

   105             if matches:

   106                 packages.append(cfg.name)

   107     return packages

   109 def resolve_dirs(pkg_cfg, dirnames):

   110     for dirname in dirnames:

   111         yield resolve_dir(pkg_cfg, dirname)

   113 def resolve_dir(pkg_cfg, dirname):

   114     return os.path.join(pkg_cfg.root_dir, dirname)

   116 def validate_permissions(perms):

   117     if (perms.get('cross-domain-content') and

   118         not isinstance(perms.get('cross-domain-content'), list)):

   119         raise ValueError("Error: `cross-domain-content` permissions in \

   120  package.json file must be an array of strings:\n  %s" % perms)

   122 def get_metadata(pkg_cfg, deps):

   123     metadata = Bunch()

   124     for pkg_name in deps:

   125         cfg = pkg_cfg.packages[pkg_name]

   126         metadata[pkg_name] = Bunch()

   127         for prop in METADATA_PROPS:

   128             if cfg.get(prop):

   129                 if prop == 'permissions':

   130                     validate_permissions(cfg[prop])

   131                 metadata[pkg_name][prop] = cfg[prop]

   132     return metadata

   134 def set_section_dir(base_json, name, base_path, dirnames, allow_root=False):

   135     resolved = compute_section_dir(base_json, base_path, dirnames, allow_root)

   136     if resolved:

   137         base_json[name] = os.path.abspath(resolved)

   139 def compute_section_dir(base_json, base_path, dirnames, allow_root):

   140     # PACKAGE_JSON.lib is highest priority

   141     # then PACKAGE_JSON.directories.lib

   142     # then lib/ (if it exists)

   143     # then . (but only if allow_root=True)

   144     for dirname in dirnames:

   145         if base_json.get(dirname):

   146             return os.path.join(base_path, base_json[dirname])

   147     if "directories" in base_json:

   148         for dirname in dirnames:

   149             if dirname in base_json.directories:

   150                 return os.path.join(base_path, base_json.directories[dirname])

   151     for dirname in dirnames:

   152         if os.path.isdir(os.path.join(base_path, dirname)):

   153             return os.path.join(base_path, dirname)

   154     if allow_root:

   155         return os.path.abspath(base_path)

   156     return None

   158 def normalize_string_or_array(base_json, key):

   159     if base_json.get(key):

   160         if isinstance(base_json[key], basestring):

   161             base_json[key] = [base_json[key]]

   163 def load_json_file(path):

   164     data = open(path, 'r').read()

   165     try:

   166         return Bunch(json.loads(data))

   167     except ValueError, e:

   168         raise MalformedJsonFileError('%s when reading "%s"' % (str(e),

   169                                                                path))

   171 def get_config_in_dir(path):

   172     package_json = os.path.join(path, MANIFEST_NAME)

   173     if not (os.path.exists(package_json) and

   174             os.path.isfile(package_json)):

   175         raise MalformedPackageError('%s not found in "%s"' % (MANIFEST_NAME,

   176                                                               path))

   177     base_json = load_json_file(package_json)

   179     if 'name' not in base_json:

   180         base_json.name = os.path.basename(path)

   182     # later processing steps will expect to see the following keys in the

   183     # base_json that we return:

   184     #

   185     #  name: name of the package

   186     #  lib: list of directories with .js files

   187     #  test: list of directories with test-*.js files

   188     #  doc: list of directories with documentation .md files

   189     #  data: list of directories with bundled arbitrary data files

   190     #  packages: ?

   192     if (not base_json.get('tests') and

   193         os.path.isdir(os.path.join(path, 'test'))):

   194         base_json['tests'] = 'test'

   196     set_section_dir(base_json, 'lib', path, ['lib'], True)

   197     set_section_dir(base_json, 'tests', path, ['test', 'tests'], False)

   198     set_section_dir(base_json, 'doc', path, ['doc', 'docs'])

   199     set_section_dir(base_json, 'data', path, ['data'])

   200     set_section_dir(base_json, 'packages', path, ['packages'])

   201     set_section_dir(base_json, 'locale', path, ['locale'])

   203     if (not base_json.get('icon') and

   204         os.path.isfile(os.path.join(path, DEFAULT_ICON))):

   205         base_json['icon'] = DEFAULT_ICON

   207     if (not base_json.get('icon64') and

   208         os.path.isfile(os.path.join(path, DEFAULT_ICON64))):

   209         base_json['icon64'] = DEFAULT_ICON64

   211     for key in ['lib', 'tests', 'dependencies', 'packages']:

   212         # TODO: lib/tests can be an array?? consider interaction with

   213         # compute_section_dir above

   214         normalize_string_or_array(base_json, key)

   216     if 'main' not in base_json and 'lib' in base_json:

   217         for dirname in base_json['lib']:

   218             program = os.path.join(path, dirname,

   219                                    '%s.js' % DEFAULT_PROGRAM_MODULE)

   220             if os.path.exists(program):

   221                 base_json['main'] = DEFAULT_PROGRAM_MODULE

   222                 break

   224     base_json.root_dir = path

   226     if "dependencies" in base_json:

   227       deps = base_json["dependencies"]

   228       deps = [x for x in deps if x not in ["addon-kit", "api-utils"]]

   229       deps.append("addon-sdk")

   230       base_json["dependencies"] = deps

   232     return base_json

   234 def _is_same_file(a, b):

   235     if hasattr(os.path, 'samefile'):

   236         return os.path.samefile(a, b)

   237     return a == b

   239 def build_config(root_dir, target_cfg, packagepath=[]):

   240     dirs_to_scan = [env_root] # root is addon-sdk dir, diff from root_dir in tests

   242     def add_packages_from_config(pkgconfig):

   243         if 'packages' in pkgconfig:

   244             for package_dir in resolve_dirs(pkgconfig, pkgconfig.packages):

   245                 dirs_to_scan.append(package_dir)

   247     add_packages_from_config(target_cfg)

   249     packages_dir = os.path.join(root_dir, 'packages')

   250     if os.path.exists(packages_dir) and os.path.isdir(packages_dir):

   251         dirs_to_scan.append(packages_dir)

   252     dirs_to_scan.extend(packagepath)

   254     packages = Bunch({target_cfg.name: target_cfg})

   256     while dirs_to_scan:

   257         packages_dir = dirs_to_scan.pop()

   258         if os.path.exists(os.path.join(packages_dir, "package.json")):

   259             package_paths = [packages_dir]

   260         else:

   261             package_paths = [os.path.join(packages_dir, dirname)

   262                              for dirname in os.listdir(packages_dir)

   263                              if not dirname.startswith('.')]

   264             package_paths = [dirname for dirname in package_paths

   265                              if os.path.isdir(dirname)]

   267         for path in package_paths:

   268             pkgconfig = get_config_in_dir(path)

   269             if pkgconfig.name in packages:

   270                 otherpkg = packages[pkgconfig.name]

   271                 if not _is_same_file(otherpkg.root_dir, path):

   272                     raise DuplicatePackageError(path, otherpkg.root_dir)

   273             else:

   274                 packages[pkgconfig.name] = pkgconfig

   275                 add_packages_from_config(pkgconfig)

   277     return Bunch(packages=packages)

   279 def get_deps_for_targets(pkg_cfg, targets):

   280     visited = []

   281     deps_left = [[dep, None] for dep in list(targets)]

   283     while deps_left:

   284         [dep, required_by] = deps_left.pop()

   285         if dep not in visited:

   286             visited.append(dep)

   287             if dep not in pkg_cfg.packages:

   288                 required_reason = ("required by '%s'" % (required_by)) \

   289                                     if required_by is not None \

   290                                     else "specified as target"

   291                 raise PackageNotFoundError(dep, required_reason)

   292             dep_cfg = pkg_cfg.packages[dep]

   293             deps_left.extend([[i, dep] for i in dep_cfg.get('dependencies', [])])

   294             deps_left.extend([[i, dep] for i in dep_cfg.get('extra_dependencies', [])])

   296     return visited

   298 def generate_build_for_target(pkg_cfg, target, deps,

   299                               include_tests=True,

   300                               include_dep_tests=False,

   301                               is_running_tests=False,

   302                               default_loader=DEFAULT_LOADER):

   304     build = Bunch(# Contains section directories for all packages:

   305                   packages=Bunch(),

   306                   locale=Bunch()

   307                   )

   309     def add_section_to_build(cfg, section, is_code=False,

   310                              is_data=False):

   311         if section in cfg:

   312             dirnames = cfg[section]

   313             if isinstance(dirnames, basestring):

   314                 # This is just for internal consistency within this

   315                 # function, it has nothing to do w/ a non-canonical

   316                 # configuration dict.

   317                 dirnames = [dirnames]

   318             for dirname in resolve_dirs(cfg, dirnames):

   319                 # ensure that package name is valid

   320                 try:

   321                     validate_resource_hostname(cfg.name)

   322                 except ValueError, err:

   323                     print err

   324                     sys.exit(1)

   325                 # ensure that this package has an entry

   326                 if not cfg.name in build.packages:

   327                     build.packages[cfg.name] = Bunch()

   328                 # detect duplicated sections

   329                 if section in build.packages[cfg.name]:

   330                     raise KeyError("package's section already defined",

   331                                    cfg.name, section)

   332                 # Register this section (lib, data, tests)

   333                 build.packages[cfg.name][section] = dirname

   335     def add_locale_to_build(cfg):

   336         # Bug 730776: Ignore locales for addon-kit, that are only for unit tests

   337         if not is_running_tests and cfg.name == "addon-sdk":

   338             return

   340         path = resolve_dir(cfg, cfg['locale'])

   341         files = os.listdir(path)

   342         for filename in files:

   343             fullpath = os.path.join(path, filename)

   344             if os.path.isfile(fullpath) and filename.endswith('.properties'):

   345                 language = filename[:-len('.properties')]

   347                 from property_parser import parse_file, MalformedLocaleFileError

   348                 try:

   349                     content = parse_file(fullpath)

   350                 except MalformedLocaleFileError, msg:

   351                     print msg[0]

   352                     sys.exit(1)

   354                 # Merge current locales into global locale hashtable.

   355                 # Locale files only contains one big JSON object

   356                 # that act as an hastable of:

   357                 # "keys to translate" => "translated keys"

   358                 if language in build.locale:

   359                     merge = (build.locale[language].items() +

   360                              content.items())

   361                     build.locale[language] = Bunch(merge)

   362                 else:

   363                     build.locale[language] = content

   365     def add_dep_to_build(dep):

   366         dep_cfg = pkg_cfg.packages[dep]

   367         add_section_to_build(dep_cfg, "lib", is_code=True)

   368         add_section_to_build(dep_cfg, "data", is_data=True)

   369         if include_tests and include_dep_tests:

   370             add_section_to_build(dep_cfg, "tests", is_code=True)

   371         if 'locale' in dep_cfg:

   372             add_locale_to_build(dep_cfg)

   373         if ("loader" in dep_cfg) and ("loader" not in build):

   374             build.loader = "%s/%s" % (dep,

   375                                                  dep_cfg.loader)

   377     target_cfg = pkg_cfg.packages[target]

   379     if include_tests and not include_dep_tests:

   380         add_section_to_build(target_cfg, "tests", is_code=True)

   382     for dep in deps:

   383         add_dep_to_build(dep)

   385     if 'loader' not in build:

   386         add_dep_to_build(DEFAULT_LOADER)

   388     if 'icon' in target_cfg:

   389         build['icon'] = os.path.join(target_cfg.root_dir, target_cfg.icon)

   390         del target_cfg['icon']

   392     if 'icon64' in target_cfg:

   393         build['icon64'] = os.path.join(target_cfg.root_dir, target_cfg.icon64)

   394         del target_cfg['icon64']

   396     if ('preferences' in target_cfg):

   397         build['preferences'] = target_cfg.preferences

   399     if 'id' in target_cfg:

   400         # NOTE: logic duplicated from buildJID()

   401         jid = target_cfg['id']

   402         if not ('@' in jid or jid.startswith('{')):

   403             jid += '@jetpack'

   404         build['preferencesBranch'] = jid

   406     if 'preferences-branch' in target_cfg:

   407         # check it's a non-empty, valid branch name

   408         preferencesBranch = target_cfg['preferences-branch']

   409         if re.match('^[\w{@}-]+$', preferencesBranch):

   410             build['preferencesBranch'] = preferencesBranch

   411         elif not is_running_tests:

   412             print >>sys.stderr, "IGNORING preferences-branch (not a valid branch name)"

   414     return build

   416 def _get_files_in_dir(path):

   417     data = {}

   418     files = os.listdir(path)

   419     for filename in files:

   420         fullpath = os.path.join(path, filename)

   421         if os.path.isdir(fullpath):

   422             data[filename] = _get_files_in_dir(fullpath)

   423         else:

   424             try:

   425                 info = os.stat(fullpath)

   426                 data[filename] = ("file", dict(size=info.st_size))

   427             except OSError:

   428                 pass

   429     return ("directory", data)

   431 def build_pkg_index(pkg_cfg):

   432     pkg_cfg = copy.deepcopy(pkg_cfg)

   433     for pkg in pkg_cfg.packages:

   434         root_dir = pkg_cfg.packages[pkg].root_dir

   435         files = _get_files_in_dir(root_dir)

   436         pkg_cfg.packages[pkg].files = files

   437         try:

   438             readme = open(root_dir + '/README.md').read()

   439             pkg_cfg.packages[pkg].readme = readme

   440         except IOError:

   441             pass

   442         del pkg_cfg.packages[pkg].root_dir

   443     return pkg_cfg.packages

   445 def build_pkg_cfg(root):

   446     pkg_cfg = build_config(root, Bunch(name='dummy'))

   447     del pkg_cfg.packages['dummy']

   448     return pkg_cfg

   450 def call_plugins(pkg_cfg, deps):

   451     for dep in deps:

   452         dep_cfg = pkg_cfg.packages[dep]

   453         dirnames = dep_cfg.get('python-lib', [])

   454         for dirname in resolve_dirs(dep_cfg, dirnames):

   455             sys.path.append(dirname)

   456         module_names = dep_cfg.get('python-plugins', [])

   457         for module_name in module_names:

   458             module = __import__(module_name)

   459             module.init(root_dir=dep_cfg.root_dir)

   461 def call_cmdline_tool(env_root, pkg_name):

   462     pkg_cfg = build_config(env_root, Bunch(name='dummy'))

   463     if pkg_name not in pkg_cfg.packages:

   464         print "This tool requires the '%s' package." % pkg_name

   465         sys.exit(1)

   466     cfg = pkg_cfg.packages[pkg_name]

   467     for dirname in resolve_dirs(cfg, cfg['python-lib']):

   468         sys.path.append(dirname)

   469     module_name = cfg.get('python-cmdline-tool')

   470     module = __import__(module_name)

   471     module.run()

The Tor Browser / file revision

addon-sdk/source/python-lib/cuddlefish/packaging.py@ac0c01689b40

addon-sdk/source/python-lib/cuddlefish/packaging.py