The Tor Browser / file comparison

comparison: addon-sdk/source/python-lib/cuddlefish/packaging.py

addon-sdk/source/python-lib/cuddlefish/packaging.py

branch
TOR_BUG_3246
changeset 7
129ffea94266
equal deleted inserted replaced
-1:000000000000 0:3288cc044c05
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 import os
6 import sys
7 import re
8 import copy
9
10 import simplejson as json
11 from cuddlefish.bunch import Bunch
12
13 MANIFEST_NAME = 'package.json'
14 DEFAULT_LOADER = 'addon-sdk'
15
16 # Is different from root_dir when running tests
17 env_root = os.environ.get('CUDDLEFISH_ROOT')
18
19 DEFAULT_PROGRAM_MODULE = 'main'
20
21 DEFAULT_ICON = 'icon.png'
22 DEFAULT_ICON64 = 'icon64.png'
23
24 METADATA_PROPS = ['name', 'description', 'keywords', 'author', 'version',
25 'translators', 'contributors', 'license', 'homepage', 'icon',
26 'icon64', 'main', 'directories', 'permissions']
27
28 RESOURCE_HOSTNAME_RE = re.compile(r'^[a-z0-9_\-]+$')
29
30 class Error(Exception):
31 pass
32
33 class MalformedPackageError(Error):
34 pass
35
36 class MalformedJsonFileError(Error):
37 pass
38
39 class DuplicatePackageError(Error):
40 pass
41
42 class PackageNotFoundError(Error):
43 def __init__(self, missing_package, reason):
44 self.missing_package = missing_package
45 self.reason = reason
46 def __str__(self):
47 return "%s (%s)" % (self.missing_package, self.reason)
48
49 class BadChromeMarkerError(Error):
50 pass
51
52 def validate_resource_hostname(name):
53 """
54 Validates the given hostname for a resource: URI.
55
56 For more information, see:
57
58 https://bugzilla.mozilla.org/show_bug.cgi?id=566812#c13
59
60 Examples:
61
62 >>> validate_resource_hostname('blarg')
63
64 >>> validate_resource_hostname('bl arg')
65 Traceback (most recent call last):
66 ...
67 ValueError: Error: the name of your package contains an invalid character.
68 Package names can contain only lower-case letters, numbers, underscores, and dashes.
69 Current package name: bl arg
70
71 >>> validate_resource_hostname('BLARG')
72 Traceback (most recent call last):
73 ...
74 ValueError: Error: the name of your package contains upper-case letters.
75 Package names can contain only lower-case letters, numbers, underscores, and dashes.
76 Current package name: BLARG
77
78 >>> validate_resource_hostname('foo@bar')
79 Traceback (most recent call last):
80 ...
81 ValueError: Error: the name of your package contains an invalid character.
82 Package names can contain only lower-case letters, numbers, underscores, and dashes.
83 Current package name: foo@bar
84 """
85
86 # See https://bugzilla.mozilla.org/show_bug.cgi?id=568131 for details.
87 if not name.islower():
88 raise ValueError("""Error: the name of your package contains upper-case letters.
89 Package names can contain only lower-case letters, numbers, underscores, and dashes.
90 Current package name: %s""" % name)
91
92 if not RESOURCE_HOSTNAME_RE.match(name):
93 raise ValueError("""Error: the name of your package contains an invalid character.
94 Package names can contain only lower-case letters, numbers, underscores, and dashes.
95 Current package name: %s""" % name)
96
97 def find_packages_with_module(pkg_cfg, name):
98 # TODO: Make this support more than just top-level modules.
99 filename = "%s.js" % name
100 packages = []
101 for cfg in pkg_cfg.packages.itervalues():
102 if 'lib' in cfg:
103 matches = [dirname for dirname in resolve_dirs(cfg, cfg.lib)
104 if os.path.exists(os.path.join(dirname, filename))]
105 if matches:
106 packages.append(cfg.name)
107 return packages
108
109 def resolve_dirs(pkg_cfg, dirnames):
110 for dirname in dirnames:
111 yield resolve_dir(pkg_cfg, dirname)
112
113 def resolve_dir(pkg_cfg, dirname):
114 return os.path.join(pkg_cfg.root_dir, dirname)
115
116 def validate_permissions(perms):
117 if (perms.get('cross-domain-content') and
118 not isinstance(perms.get('cross-domain-content'), list)):
119 raise ValueError("Error: `cross-domain-content` permissions in \
120 package.json file must be an array of strings:\n %s" % perms)
121
122 def get_metadata(pkg_cfg, deps):
123 metadata = Bunch()
124 for pkg_name in deps:
125 cfg = pkg_cfg.packages[pkg_name]
126 metadata[pkg_name] = Bunch()
127 for prop in METADATA_PROPS:
128 if cfg.get(prop):
129 if prop == 'permissions':
130 validate_permissions(cfg[prop])
131 metadata[pkg_name][prop] = cfg[prop]
132 return metadata
133
134 def set_section_dir(base_json, name, base_path, dirnames, allow_root=False):
135 resolved = compute_section_dir(base_json, base_path, dirnames, allow_root)
136 if resolved:
137 base_json[name] = os.path.abspath(resolved)
138
139 def compute_section_dir(base_json, base_path, dirnames, allow_root):
140 # PACKAGE_JSON.lib is highest priority
141 # then PACKAGE_JSON.directories.lib
142 # then lib/ (if it exists)
143 # then . (but only if allow_root=True)
144 for dirname in dirnames:
145 if base_json.get(dirname):
146 return os.path.join(base_path, base_json[dirname])
147 if "directories" in base_json:
148 for dirname in dirnames:
149 if dirname in base_json.directories:
150 return os.path.join(base_path, base_json.directories[dirname])
151 for dirname in dirnames:
152 if os.path.isdir(os.path.join(base_path, dirname)):
153 return os.path.join(base_path, dirname)
154 if allow_root:
155 return os.path.abspath(base_path)
156 return None
157
158 def normalize_string_or_array(base_json, key):
159 if base_json.get(key):
160 if isinstance(base_json[key], basestring):
161 base_json[key] = [base_json[key]]
162
163 def load_json_file(path):
164 data = open(path, 'r').read()
165 try:
166 return Bunch(json.loads(data))
167 except ValueError, e:
168 raise MalformedJsonFileError('%s when reading "%s"' % (str(e),
169 path))
170
171 def get_config_in_dir(path):
172 package_json = os.path.join(path, MANIFEST_NAME)
173 if not (os.path.exists(package_json) and
174 os.path.isfile(package_json)):
175 raise MalformedPackageError('%s not found in "%s"' % (MANIFEST_NAME,
176 path))
177 base_json = load_json_file(package_json)
178
179 if 'name' not in base_json:
180 base_json.name = os.path.basename(path)
181
182 # later processing steps will expect to see the following keys in the
183 # base_json that we return:
184 #
185 # name: name of the package
186 # lib: list of directories with .js files
187 # test: list of directories with test-*.js files
188 # doc: list of directories with documentation .md files
189 # data: list of directories with bundled arbitrary data files
190 # packages: ?
191
192 if (not base_json.get('tests') and
193 os.path.isdir(os.path.join(path, 'test'))):
194 base_json['tests'] = 'test'
195
196 set_section_dir(base_json, 'lib', path, ['lib'], True)
197 set_section_dir(base_json, 'tests', path, ['test', 'tests'], False)
198 set_section_dir(base_json, 'doc', path, ['doc', 'docs'])
199 set_section_dir(base_json, 'data', path, ['data'])
200 set_section_dir(base_json, 'packages', path, ['packages'])
201 set_section_dir(base_json, 'locale', path, ['locale'])
202
203 if (not base_json.get('icon') and
204 os.path.isfile(os.path.join(path, DEFAULT_ICON))):
205 base_json['icon'] = DEFAULT_ICON
206
207 if (not base_json.get('icon64') and
208 os.path.isfile(os.path.join(path, DEFAULT_ICON64))):
209 base_json['icon64'] = DEFAULT_ICON64
210
211 for key in ['lib', 'tests', 'dependencies', 'packages']:
212 # TODO: lib/tests can be an array?? consider interaction with
213 # compute_section_dir above
214 normalize_string_or_array(base_json, key)
215
216 if 'main' not in base_json and 'lib' in base_json:
217 for dirname in base_json['lib']:
218 program = os.path.join(path, dirname,
219 '%s.js' % DEFAULT_PROGRAM_MODULE)
220 if os.path.exists(program):
221 base_json['main'] = DEFAULT_PROGRAM_MODULE
222 break
223
224 base_json.root_dir = path
225
226 if "dependencies" in base_json:
227 deps = base_json["dependencies"]
228 deps = [x for x in deps if x not in ["addon-kit", "api-utils"]]
229 deps.append("addon-sdk")
230 base_json["dependencies"] = deps
231
232 return base_json
233
234 def _is_same_file(a, b):
235 if hasattr(os.path, 'samefile'):
236 return os.path.samefile(a, b)
237 return a == b
238
239 def build_config(root_dir, target_cfg, packagepath=[]):
240 dirs_to_scan = [env_root] # root is addon-sdk dir, diff from root_dir in tests
241
242 def add_packages_from_config(pkgconfig):
243 if 'packages' in pkgconfig:
244 for package_dir in resolve_dirs(pkgconfig, pkgconfig.packages):
245 dirs_to_scan.append(package_dir)
246
247 add_packages_from_config(target_cfg)
248
249 packages_dir = os.path.join(root_dir, 'packages')
250 if os.path.exists(packages_dir) and os.path.isdir(packages_dir):
251 dirs_to_scan.append(packages_dir)
252 dirs_to_scan.extend(packagepath)
253
254 packages = Bunch({target_cfg.name: target_cfg})
255
256 while dirs_to_scan:
257 packages_dir = dirs_to_scan.pop()
258 if os.path.exists(os.path.join(packages_dir, "package.json")):
259 package_paths = [packages_dir]
260 else:
261 package_paths = [os.path.join(packages_dir, dirname)
262 for dirname in os.listdir(packages_dir)
263 if not dirname.startswith('.')]
264 package_paths = [dirname for dirname in package_paths
265 if os.path.isdir(dirname)]
266
267 for path in package_paths:
268 pkgconfig = get_config_in_dir(path)
269 if pkgconfig.name in packages:
270 otherpkg = packages[pkgconfig.name]
271 if not _is_same_file(otherpkg.root_dir, path):
272 raise DuplicatePackageError(path, otherpkg.root_dir)
273 else:
274 packages[pkgconfig.name] = pkgconfig
275 add_packages_from_config(pkgconfig)
276
277 return Bunch(packages=packages)
278
279 def get_deps_for_targets(pkg_cfg, targets):
280 visited = []
281 deps_left = [[dep, None] for dep in list(targets)]
282
283 while deps_left:
284 [dep, required_by] = deps_left.pop()
285 if dep not in visited:
286 visited.append(dep)
287 if dep not in pkg_cfg.packages:
288 required_reason = ("required by '%s'" % (required_by)) \
289 if required_by is not None \
290 else "specified as target"
291 raise PackageNotFoundError(dep, required_reason)
292 dep_cfg = pkg_cfg.packages[dep]
293 deps_left.extend([[i, dep] for i in dep_cfg.get('dependencies', [])])
294 deps_left.extend([[i, dep] for i in dep_cfg.get('extra_dependencies', [])])
295
296 return visited
297
298 def generate_build_for_target(pkg_cfg, target, deps,
299 include_tests=True,
300 include_dep_tests=False,
301 is_running_tests=False,
302 default_loader=DEFAULT_LOADER):
303
304 build = Bunch(# Contains section directories for all packages:
305 packages=Bunch(),
306 locale=Bunch()
307 )
308
309 def add_section_to_build(cfg, section, is_code=False,
310 is_data=False):
311 if section in cfg:
312 dirnames = cfg[section]
313 if isinstance(dirnames, basestring):
314 # This is just for internal consistency within this
315 # function, it has nothing to do w/ a non-canonical
316 # configuration dict.
317 dirnames = [dirnames]
318 for dirname in resolve_dirs(cfg, dirnames):
319 # ensure that package name is valid
320 try:
321 validate_resource_hostname(cfg.name)
322 except ValueError, err:
323 print err
324 sys.exit(1)
325 # ensure that this package has an entry
326 if not cfg.name in build.packages:
327 build.packages[cfg.name] = Bunch()
328 # detect duplicated sections
329 if section in build.packages[cfg.name]:
330 raise KeyError("package's section already defined",
331 cfg.name, section)
332 # Register this section (lib, data, tests)
333 build.packages[cfg.name][section] = dirname
334
335 def add_locale_to_build(cfg):
336 # Bug 730776: Ignore locales for addon-kit, that are only for unit tests
337 if not is_running_tests and cfg.name == "addon-sdk":
338 return
339
340 path = resolve_dir(cfg, cfg['locale'])
341 files = os.listdir(path)
342 for filename in files:
343 fullpath = os.path.join(path, filename)
344 if os.path.isfile(fullpath) and filename.endswith('.properties'):
345 language = filename[:-len('.properties')]
346
347 from property_parser import parse_file, MalformedLocaleFileError
348 try:
349 content = parse_file(fullpath)
350 except MalformedLocaleFileError, msg:
351 print msg[0]
352 sys.exit(1)
353
354 # Merge current locales into global locale hashtable.
355 # Locale files only contains one big JSON object
356 # that act as an hastable of:
357 # "keys to translate" => "translated keys"
358 if language in build.locale:
359 merge = (build.locale[language].items() +
360 content.items())
361 build.locale[language] = Bunch(merge)
362 else:
363 build.locale[language] = content
364
365 def add_dep_to_build(dep):
366 dep_cfg = pkg_cfg.packages[dep]
367 add_section_to_build(dep_cfg, "lib", is_code=True)
368 add_section_to_build(dep_cfg, "data", is_data=True)
369 if include_tests and include_dep_tests:
370 add_section_to_build(dep_cfg, "tests", is_code=True)
371 if 'locale' in dep_cfg:
372 add_locale_to_build(dep_cfg)
373 if ("loader" in dep_cfg) and ("loader" not in build):
374 build.loader = "%s/%s" % (dep,
375 dep_cfg.loader)
376
377 target_cfg = pkg_cfg.packages[target]
378
379 if include_tests and not include_dep_tests:
380 add_section_to_build(target_cfg, "tests", is_code=True)
381
382 for dep in deps:
383 add_dep_to_build(dep)
384
385 if 'loader' not in build:
386 add_dep_to_build(DEFAULT_LOADER)
387
388 if 'icon' in target_cfg:
389 build['icon'] = os.path.join(target_cfg.root_dir, target_cfg.icon)
390 del target_cfg['icon']
391
392 if 'icon64' in target_cfg:
393 build['icon64'] = os.path.join(target_cfg.root_dir, target_cfg.icon64)
394 del target_cfg['icon64']
395
396 if ('preferences' in target_cfg):
397 build['preferences'] = target_cfg.preferences
398
399 if 'id' in target_cfg:
400 # NOTE: logic duplicated from buildJID()
401 jid = target_cfg['id']
402 if not ('@' in jid or jid.startswith('{')):
403 jid += '@jetpack'
404 build['preferencesBranch'] = jid
405
406 if 'preferences-branch' in target_cfg:
407 # check it's a non-empty, valid branch name
408 preferencesBranch = target_cfg['preferences-branch']
409 if re.match('^[\w{@}-]+$', preferencesBranch):
410 build['preferencesBranch'] = preferencesBranch
411 elif not is_running_tests:
412 print >>sys.stderr, "IGNORING preferences-branch (not a valid branch name)"
413
414 return build
415
416 def _get_files_in_dir(path):
417 data = {}
418 files = os.listdir(path)
419 for filename in files:
420 fullpath = os.path.join(path, filename)
421 if os.path.isdir(fullpath):
422 data[filename] = _get_files_in_dir(fullpath)
423 else:
424 try:
425 info = os.stat(fullpath)
426 data[filename] = ("file", dict(size=info.st_size))
427 except OSError:
428 pass
429 return ("directory", data)
430
431 def build_pkg_index(pkg_cfg):
432 pkg_cfg = copy.deepcopy(pkg_cfg)
433 for pkg in pkg_cfg.packages:
434 root_dir = pkg_cfg.packages[pkg].root_dir
435 files = _get_files_in_dir(root_dir)
436 pkg_cfg.packages[pkg].files = files
437 try:
438 readme = open(root_dir + '/README.md').read()
439 pkg_cfg.packages[pkg].readme = readme
440 except IOError:
441 pass
442 del pkg_cfg.packages[pkg].root_dir
443 return pkg_cfg.packages
444
445 def build_pkg_cfg(root):
446 pkg_cfg = build_config(root, Bunch(name='dummy'))
447 del pkg_cfg.packages['dummy']
448 return pkg_cfg
449
450 def call_plugins(pkg_cfg, deps):
451 for dep in deps:
452 dep_cfg = pkg_cfg.packages[dep]
453 dirnames = dep_cfg.get('python-lib', [])
454 for dirname in resolve_dirs(dep_cfg, dirnames):
455 sys.path.append(dirname)
456 module_names = dep_cfg.get('python-plugins', [])
457 for module_name in module_names:
458 module = __import__(module_name)
459 module.init(root_dir=dep_cfg.root_dir)
460
461 def call_cmdline_tool(env_root, pkg_name):
462 pkg_cfg = build_config(env_root, Bunch(name='dummy'))
463 if pkg_name not in pkg_cfg.packages:
464 print "This tool requires the '%s' package." % pkg_name
465 sys.exit(1)
466 cfg = pkg_cfg.packages[pkg_name]
467 for dirname in resolve_dirs(cfg, cfg['python-lib']):
468 sys.path.append(dirname)
469 module_name = cfg.get('python-cmdline-tool')
470 module = __import__(module_name)
471 module.run()

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial