The Tor Browser / file comparison
comparison: toolkit/mozapps/installer/packager.py
toolkit/mozapps/installer/packager.py
- branch
- TOR_BUG_3246
- changeset 7
- 129ffea94266
equal
deleted
inserted
replaced
| -1:000000000000 | 0:de9149baeb9f |
|---|---|
| 1 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 2 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 4 | |
| 5 from mozpack.packager.formats import ( | |
| 6 FlatFormatter, | |
| 7 JarFormatter, | |
| 8 OmniJarFormatter, | |
| 9 ) | |
| 10 from mozpack.packager import ( | |
| 11 preprocess_manifest, | |
| 12 preprocess, | |
| 13 Component, | |
| 14 SimpleManifestSink, | |
| 15 ) | |
| 16 from mozpack.files import ( | |
| 17 GeneratedFile, | |
| 18 FileFinder, | |
| 19 File, | |
| 20 ) | |
| 21 from mozpack.copier import ( | |
| 22 FileCopier, | |
| 23 Jarrer, | |
| 24 ) | |
| 25 from mozpack.errors import errors | |
| 26 from mozpack.unify import UnifiedBuildFinder | |
| 27 import mozpack.path | |
| 28 import buildconfig | |
| 29 from argparse import ArgumentParser | |
| 30 from createprecomplete import generate_precomplete | |
| 31 import os | |
| 32 from StringIO import StringIO | |
| 33 import subprocess | |
| 34 import platform | |
| 35 | |
| 36 # List of libraries to shlibsign. | |
| 37 SIGN_LIBS = [ | |
| 38 'softokn3', | |
| 39 'nssdbm3', | |
| 40 'freebl3', | |
| 41 'freebl_32fpu_3', | |
| 42 'freebl_32int_3', | |
| 43 'freebl_32int64_3', | |
| 44 'freebl_64fpu_3', | |
| 45 'freebl_64int_3', | |
| 46 ] | |
| 47 | |
| 48 | |
| 49 class ToolLauncher(object): | |
| 50 ''' | |
| 51 Helper to execute tools like xpcshell with the appropriate environment. | |
| 52 launcher = ToolLauncher() | |
| 53 launcher.tooldir = '/path/to/tools' | |
| 54 launcher.launch(['xpcshell', '-e', 'foo.js']) | |
| 55 ''' | |
| 56 def __init__(self): | |
| 57 self.tooldir = None | |
| 58 | |
| 59 def launch(self, cmd, extra_linker_path=None, extra_env={}): | |
| 60 ''' | |
| 61 Launch the given command, passed as a list. The first item in the | |
| 62 command list is the program name, without a path and without a suffix. | |
| 63 These are determined from the tooldir member and the BIN_SUFFIX value. | |
| 64 An extra_linker_path may be passed to give an additional directory | |
| 65 to add to the search paths for the dynamic linker. | |
| 66 An extra_env dict may be passed to give additional environment | |
| 67 variables to export when running the command. | |
| 68 ''' | |
| 69 assert self.tooldir | |
| 70 cmd[0] = os.path.join(self.tooldir, 'bin', | |
| 71 cmd[0] + buildconfig.substs['BIN_SUFFIX']) | |
| 72 if not extra_linker_path: | |
| 73 extra_linker_path = os.path.join(self.tooldir, 'bin') | |
| 74 env = dict(os.environ) | |
| 75 for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']: | |
| 76 if p in env: | |
| 77 env[p] = extra_linker_path + ':' + env[p] | |
| 78 else: | |
| 79 env[p] = extra_linker_path | |
| 80 for e in extra_env: | |
| 81 env[e] = extra_env[e] | |
| 82 | |
| 83 # Work around a bug in Python 2.7.2 and lower where unicode types in | |
| 84 # environment variables aren't handled by subprocess. | |
| 85 for k, v in env.items(): | |
| 86 if isinstance(v, unicode): | |
| 87 env[k] = v.encode('utf-8') | |
| 88 | |
| 89 print >>errors.out, 'Executing', ' '.join(cmd) | |
| 90 errors.out.flush() | |
| 91 return subprocess.call(cmd, env=env) | |
| 92 | |
| 93 def can_launch(self): | |
| 94 return self.tooldir is not None | |
| 95 | |
| 96 launcher = ToolLauncher() | |
| 97 | |
| 98 | |
| 99 class LibSignFile(File): | |
| 100 ''' | |
| 101 File class for shlibsign signatures. | |
| 102 ''' | |
| 103 def copy(self, dest, skip_if_older=True): | |
| 104 assert isinstance(dest, basestring) | |
| 105 # os.path.getmtime returns a result in seconds with precision up to the | |
| 106 # microsecond. But microsecond is too precise because shutil.copystat | |
| 107 # only copies milliseconds, and seconds is not enough precision. | |
| 108 if os.path.exists(dest) and skip_if_older and \ | |
| 109 int(os.path.getmtime(self.path) * 1000) <= \ | |
| 110 int(os.path.getmtime(dest) * 1000): | |
| 111 return False | |
| 112 if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]): | |
| 113 errors.fatal('Error while signing %s' % self.path) | |
| 114 | |
| 115 | |
| 116 def precompile_cache(formatter, source_path, gre_path, app_path): | |
| 117 ''' | |
| 118 Create startup cache for the given application directory, using the | |
| 119 given GRE path. | |
| 120 - formatter is a Formatter instance where to add the startup cache. | |
| 121 - source_path is the base path of the package. | |
| 122 - gre_path is the GRE path, relative to source_path. | |
| 123 - app_path is the application path, relative to source_path. | |
| 124 Startup cache for all resources under resource://app/ are generated, | |
| 125 except when gre_path == app_path, in which case it's under | |
| 126 resource://gre/. | |
| 127 ''' | |
| 128 from tempfile import mkstemp | |
| 129 source_path = os.path.abspath(source_path) | |
| 130 if app_path != gre_path: | |
| 131 resource = 'app' | |
| 132 else: | |
| 133 resource = 'gre' | |
| 134 app_path = os.path.join(source_path, app_path) | |
| 135 gre_path = os.path.join(source_path, gre_path) | |
| 136 | |
| 137 fd, cache = mkstemp('.zip') | |
| 138 os.close(fd) | |
| 139 os.remove(cache) | |
| 140 | |
| 141 # For VC12, make sure we can find the right bitness of pgort120.dll | |
| 142 env = os.environ.copy() | |
| 143 if 'VS120COMNTOOLS' in env and not buildconfig.substs['HAVE_64BIT_OS']: | |
| 144 vc12dir = os.path.abspath(os.path.join(env['VS120COMNTOOLS'], | |
| 145 '../../VC/bin')) | |
| 146 if os.path.exists(vc12dir): | |
| 147 env['PATH'] = vc12dir + ';' + env['PATH'] | |
| 148 | |
| 149 try: | |
| 150 if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path, | |
| 151 '-f', os.path.join(os.path.dirname(__file__), | |
| 152 'precompile_cache.js'), | |
| 153 '-e', 'precompile_startupcache("resource://%s/");' | |
| 154 % resource], | |
| 155 extra_linker_path=gre_path, | |
| 156 extra_env={'MOZ_STARTUP_CACHE': cache, | |
| 157 'PATH': env['PATH']}): | |
| 158 errors.fatal('Error while running startup cache precompilation') | |
| 159 return | |
| 160 from mozpack.mozjar import JarReader | |
| 161 jar = JarReader(cache) | |
| 162 resource = '/resource/%s/' % resource | |
| 163 for f in jar: | |
| 164 if resource in f.filename: | |
| 165 path = f.filename[f.filename.index(resource) + len(resource):] | |
| 166 if formatter.contains(path): | |
| 167 formatter.add(f.filename, GeneratedFile(f.read())) | |
| 168 jar.close() | |
| 169 finally: | |
| 170 if os.path.exists(cache): | |
| 171 os.remove(cache) | |
| 172 | |
| 173 | |
| 174 class RemovedFiles(GeneratedFile): | |
| 175 ''' | |
| 176 File class for removed-files. Is used as a preprocessor parser. | |
| 177 ''' | |
| 178 def __init__(self, copier): | |
| 179 self.copier = copier | |
| 180 GeneratedFile.__init__(self, '') | |
| 181 | |
| 182 def handle_line(self, str): | |
| 183 f = str.strip() | |
| 184 if self.copier.contains(f): | |
| 185 errors.error('Removal of packaged file(s): %s' % f) | |
| 186 self.content += f + '\n' | |
| 187 | |
| 188 | |
| 189 def split_define(define): | |
| 190 ''' | |
| 191 Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to | |
| 192 1. Numeric VALs are returned as ints. | |
| 193 ''' | |
| 194 if '=' in define: | |
| 195 name, value = define.split('=', 1) | |
| 196 try: | |
| 197 value = int(value) | |
| 198 except ValueError: | |
| 199 pass | |
| 200 return (name, value) | |
| 201 return (define, 1) | |
| 202 | |
| 203 | |
| 204 class NoPkgFilesRemover(object): | |
| 205 ''' | |
| 206 Formatter wrapper to handle NO_PKG_FILES. | |
| 207 ''' | |
| 208 def __init__(self, formatter, has_manifest): | |
| 209 assert 'NO_PKG_FILES' in os.environ | |
| 210 self._formatter = formatter | |
| 211 self._files = os.environ['NO_PKG_FILES'].split() | |
| 212 if has_manifest: | |
| 213 self._error = errors.error | |
| 214 self._msg = 'NO_PKG_FILES contains file listed in manifest: %s' | |
| 215 else: | |
| 216 self._error = errors.warn | |
| 217 self._msg = 'Skipping %s' | |
| 218 | |
| 219 def add_base(self, base): | |
| 220 self._formatter.add_base(base) | |
| 221 | |
| 222 def add(self, path, content): | |
| 223 if not any(mozpack.path.match(path, spec) for spec in self._files): | |
| 224 self._formatter.add(path, content) | |
| 225 else: | |
| 226 self._error(self._msg % path) | |
| 227 | |
| 228 def add_manifest(self, entry): | |
| 229 self._formatter.add_manifest(entry) | |
| 230 | |
| 231 def add_interfaces(self, path, content): | |
| 232 self._formatter.add_interfaces(path, content) | |
| 233 | |
| 234 def contains(self, path): | |
| 235 return self._formatter.contains(path) | |
| 236 | |
| 237 | |
| 238 def main(): | |
| 239 parser = ArgumentParser() | |
| 240 parser.add_argument('-D', dest='defines', action='append', | |
| 241 metavar="VAR[=VAL]", help='Define a variable') | |
| 242 parser.add_argument('--format', default='omni', | |
| 243 help='Choose the chrome format for packaging ' + | |
| 244 '(omni, jar or flat ; default: %(default)s)') | |
| 245 parser.add_argument('--removals', default=None, | |
| 246 help='removed-files source file') | |
| 247 parser.add_argument('--ignore-errors', action='store_true', default=False, | |
| 248 help='Transform errors into warnings.') | |
| 249 parser.add_argument('--minify', action='store_true', default=False, | |
| 250 help='Make some files more compact while packaging') | |
| 251 parser.add_argument('--minify-js', action='store_true', | |
| 252 help='Minify JavaScript files while packaging.') | |
| 253 parser.add_argument('--js-binary', | |
| 254 help='Path to js binary. This is used to verify ' | |
| 255 'minified JavaScript. If this is not defined, ' | |
| 256 'minification verification will not be performed.') | |
| 257 parser.add_argument('--jarlog', default='', help='File containing jar ' + | |
| 258 'access logs') | |
| 259 parser.add_argument('--optimizejars', action='store_true', default=False, | |
| 260 help='Enable jar optimizations') | |
| 261 parser.add_argument('--unify', default='', | |
| 262 help='Base directory of another build to unify with') | |
| 263 parser.add_argument('manifest', default=None, nargs='?', | |
| 264 help='Manifest file name') | |
| 265 parser.add_argument('source', help='Source directory') | |
| 266 parser.add_argument('destination', help='Destination directory') | |
| 267 parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', | |
| 268 default=[], | |
| 269 help='Extra files not to be considered as resources') | |
| 270 args = parser.parse_args() | |
| 271 | |
| 272 defines = dict(buildconfig.defines) | |
| 273 if args.ignore_errors: | |
| 274 errors.ignore_errors() | |
| 275 | |
| 276 if args.defines: | |
| 277 for name, value in [split_define(d) for d in args.defines]: | |
| 278 defines[name] = value | |
| 279 | |
| 280 copier = FileCopier() | |
| 281 if args.format == 'flat': | |
| 282 formatter = FlatFormatter(copier) | |
| 283 elif args.format == 'jar': | |
| 284 formatter = JarFormatter(copier, optimize=args.optimizejars) | |
| 285 elif args.format == 'omni': | |
| 286 formatter = OmniJarFormatter(copier, | |
| 287 buildconfig.substs['OMNIJAR_NAME'], | |
| 288 optimize=args.optimizejars, | |
| 289 non_resources=args.non_resource) | |
| 290 else: | |
| 291 errors.fatal('Unknown format: %s' % args.format) | |
| 292 | |
| 293 # Adjust defines according to the requested format. | |
| 294 if isinstance(formatter, OmniJarFormatter): | |
| 295 defines['MOZ_OMNIJAR'] = 1 | |
| 296 elif 'MOZ_OMNIJAR' in defines: | |
| 297 del defines['MOZ_OMNIJAR'] | |
| 298 | |
| 299 binpath = '' | |
| 300 if 'BINPATH' in defines: | |
| 301 binpath = SimpleManifestSink.normalize_path(defines['BINPATH']) | |
| 302 while binpath.startswith('/'): | |
| 303 binpath = binpath[1:] | |
| 304 | |
| 305 if args.unify: | |
| 306 def is_native(path): | |
| 307 path = os.path.abspath(path) | |
| 308 return platform.machine() in mozpack.path.split(path) | |
| 309 | |
| 310 # Invert args.unify and args.source if args.unify points to the | |
| 311 # native architecture. | |
| 312 args.source, args.unify = sorted([args.source, args.unify], | |
| 313 key=is_native, reverse=True) | |
| 314 if is_native(args.source): | |
| 315 launcher.tooldir = args.source | |
| 316 elif not buildconfig.substs['CROSS_COMPILE']: | |
| 317 launcher.tooldir = buildconfig.substs['LIBXUL_DIST'] | |
| 318 | |
| 319 with errors.accumulate(): | |
| 320 finder_args = dict( | |
| 321 minify=args.minify, | |
| 322 minify_js=args.minify_js, | |
| 323 ) | |
| 324 if args.js_binary: | |
| 325 finder_args['minify_js_verify_command'] = [ | |
| 326 args.js_binary, | |
| 327 os.path.join(os.path.abspath(os.path.dirname(__file__)), | |
| 328 'js-compare-ast.js') | |
| 329 ] | |
| 330 if args.unify: | |
| 331 finder = UnifiedBuildFinder(FileFinder(args.source), | |
| 332 FileFinder(args.unify), | |
| 333 **finder_args) | |
| 334 else: | |
| 335 finder = FileFinder(args.source, **finder_args) | |
| 336 if 'NO_PKG_FILES' in os.environ: | |
| 337 sinkformatter = NoPkgFilesRemover(formatter, | |
| 338 args.manifest is not None) | |
| 339 else: | |
| 340 sinkformatter = formatter | |
| 341 sink = SimpleManifestSink(finder, sinkformatter) | |
| 342 if args.manifest: | |
| 343 preprocess_manifest(sink, args.manifest, defines) | |
| 344 else: | |
| 345 sink.add(Component(''), 'bin/*') | |
| 346 sink.close(args.manifest is not None) | |
| 347 | |
| 348 if args.removals: | |
| 349 lines = [l.lstrip() for l in open(args.removals).readlines()] | |
| 350 removals_in = StringIO(''.join(lines)) | |
| 351 removals_in.name = args.removals | |
| 352 removals = RemovedFiles(copier) | |
| 353 preprocess(removals_in, removals, defines) | |
| 354 copier.add(mozpack.path.join(binpath, 'removed-files'), removals) | |
| 355 | |
| 356 # shlibsign libraries | |
| 357 if launcher.can_launch(): | |
| 358 for lib in SIGN_LIBS: | |
| 359 libbase = mozpack.path.join(binpath, '%s%s') \ | |
| 360 % (buildconfig.substs['DLL_PREFIX'], lib) | |
| 361 libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) | |
| 362 if copier.contains(libname): | |
| 363 copier.add(libbase + '.chk', | |
| 364 LibSignFile(os.path.join(args.destination, | |
| 365 libname))) | |
| 366 | |
| 367 # Setup preloading | |
| 368 if args.jarlog and os.path.exists(args.jarlog): | |
| 369 from mozpack.mozjar import JarLog | |
| 370 log = JarLog(args.jarlog) | |
| 371 for p, f in copier: | |
| 372 if not isinstance(f, Jarrer): | |
| 373 continue | |
| 374 key = JarLog.canonicalize(os.path.join(args.destination, p)) | |
| 375 if key in log: | |
| 376 f.preload(log[key]) | |
| 377 | |
| 378 # Fill startup cache | |
| 379 if isinstance(formatter, OmniJarFormatter) and launcher.can_launch(): | |
| 380 if buildconfig.substs['LIBXUL_SDK']: | |
| 381 gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'], | |
| 382 'bin') | |
| 383 else: | |
| 384 gre_path = None | |
| 385 for base in sorted([[p for p in [mozpack.path.join('bin', b), b] | |
| 386 if os.path.exists(os.path.join(args.source, p))][0] | |
| 387 for b in sink.packager.get_bases()]): | |
| 388 if not gre_path: | |
| 389 gre_path = base | |
| 390 base_path = sink.normalize_path(base) | |
| 391 if base_path in formatter.omnijars: | |
| 392 precompile_cache(formatter.omnijars[base_path], | |
| 393 args.source, gre_path, base) | |
| 394 | |
| 395 copier.copy(args.destination) | |
| 396 generate_precomplete(os.path.normpath(os.path.join(args.destination, | |
| 397 binpath))) | |
| 398 | |
| 399 | |
| 400 if __name__ == '__main__': | |
| 401 main() |
