1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/toolkit/mozapps/installer/packager.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,401 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +from mozpack.packager.formats import (
1.9 + FlatFormatter,
1.10 + JarFormatter,
1.11 + OmniJarFormatter,
1.12 +)
1.13 +from mozpack.packager import (
1.14 + preprocess_manifest,
1.15 + preprocess,
1.16 + Component,
1.17 + SimpleManifestSink,
1.18 +)
1.19 +from mozpack.files import (
1.20 + GeneratedFile,
1.21 + FileFinder,
1.22 + File,
1.23 +)
1.24 +from mozpack.copier import (
1.25 + FileCopier,
1.26 + Jarrer,
1.27 +)
1.28 +from mozpack.errors import errors
1.29 +from mozpack.unify import UnifiedBuildFinder
1.30 +import mozpack.path
1.31 +import buildconfig
1.32 +from argparse import ArgumentParser
1.33 +from createprecomplete import generate_precomplete
1.34 +import os
1.35 +from StringIO import StringIO
1.36 +import subprocess
1.37 +import platform
1.38 +
1.39 +# List of libraries to shlibsign.
1.40 +SIGN_LIBS = [
1.41 + 'softokn3',
1.42 + 'nssdbm3',
1.43 + 'freebl3',
1.44 + 'freebl_32fpu_3',
1.45 + 'freebl_32int_3',
1.46 + 'freebl_32int64_3',
1.47 + 'freebl_64fpu_3',
1.48 + 'freebl_64int_3',
1.49 +]
1.50 +
1.51 +
1.52 +class ToolLauncher(object):
1.53 + '''
1.54 + Helper to execute tools like xpcshell with the appropriate environment.
1.55 + launcher = ToolLauncher()
1.56 + launcher.tooldir = '/path/to/tools'
1.57 + launcher.launch(['xpcshell', '-e', 'foo.js'])
1.58 + '''
1.59 + def __init__(self):
1.60 + self.tooldir = None
1.61 +
1.62 + def launch(self, cmd, extra_linker_path=None, extra_env={}):
1.63 + '''
1.64 + Launch the given command, passed as a list. The first item in the
1.65 + command list is the program name, without a path and without a suffix.
1.66 + These are determined from the tooldir member and the BIN_SUFFIX value.
1.67 + An extra_linker_path may be passed to give an additional directory
1.68 + to add to the search paths for the dynamic linker.
1.69 + An extra_env dict may be passed to give additional environment
1.70 + variables to export when running the command.
1.71 + '''
1.72 + assert self.tooldir
1.73 + cmd[0] = os.path.join(self.tooldir, 'bin',
1.74 + cmd[0] + buildconfig.substs['BIN_SUFFIX'])
1.75 + if not extra_linker_path:
1.76 + extra_linker_path = os.path.join(self.tooldir, 'bin')
1.77 + env = dict(os.environ)
1.78 + for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
1.79 + if p in env:
1.80 + env[p] = extra_linker_path + ':' + env[p]
1.81 + else:
1.82 + env[p] = extra_linker_path
1.83 + for e in extra_env:
1.84 + env[e] = extra_env[e]
1.85 +
1.86 + # Work around a bug in Python 2.7.2 and lower where unicode types in
1.87 + # environment variables aren't handled by subprocess.
1.88 + for k, v in env.items():
1.89 + if isinstance(v, unicode):
1.90 + env[k] = v.encode('utf-8')
1.91 +
1.92 + print >>errors.out, 'Executing', ' '.join(cmd)
1.93 + errors.out.flush()
1.94 + return subprocess.call(cmd, env=env)
1.95 +
1.96 + def can_launch(self):
1.97 + return self.tooldir is not None
1.98 +
1.99 +launcher = ToolLauncher()
1.100 +
1.101 +
1.102 +class LibSignFile(File):
1.103 + '''
1.104 + File class for shlibsign signatures.
1.105 + '''
1.106 + def copy(self, dest, skip_if_older=True):
1.107 + assert isinstance(dest, basestring)
1.108 + # os.path.getmtime returns a result in seconds with precision up to the
1.109 + # microsecond. But microsecond is too precise because shutil.copystat
1.110 + # only copies milliseconds, and seconds is not enough precision.
1.111 + if os.path.exists(dest) and skip_if_older and \
1.112 + int(os.path.getmtime(self.path) * 1000) <= \
1.113 + int(os.path.getmtime(dest) * 1000):
1.114 + return False
1.115 + if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
1.116 + errors.fatal('Error while signing %s' % self.path)
1.117 +
1.118 +
1.119 +def precompile_cache(formatter, source_path, gre_path, app_path):
1.120 + '''
1.121 + Create startup cache for the given application directory, using the
1.122 + given GRE path.
1.123 + - formatter is a Formatter instance where to add the startup cache.
1.124 + - source_path is the base path of the package.
1.125 + - gre_path is the GRE path, relative to source_path.
1.126 + - app_path is the application path, relative to source_path.
1.127 + Startup cache for all resources under resource://app/ are generated,
1.128 + except when gre_path == app_path, in which case it's under
1.129 + resource://gre/.
1.130 + '''
1.131 + from tempfile import mkstemp
1.132 + source_path = os.path.abspath(source_path)
1.133 + if app_path != gre_path:
1.134 + resource = 'app'
1.135 + else:
1.136 + resource = 'gre'
1.137 + app_path = os.path.join(source_path, app_path)
1.138 + gre_path = os.path.join(source_path, gre_path)
1.139 +
1.140 + fd, cache = mkstemp('.zip')
1.141 + os.close(fd)
1.142 + os.remove(cache)
1.143 +
1.144 + # For VC12, make sure we can find the right bitness of pgort120.dll
1.145 + env = os.environ.copy()
1.146 + if 'VS120COMNTOOLS' in env and not buildconfig.substs['HAVE_64BIT_OS']:
1.147 + vc12dir = os.path.abspath(os.path.join(env['VS120COMNTOOLS'],
1.148 + '../../VC/bin'))
1.149 + if os.path.exists(vc12dir):
1.150 + env['PATH'] = vc12dir + ';' + env['PATH']
1.151 +
1.152 + try:
1.153 + if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
1.154 + '-f', os.path.join(os.path.dirname(__file__),
1.155 + 'precompile_cache.js'),
1.156 + '-e', 'precompile_startupcache("resource://%s/");'
1.157 + % resource],
1.158 + extra_linker_path=gre_path,
1.159 + extra_env={'MOZ_STARTUP_CACHE': cache,
1.160 + 'PATH': env['PATH']}):
1.161 + errors.fatal('Error while running startup cache precompilation')
1.162 + return
1.163 + from mozpack.mozjar import JarReader
1.164 + jar = JarReader(cache)
1.165 + resource = '/resource/%s/' % resource
1.166 + for f in jar:
1.167 + if resource in f.filename:
1.168 + path = f.filename[f.filename.index(resource) + len(resource):]
1.169 + if formatter.contains(path):
1.170 + formatter.add(f.filename, GeneratedFile(f.read()))
1.171 + jar.close()
1.172 + finally:
1.173 + if os.path.exists(cache):
1.174 + os.remove(cache)
1.175 +
1.176 +
1.177 +class RemovedFiles(GeneratedFile):
1.178 + '''
1.179 + File class for removed-files. Is used as a preprocessor parser.
1.180 + '''
1.181 + def __init__(self, copier):
1.182 + self.copier = copier
1.183 + GeneratedFile.__init__(self, '')
1.184 +
1.185 + def handle_line(self, str):
1.186 + f = str.strip()
1.187 + if self.copier.contains(f):
1.188 + errors.error('Removal of packaged file(s): %s' % f)
1.189 + self.content += f + '\n'
1.190 +
1.191 +
1.192 +def split_define(define):
1.193 + '''
1.194 + Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
1.195 + 1. Numeric VALs are returned as ints.
1.196 + '''
1.197 + if '=' in define:
1.198 + name, value = define.split('=', 1)
1.199 + try:
1.200 + value = int(value)
1.201 + except ValueError:
1.202 + pass
1.203 + return (name, value)
1.204 + return (define, 1)
1.205 +
1.206 +
1.207 +class NoPkgFilesRemover(object):
1.208 + '''
1.209 + Formatter wrapper to handle NO_PKG_FILES.
1.210 + '''
1.211 + def __init__(self, formatter, has_manifest):
1.212 + assert 'NO_PKG_FILES' in os.environ
1.213 + self._formatter = formatter
1.214 + self._files = os.environ['NO_PKG_FILES'].split()
1.215 + if has_manifest:
1.216 + self._error = errors.error
1.217 + self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
1.218 + else:
1.219 + self._error = errors.warn
1.220 + self._msg = 'Skipping %s'
1.221 +
1.222 + def add_base(self, base):
1.223 + self._formatter.add_base(base)
1.224 +
1.225 + def add(self, path, content):
1.226 + if not any(mozpack.path.match(path, spec) for spec in self._files):
1.227 + self._formatter.add(path, content)
1.228 + else:
1.229 + self._error(self._msg % path)
1.230 +
1.231 + def add_manifest(self, entry):
1.232 + self._formatter.add_manifest(entry)
1.233 +
1.234 + def add_interfaces(self, path, content):
1.235 + self._formatter.add_interfaces(path, content)
1.236 +
1.237 + def contains(self, path):
1.238 + return self._formatter.contains(path)
1.239 +
1.240 +
1.241 +def main():
1.242 + parser = ArgumentParser()
1.243 + parser.add_argument('-D', dest='defines', action='append',
1.244 + metavar="VAR[=VAL]", help='Define a variable')
1.245 + parser.add_argument('--format', default='omni',
1.246 + help='Choose the chrome format for packaging ' +
1.247 + '(omni, jar or flat ; default: %(default)s)')
1.248 + parser.add_argument('--removals', default=None,
1.249 + help='removed-files source file')
1.250 + parser.add_argument('--ignore-errors', action='store_true', default=False,
1.251 + help='Transform errors into warnings.')
1.252 + parser.add_argument('--minify', action='store_true', default=False,
1.253 + help='Make some files more compact while packaging')
1.254 + parser.add_argument('--minify-js', action='store_true',
1.255 + help='Minify JavaScript files while packaging.')
1.256 + parser.add_argument('--js-binary',
1.257 + help='Path to js binary. This is used to verify '
1.258 + 'minified JavaScript. If this is not defined, '
1.259 + 'minification verification will not be performed.')
1.260 + parser.add_argument('--jarlog', default='', help='File containing jar ' +
1.261 + 'access logs')
1.262 + parser.add_argument('--optimizejars', action='store_true', default=False,
1.263 + help='Enable jar optimizations')
1.264 + parser.add_argument('--unify', default='',
1.265 + help='Base directory of another build to unify with')
1.266 + parser.add_argument('manifest', default=None, nargs='?',
1.267 + help='Manifest file name')
1.268 + parser.add_argument('source', help='Source directory')
1.269 + parser.add_argument('destination', help='Destination directory')
1.270 + parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
1.271 + default=[],
1.272 + help='Extra files not to be considered as resources')
1.273 + args = parser.parse_args()
1.274 +
1.275 + defines = dict(buildconfig.defines)
1.276 + if args.ignore_errors:
1.277 + errors.ignore_errors()
1.278 +
1.279 + if args.defines:
1.280 + for name, value in [split_define(d) for d in args.defines]:
1.281 + defines[name] = value
1.282 +
1.283 + copier = FileCopier()
1.284 + if args.format == 'flat':
1.285 + formatter = FlatFormatter(copier)
1.286 + elif args.format == 'jar':
1.287 + formatter = JarFormatter(copier, optimize=args.optimizejars)
1.288 + elif args.format == 'omni':
1.289 + formatter = OmniJarFormatter(copier,
1.290 + buildconfig.substs['OMNIJAR_NAME'],
1.291 + optimize=args.optimizejars,
1.292 + non_resources=args.non_resource)
1.293 + else:
1.294 + errors.fatal('Unknown format: %s' % args.format)
1.295 +
1.296 + # Adjust defines according to the requested format.
1.297 + if isinstance(formatter, OmniJarFormatter):
1.298 + defines['MOZ_OMNIJAR'] = 1
1.299 + elif 'MOZ_OMNIJAR' in defines:
1.300 + del defines['MOZ_OMNIJAR']
1.301 +
1.302 + binpath = ''
1.303 + if 'BINPATH' in defines:
1.304 + binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
1.305 + while binpath.startswith('/'):
1.306 + binpath = binpath[1:]
1.307 +
1.308 + if args.unify:
1.309 + def is_native(path):
1.310 + path = os.path.abspath(path)
1.311 + return platform.machine() in mozpack.path.split(path)
1.312 +
1.313 + # Invert args.unify and args.source if args.unify points to the
1.314 + # native architecture.
1.315 + args.source, args.unify = sorted([args.source, args.unify],
1.316 + key=is_native, reverse=True)
1.317 + if is_native(args.source):
1.318 + launcher.tooldir = args.source
1.319 + elif not buildconfig.substs['CROSS_COMPILE']:
1.320 + launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
1.321 +
1.322 + with errors.accumulate():
1.323 + finder_args = dict(
1.324 + minify=args.minify,
1.325 + minify_js=args.minify_js,
1.326 + )
1.327 + if args.js_binary:
1.328 + finder_args['minify_js_verify_command'] = [
1.329 + args.js_binary,
1.330 + os.path.join(os.path.abspath(os.path.dirname(__file__)),
1.331 + 'js-compare-ast.js')
1.332 + ]
1.333 + if args.unify:
1.334 + finder = UnifiedBuildFinder(FileFinder(args.source),
1.335 + FileFinder(args.unify),
1.336 + **finder_args)
1.337 + else:
1.338 + finder = FileFinder(args.source, **finder_args)
1.339 + if 'NO_PKG_FILES' in os.environ:
1.340 + sinkformatter = NoPkgFilesRemover(formatter,
1.341 + args.manifest is not None)
1.342 + else:
1.343 + sinkformatter = formatter
1.344 + sink = SimpleManifestSink(finder, sinkformatter)
1.345 + if args.manifest:
1.346 + preprocess_manifest(sink, args.manifest, defines)
1.347 + else:
1.348 + sink.add(Component(''), 'bin/*')
1.349 + sink.close(args.manifest is not None)
1.350 +
1.351 + if args.removals:
1.352 + lines = [l.lstrip() for l in open(args.removals).readlines()]
1.353 + removals_in = StringIO(''.join(lines))
1.354 + removals_in.name = args.removals
1.355 + removals = RemovedFiles(copier)
1.356 + preprocess(removals_in, removals, defines)
1.357 + copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
1.358 +
1.359 + # shlibsign libraries
1.360 + if launcher.can_launch():
1.361 + for lib in SIGN_LIBS:
1.362 + libbase = mozpack.path.join(binpath, '%s%s') \
1.363 + % (buildconfig.substs['DLL_PREFIX'], lib)
1.364 + libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
1.365 + if copier.contains(libname):
1.366 + copier.add(libbase + '.chk',
1.367 + LibSignFile(os.path.join(args.destination,
1.368 + libname)))
1.369 +
1.370 + # Setup preloading
1.371 + if args.jarlog and os.path.exists(args.jarlog):
1.372 + from mozpack.mozjar import JarLog
1.373 + log = JarLog(args.jarlog)
1.374 + for p, f in copier:
1.375 + if not isinstance(f, Jarrer):
1.376 + continue
1.377 + key = JarLog.canonicalize(os.path.join(args.destination, p))
1.378 + if key in log:
1.379 + f.preload(log[key])
1.380 +
1.381 + # Fill startup cache
1.382 + if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
1.383 + if buildconfig.substs['LIBXUL_SDK']:
1.384 + gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
1.385 + 'bin')
1.386 + else:
1.387 + gre_path = None
1.388 + for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
1.389 + if os.path.exists(os.path.join(args.source, p))][0]
1.390 + for b in sink.packager.get_bases()]):
1.391 + if not gre_path:
1.392 + gre_path = base
1.393 + base_path = sink.normalize_path(base)
1.394 + if base_path in formatter.omnijars:
1.395 + precompile_cache(formatter.omnijars[base_path],
1.396 + args.source, gre_path, base)
1.397 +
1.398 + copier.copy(args.destination)
1.399 + generate_precomplete(os.path.normpath(os.path.join(args.destination,
1.400 + binpath)))
1.401 +
1.402 +
1.403 +if __name__ == '__main__':
1.404 + main()
