michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: from mozpack.packager.formats import (
michael@0:     FlatFormatter,
michael@0:     JarFormatter,
michael@0:     OmniJarFormatter,
michael@0: )
michael@0: from mozpack.packager import (
michael@0:     preprocess_manifest,
michael@0:     preprocess,
michael@0:     Component,
michael@0:     SimpleManifestSink,
michael@0: )
michael@0: from mozpack.files import (
michael@0:     GeneratedFile,
michael@0:     FileFinder,
michael@0:     File,
michael@0: )
michael@0: from mozpack.copier import (
michael@0:     FileCopier,
michael@0:     Jarrer,
michael@0: )
michael@0: from mozpack.errors import errors
michael@0: from mozpack.unify import UnifiedBuildFinder
michael@0: import mozpack.path
michael@0: import buildconfig
michael@0: from argparse import ArgumentParser
michael@0: from createprecomplete import generate_precomplete
michael@0: import os
michael@0: from StringIO import StringIO
michael@0: import subprocess
michael@0: import platform
michael@0: 
michael@0: # List of libraries to shlibsign.
michael@0: SIGN_LIBS = [
michael@0:     'softokn3',
michael@0:     'nssdbm3',
michael@0:     'freebl3',
michael@0:     'freebl_32fpu_3',
michael@0:     'freebl_32int_3',
michael@0:     'freebl_32int64_3',
michael@0:     'freebl_64fpu_3',
michael@0:     'freebl_64int_3',
michael@0: ]
michael@0: 
michael@0: 
michael@0: class ToolLauncher(object):
michael@0:     '''
michael@0:     Helper to execute tools like xpcshell with the appropriate environment.
michael@0:         launcher = ToolLauncher()
michael@0:         launcher.tooldir = '/path/to/tools'
michael@0:         launcher.launch(['xpcshell', '-e', 'foo.js'])
michael@0:     '''
michael@0:     def __init__(self):
michael@0:         self.tooldir = None
michael@0: 
michael@0:     def launch(self, cmd, extra_linker_path=None, extra_env={}):
michael@0:         '''
michael@0:         Launch the given command, passed as a list. The first item in the
michael@0:         command list is the program name, without a path and without a suffix.
michael@0:         These are determined from the tooldir member and the BIN_SUFFIX value.
michael@0:         An extra_linker_path may be passed to give an additional directory
michael@0:         to add to the search paths for the dynamic linker.
michael@0:         An extra_env dict may be passed to give additional environment
michael@0:         variables to export when running the command.
michael@0:         '''
michael@0:         assert self.tooldir
michael@0:         cmd[0] = os.path.join(self.tooldir, 'bin',
michael@0:                               cmd[0] + buildconfig.substs['BIN_SUFFIX'])
michael@0:         if not extra_linker_path:
michael@0:             extra_linker_path = os.path.join(self.tooldir, 'bin')
michael@0:         env = dict(os.environ)
michael@0:         for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
michael@0:             if p in env:
michael@0:                 env[p] = extra_linker_path + ':' + env[p]
michael@0:             else:
michael@0:                 env[p] = extra_linker_path
michael@0:         for e in extra_env:
michael@0:             env[e] = extra_env[e]
michael@0: 
michael@0:         # Work around a bug in Python 2.7.2 and lower where unicode types in
michael@0:         # environment variables aren't handled by subprocess.
michael@0:         for k, v in env.items():
michael@0:             if isinstance(v, unicode):
michael@0:                 env[k] = v.encode('utf-8')
michael@0: 
michael@0:         print >>errors.out, 'Executing', ' '.join(cmd)
michael@0:         errors.out.flush()
michael@0:         return subprocess.call(cmd, env=env)
michael@0: 
michael@0:     def can_launch(self):
michael@0:         return self.tooldir is not None
michael@0: 
michael@0: launcher = ToolLauncher()
michael@0: 
michael@0: 
michael@0: class LibSignFile(File):
michael@0:     '''
michael@0:     File class for shlibsign signatures.
michael@0:     '''
michael@0:     def copy(self, dest, skip_if_older=True):
michael@0:         assert isinstance(dest, basestring)
michael@0:         # os.path.getmtime returns a result in seconds with precision up to the
michael@0:         # microsecond. But microsecond is too precise because shutil.copystat
michael@0:         # only copies milliseconds, and seconds is not enough precision.
michael@0:         if os.path.exists(dest) and skip_if_older and \
michael@0:                 int(os.path.getmtime(self.path) * 1000) <= \
michael@0:                 int(os.path.getmtime(dest) * 1000):
michael@0:             return False
michael@0:         if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
michael@0:             errors.fatal('Error while signing %s' % self.path)
michael@0: 
michael@0: 
michael@0: def precompile_cache(formatter, source_path, gre_path, app_path):
michael@0:     '''
michael@0:     Create startup cache for the given application directory, using the
michael@0:     given GRE path.
michael@0:     - formatter is a Formatter instance where to add the startup cache.
michael@0:     - source_path is the base path of the package.
michael@0:     - gre_path is the GRE path, relative to source_path.
michael@0:     - app_path is the application path, relative to source_path.
michael@0:     Startup cache for all resources under resource://app/ are generated,
michael@0:     except when gre_path == app_path, in which case it's under
michael@0:     resource://gre/.
michael@0:     '''
michael@0:     from tempfile import mkstemp
michael@0:     source_path = os.path.abspath(source_path)
michael@0:     if app_path != gre_path:
michael@0:         resource = 'app'
michael@0:     else:
michael@0:         resource = 'gre'
michael@0:     app_path = os.path.join(source_path, app_path)
michael@0:     gre_path = os.path.join(source_path, gre_path)
michael@0: 
michael@0:     fd, cache = mkstemp('.zip')
michael@0:     os.close(fd)
michael@0:     os.remove(cache)
michael@0: 
michael@0:     # For VC12, make sure we can find the right bitness of pgort120.dll
michael@0:     env = os.environ.copy()
michael@0:     if 'VS120COMNTOOLS' in env and not buildconfig.substs['HAVE_64BIT_OS']:
michael@0:       vc12dir = os.path.abspath(os.path.join(env['VS120COMNTOOLS'],
michael@0:                                              '../../VC/bin'))
michael@0:       if os.path.exists(vc12dir):
michael@0:         env['PATH'] = vc12dir + ';' + env['PATH']
michael@0: 
michael@0:     try:
michael@0:         if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
michael@0:                             '-f', os.path.join(os.path.dirname(__file__),
michael@0:                             'precompile_cache.js'),
michael@0:                             '-e', 'precompile_startupcache("resource://%s/");'
michael@0:                                   % resource],
michael@0:                            extra_linker_path=gre_path,
michael@0:                            extra_env={'MOZ_STARTUP_CACHE': cache,
michael@0:                                       'PATH': env['PATH']}):
michael@0:             errors.fatal('Error while running startup cache precompilation')
michael@0:             return
michael@0:         from mozpack.mozjar import JarReader
michael@0:         jar = JarReader(cache)
michael@0:         resource = '/resource/%s/' % resource
michael@0:         for f in jar:
michael@0:             if resource in f.filename:
michael@0:                 path = f.filename[f.filename.index(resource) + len(resource):]
michael@0:                 if formatter.contains(path):
michael@0:                     formatter.add(f.filename, GeneratedFile(f.read()))
michael@0:         jar.close()
michael@0:     finally:
michael@0:         if os.path.exists(cache):
michael@0:             os.remove(cache)
michael@0: 
michael@0: 
michael@0: class RemovedFiles(GeneratedFile):
michael@0:     '''
michael@0:     File class for removed-files. Is used as a preprocessor parser.
michael@0:     '''
michael@0:     def __init__(self, copier):
michael@0:         self.copier = copier
michael@0:         GeneratedFile.__init__(self, '')
michael@0: 
michael@0:     def handle_line(self, str):
michael@0:         f = str.strip()
michael@0:         if self.copier.contains(f):
michael@0:             errors.error('Removal of packaged file(s): %s' % f)
michael@0:         self.content += f + '\n'
michael@0: 
michael@0: 
michael@0: def split_define(define):
michael@0:     '''
michael@0:     Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
michael@0:     1. Numeric VALs are returned as ints.
michael@0:     '''
michael@0:     if '=' in define:
michael@0:         name, value = define.split('=', 1)
michael@0:         try:
michael@0:             value = int(value)
michael@0:         except ValueError:
michael@0:             pass
michael@0:         return (name, value)
michael@0:     return (define, 1)
michael@0: 
michael@0: 
michael@0: class NoPkgFilesRemover(object):
michael@0:     '''
michael@0:     Formatter wrapper to handle NO_PKG_FILES.
michael@0:     '''
michael@0:     def __init__(self, formatter, has_manifest):
michael@0:         assert 'NO_PKG_FILES' in os.environ
michael@0:         self._formatter = formatter
michael@0:         self._files = os.environ['NO_PKG_FILES'].split()
michael@0:         if has_manifest:
michael@0:             self._error = errors.error
michael@0:             self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
michael@0:         else:
michael@0:             self._error = errors.warn
michael@0:             self._msg = 'Skipping %s'
michael@0: 
michael@0:     def add_base(self, base):
michael@0:         self._formatter.add_base(base)
michael@0: 
michael@0:     def add(self, path, content):
michael@0:         if not any(mozpack.path.match(path, spec) for spec in self._files):
michael@0:             self._formatter.add(path, content)
michael@0:         else:
michael@0:             self._error(self._msg % path)
michael@0: 
michael@0:     def add_manifest(self, entry):
michael@0:         self._formatter.add_manifest(entry)
michael@0: 
michael@0:     def add_interfaces(self, path, content):
michael@0:         self._formatter.add_interfaces(path, content)
michael@0: 
michael@0:     def contains(self, path):
michael@0:         return self._formatter.contains(path)
michael@0: 
michael@0: 
michael@0: def main():
michael@0:     parser = ArgumentParser()
michael@0:     parser.add_argument('-D', dest='defines', action='append',
michael@0:                         metavar="VAR[=VAL]", help='Define a variable')
michael@0:     parser.add_argument('--format', default='omni',
michael@0:                         help='Choose the chrome format for packaging ' +
michael@0:                         '(omni, jar or flat ; default: %(default)s)')
michael@0:     parser.add_argument('--removals', default=None,
michael@0:                         help='removed-files source file')
michael@0:     parser.add_argument('--ignore-errors', action='store_true', default=False,
michael@0:                         help='Transform errors into warnings.')
michael@0:     parser.add_argument('--minify', action='store_true', default=False,
michael@0:                         help='Make some files more compact while packaging')
michael@0:     parser.add_argument('--minify-js', action='store_true',
michael@0:                         help='Minify JavaScript files while packaging.')
michael@0:     parser.add_argument('--js-binary',
michael@0:                         help='Path to js binary. This is used to verify '
michael@0:                         'minified JavaScript. If this is not defined, '
michael@0:                         'minification verification will not be performed.')
michael@0:     parser.add_argument('--jarlog', default='', help='File containing jar ' +
michael@0:                         'access logs')
michael@0:     parser.add_argument('--optimizejars', action='store_true', default=False,
michael@0:                         help='Enable jar optimizations')
michael@0:     parser.add_argument('--unify', default='',
michael@0:                         help='Base directory of another build to unify with')
michael@0:     parser.add_argument('manifest', default=None, nargs='?',
michael@0:                         help='Manifest file name')
michael@0:     parser.add_argument('source', help='Source directory')
michael@0:     parser.add_argument('destination', help='Destination directory')
michael@0:     parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
michael@0:                         default=[],
michael@0:                         help='Extra files not to be considered as resources')
michael@0:     args = parser.parse_args()
michael@0: 
michael@0:     defines = dict(buildconfig.defines)
michael@0:     if args.ignore_errors:
michael@0:         errors.ignore_errors()
michael@0: 
michael@0:     if args.defines:
michael@0:         for name, value in [split_define(d) for d in args.defines]:
michael@0:             defines[name] = value
michael@0: 
michael@0:     copier = FileCopier()
michael@0:     if args.format == 'flat':
michael@0:         formatter = FlatFormatter(copier)
michael@0:     elif args.format == 'jar':
michael@0:         formatter = JarFormatter(copier, optimize=args.optimizejars)
michael@0:     elif args.format == 'omni':
michael@0:         formatter = OmniJarFormatter(copier,
michael@0:                                      buildconfig.substs['OMNIJAR_NAME'],
michael@0:                                      optimize=args.optimizejars,
michael@0:                                      non_resources=args.non_resource)
michael@0:     else:
michael@0:         errors.fatal('Unknown format: %s' % args.format)
michael@0: 
michael@0:     # Adjust defines according to the requested format.
michael@0:     if isinstance(formatter, OmniJarFormatter):
michael@0:         defines['MOZ_OMNIJAR'] = 1
michael@0:     elif 'MOZ_OMNIJAR' in defines:
michael@0:         del defines['MOZ_OMNIJAR']
michael@0: 
michael@0:     binpath = ''
michael@0:     if 'BINPATH' in defines:
michael@0:         binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
michael@0:     while binpath.startswith('/'):
michael@0:         binpath = binpath[1:]
michael@0: 
michael@0:     if args.unify:
michael@0:         def is_native(path):
michael@0:             path = os.path.abspath(path)
michael@0:             return platform.machine() in mozpack.path.split(path)
michael@0: 
michael@0:         # Invert args.unify and args.source if args.unify points to the
michael@0:         # native architecture.
michael@0:         args.source, args.unify = sorted([args.source, args.unify],
michael@0:                                          key=is_native, reverse=True)
michael@0:         if is_native(args.source):
michael@0:             launcher.tooldir = args.source
michael@0:     elif not buildconfig.substs['CROSS_COMPILE']:
michael@0:         launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
michael@0: 
michael@0:     with errors.accumulate():
michael@0:         finder_args = dict(
michael@0:             minify=args.minify,
michael@0:             minify_js=args.minify_js,
michael@0:         )
michael@0:         if args.js_binary:
michael@0:             finder_args['minify_js_verify_command'] = [
michael@0:                 args.js_binary,
michael@0:                 os.path.join(os.path.abspath(os.path.dirname(__file__)),
michael@0:                     'js-compare-ast.js')
michael@0:             ]
michael@0:         if args.unify:
michael@0:             finder = UnifiedBuildFinder(FileFinder(args.source),
michael@0:                                         FileFinder(args.unify),
michael@0:                                         **finder_args)
michael@0:         else:
michael@0:             finder = FileFinder(args.source, **finder_args)
michael@0:         if 'NO_PKG_FILES' in os.environ:
michael@0:             sinkformatter = NoPkgFilesRemover(formatter,
michael@0:                                               args.manifest is not None)
michael@0:         else:
michael@0:             sinkformatter = formatter
michael@0:         sink = SimpleManifestSink(finder, sinkformatter)
michael@0:         if args.manifest:
michael@0:             preprocess_manifest(sink, args.manifest, defines)
michael@0:         else:
michael@0:             sink.add(Component(''), 'bin/*')
michael@0:         sink.close(args.manifest is not None)
michael@0: 
michael@0:         if args.removals:
michael@0:             lines = [l.lstrip() for l in open(args.removals).readlines()]
michael@0:             removals_in = StringIO(''.join(lines))
michael@0:             removals_in.name = args.removals
michael@0:             removals = RemovedFiles(copier)
michael@0:             preprocess(removals_in, removals, defines)
michael@0:             copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
michael@0: 
michael@0:     # shlibsign libraries
michael@0:     if launcher.can_launch():
michael@0:         for lib in SIGN_LIBS:
michael@0:             libbase = mozpack.path.join(binpath, '%s%s') \
michael@0:                 % (buildconfig.substs['DLL_PREFIX'], lib)
michael@0:             libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
michael@0:             if copier.contains(libname):
michael@0:                 copier.add(libbase + '.chk',
michael@0:                            LibSignFile(os.path.join(args.destination,
michael@0:                                                     libname)))
michael@0: 
michael@0:     # Setup preloading
michael@0:     if args.jarlog and os.path.exists(args.jarlog):
michael@0:         from mozpack.mozjar import JarLog
michael@0:         log = JarLog(args.jarlog)
michael@0:         for p, f in copier:
michael@0:             if not isinstance(f, Jarrer):
michael@0:                 continue
michael@0:             key = JarLog.canonicalize(os.path.join(args.destination, p))
michael@0:             if key in log:
michael@0:                 f.preload(log[key])
michael@0: 
michael@0:     # Fill startup cache
michael@0:     if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
michael@0:         if buildconfig.substs['LIBXUL_SDK']:
michael@0:             gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
michael@0:                                          'bin')
michael@0:         else:
michael@0:             gre_path = None
michael@0:         for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
michael@0:                             if os.path.exists(os.path.join(args.source, p))][0]
michael@0:                            for b in sink.packager.get_bases()]):
michael@0:             if not gre_path:
michael@0:                 gre_path = base
michael@0:             base_path = sink.normalize_path(base)
michael@0:             if base_path in formatter.omnijars:
michael@0:                 precompile_cache(formatter.omnijars[base_path],
michael@0:                                  args.source, gre_path, base)
michael@0: 
michael@0:     copier.copy(args.destination)
michael@0:     generate_precomplete(os.path.normpath(os.path.join(args.destination,
michael@0:                                                        binpath)))
michael@0: 
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     main()