The Tor Browser: toolkit/mozapps/installer/packager.py@129ffea94266

Conditionally enable double key logic according to:
private browsing mode or privacy.thirdparty.isolate preference and
implement in GetCookieStringCommon and FindCookie where it counts...
With some reservations of how to convince FindCookie users to test
condition and pass a nullptr when disabling double key logic.

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 from mozpack.packager.formats import (

     6     FlatFormatter,

     7     JarFormatter,

     8     OmniJarFormatter,

9 )

    10 from mozpack.packager import (

    11     preprocess_manifest,

    12     preprocess,

    13     Component,

    14     SimpleManifestSink,

    15 )

    16 from mozpack.files import (

    17     GeneratedFile,

    18     FileFinder,

    19     File,

    20 )

    21 from mozpack.copier import (

    22     FileCopier,

    23     Jarrer,

    24 )

    25 from mozpack.errors import errors

    26 from mozpack.unify import UnifiedBuildFinder

    27 import mozpack.path

    28 import buildconfig

    29 from argparse import ArgumentParser

    30 from createprecomplete import generate_precomplete

    31 import os

    32 from StringIO import StringIO

    33 import subprocess

    34 import platform

    36 # List of libraries to shlibsign.

    37 SIGN_LIBS = [

    38     'softokn3',

    39     'nssdbm3',

    40     'freebl3',

    41     'freebl_32fpu_3',

    42     'freebl_32int_3',

    43     'freebl_32int64_3',

    44     'freebl_64fpu_3',

    45     'freebl_64int_3',

    46 ]

    49 class ToolLauncher(object):

    50     '''

    51     Helper to execute tools like xpcshell with the appropriate environment.

    52         launcher = ToolLauncher()

    53         launcher.tooldir = '/path/to/tools'

    54         launcher.launch(['xpcshell', '-e', 'foo.js'])

    55     '''

    56     def __init__(self):

    57         self.tooldir = None

    59     def launch(self, cmd, extra_linker_path=None, extra_env={}):

    60         '''

    61         Launch the given command, passed as a list. The first item in the

    62         command list is the program name, without a path and without a suffix.

    63         These are determined from the tooldir member and the BIN_SUFFIX value.

    64         An extra_linker_path may be passed to give an additional directory

    65         to add to the search paths for the dynamic linker.

    66         An extra_env dict may be passed to give additional environment

    67         variables to export when running the command.

    68         '''

    69         assert self.tooldir

    70         cmd[0] = os.path.join(self.tooldir, 'bin',

    71                               cmd[0] + buildconfig.substs['BIN_SUFFIX'])

    72         if not extra_linker_path:

    73             extra_linker_path = os.path.join(self.tooldir, 'bin')

    74         env = dict(os.environ)

    75         for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:

    76             if p in env:

    77                 env[p] = extra_linker_path + ':' + env[p]

    78             else:

    79                 env[p] = extra_linker_path

    80         for e in extra_env:

    81             env[e] = extra_env[e]

    83         # Work around a bug in Python 2.7.2 and lower where unicode types in

    84         # environment variables aren't handled by subprocess.

    85         for k, v in env.items():

    86             if isinstance(v, unicode):

    87                 env[k] = v.encode('utf-8')

    89         print >>errors.out, 'Executing', ' '.join(cmd)

    90         errors.out.flush()

    91         return subprocess.call(cmd, env=env)

    93     def can_launch(self):

    94         return self.tooldir is not None

    96 launcher = ToolLauncher()

    99 class LibSignFile(File):

   100     '''

   101     File class for shlibsign signatures.

   102     '''

   103     def copy(self, dest, skip_if_older=True):

   104         assert isinstance(dest, basestring)

   105         # os.path.getmtime returns a result in seconds with precision up to the

   106         # microsecond. But microsecond is too precise because shutil.copystat

   107         # only copies milliseconds, and seconds is not enough precision.

   108         if os.path.exists(dest) and skip_if_older and \

   109                 int(os.path.getmtime(self.path) * 1000) <= \

   110                 int(os.path.getmtime(dest) * 1000):

   111             return False

   112         if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):

   113             errors.fatal('Error while signing %s' % self.path)

   116 def precompile_cache(formatter, source_path, gre_path, app_path):

   117     '''

   118     Create startup cache for the given application directory, using the

   119     given GRE path.

   120     - formatter is a Formatter instance where to add the startup cache.

   121     - source_path is the base path of the package.

   122     - gre_path is the GRE path, relative to source_path.

   123     - app_path is the application path, relative to source_path.

   124     Startup cache for all resources under resource://app/ are generated,

   125     except when gre_path == app_path, in which case it's under

   126     resource://gre/.

   127     '''

   128     from tempfile import mkstemp

   129     source_path = os.path.abspath(source_path)

   130     if app_path != gre_path:

   131         resource = 'app'

   132     else:

   133         resource = 'gre'

   134     app_path = os.path.join(source_path, app_path)

   135     gre_path = os.path.join(source_path, gre_path)

   137     fd, cache = mkstemp('.zip')

   138     os.close(fd)

   139     os.remove(cache)

   141     # For VC12, make sure we can find the right bitness of pgort120.dll

   142     env = os.environ.copy()

   143     if 'VS120COMNTOOLS' in env and not buildconfig.substs['HAVE_64BIT_OS']:

   144       vc12dir = os.path.abspath(os.path.join(env['VS120COMNTOOLS'],

   145                                              '../../VC/bin'))

   146       if os.path.exists(vc12dir):

   147         env['PATH'] = vc12dir + ';' + env['PATH']

   149     try:

   150         if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,

   151                             '-f', os.path.join(os.path.dirname(__file__),

   152                             'precompile_cache.js'),

   153                             '-e', 'precompile_startupcache("resource://%s/");'

   154                                   % resource],

   155                            extra_linker_path=gre_path,

   156                            extra_env={'MOZ_STARTUP_CACHE': cache,

   157                                       'PATH': env['PATH']}):

   158             errors.fatal('Error while running startup cache precompilation')

   159             return

   160         from mozpack.mozjar import JarReader

   161         jar = JarReader(cache)

   162         resource = '/resource/%s/' % resource

   163         for f in jar:

   164             if resource in f.filename:

   165                 path = f.filename[f.filename.index(resource) + len(resource):]

   166                 if formatter.contains(path):

   167                     formatter.add(f.filename, GeneratedFile(f.read()))

   168         jar.close()

   169     finally:

   170         if os.path.exists(cache):

   171             os.remove(cache)

   174 class RemovedFiles(GeneratedFile):

   175     '''

   176     File class for removed-files. Is used as a preprocessor parser.

   177     '''

   178     def __init__(self, copier):

   179         self.copier = copier

   180         GeneratedFile.__init__(self, '')

   182     def handle_line(self, str):

   183         f = str.strip()

   184         if self.copier.contains(f):

   185             errors.error('Removal of packaged file(s): %s' % f)

   186         self.content += f + '\n'

   189 def split_define(define):

   190     '''

   191     Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to

   192     1. Numeric VALs are returned as ints.

   193     '''

   194     if '=' in define:

   195         name, value = define.split('=', 1)

   196         try:

   197             value = int(value)

   198         except ValueError:

   199             pass

   200         return (name, value)

   201     return (define, 1)

   204 class NoPkgFilesRemover(object):

   205     '''

   206     Formatter wrapper to handle NO_PKG_FILES.

   207     '''

   208     def __init__(self, formatter, has_manifest):

   209         assert 'NO_PKG_FILES' in os.environ

   210         self._formatter = formatter

   211         self._files = os.environ['NO_PKG_FILES'].split()

   212         if has_manifest:

   213             self._error = errors.error

   214             self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'

   215         else:

   216             self._error = errors.warn

   217             self._msg = 'Skipping %s'

   219     def add_base(self, base):

   220         self._formatter.add_base(base)

   222     def add(self, path, content):

   223         if not any(mozpack.path.match(path, spec) for spec in self._files):

   224             self._formatter.add(path, content)

   225         else:

   226             self._error(self._msg % path)

   228     def add_manifest(self, entry):

   229         self._formatter.add_manifest(entry)

   231     def add_interfaces(self, path, content):

   232         self._formatter.add_interfaces(path, content)

   234     def contains(self, path):

   235         return self._formatter.contains(path)

   238 def main():

   239     parser = ArgumentParser()

   240     parser.add_argument('-D', dest='defines', action='append',

   241                         metavar="VAR[=VAL]", help='Define a variable')

   242     parser.add_argument('--format', default='omni',

   243                         help='Choose the chrome format for packaging ' +

   244                         '(omni, jar or flat ; default: %(default)s)')

   245     parser.add_argument('--removals', default=None,

   246                         help='removed-files source file')

   247     parser.add_argument('--ignore-errors', action='store_true', default=False,

   248                         help='Transform errors into warnings.')

   249     parser.add_argument('--minify', action='store_true', default=False,

   250                         help='Make some files more compact while packaging')

   251     parser.add_argument('--minify-js', action='store_true',

   252                         help='Minify JavaScript files while packaging.')

   253     parser.add_argument('--js-binary',

   254                         help='Path to js binary. This is used to verify '

   255                         'minified JavaScript. If this is not defined, '

   256                         'minification verification will not be performed.')

   257     parser.add_argument('--jarlog', default='', help='File containing jar ' +

   258                         'access logs')

   259     parser.add_argument('--optimizejars', action='store_true', default=False,

   260                         help='Enable jar optimizations')

   261     parser.add_argument('--unify', default='',

   262                         help='Base directory of another build to unify with')

   263     parser.add_argument('manifest', default=None, nargs='?',

   264                         help='Manifest file name')

   265     parser.add_argument('source', help='Source directory')

   266     parser.add_argument('destination', help='Destination directory')

   267     parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',

   268                         default=[],

   269                         help='Extra files not to be considered as resources')

   270     args = parser.parse_args()

   272     defines = dict(buildconfig.defines)

   273     if args.ignore_errors:

   274         errors.ignore_errors()

   276     if args.defines:

   277         for name, value in [split_define(d) for d in args.defines]:

   278             defines[name] = value

   280     copier = FileCopier()

   281     if args.format == 'flat':

   282         formatter = FlatFormatter(copier)

   283     elif args.format == 'jar':

   284         formatter = JarFormatter(copier, optimize=args.optimizejars)

   285     elif args.format == 'omni':

   286         formatter = OmniJarFormatter(copier,

   287                                      buildconfig.substs['OMNIJAR_NAME'],

   288                                      optimize=args.optimizejars,

   289                                      non_resources=args.non_resource)

   290     else:

   291         errors.fatal('Unknown format: %s' % args.format)

   293     # Adjust defines according to the requested format.

   294     if isinstance(formatter, OmniJarFormatter):

   295         defines['MOZ_OMNIJAR'] = 1

   296     elif 'MOZ_OMNIJAR' in defines:

   297         del defines['MOZ_OMNIJAR']

   299     binpath = ''

   300     if 'BINPATH' in defines:

   301         binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])

   302     while binpath.startswith('/'):

   303         binpath = binpath[1:]

   305     if args.unify:

   306         def is_native(path):

   307             path = os.path.abspath(path)

   308             return platform.machine() in mozpack.path.split(path)

   310         # Invert args.unify and args.source if args.unify points to the

   311         # native architecture.

   312         args.source, args.unify = sorted([args.source, args.unify],

   313                                          key=is_native, reverse=True)

   314         if is_native(args.source):

   315             launcher.tooldir = args.source

   316     elif not buildconfig.substs['CROSS_COMPILE']:

   317         launcher.tooldir = buildconfig.substs['LIBXUL_DIST']

   319     with errors.accumulate():

   320         finder_args = dict(

   321             minify=args.minify,

   322             minify_js=args.minify_js,

   323         )

   324         if args.js_binary:

   325             finder_args['minify_js_verify_command'] = [

   326                 args.js_binary,

   327                 os.path.join(os.path.abspath(os.path.dirname(__file__)),

   328                     'js-compare-ast.js')

   329             ]

   330         if args.unify:

   331             finder = UnifiedBuildFinder(FileFinder(args.source),

   332                                         FileFinder(args.unify),

   333                                         **finder_args)

   334         else:

   335             finder = FileFinder(args.source, **finder_args)

   336         if 'NO_PKG_FILES' in os.environ:

   337             sinkformatter = NoPkgFilesRemover(formatter,

   338                                               args.manifest is not None)

   339         else:

   340             sinkformatter = formatter

   341         sink = SimpleManifestSink(finder, sinkformatter)

   342         if args.manifest:

   343             preprocess_manifest(sink, args.manifest, defines)

   344         else:

   345             sink.add(Component(''), 'bin/*')

   346         sink.close(args.manifest is not None)

   348         if args.removals:

   349             lines = [l.lstrip() for l in open(args.removals).readlines()]

   350             removals_in = StringIO(''.join(lines))

   351             removals_in.name = args.removals

   352             removals = RemovedFiles(copier)

   353             preprocess(removals_in, removals, defines)

   354             copier.add(mozpack.path.join(binpath, 'removed-files'), removals)

   356     # shlibsign libraries

   357     if launcher.can_launch():

   358         for lib in SIGN_LIBS:

   359             libbase = mozpack.path.join(binpath, '%s%s') \

   360                 % (buildconfig.substs['DLL_PREFIX'], lib)

   361             libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])

   362             if copier.contains(libname):

   363                 copier.add(libbase + '.chk',

   364                            LibSignFile(os.path.join(args.destination,

   365                                                     libname)))

   367     # Setup preloading

   368     if args.jarlog and os.path.exists(args.jarlog):

   369         from mozpack.mozjar import JarLog

   370         log = JarLog(args.jarlog)

   371         for p, f in copier:

   372             if not isinstance(f, Jarrer):

   373                 continue

   374             key = JarLog.canonicalize(os.path.join(args.destination, p))

   375             if key in log:

   376                 f.preload(log[key])

   378     # Fill startup cache

   379     if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():

   380         if buildconfig.substs['LIBXUL_SDK']:

   381             gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],

   382                                          'bin')

   383         else:

   384             gre_path = None

   385         for base in sorted([[p for p in [mozpack.path.join('bin', b), b]

   386                             if os.path.exists(os.path.join(args.source, p))][0]

   387                            for b in sink.packager.get_bases()]):

   388             if not gre_path:

   389                 gre_path = base

   390             base_path = sink.normalize_path(base)

   391             if base_path in formatter.omnijars:

   392                 precompile_cache(formatter.omnijars[base_path],

   393                                  args.source, gre_path, base)

   395     copier.copy(args.destination)

   396     generate_precomplete(os.path.normpath(os.path.join(args.destination,

   397                                                        binpath)))

   400 if __name__ == '__main__':

   401     main()

The Tor Browser / file revision

toolkit/mozapps/installer/packager.py@129ffea94266

toolkit/mozapps/installer/packager.py