The Tor Browser: media/libvpx/update.py@ac0c01689b40

Implement a real Private Browsing Mode condition by changing the API/ABI;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

     1 #!/usr/bin/env python

     2 # This Source Code Form is subject to the terms of the Mozilla Public

     3 # License, v. 2.0. If a copy of the MPL was not distributed with this

     4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 import argparse

     6 import os

     7 import re

     8 import shutil

     9 import sys

    10 import subprocess

    11 from pprint import pprint

    12 from StringIO import StringIO

    14 PLATFORMS= [

    15   'x86-win32-vs8',

    16   'x86_64-win64-vs8',

    17   'x86-linux-gcc',

    18   'x86_64-linux-gcc',

    19   'generic-gnu',

    20   'x86-darwin9-gcc',

    21   'x86_64-darwin9-gcc',

    22   'armv7-android-gcc',

    23   'x86-win32-gcc',

    24   'x86_64-win64-gcc',

    25 ]

    28 mk_files = [

    29     'vp8/vp8_common.mk',

    30     'vp8/vp8cx_arm.mk',

    31     'vp8/vp8cx.mk',

    32     'vp8/vp8dx.mk',

    33     'vp9/vp9_common.mk',

    34     'vp9/vp9cx.mk',

    35     'vp9/vp9dx.mk',

    36     'vpx_mem/vpx_mem.mk',

    37     'vpx_ports/vpx_ports.mk',

    38     'vpx_scale/vpx_scale.mk',

    39     'vpx/vpx_codec.mk',

    40 ]

    42 extensions = ['.asm', '.c', '.h']

    44 MODULES = {

    45     'UNIFIED_SOURCES': [

    46         'API_DOC_SRCS-$(CONFIG_VP8_DECODER)',

    47         'API_DOC_SRCS-yes',

    48         'API_EXPORTS',

    49         'API_SRCS-$(CONFIG_VP8_DECODER)',

    50         'API_SRCS-yes',

    51         'MEM_SRCS-yes',

    52         'PORTS_SRCS-yes',

    53         'SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING)',

    54         'SCALE_SRCS-no',

    55         'SCALE_SRCS-yes',

    56         'VP8_COMMON_SRCS-yes',

    57         'VP8_DX_EXPORTS',

    58         'VP8_DX_SRCS-$(CONFIG_MULTITHREAD)',

    59         'VP8_DX_SRCS-no',

    60         'VP8_DX_SRCS_REMOVE-no',

    61         'VP8_DX_SRCS_REMOVE-yes',

    62         'VP8_DX_SRCS-yes',

    63         'VP9_COMMON_SRCS-yes',

    64         'VP9_DX_EXPORTS',

    65         'VP9_DX_SRCS-no',

    66         'VP9_DX_SRCS_REMOVE-no',

    67         'VP9_DX_SRCS_REMOVE-yes',

    68         'VP9_DX_SRCS-yes',

    69         'API_DOC_SRCS-$(CONFIG_VP8_ENCODER)',

    70         'API_SRCS-$(BUILD_LIBVPX)',

    71         'API_SRCS-$(CONFIG_VP8_ENCODER)',

    72         'API_SRCS-$(CONFIG_VP9_ENCODER)',

    73         'VP8_CX_EXPORTS',

    74         'VP8_CX_SRCS-$(CONFIG_MULTI_RES_ENCODING)',

    75         'VP8_CX_SRCS-$(CONFIG_MULTITHREAD)',

    76         'VP8_CX_SRCS-$(CONFIG_TEMPORAL_DENOISING)',

    77         'VP8_CX_SRCS-no',

    78         'VP8_CX_SRCS_REMOVE-no',

    79         'VP8_CX_SRCS_REMOVE-yes',

    80         'VP8_CX_SRCS-yes',

    81         'VP9_CX_EXPORTS',

    82         'VP9_CX_SRCS-no',

    83         'VP9_CX_SRCS_REMOVE-no',

    84         'VP9_CX_SRCS_REMOVE-yes',

    85         'VP9_CX_SRCS-yes',

    86     ],

    87     'X86_ASM': [

    88         'PORTS_SRCS-$(BUILD_LIBVPX)',

    89         'VP8_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',

    90         'VP8_COMMON_SRCS-$(HAVE_MMX)',

    91         'VP8_COMMON_SRCS-$(HAVE_SSE2)',

    92         'VP8_COMMON_SRCS-$(HAVE_SSE3)',

    93         'VP8_COMMON_SRCS-$(HAVE_SSE4_1)',

    94         'VP8_COMMON_SRCS-$(HAVE_SSSE3)',

    95         'VP9_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',

    96         'VP9_COMMON_SRCS-$(HAVE_MMX)',

    97         'VP9_COMMON_SRCS-$(HAVE_SSE2)',

    98         'VP9_COMMON_SRCS-$(HAVE_SSSE3)',

    99         'VP8_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)',

   100         'VP8_CX_SRCS-$(HAVE_MMX)',

   101         'VP8_CX_SRCS-$(HAVE_SSE2)',

   102         'VP8_CX_SRCS-$(HAVE_SSE4_1)',

   103         'VP8_CX_SRCS-$(HAVE_SSSE3)',

   104         'VP8_CX_SRCS_REMOVE-$(HAVE_SSE2)',

   105         'VP9_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)',

   106         'VP9_CX_SRCS-$(HAVE_MMX)',

   107         'VP9_CX_SRCS-$(HAVE_SSE2)',

   108         'VP9_CX_SRCS-$(HAVE_SSE3)',

   109         'VP9_CX_SRCS-$(HAVE_SSE4_1)',

   110         'VP9_CX_SRCS-$(HAVE_SSSE3)',

   111     ],

   112     'X86-64_ASM': [

   113         'VP8_CX_SRCS-$(ARCH_X86_64)',

   114         'VP9_CX_SRCS-$(ARCH_X86_64)',

   115     ],

   116     'ARM_ASM': [

   117         'PORTS_SRCS-$(ARCH_ARM)',

   118         'SCALE_SRCS-$(HAVE_NEON)',

   119         'VP8_COMMON_SRCS-$(ARCH_ARM)',

   120         'VP8_COMMON_SRCS-$(HAVE_MEDIA)',

   121         'VP8_COMMON_SRCS-$(HAVE_NEON)',

   122         'VP9_COMMON_SRCS-$(HAVE_NEON)',

   123         'VP8_CX_SRCS-$(ARCH_ARM)',

   124         'VP8_CX_SRCS-$(HAVE_EDSP)',

   125         'VP8_CX_SRCS-$(HAVE_MEDIA)',

   126         'VP8_CX_SRCS-$(HAVE_NEON)',

   127     ],

   128     'ERROR_CONCEALMENT': [

   129         'VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT)',

   130     ],

   131     'AVX2': [

   132         'VP9_COMMON_SRCS-$(HAVE_AVX2)',

   133     ],

   134     'VP8_POSTPROC': [

   135         'VP8_COMMON_SRCS-$(CONFIG_POSTPROC)',

   136     ],

   137     'VP9_POSTPROC': [

   138         'VP9_COMMON_SRCS-$(CONFIG_VP9_POSTPROC)',

   139     ]

   140 }

   142 DISABLED_MODULES = [

   143     'MEM_SRCS-$(CONFIG_MEM_MANAGER)',

   144     'MEM_SRCS-$(CONFIG_MEM_TRACKER)',

   145     'VP8_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)',

   146     'VP9_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)',

   147     'VP8_CX_SRCS-$(CONFIG_INTERNAL_STATS)',

   148     'VP9_CX_SRCS-$(CONFIG_INTERNAL_STATS)',

   150     # mips files are also ignored via ignored_folders

   151     'SCALE_SRCS-$(HAVE_DSPR2)',

   152     'VP8_COMMON_SRCS-$(HAVE_DSPR2)',

   153     'VP9_COMMON_SRCS-$(HAVE_DSPR2)',

   154     'VP8_CX_SRCS_REMOVE-$(HAVE_EDSP)',

   155 ]

   157 libvpx_files = [

   158     'build/make/obj_int_extract.c',

   159     'build/make/ads2gas.pl',

   160     'build/make/thumb.pm',

   161     'LICENSE',

   162     'PATENTS',

   163 ]

   165 ignore_files = [

   166     'vp8/common/context.c',

   167     'vp8/common/textblit.c',

   168     'vp8/encoder/ssim.c',

   169     'vp8/encoder/x86/ssim_opt.asm',

   170     'vp9/common/vp9_textblit.c',

   171     'vp9/common/vp9_textblit.h',

   172     'vp9/encoder/vp9_ssim.c',

   173     'vp9/encoder/x86/vp9_ssim_opt.asm',

   174     'vpx_mem/vpx_mem_tracker.c',

   175     'vpx_scale/generic/bicubic_scaler.c',

   176     'vpx_scale/win32/scaleopt.c',

   177     'vpx_scale/win32/scalesystemdependent.c',

   178 ]

   180 ignore_folders = [

   181     'examples/',

   182     'googletest/',

   183     'libmkv/',

   184     'libyuv/',

   185     'mips/',

   186     'nestegg/',

   187     'objdir/',

   188     'ppc/',

   189     'test/',

   190     'vpx_mem/memory_manager/',

   191 ]

   192 files = {

   193     'EXPORTS': [

   194         'vpx_mem/include/vpx_mem_intrnl.h',

   195         'vpx_mem/vpx_mem.h',

   196         'vpx_ports/arm.h',

   197         'vpx_ports/mem.h',

   198         'vpx_ports/vpx_timer.h',

   199         'vpx_ports/x86.h',

   200         'vpx_scale/vpx_scale.h',

   201         'vpx_scale/yv12config.h',

   202         'vpx/vp8cx.h',

   203         'vpx/vp8dx.h',

   204         'vpx/vp8.h',

   205         'vpx/vpx_codec.h',

   206         'vpx/vpx_decoder.h',

   207         'vpx/vpx_encoder.h',

   208         'vpx/vpx_image.h',

   209         'vpx/vpx_integer.h',

   210     ],

   211     'X86-64_ASM': [

   212         'third_party/x86inc/x86inc.asm',

   213         'vp8/common/x86/loopfilter_block_sse2.asm',

   214         'vp9/encoder/x86/vp9_quantize_ssse3.asm',

   215     ],

   216     'SOURCES': [

   217         'vp8/common/rtcd.c',

   218         'vp8/common/sad_c.c',

   219         'vp8/vp8_dx_iface.c',

   220         'vp9/common/vp9_entropymv.c',

   221         'vp9/common/vp9_rtcd.c',

   222         'vp9/encoder/vp9_bitstream.c',

   223         'vpx/src/svc_encodeframe.c',

   224         'vpx_mem/vpx_mem.c',

   225     ]

   226 }

   228 manual = [

   229     # special case in moz.build

   230     'vp8/encoder/boolhuff.c',

   232     # 64bit only

   233     'vp8/common/x86/loopfilter_block_sse2.asm',

   234     'vp9/encoder/x86/vp9_quantize_ssse3.asm',

   236     # offsets are special cased in Makefile.in

   237     'vp8/encoder/vp8_asm_enc_offsets.c',

   238     'vpx_scale/vpx_scale_asm_offsets.c',

   240     # ignore while vp9 postproc is not enabled

   241     'vp9/common/x86/vp9_postproc_mmx.asm',

   242     'vp9/common/x86/vp9_postproc_sse2.asm',

   244     # ssim_opt is not enabled

   245     'vp8/encoder/x86/ssim_opt.asm',

   246     'vp9/encoder/x86/vp9_ssim_opt.asm',

   248     # asm includes

   249     'vpx_ports/x86_abi_support.asm',

   250 ]

   252 platform_files = [

   253     'vp8_rtcd.h',

   254     'vp9_rtcd.h',

   255     'vpx_config.asm',

   256     'vpx_config.h',

   257     'vpx_scale_rtcd.h',

   258 ]

   260 def prepare_upstream(prefix, commit=None):

   261     if os.path.exists(prefix):

   262         print "Please remove '%s' folder before running %s" % (prefix, sys.argv[0])

   263         sys.exit(1)

   265     upstream_url = 'https://gerrit.chromium.org/gerrit/webm/libvpx'

   266     subprocess.call(['git', 'clone', upstream_url, prefix])

   267     if commit:

   268         os.chdir(prefix)

   269         subprocess.call(['git', 'checkout', commit])

   270     else:

   271         os.chdir(prefix)

   272         p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)

   273         stdout, stderr = p.communicate()

   274         commit = stdout.strip()

   276     for target in PLATFORMS:

   277         target_objdir = os.path.join(prefix, 'objdir', target)

   278         os.makedirs(target_objdir)

   279         os.chdir(target_objdir)

   280         configure = ['../../configure', '--target=%s' % target,

   281             '--disable-examples', '--disable-install-docs',

   282             '--enable-multi-res-encoding',

   283         ]

   285         if 'darwin9' in target:

   286             configure += ['--enable-pic']

   287         if 'linux' in target:

   288             configure += ['--enable-pic']

   289         # x86inc.asm is not compatible with pic 32bit builds

   290         if target == 'x86-linux-gcc':

   291             configure += ['--disable-use-x86inc']

   293         if target == 'armv7-android-gcc':

   294             configure += ['--sdk-path=%s' % ndk_path]

   296         subprocess.call(configure)

   297         make_targets = [f for f in platform_files if not os.path.exists(f)]

   298         if make_targets:

   299             subprocess.call(['make'] + make_targets)

   300         for f in make_targets:

   301             if not os.path.exists(f):

   302                 print "%s missing from %s, check toolchain" % (f, target)

   303                 sys.exit(1)

   305     os.chdir(base)

   306     return commit

   308 def cleanup_upstream():

   309     shutil.rmtree(os.path.join(base, 'upstream'))

   311 def get_module(key):

   312     for module in MODULES:

   313         if key in MODULES[module]:

   314             return module

   316 def get_libvpx_files(prefix):

   317     for root, folders, files in os.walk(prefix):

   318         for f in files:

   319             f = os.path.join(root, f)[len(prefix):]

   320             if os.path.splitext(f)[-1] in extensions \

   321               and os.sep in f \

   322               and f not in ignore_files \

   323               and not any(folder in f for folder in ignore_folders):

   324                 libvpx_files.append(f)

   325     return libvpx_files

   327 def get_sources(prefix):

   328     source = {}

   329     unknown = {}

   330     disabled = {}

   332     for mk in mk_files:

   333         with open(os.path.join(prefix, mk)) as f:

   334             base = os.path.dirname(mk)

   335             for l in f:

   336                 if '+=' in l:

   337                     l = l.split('+=')

   338                     key = l[0].strip()

   339                     value = l[1].strip().replace('$(ASM)', '.asm')

   340                     value = os.path.join(base, value)

   341                     if not key.startswith('#') and os.path.splitext(value)[-1] in extensions:

   342                         if key not in source:

   343                             source[key] = []

   344                         source[key].append(value)

   346     for key in source:

   347         for f in source[key]:

   348             if key.endswith('EXPORTS') and f.endswith('.h'):

   349                 files['EXPORTS'].append(f)

   350             if os.path.splitext(f)[-1] in ('.c', '.asm') and not f in manual:

   351                 module = get_module(key)

   352                 if module:

   353                     if not module in files:

   354                         files[module] = []

   355                     t = files[module]

   356                 elif key in DISABLED_MODULES:

   357                     if not key in disabled:

   358                         disabled[key] = []

   359                     t = disabled[key]

   360                 else:

   361                     if not key in unknown:

   362                         unknown[key] = []

   363                     t = unknown[key]

   364                 t.append(f)

   366     files['UNIFIED_SOURCES'] = [f for f in files['UNIFIED_SOURCES'] if f not in files['SOURCES']]

   368     for key in files:

   369         files[key] = list(sorted(set(files[key])))

   371     return source, files, disabled, unknown

   373 def update_sources_mozbuild(files, sources_mozbuild):

   374     f = StringIO()

   375     pprint(files, stream=f)

   376     sources_mozbuild_new = "files = {\n %s\n}\n" % f.getvalue().strip()[1:-1]

   377     if sources_mozbuild != sources_mozbuild_new:

   378         print 'updating sources.mozbuild'

   379         with open('sources.mozbuild', 'w') as f:

   380             f.write(sources_mozbuild_new)

   382 def get_current_files():

   383     current_files = []

   384     for root, folders, files in os.walk('.'):

   385         for f in files:

   386             f = os.path.join(root, f)[len('.%s'%os.sep):]

   387             if 'upstream%s'%os.sep in f or not os.sep in f:

   388                 continue

   389             if os.path.splitext(f)[-1] in extensions:

   390                 current_files.append(f)

   391     return current_files

   393 def is_new(a, b):

   394     return not os.path.exists(a) \

   395         or not os.path.exists(b) \

   396         or open(a).read() != open(b).read()

   398 def get_sources_mozbuild():

   399     with open('sources.mozbuild') as f:

   400         sources_mozbuild = f.read()

   401     exec(sources_mozbuild)

   402     return sources_mozbuild, files

   404 def update_and_remove_files(prefix, libvpx_files, files):

   405     current_files = get_current_files()

   407     def copy(src, dst):

   408         print '    ', dst

   409         shutil.copy(src, dst)

   411     # Update files

   412     first = True

   413     for f in libvpx_files:

   414         fdir = os.path.dirname(f)

   415         if fdir and not os.path.exists(fdir):

   416             os.makedirs(fdir)

   417         s = os.path.join(prefix, f)

   418         if is_new(f, s):

   419             if first:

   420                 print "Copy files:"

   421                 first = False

   422             copy(s, f)

   424     # Copy configuration files for each platform

   425     for target in PLATFORMS:

   426         first = True

   427         for f in platform_files:

   428             t = os.path.splitext(f)

   429             t = '%s_%s%s' % (t[0], target, t[1])

   430             f = os.path.join(prefix, 'objdir', target, f)

   431             if is_new(f, t):

   432                 if first:

   433                     print "Copy files for %s:" % target

   434                     first = False

   435                 copy(f, t)

   437     # Copy vpx_version.h from one of the build targets

   438     s = os.path.join(prefix, 'objdir/x86-linux-gcc/vpx_version.h')

   439     f = 'vpx_version.h'

   440     if is_new(s, f):

   441         copy(s, f)

   443     # Remove unknown files from tree

   444     removed_files = [f for f in current_files if f not in libvpx_files]

   445     if removed_files:

   446         print "Remove files:"

   447         for f in removed_files:

   448             os.unlink(f)

   449             print '    ', f

   451 def apply_patches():

   452     # Patch to permit vpx users to specify their own <stdint.h> types.

   453     os.system("patch -p3 < stdint.patch")

   454     os.system("patch -p3 < unified.patch")

   455     os.system("patch -p3 < mingw.patch")

   457 def update_readme(commit):

   458     with open('README_MOZILLA') as f:

   459         readme = f.read()

   461     if 'The git commit ID used was' in readme:

   462         new_readme = re.sub('The git commit ID used was [a-f0-9]+',

   463             'The git commit ID used was %s' % commit, readme)

   464     else:

   465         new_readme = "%s\n\nThe git commit ID used was %s\n" % (readme, commit)

   467     if readme != new_readme:

   468         with open('README_MOZILLA', 'w') as f:

   469             f.write(new_readme)

   471 def print_info(source, files, disabled, unknown, moz_build_files):

   472     for key in moz_build_files:

   473         if key not in files:

   474             print key, 'MISSING'

   475         else:

   476             gone = set(moz_build_files[key]) - set(files[key])

   477             new = set(files[key]) - set(moz_build_files[key])

   478             if gone:

   479                 print key, 'GONE:'

   480                 print '    '+ '\n    '.join(gone)

   481             if new:

   482                 print key, 'NEW:'

   483                 print '    '+ '\n    '.join(new)

   485     if unknown:

   486         print "Please update this script, the following modules are unknown"

   487         pprint(unknown)

   489     if DEBUG:

   490         print "===== SOURCE"

   491         pprint(source)

   492         print "===== FILES"

   493         pprint(files)

   494         print "===== DISABLED"

   495         pprint(disabled)

   496         print "===== UNKNOWN"

   497         pprint(unknown)

   500 if __name__ == '__main__':

   501     parser = argparse.ArgumentParser(description='''This script only works on Mac OS X since the OS X Toolchain is not available on other platforms.

   502 In addition you need XCode and the Android NDK installed.

   503 If commit hash is not provided, current git master is used.''')

   504     parser.add_argument('--debug', dest='debug', action="store_true")

   505     parser.add_argument('--ndk', dest='ndk', type=str)

   506     parser.add_argument('--commit', dest='commit', type=str, default=None)

   508     args = parser.parse_args()

   510     if sys.platform != 'darwin' or not args.ndk:

   511         parser.print_help()

   512         sys.exit(1)

   514     ndk_path = args.ndk

   515     commit = args.commit

   516     DEBUG = args.debug

   518     base = os.path.abspath(os.curdir)

   519     prefix = os.path.join(base, 'upstream/')

   521     commit = prepare_upstream(prefix, commit)

   523     libvpx_files = get_libvpx_files(prefix)

   524     source, files, disabled, unknown = get_sources(prefix)

   526     sources_mozbuild, moz_build_files = get_sources_mozbuild()

   528     print_info(source, files, disabled, unknown, moz_build_files)

   529     update_sources_mozbuild(files, sources_mozbuild)

   530     update_and_remove_files(prefix, libvpx_files, files)

   531     apply_patches()

   532     update_readme(commit)

   534     cleanup_upstream()

The Tor Browser / file revision

media/libvpx/update.py@ac0c01689b40

media/libvpx/update.py