The Tor Browser: media/libvpx/update.py@ac0c01689b40 (annotated)

media/libvpx/update.py@ac0c01689b40 (annotated)

media/libvpx/update.py

Thu, 15 Jan 2015 15:59:08 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 15 Jan 2015 15:59:08 +0100
branch: TOR_BUG_9701
changeset 10: ac0c01689b40
permissions: -rwxr-xr-x

Implement a real Private Browsing Mode condition by changing the API/ABI;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 #!/usr/bin/env python
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import argparse
 import os
 import re
 import shutil
 import sys
 import subprocess
 from pprint import pprint
 from StringIO import StringIO
 PLATFORMS= [
   'x86-win32-vs8',
   'x86_64-win64-vs8',
   'x86-linux-gcc',
   'x86_64-linux-gcc',
   'generic-gnu',
   'x86-darwin9-gcc',
   'x86_64-darwin9-gcc',
   'armv7-android-gcc',
   'x86-win32-gcc',
   'x86_64-win64-gcc',
 ]
 mk_files = [
     'vp8/vp8_common.mk',
     'vp8/vp8cx_arm.mk',
     'vp8/vp8cx.mk',
     'vp8/vp8dx.mk',
     'vp9/vp9_common.mk',
     'vp9/vp9cx.mk',
     'vp9/vp9dx.mk',
     'vpx_mem/vpx_mem.mk',
     'vpx_ports/vpx_ports.mk',
     'vpx_scale/vpx_scale.mk',
     'vpx/vpx_codec.mk',
 ]
 extensions = ['.asm', '.c', '.h']
 MODULES = {
     'UNIFIED_SOURCES': [
         'API_DOC_SRCS-$(CONFIG_VP8_DECODER)',
         'API_DOC_SRCS-yes',
         'API_EXPORTS',
         'API_SRCS-$(CONFIG_VP8_DECODER)',
         'API_SRCS-yes',
         'MEM_SRCS-yes',
         'PORTS_SRCS-yes',
         'SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING)',
         'SCALE_SRCS-no',
         'SCALE_SRCS-yes',
         'VP8_COMMON_SRCS-yes',
         'VP8_DX_EXPORTS',
         'VP8_DX_SRCS-$(CONFIG_MULTITHREAD)',
         'VP8_DX_SRCS-no',
         'VP8_DX_SRCS_REMOVE-no',
         'VP8_DX_SRCS_REMOVE-yes',
         'VP8_DX_SRCS-yes',
         'VP9_COMMON_SRCS-yes',
         'VP9_DX_EXPORTS',
         'VP9_DX_SRCS-no',
         'VP9_DX_SRCS_REMOVE-no',
         'VP9_DX_SRCS_REMOVE-yes',
         'VP9_DX_SRCS-yes',
         'API_DOC_SRCS-$(CONFIG_VP8_ENCODER)',
         'API_SRCS-$(BUILD_LIBVPX)',
         'API_SRCS-$(CONFIG_VP8_ENCODER)',
         'API_SRCS-$(CONFIG_VP9_ENCODER)',
         'VP8_CX_EXPORTS',
         'VP8_CX_SRCS-$(CONFIG_MULTI_RES_ENCODING)',
         'VP8_CX_SRCS-$(CONFIG_MULTITHREAD)',
         'VP8_CX_SRCS-$(CONFIG_TEMPORAL_DENOISING)',
         'VP8_CX_SRCS-no',
         'VP8_CX_SRCS_REMOVE-no',
         'VP8_CX_SRCS_REMOVE-yes',
         'VP8_CX_SRCS-yes',
         'VP9_CX_EXPORTS',
         'VP9_CX_SRCS-no',
         'VP9_CX_SRCS_REMOVE-no',
         'VP9_CX_SRCS_REMOVE-yes',
         'VP9_CX_SRCS-yes',
     ],
     'X86_ASM': [
         'PORTS_SRCS-$(BUILD_LIBVPX)',
         'VP8_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
         'VP8_COMMON_SRCS-$(HAVE_MMX)',
         'VP8_COMMON_SRCS-$(HAVE_SSE2)',
         'VP8_COMMON_SRCS-$(HAVE_SSE3)',
         'VP8_COMMON_SRCS-$(HAVE_SSE4_1)',
         'VP8_COMMON_SRCS-$(HAVE_SSSE3)',
         'VP9_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
         'VP9_COMMON_SRCS-$(HAVE_MMX)',
         'VP9_COMMON_SRCS-$(HAVE_SSE2)',
         'VP9_COMMON_SRCS-$(HAVE_SSSE3)',
         'VP8_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
         'VP8_CX_SRCS-$(HAVE_MMX)',
         'VP8_CX_SRCS-$(HAVE_SSE2)',
         'VP8_CX_SRCS-$(HAVE_SSE4_1)',
         'VP8_CX_SRCS-$(HAVE_SSSE3)',
         'VP8_CX_SRCS_REMOVE-$(HAVE_SSE2)',
         'VP9_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
         'VP9_CX_SRCS-$(HAVE_MMX)',
         'VP9_CX_SRCS-$(HAVE_SSE2)',
         'VP9_CX_SRCS-$(HAVE_SSE3)',
         'VP9_CX_SRCS-$(HAVE_SSE4_1)',
         'VP9_CX_SRCS-$(HAVE_SSSE3)',
     ],
     'X86-64_ASM': [
         'VP8_CX_SRCS-$(ARCH_X86_64)',
         'VP9_CX_SRCS-$(ARCH_X86_64)',
     ],
     'ARM_ASM': [
         'PORTS_SRCS-$(ARCH_ARM)',
         'SCALE_SRCS-$(HAVE_NEON)',
         'VP8_COMMON_SRCS-$(ARCH_ARM)',
         'VP8_COMMON_SRCS-$(HAVE_MEDIA)',
         'VP8_COMMON_SRCS-$(HAVE_NEON)',
         'VP9_COMMON_SRCS-$(HAVE_NEON)',
         'VP8_CX_SRCS-$(ARCH_ARM)',
         'VP8_CX_SRCS-$(HAVE_EDSP)',
         'VP8_CX_SRCS-$(HAVE_MEDIA)',
         'VP8_CX_SRCS-$(HAVE_NEON)',
     ],
     'ERROR_CONCEALMENT': [
         'VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT)',
     ],
     'AVX2': [
         'VP9_COMMON_SRCS-$(HAVE_AVX2)',
     ],
     'VP8_POSTPROC': [
         'VP8_COMMON_SRCS-$(CONFIG_POSTPROC)',
     ],
     'VP9_POSTPROC': [
         'VP9_COMMON_SRCS-$(CONFIG_VP9_POSTPROC)',
     ]
 }
 DISABLED_MODULES = [
     'MEM_SRCS-$(CONFIG_MEM_MANAGER)',
     'MEM_SRCS-$(CONFIG_MEM_TRACKER)',
     'VP8_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)',
     'VP9_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)',
     'VP8_CX_SRCS-$(CONFIG_INTERNAL_STATS)',
     'VP9_CX_SRCS-$(CONFIG_INTERNAL_STATS)',
     # mips files are also ignored via ignored_folders
     'SCALE_SRCS-$(HAVE_DSPR2)',
     'VP8_COMMON_SRCS-$(HAVE_DSPR2)',
     'VP9_COMMON_SRCS-$(HAVE_DSPR2)',
     'VP8_CX_SRCS_REMOVE-$(HAVE_EDSP)',
 ]
 libvpx_files = [
     'build/make/obj_int_extract.c',
     'build/make/ads2gas.pl',
     'build/make/thumb.pm',
     'LICENSE',
     'PATENTS',
 ]
 ignore_files = [
     'vp8/common/context.c',
     'vp8/common/textblit.c',
     'vp8/encoder/ssim.c',
     'vp8/encoder/x86/ssim_opt.asm',
     'vp9/common/vp9_textblit.c',
     'vp9/common/vp9_textblit.h',
     'vp9/encoder/vp9_ssim.c',
     'vp9/encoder/x86/vp9_ssim_opt.asm',
     'vpx_mem/vpx_mem_tracker.c',
     'vpx_scale/generic/bicubic_scaler.c',
     'vpx_scale/win32/scaleopt.c',
     'vpx_scale/win32/scalesystemdependent.c',
 ]
 ignore_folders = [
     'examples/',
     'googletest/',
     'libmkv/',
     'libyuv/',
     'mips/',
     'nestegg/',
     'objdir/',
     'ppc/',
     'test/',
     'vpx_mem/memory_manager/',
 ]
 files = {
     'EXPORTS': [
         'vpx_mem/include/vpx_mem_intrnl.h',
         'vpx_mem/vpx_mem.h',
         'vpx_ports/arm.h',
         'vpx_ports/mem.h',
         'vpx_ports/vpx_timer.h',
         'vpx_ports/x86.h',
         'vpx_scale/vpx_scale.h',
         'vpx_scale/yv12config.h',
         'vpx/vp8cx.h',
         'vpx/vp8dx.h',
         'vpx/vp8.h',
         'vpx/vpx_codec.h',
         'vpx/vpx_decoder.h',
         'vpx/vpx_encoder.h',
         'vpx/vpx_image.h',
         'vpx/vpx_integer.h',
     ],
     'X86-64_ASM': [
         'third_party/x86inc/x86inc.asm',
         'vp8/common/x86/loopfilter_block_sse2.asm',
         'vp9/encoder/x86/vp9_quantize_ssse3.asm',
     ],
     'SOURCES': [
         'vp8/common/rtcd.c',
         'vp8/common/sad_c.c',
         'vp8/vp8_dx_iface.c',
         'vp9/common/vp9_entropymv.c',
         'vp9/common/vp9_rtcd.c',
         'vp9/encoder/vp9_bitstream.c',
         'vpx/src/svc_encodeframe.c',
         'vpx_mem/vpx_mem.c',
     ]
 }
 manual = [
     # special case in moz.build
     'vp8/encoder/boolhuff.c',
     # 64bit only
     'vp8/common/x86/loopfilter_block_sse2.asm',
     'vp9/encoder/x86/vp9_quantize_ssse3.asm',
     # offsets are special cased in Makefile.in
     'vp8/encoder/vp8_asm_enc_offsets.c',
     'vpx_scale/vpx_scale_asm_offsets.c',
     # ignore while vp9 postproc is not enabled
     'vp9/common/x86/vp9_postproc_mmx.asm',
     'vp9/common/x86/vp9_postproc_sse2.asm',
     # ssim_opt is not enabled
     'vp8/encoder/x86/ssim_opt.asm',
     'vp9/encoder/x86/vp9_ssim_opt.asm',
     # asm includes
     'vpx_ports/x86_abi_support.asm',
 ]
 platform_files = [
     'vp8_rtcd.h',
     'vp9_rtcd.h',
     'vpx_config.asm',
     'vpx_config.h',
     'vpx_scale_rtcd.h',
 ]
 def prepare_upstream(prefix, commit=None):
     if os.path.exists(prefix):
         print "Please remove '%s' folder before running %s" % (prefix, sys.argv[0])
         sys.exit(1)
     upstream_url = 'https://gerrit.chromium.org/gerrit/webm/libvpx'
     subprocess.call(['git', 'clone', upstream_url, prefix])
     if commit:
         os.chdir(prefix)
         subprocess.call(['git', 'checkout', commit])
     else:
         os.chdir(prefix)
         p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
         stdout, stderr = p.communicate()
         commit = stdout.strip()
     for target in PLATFORMS:
         target_objdir = os.path.join(prefix, 'objdir', target)
         os.makedirs(target_objdir)
         os.chdir(target_objdir)
         configure = ['../../configure', '--target=%s' % target,
             '--disable-examples', '--disable-install-docs',
             '--enable-multi-res-encoding',
         ]
         if 'darwin9' in target:
             configure += ['--enable-pic']
         if 'linux' in target:
             configure += ['--enable-pic']
         # x86inc.asm is not compatible with pic 32bit builds
         if target == 'x86-linux-gcc':
             configure += ['--disable-use-x86inc']
         if target == 'armv7-android-gcc':
             configure += ['--sdk-path=%s' % ndk_path]
         subprocess.call(configure)
         make_targets = [f for f in platform_files if not os.path.exists(f)]
         if make_targets:
             subprocess.call(['make'] + make_targets)
         for f in make_targets:
             if not os.path.exists(f):
                 print "%s missing from %s, check toolchain" % (f, target)
                 sys.exit(1)
     os.chdir(base)
     return commit
 def cleanup_upstream():
     shutil.rmtree(os.path.join(base, 'upstream'))
 def get_module(key):
     for module in MODULES:
         if key in MODULES[module]:
             return module
 def get_libvpx_files(prefix):
     for root, folders, files in os.walk(prefix):
         for f in files:
             f = os.path.join(root, f)[len(prefix):]
             if os.path.splitext(f)[-1] in extensions \
               and os.sep in f \
               and f not in ignore_files \
               and not any(folder in f for folder in ignore_folders):
                 libvpx_files.append(f)
     return libvpx_files
 def get_sources(prefix):
     source = {}
     unknown = {}
     disabled = {}
     for mk in mk_files:
         with open(os.path.join(prefix, mk)) as f:
             base = os.path.dirname(mk)
             for l in f:
                 if '+=' in l:
                     l = l.split('+=')
                     key = l[0].strip()
                     value = l[1].strip().replace('$(ASM)', '.asm')
                     value = os.path.join(base, value)
                     if not key.startswith('#') and os.path.splitext(value)[-1] in extensions:
                         if key not in source:
                             source[key] = []
                         source[key].append(value)
     for key in source:
         for f in source[key]:
             if key.endswith('EXPORTS') and f.endswith('.h'):
                 files['EXPORTS'].append(f)
             if os.path.splitext(f)[-1] in ('.c', '.asm') and not f in manual:
                 module = get_module(key)
                 if module:
                     if not module in files:
                         files[module] = []
                     t = files[module]
                 elif key in DISABLED_MODULES:
                     if not key in disabled:
                         disabled[key] = []
                     t = disabled[key]
                 else:
                     if not key in unknown:
                         unknown[key] = []
                     t = unknown[key]
                 t.append(f)
     files['UNIFIED_SOURCES'] = [f for f in files['UNIFIED_SOURCES'] if f not in files['SOURCES']]
     for key in files:
         files[key] = list(sorted(set(files[key])))
     return source, files, disabled, unknown
 def update_sources_mozbuild(files, sources_mozbuild):
     f = StringIO()
     pprint(files, stream=f)
     sources_mozbuild_new = "files = {\n %s\n}\n" % f.getvalue().strip()[1:-1]
     if sources_mozbuild != sources_mozbuild_new:
         print 'updating sources.mozbuild'
         with open('sources.mozbuild', 'w') as f:
             f.write(sources_mozbuild_new)
 def get_current_files():
     current_files = []
     for root, folders, files in os.walk('.'):
         for f in files:
             f = os.path.join(root, f)[len('.%s'%os.sep):]
             if 'upstream%s'%os.sep in f or not os.sep in f:
                 continue
             if os.path.splitext(f)[-1] in extensions:
                 current_files.append(f)
     return current_files
 def is_new(a, b):
     return not os.path.exists(a) \
         or not os.path.exists(b) \
         or open(a).read() != open(b).read()
 def get_sources_mozbuild():
     with open('sources.mozbuild') as f:
         sources_mozbuild = f.read()
     exec(sources_mozbuild)
     return sources_mozbuild, files
 def update_and_remove_files(prefix, libvpx_files, files):
     current_files = get_current_files()
     def copy(src, dst):
         print '    ', dst
         shutil.copy(src, dst)
     # Update files
     first = True
     for f in libvpx_files:
         fdir = os.path.dirname(f)
         if fdir and not os.path.exists(fdir):
             os.makedirs(fdir)
         s = os.path.join(prefix, f)
         if is_new(f, s):
             if first:
                 print "Copy files:"
                 first = False
             copy(s, f)
     # Copy configuration files for each platform
     for target in PLATFORMS:
         first = True
         for f in platform_files:
             t = os.path.splitext(f)
             t = '%s_%s%s' % (t[0], target, t[1])
             f = os.path.join(prefix, 'objdir', target, f)
             if is_new(f, t):
                 if first:
                     print "Copy files for %s:" % target
                     first = False
                 copy(f, t)
     # Copy vpx_version.h from one of the build targets
     s = os.path.join(prefix, 'objdir/x86-linux-gcc/vpx_version.h')
     f = 'vpx_version.h'
     if is_new(s, f):
         copy(s, f)
     # Remove unknown files from tree
     removed_files = [f for f in current_files if f not in libvpx_files]
     if removed_files:
         print "Remove files:"
         for f in removed_files:
             os.unlink(f)
             print '    ', f
 def apply_patches():
     # Patch to permit vpx users to specify their own <stdint.h> types.
     os.system("patch -p3 < stdint.patch")
     os.system("patch -p3 < unified.patch")
     os.system("patch -p3 < mingw.patch")
 def update_readme(commit):
     with open('README_MOZILLA') as f:
         readme = f.read()
     if 'The git commit ID used was' in readme:
         new_readme = re.sub('The git commit ID used was [a-f0-9]+',
             'The git commit ID used was %s' % commit, readme)
     else:
         new_readme = "%s\n\nThe git commit ID used was %s\n" % (readme, commit)
     if readme != new_readme:
         with open('README_MOZILLA', 'w') as f:
             f.write(new_readme)
 def print_info(source, files, disabled, unknown, moz_build_files):
     for key in moz_build_files:
         if key not in files:
             print key, 'MISSING'
         else:
             gone = set(moz_build_files[key]) - set(files[key])
             new = set(files[key]) - set(moz_build_files[key])
             if gone:
                 print key, 'GONE:'
                 print '    '+ '\n    '.join(gone)
             if new:
                 print key, 'NEW:'
                 print '    '+ '\n    '.join(new)
     if unknown:
         print "Please update this script, the following modules are unknown"
         pprint(unknown)
     if DEBUG:
         print "===== SOURCE"
         pprint(source)
         print "===== FILES"
         pprint(files)
         print "===== DISABLED"
         pprint(disabled)
         print "===== UNKNOWN"
         pprint(unknown)
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='''This script only works on Mac OS X since the OS X Toolchain is not available on other platforms.
 In addition you need XCode and the Android NDK installed.
 If commit hash is not provided, current git master is used.''')
     parser.add_argument('--debug', dest='debug', action="store_true")
     parser.add_argument('--ndk', dest='ndk', type=str)
     parser.add_argument('--commit', dest='commit', type=str, default=None)
     args = parser.parse_args()
     if sys.platform != 'darwin' or not args.ndk:
         parser.print_help()
         sys.exit(1)
     ndk_path = args.ndk
     commit = args.commit
     DEBUG = args.debug
     base = os.path.abspath(os.curdir)
     prefix = os.path.join(base, 'upstream/')
     commit = prepare_upstream(prefix, commit)
     libvpx_files = get_libvpx_files(prefix)
     source, files, disabled, unknown = get_sources(prefix)
     sources_mozbuild, moz_build_files = get_sources_mozbuild()
     print_info(source, files, disabled, unknown, moz_build_files)
     update_sources_mozbuild(files, sources_mozbuild)
     update_and_remove_files(prefix, libvpx_files, files)
     apply_patches()
     update_readme(commit)
     cleanup_upstream()

The Tor Browser / annotate

media/libvpx/update.py@ac0c01689b40 (annotated)

media/libvpx/update.py