michael@0: #!/usr/bin/env python michael@0: # This Source Code Form is subject to the terms of the Mozilla Public michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/. michael@0: import argparse michael@0: import os michael@0: import re michael@0: import shutil michael@0: import sys michael@0: import subprocess michael@0: from pprint import pprint michael@0: from StringIO import StringIO michael@0: michael@0: PLATFORMS= [ michael@0: 'x86-win32-vs8', michael@0: 'x86_64-win64-vs8', michael@0: 'x86-linux-gcc', michael@0: 'x86_64-linux-gcc', michael@0: 'generic-gnu', michael@0: 'x86-darwin9-gcc', michael@0: 'x86_64-darwin9-gcc', michael@0: 'armv7-android-gcc', michael@0: 'x86-win32-gcc', michael@0: 'x86_64-win64-gcc', michael@0: ] michael@0: michael@0: michael@0: mk_files = [ michael@0: 'vp8/vp8_common.mk', michael@0: 'vp8/vp8cx_arm.mk', michael@0: 'vp8/vp8cx.mk', michael@0: 'vp8/vp8dx.mk', michael@0: 'vp9/vp9_common.mk', michael@0: 'vp9/vp9cx.mk', michael@0: 'vp9/vp9dx.mk', michael@0: 'vpx_mem/vpx_mem.mk', michael@0: 'vpx_ports/vpx_ports.mk', michael@0: 'vpx_scale/vpx_scale.mk', michael@0: 'vpx/vpx_codec.mk', michael@0: ] michael@0: michael@0: extensions = ['.asm', '.c', '.h'] michael@0: michael@0: MODULES = { michael@0: 'UNIFIED_SOURCES': [ michael@0: 'API_DOC_SRCS-$(CONFIG_VP8_DECODER)', michael@0: 'API_DOC_SRCS-yes', michael@0: 'API_EXPORTS', michael@0: 'API_SRCS-$(CONFIG_VP8_DECODER)', michael@0: 'API_SRCS-yes', michael@0: 'MEM_SRCS-yes', michael@0: 'PORTS_SRCS-yes', michael@0: 'SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING)', michael@0: 'SCALE_SRCS-no', michael@0: 'SCALE_SRCS-yes', michael@0: 'VP8_COMMON_SRCS-yes', michael@0: 'VP8_DX_EXPORTS', michael@0: 'VP8_DX_SRCS-$(CONFIG_MULTITHREAD)', michael@0: 'VP8_DX_SRCS-no', michael@0: 'VP8_DX_SRCS_REMOVE-no', michael@0: 'VP8_DX_SRCS_REMOVE-yes', michael@0: 'VP8_DX_SRCS-yes', michael@0: 'VP9_COMMON_SRCS-yes', michael@0: 'VP9_DX_EXPORTS', michael@0: 'VP9_DX_SRCS-no', michael@0: 'VP9_DX_SRCS_REMOVE-no', michael@0: 'VP9_DX_SRCS_REMOVE-yes', michael@0: 'VP9_DX_SRCS-yes', michael@0: 'API_DOC_SRCS-$(CONFIG_VP8_ENCODER)', michael@0: 'API_SRCS-$(BUILD_LIBVPX)', michael@0: 'API_SRCS-$(CONFIG_VP8_ENCODER)', michael@0: 'API_SRCS-$(CONFIG_VP9_ENCODER)', michael@0: 'VP8_CX_EXPORTS', michael@0: 'VP8_CX_SRCS-$(CONFIG_MULTI_RES_ENCODING)', michael@0: 'VP8_CX_SRCS-$(CONFIG_MULTITHREAD)', michael@0: 'VP8_CX_SRCS-$(CONFIG_TEMPORAL_DENOISING)', michael@0: 'VP8_CX_SRCS-no', michael@0: 'VP8_CX_SRCS_REMOVE-no', michael@0: 'VP8_CX_SRCS_REMOVE-yes', michael@0: 'VP8_CX_SRCS-yes', michael@0: 'VP9_CX_EXPORTS', michael@0: 'VP9_CX_SRCS-no', michael@0: 'VP9_CX_SRCS_REMOVE-no', michael@0: 'VP9_CX_SRCS_REMOVE-yes', michael@0: 'VP9_CX_SRCS-yes', michael@0: ], michael@0: 'X86_ASM': [ michael@0: 'PORTS_SRCS-$(BUILD_LIBVPX)', michael@0: 'VP8_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_MMX)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_SSE2)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_SSE3)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_SSE4_1)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_SSSE3)', michael@0: 'VP9_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)', michael@0: 'VP9_COMMON_SRCS-$(HAVE_MMX)', michael@0: 'VP9_COMMON_SRCS-$(HAVE_SSE2)', michael@0: 'VP9_COMMON_SRCS-$(HAVE_SSSE3)', michael@0: 'VP8_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)', michael@0: 'VP8_CX_SRCS-$(HAVE_MMX)', michael@0: 'VP8_CX_SRCS-$(HAVE_SSE2)', michael@0: 'VP8_CX_SRCS-$(HAVE_SSE4_1)', michael@0: 'VP8_CX_SRCS-$(HAVE_SSSE3)', michael@0: 'VP8_CX_SRCS_REMOVE-$(HAVE_SSE2)', michael@0: 'VP9_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)', michael@0: 'VP9_CX_SRCS-$(HAVE_MMX)', michael@0: 'VP9_CX_SRCS-$(HAVE_SSE2)', michael@0: 'VP9_CX_SRCS-$(HAVE_SSE3)', michael@0: 'VP9_CX_SRCS-$(HAVE_SSE4_1)', michael@0: 'VP9_CX_SRCS-$(HAVE_SSSE3)', michael@0: ], michael@0: 'X86-64_ASM': [ michael@0: 'VP8_CX_SRCS-$(ARCH_X86_64)', michael@0: 'VP9_CX_SRCS-$(ARCH_X86_64)', michael@0: ], michael@0: 'ARM_ASM': [ michael@0: 'PORTS_SRCS-$(ARCH_ARM)', michael@0: 'SCALE_SRCS-$(HAVE_NEON)', michael@0: 'VP8_COMMON_SRCS-$(ARCH_ARM)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_MEDIA)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_NEON)', michael@0: 'VP9_COMMON_SRCS-$(HAVE_NEON)', michael@0: 'VP8_CX_SRCS-$(ARCH_ARM)', michael@0: 'VP8_CX_SRCS-$(HAVE_EDSP)', michael@0: 'VP8_CX_SRCS-$(HAVE_MEDIA)', michael@0: 'VP8_CX_SRCS-$(HAVE_NEON)', michael@0: ], michael@0: 'ERROR_CONCEALMENT': [ michael@0: 'VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT)', michael@0: ], michael@0: 'AVX2': [ michael@0: 'VP9_COMMON_SRCS-$(HAVE_AVX2)', michael@0: ], michael@0: 'VP8_POSTPROC': [ michael@0: 'VP8_COMMON_SRCS-$(CONFIG_POSTPROC)', michael@0: ], michael@0: 'VP9_POSTPROC': [ michael@0: 'VP9_COMMON_SRCS-$(CONFIG_VP9_POSTPROC)', michael@0: ] michael@0: } michael@0: michael@0: DISABLED_MODULES = [ michael@0: 'MEM_SRCS-$(CONFIG_MEM_MANAGER)', michael@0: 'MEM_SRCS-$(CONFIG_MEM_TRACKER)', michael@0: 'VP8_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)', michael@0: 'VP9_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)', michael@0: 'VP8_CX_SRCS-$(CONFIG_INTERNAL_STATS)', michael@0: 'VP9_CX_SRCS-$(CONFIG_INTERNAL_STATS)', michael@0: michael@0: # mips files are also ignored via ignored_folders michael@0: 'SCALE_SRCS-$(HAVE_DSPR2)', michael@0: 'VP8_COMMON_SRCS-$(HAVE_DSPR2)', michael@0: 'VP9_COMMON_SRCS-$(HAVE_DSPR2)', michael@0: 'VP8_CX_SRCS_REMOVE-$(HAVE_EDSP)', michael@0: ] michael@0: michael@0: libvpx_files = [ michael@0: 'build/make/obj_int_extract.c', michael@0: 'build/make/ads2gas.pl', michael@0: 'build/make/thumb.pm', michael@0: 'LICENSE', michael@0: 'PATENTS', michael@0: ] michael@0: michael@0: ignore_files = [ michael@0: 'vp8/common/context.c', michael@0: 'vp8/common/textblit.c', michael@0: 'vp8/encoder/ssim.c', michael@0: 'vp8/encoder/x86/ssim_opt.asm', michael@0: 'vp9/common/vp9_textblit.c', michael@0: 'vp9/common/vp9_textblit.h', michael@0: 'vp9/encoder/vp9_ssim.c', michael@0: 'vp9/encoder/x86/vp9_ssim_opt.asm', michael@0: 'vpx_mem/vpx_mem_tracker.c', michael@0: 'vpx_scale/generic/bicubic_scaler.c', michael@0: 'vpx_scale/win32/scaleopt.c', michael@0: 'vpx_scale/win32/scalesystemdependent.c', michael@0: ] michael@0: michael@0: ignore_folders = [ michael@0: 'examples/', michael@0: 'googletest/', michael@0: 'libmkv/', michael@0: 'libyuv/', michael@0: 'mips/', michael@0: 'nestegg/', michael@0: 'objdir/', michael@0: 'ppc/', michael@0: 'test/', michael@0: 'vpx_mem/memory_manager/', michael@0: ] michael@0: files = { michael@0: 'EXPORTS': [ michael@0: 'vpx_mem/include/vpx_mem_intrnl.h', michael@0: 'vpx_mem/vpx_mem.h', michael@0: 'vpx_ports/arm.h', michael@0: 'vpx_ports/mem.h', michael@0: 'vpx_ports/vpx_timer.h', michael@0: 'vpx_ports/x86.h', michael@0: 'vpx_scale/vpx_scale.h', michael@0: 'vpx_scale/yv12config.h', michael@0: 'vpx/vp8cx.h', michael@0: 'vpx/vp8dx.h', michael@0: 'vpx/vp8.h', michael@0: 'vpx/vpx_codec.h', michael@0: 'vpx/vpx_decoder.h', michael@0: 'vpx/vpx_encoder.h', michael@0: 'vpx/vpx_image.h', michael@0: 'vpx/vpx_integer.h', michael@0: ], michael@0: 'X86-64_ASM': [ michael@0: 'third_party/x86inc/x86inc.asm', michael@0: 'vp8/common/x86/loopfilter_block_sse2.asm', michael@0: 'vp9/encoder/x86/vp9_quantize_ssse3.asm', michael@0: ], michael@0: 'SOURCES': [ michael@0: 'vp8/common/rtcd.c', michael@0: 'vp8/common/sad_c.c', michael@0: 'vp8/vp8_dx_iface.c', michael@0: 'vp9/common/vp9_entropymv.c', michael@0: 'vp9/common/vp9_rtcd.c', michael@0: 'vp9/encoder/vp9_bitstream.c', michael@0: 'vpx/src/svc_encodeframe.c', michael@0: 'vpx_mem/vpx_mem.c', michael@0: ] michael@0: } michael@0: michael@0: manual = [ michael@0: # special case in moz.build michael@0: 'vp8/encoder/boolhuff.c', michael@0: michael@0: # 64bit only michael@0: 'vp8/common/x86/loopfilter_block_sse2.asm', michael@0: 'vp9/encoder/x86/vp9_quantize_ssse3.asm', michael@0: michael@0: # offsets are special cased in Makefile.in michael@0: 'vp8/encoder/vp8_asm_enc_offsets.c', michael@0: 'vpx_scale/vpx_scale_asm_offsets.c', michael@0: michael@0: # ignore while vp9 postproc is not enabled michael@0: 'vp9/common/x86/vp9_postproc_mmx.asm', michael@0: 'vp9/common/x86/vp9_postproc_sse2.asm', michael@0: michael@0: # ssim_opt is not enabled michael@0: 'vp8/encoder/x86/ssim_opt.asm', michael@0: 'vp9/encoder/x86/vp9_ssim_opt.asm', michael@0: michael@0: # asm includes michael@0: 'vpx_ports/x86_abi_support.asm', michael@0: ] michael@0: michael@0: platform_files = [ michael@0: 'vp8_rtcd.h', michael@0: 'vp9_rtcd.h', michael@0: 'vpx_config.asm', michael@0: 'vpx_config.h', michael@0: 'vpx_scale_rtcd.h', michael@0: ] michael@0: michael@0: def prepare_upstream(prefix, commit=None): michael@0: if os.path.exists(prefix): michael@0: print "Please remove '%s' folder before running %s" % (prefix, sys.argv[0]) michael@0: sys.exit(1) michael@0: michael@0: upstream_url = 'https://gerrit.chromium.org/gerrit/webm/libvpx' michael@0: subprocess.call(['git', 'clone', upstream_url, prefix]) michael@0: if commit: michael@0: os.chdir(prefix) michael@0: subprocess.call(['git', 'checkout', commit]) michael@0: else: michael@0: os.chdir(prefix) michael@0: p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE) michael@0: stdout, stderr = p.communicate() michael@0: commit = stdout.strip() michael@0: michael@0: for target in PLATFORMS: michael@0: target_objdir = os.path.join(prefix, 'objdir', target) michael@0: os.makedirs(target_objdir) michael@0: os.chdir(target_objdir) michael@0: configure = ['../../configure', '--target=%s' % target, michael@0: '--disable-examples', '--disable-install-docs', michael@0: '--enable-multi-res-encoding', michael@0: ] michael@0: michael@0: if 'darwin9' in target: michael@0: configure += ['--enable-pic'] michael@0: if 'linux' in target: michael@0: configure += ['--enable-pic'] michael@0: # x86inc.asm is not compatible with pic 32bit builds michael@0: if target == 'x86-linux-gcc': michael@0: configure += ['--disable-use-x86inc'] michael@0: michael@0: if target == 'armv7-android-gcc': michael@0: configure += ['--sdk-path=%s' % ndk_path] michael@0: michael@0: subprocess.call(configure) michael@0: make_targets = [f for f in platform_files if not os.path.exists(f)] michael@0: if make_targets: michael@0: subprocess.call(['make'] + make_targets) michael@0: for f in make_targets: michael@0: if not os.path.exists(f): michael@0: print "%s missing from %s, check toolchain" % (f, target) michael@0: sys.exit(1) michael@0: michael@0: os.chdir(base) michael@0: return commit michael@0: michael@0: def cleanup_upstream(): michael@0: shutil.rmtree(os.path.join(base, 'upstream')) michael@0: michael@0: def get_module(key): michael@0: for module in MODULES: michael@0: if key in MODULES[module]: michael@0: return module michael@0: michael@0: def get_libvpx_files(prefix): michael@0: for root, folders, files in os.walk(prefix): michael@0: for f in files: michael@0: f = os.path.join(root, f)[len(prefix):] michael@0: if os.path.splitext(f)[-1] in extensions \ michael@0: and os.sep in f \ michael@0: and f not in ignore_files \ michael@0: and not any(folder in f for folder in ignore_folders): michael@0: libvpx_files.append(f) michael@0: return libvpx_files michael@0: michael@0: def get_sources(prefix): michael@0: source = {} michael@0: unknown = {} michael@0: disabled = {} michael@0: michael@0: for mk in mk_files: michael@0: with open(os.path.join(prefix, mk)) as f: michael@0: base = os.path.dirname(mk) michael@0: for l in f: michael@0: if '+=' in l: michael@0: l = l.split('+=') michael@0: key = l[0].strip() michael@0: value = l[1].strip().replace('$(ASM)', '.asm') michael@0: value = os.path.join(base, value) michael@0: if not key.startswith('#') and os.path.splitext(value)[-1] in extensions: michael@0: if key not in source: michael@0: source[key] = [] michael@0: source[key].append(value) michael@0: michael@0: for key in source: michael@0: for f in source[key]: michael@0: if key.endswith('EXPORTS') and f.endswith('.h'): michael@0: files['EXPORTS'].append(f) michael@0: if os.path.splitext(f)[-1] in ('.c', '.asm') and not f in manual: michael@0: module = get_module(key) michael@0: if module: michael@0: if not module in files: michael@0: files[module] = [] michael@0: t = files[module] michael@0: elif key in DISABLED_MODULES: michael@0: if not key in disabled: michael@0: disabled[key] = [] michael@0: t = disabled[key] michael@0: else: michael@0: if not key in unknown: michael@0: unknown[key] = [] michael@0: t = unknown[key] michael@0: t.append(f) michael@0: michael@0: files['UNIFIED_SOURCES'] = [f for f in files['UNIFIED_SOURCES'] if f not in files['SOURCES']] michael@0: michael@0: for key in files: michael@0: files[key] = list(sorted(set(files[key]))) michael@0: michael@0: return source, files, disabled, unknown michael@0: michael@0: def update_sources_mozbuild(files, sources_mozbuild): michael@0: f = StringIO() michael@0: pprint(files, stream=f) michael@0: sources_mozbuild_new = "files = {\n %s\n}\n" % f.getvalue().strip()[1:-1] michael@0: if sources_mozbuild != sources_mozbuild_new: michael@0: print 'updating sources.mozbuild' michael@0: with open('sources.mozbuild', 'w') as f: michael@0: f.write(sources_mozbuild_new) michael@0: michael@0: def get_current_files(): michael@0: current_files = [] michael@0: for root, folders, files in os.walk('.'): michael@0: for f in files: michael@0: f = os.path.join(root, f)[len('.%s'%os.sep):] michael@0: if 'upstream%s'%os.sep in f or not os.sep in f: michael@0: continue michael@0: if os.path.splitext(f)[-1] in extensions: michael@0: current_files.append(f) michael@0: return current_files michael@0: michael@0: def is_new(a, b): michael@0: return not os.path.exists(a) \ michael@0: or not os.path.exists(b) \ michael@0: or open(a).read() != open(b).read() michael@0: michael@0: def get_sources_mozbuild(): michael@0: with open('sources.mozbuild') as f: michael@0: sources_mozbuild = f.read() michael@0: exec(sources_mozbuild) michael@0: return sources_mozbuild, files michael@0: michael@0: def update_and_remove_files(prefix, libvpx_files, files): michael@0: current_files = get_current_files() michael@0: michael@0: def copy(src, dst): michael@0: print ' ', dst michael@0: shutil.copy(src, dst) michael@0: michael@0: # Update files michael@0: first = True michael@0: for f in libvpx_files: michael@0: fdir = os.path.dirname(f) michael@0: if fdir and not os.path.exists(fdir): michael@0: os.makedirs(fdir) michael@0: s = os.path.join(prefix, f) michael@0: if is_new(f, s): michael@0: if first: michael@0: print "Copy files:" michael@0: first = False michael@0: copy(s, f) michael@0: michael@0: # Copy configuration files for each platform michael@0: for target in PLATFORMS: michael@0: first = True michael@0: for f in platform_files: michael@0: t = os.path.splitext(f) michael@0: t = '%s_%s%s' % (t[0], target, t[1]) michael@0: f = os.path.join(prefix, 'objdir', target, f) michael@0: if is_new(f, t): michael@0: if first: michael@0: print "Copy files for %s:" % target michael@0: first = False michael@0: copy(f, t) michael@0: michael@0: # Copy vpx_version.h from one of the build targets michael@0: s = os.path.join(prefix, 'objdir/x86-linux-gcc/vpx_version.h') michael@0: f = 'vpx_version.h' michael@0: if is_new(s, f): michael@0: copy(s, f) michael@0: michael@0: # Remove unknown files from tree michael@0: removed_files = [f for f in current_files if f not in libvpx_files] michael@0: if removed_files: michael@0: print "Remove files:" michael@0: for f in removed_files: michael@0: os.unlink(f) michael@0: print ' ', f michael@0: michael@0: def apply_patches(): michael@0: # Patch to permit vpx users to specify their own types. michael@0: os.system("patch -p3 < stdint.patch") michael@0: os.system("patch -p3 < unified.patch") michael@0: os.system("patch -p3 < mingw.patch") michael@0: michael@0: def update_readme(commit): michael@0: with open('README_MOZILLA') as f: michael@0: readme = f.read() michael@0: michael@0: if 'The git commit ID used was' in readme: michael@0: new_readme = re.sub('The git commit ID used was [a-f0-9]+', michael@0: 'The git commit ID used was %s' % commit, readme) michael@0: else: michael@0: new_readme = "%s\n\nThe git commit ID used was %s\n" % (readme, commit) michael@0: michael@0: if readme != new_readme: michael@0: with open('README_MOZILLA', 'w') as f: michael@0: f.write(new_readme) michael@0: michael@0: def print_info(source, files, disabled, unknown, moz_build_files): michael@0: for key in moz_build_files: michael@0: if key not in files: michael@0: print key, 'MISSING' michael@0: else: michael@0: gone = set(moz_build_files[key]) - set(files[key]) michael@0: new = set(files[key]) - set(moz_build_files[key]) michael@0: if gone: michael@0: print key, 'GONE:' michael@0: print ' '+ '\n '.join(gone) michael@0: if new: michael@0: print key, 'NEW:' michael@0: print ' '+ '\n '.join(new) michael@0: michael@0: if unknown: michael@0: print "Please update this script, the following modules are unknown" michael@0: pprint(unknown) michael@0: michael@0: if DEBUG: michael@0: print "===== SOURCE" michael@0: pprint(source) michael@0: print "===== FILES" michael@0: pprint(files) michael@0: print "===== DISABLED" michael@0: pprint(disabled) michael@0: print "===== UNKNOWN" michael@0: pprint(unknown) michael@0: michael@0: michael@0: if __name__ == '__main__': michael@0: parser = argparse.ArgumentParser(description='''This script only works on Mac OS X since the OS X Toolchain is not available on other platforms. michael@0: In addition you need XCode and the Android NDK installed. michael@0: If commit hash is not provided, current git master is used.''') michael@0: parser.add_argument('--debug', dest='debug', action="store_true") michael@0: parser.add_argument('--ndk', dest='ndk', type=str) michael@0: parser.add_argument('--commit', dest='commit', type=str, default=None) michael@0: michael@0: args = parser.parse_args() michael@0: michael@0: if sys.platform != 'darwin' or not args.ndk: michael@0: parser.print_help() michael@0: sys.exit(1) michael@0: michael@0: ndk_path = args.ndk michael@0: commit = args.commit michael@0: DEBUG = args.debug michael@0: michael@0: base = os.path.abspath(os.curdir) michael@0: prefix = os.path.join(base, 'upstream/') michael@0: michael@0: commit = prepare_upstream(prefix, commit) michael@0: michael@0: libvpx_files = get_libvpx_files(prefix) michael@0: source, files, disabled, unknown = get_sources(prefix) michael@0: michael@0: sources_mozbuild, moz_build_files = get_sources_mozbuild() michael@0: michael@0: print_info(source, files, disabled, unknown, moz_build_files) michael@0: update_sources_mozbuild(files, sources_mozbuild) michael@0: update_and_remove_files(prefix, libvpx_files, files) michael@0: apply_patches() michael@0: update_readme(commit) michael@0: michael@0: cleanup_upstream()