The Tor Browser / file comparison

comparison: media/libvpx/update.py

media/libvpx/update.py

branch
TOR_BUG_9701
changeset 10
ac0c01689b40
equal deleted inserted replaced
-1:000000000000 0:93ec1890811b
1 #!/usr/bin/env python
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this
4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 import argparse
6 import os
7 import re
8 import shutil
9 import sys
10 import subprocess
11 from pprint import pprint
12 from StringIO import StringIO
13
14 PLATFORMS= [
15 'x86-win32-vs8',
16 'x86_64-win64-vs8',
17 'x86-linux-gcc',
18 'x86_64-linux-gcc',
19 'generic-gnu',
20 'x86-darwin9-gcc',
21 'x86_64-darwin9-gcc',
22 'armv7-android-gcc',
23 'x86-win32-gcc',
24 'x86_64-win64-gcc',
25 ]
26
27
28 mk_files = [
29 'vp8/vp8_common.mk',
30 'vp8/vp8cx_arm.mk',
31 'vp8/vp8cx.mk',
32 'vp8/vp8dx.mk',
33 'vp9/vp9_common.mk',
34 'vp9/vp9cx.mk',
35 'vp9/vp9dx.mk',
36 'vpx_mem/vpx_mem.mk',
37 'vpx_ports/vpx_ports.mk',
38 'vpx_scale/vpx_scale.mk',
39 'vpx/vpx_codec.mk',
40 ]
41
42 extensions = ['.asm', '.c', '.h']
43
44 MODULES = {
45 'UNIFIED_SOURCES': [
46 'API_DOC_SRCS-$(CONFIG_VP8_DECODER)',
47 'API_DOC_SRCS-yes',
48 'API_EXPORTS',
49 'API_SRCS-$(CONFIG_VP8_DECODER)',
50 'API_SRCS-yes',
51 'MEM_SRCS-yes',
52 'PORTS_SRCS-yes',
53 'SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING)',
54 'SCALE_SRCS-no',
55 'SCALE_SRCS-yes',
56 'VP8_COMMON_SRCS-yes',
57 'VP8_DX_EXPORTS',
58 'VP8_DX_SRCS-$(CONFIG_MULTITHREAD)',
59 'VP8_DX_SRCS-no',
60 'VP8_DX_SRCS_REMOVE-no',
61 'VP8_DX_SRCS_REMOVE-yes',
62 'VP8_DX_SRCS-yes',
63 'VP9_COMMON_SRCS-yes',
64 'VP9_DX_EXPORTS',
65 'VP9_DX_SRCS-no',
66 'VP9_DX_SRCS_REMOVE-no',
67 'VP9_DX_SRCS_REMOVE-yes',
68 'VP9_DX_SRCS-yes',
69 'API_DOC_SRCS-$(CONFIG_VP8_ENCODER)',
70 'API_SRCS-$(BUILD_LIBVPX)',
71 'API_SRCS-$(CONFIG_VP8_ENCODER)',
72 'API_SRCS-$(CONFIG_VP9_ENCODER)',
73 'VP8_CX_EXPORTS',
74 'VP8_CX_SRCS-$(CONFIG_MULTI_RES_ENCODING)',
75 'VP8_CX_SRCS-$(CONFIG_MULTITHREAD)',
76 'VP8_CX_SRCS-$(CONFIG_TEMPORAL_DENOISING)',
77 'VP8_CX_SRCS-no',
78 'VP8_CX_SRCS_REMOVE-no',
79 'VP8_CX_SRCS_REMOVE-yes',
80 'VP8_CX_SRCS-yes',
81 'VP9_CX_EXPORTS',
82 'VP9_CX_SRCS-no',
83 'VP9_CX_SRCS_REMOVE-no',
84 'VP9_CX_SRCS_REMOVE-yes',
85 'VP9_CX_SRCS-yes',
86 ],
87 'X86_ASM': [
88 'PORTS_SRCS-$(BUILD_LIBVPX)',
89 'VP8_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
90 'VP8_COMMON_SRCS-$(HAVE_MMX)',
91 'VP8_COMMON_SRCS-$(HAVE_SSE2)',
92 'VP8_COMMON_SRCS-$(HAVE_SSE3)',
93 'VP8_COMMON_SRCS-$(HAVE_SSE4_1)',
94 'VP8_COMMON_SRCS-$(HAVE_SSSE3)',
95 'VP9_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
96 'VP9_COMMON_SRCS-$(HAVE_MMX)',
97 'VP9_COMMON_SRCS-$(HAVE_SSE2)',
98 'VP9_COMMON_SRCS-$(HAVE_SSSE3)',
99 'VP8_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
100 'VP8_CX_SRCS-$(HAVE_MMX)',
101 'VP8_CX_SRCS-$(HAVE_SSE2)',
102 'VP8_CX_SRCS-$(HAVE_SSE4_1)',
103 'VP8_CX_SRCS-$(HAVE_SSSE3)',
104 'VP8_CX_SRCS_REMOVE-$(HAVE_SSE2)',
105 'VP9_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
106 'VP9_CX_SRCS-$(HAVE_MMX)',
107 'VP9_CX_SRCS-$(HAVE_SSE2)',
108 'VP9_CX_SRCS-$(HAVE_SSE3)',
109 'VP9_CX_SRCS-$(HAVE_SSE4_1)',
110 'VP9_CX_SRCS-$(HAVE_SSSE3)',
111 ],
112 'X86-64_ASM': [
113 'VP8_CX_SRCS-$(ARCH_X86_64)',
114 'VP9_CX_SRCS-$(ARCH_X86_64)',
115 ],
116 'ARM_ASM': [
117 'PORTS_SRCS-$(ARCH_ARM)',
118 'SCALE_SRCS-$(HAVE_NEON)',
119 'VP8_COMMON_SRCS-$(ARCH_ARM)',
120 'VP8_COMMON_SRCS-$(HAVE_MEDIA)',
121 'VP8_COMMON_SRCS-$(HAVE_NEON)',
122 'VP9_COMMON_SRCS-$(HAVE_NEON)',
123 'VP8_CX_SRCS-$(ARCH_ARM)',
124 'VP8_CX_SRCS-$(HAVE_EDSP)',
125 'VP8_CX_SRCS-$(HAVE_MEDIA)',
126 'VP8_CX_SRCS-$(HAVE_NEON)',
127 ],
128 'ERROR_CONCEALMENT': [
129 'VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT)',
130 ],
131 'AVX2': [
132 'VP9_COMMON_SRCS-$(HAVE_AVX2)',
133 ],
134 'VP8_POSTPROC': [
135 'VP8_COMMON_SRCS-$(CONFIG_POSTPROC)',
136 ],
137 'VP9_POSTPROC': [
138 'VP9_COMMON_SRCS-$(CONFIG_VP9_POSTPROC)',
139 ]
140 }
141
142 DISABLED_MODULES = [
143 'MEM_SRCS-$(CONFIG_MEM_MANAGER)',
144 'MEM_SRCS-$(CONFIG_MEM_TRACKER)',
145 'VP8_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)',
146 'VP9_COMMON_SRCS-$(CONFIG_POSTPROC_VISUALIZER)',
147 'VP8_CX_SRCS-$(CONFIG_INTERNAL_STATS)',
148 'VP9_CX_SRCS-$(CONFIG_INTERNAL_STATS)',
149
150 # mips files are also ignored via ignored_folders
151 'SCALE_SRCS-$(HAVE_DSPR2)',
152 'VP8_COMMON_SRCS-$(HAVE_DSPR2)',
153 'VP9_COMMON_SRCS-$(HAVE_DSPR2)',
154 'VP8_CX_SRCS_REMOVE-$(HAVE_EDSP)',
155 ]
156
157 libvpx_files = [
158 'build/make/obj_int_extract.c',
159 'build/make/ads2gas.pl',
160 'build/make/thumb.pm',
161 'LICENSE',
162 'PATENTS',
163 ]
164
165 ignore_files = [
166 'vp8/common/context.c',
167 'vp8/common/textblit.c',
168 'vp8/encoder/ssim.c',
169 'vp8/encoder/x86/ssim_opt.asm',
170 'vp9/common/vp9_textblit.c',
171 'vp9/common/vp9_textblit.h',
172 'vp9/encoder/vp9_ssim.c',
173 'vp9/encoder/x86/vp9_ssim_opt.asm',
174 'vpx_mem/vpx_mem_tracker.c',
175 'vpx_scale/generic/bicubic_scaler.c',
176 'vpx_scale/win32/scaleopt.c',
177 'vpx_scale/win32/scalesystemdependent.c',
178 ]
179
180 ignore_folders = [
181 'examples/',
182 'googletest/',
183 'libmkv/',
184 'libyuv/',
185 'mips/',
186 'nestegg/',
187 'objdir/',
188 'ppc/',
189 'test/',
190 'vpx_mem/memory_manager/',
191 ]
192 files = {
193 'EXPORTS': [
194 'vpx_mem/include/vpx_mem_intrnl.h',
195 'vpx_mem/vpx_mem.h',
196 'vpx_ports/arm.h',
197 'vpx_ports/mem.h',
198 'vpx_ports/vpx_timer.h',
199 'vpx_ports/x86.h',
200 'vpx_scale/vpx_scale.h',
201 'vpx_scale/yv12config.h',
202 'vpx/vp8cx.h',
203 'vpx/vp8dx.h',
204 'vpx/vp8.h',
205 'vpx/vpx_codec.h',
206 'vpx/vpx_decoder.h',
207 'vpx/vpx_encoder.h',
208 'vpx/vpx_image.h',
209 'vpx/vpx_integer.h',
210 ],
211 'X86-64_ASM': [
212 'third_party/x86inc/x86inc.asm',
213 'vp8/common/x86/loopfilter_block_sse2.asm',
214 'vp9/encoder/x86/vp9_quantize_ssse3.asm',
215 ],
216 'SOURCES': [
217 'vp8/common/rtcd.c',
218 'vp8/common/sad_c.c',
219 'vp8/vp8_dx_iface.c',
220 'vp9/common/vp9_entropymv.c',
221 'vp9/common/vp9_rtcd.c',
222 'vp9/encoder/vp9_bitstream.c',
223 'vpx/src/svc_encodeframe.c',
224 'vpx_mem/vpx_mem.c',
225 ]
226 }
227
228 manual = [
229 # special case in moz.build
230 'vp8/encoder/boolhuff.c',
231
232 # 64bit only
233 'vp8/common/x86/loopfilter_block_sse2.asm',
234 'vp9/encoder/x86/vp9_quantize_ssse3.asm',
235
236 # offsets are special cased in Makefile.in
237 'vp8/encoder/vp8_asm_enc_offsets.c',
238 'vpx_scale/vpx_scale_asm_offsets.c',
239
240 # ignore while vp9 postproc is not enabled
241 'vp9/common/x86/vp9_postproc_mmx.asm',
242 'vp9/common/x86/vp9_postproc_sse2.asm',
243
244 # ssim_opt is not enabled
245 'vp8/encoder/x86/ssim_opt.asm',
246 'vp9/encoder/x86/vp9_ssim_opt.asm',
247
248 # asm includes
249 'vpx_ports/x86_abi_support.asm',
250 ]
251
252 platform_files = [
253 'vp8_rtcd.h',
254 'vp9_rtcd.h',
255 'vpx_config.asm',
256 'vpx_config.h',
257 'vpx_scale_rtcd.h',
258 ]
259
260 def prepare_upstream(prefix, commit=None):
261 if os.path.exists(prefix):
262 print "Please remove '%s' folder before running %s" % (prefix, sys.argv[0])
263 sys.exit(1)
264
265 upstream_url = 'https://gerrit.chromium.org/gerrit/webm/libvpx'
266 subprocess.call(['git', 'clone', upstream_url, prefix])
267 if commit:
268 os.chdir(prefix)
269 subprocess.call(['git', 'checkout', commit])
270 else:
271 os.chdir(prefix)
272 p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
273 stdout, stderr = p.communicate()
274 commit = stdout.strip()
275
276 for target in PLATFORMS:
277 target_objdir = os.path.join(prefix, 'objdir', target)
278 os.makedirs(target_objdir)
279 os.chdir(target_objdir)
280 configure = ['../../configure', '--target=%s' % target,
281 '--disable-examples', '--disable-install-docs',
282 '--enable-multi-res-encoding',
283 ]
284
285 if 'darwin9' in target:
286 configure += ['--enable-pic']
287 if 'linux' in target:
288 configure += ['--enable-pic']
289 # x86inc.asm is not compatible with pic 32bit builds
290 if target == 'x86-linux-gcc':
291 configure += ['--disable-use-x86inc']
292
293 if target == 'armv7-android-gcc':
294 configure += ['--sdk-path=%s' % ndk_path]
295
296 subprocess.call(configure)
297 make_targets = [f for f in platform_files if not os.path.exists(f)]
298 if make_targets:
299 subprocess.call(['make'] + make_targets)
300 for f in make_targets:
301 if not os.path.exists(f):
302 print "%s missing from %s, check toolchain" % (f, target)
303 sys.exit(1)
304
305 os.chdir(base)
306 return commit
307
308 def cleanup_upstream():
309 shutil.rmtree(os.path.join(base, 'upstream'))
310
311 def get_module(key):
312 for module in MODULES:
313 if key in MODULES[module]:
314 return module
315
316 def get_libvpx_files(prefix):
317 for root, folders, files in os.walk(prefix):
318 for f in files:
319 f = os.path.join(root, f)[len(prefix):]
320 if os.path.splitext(f)[-1] in extensions \
321 and os.sep in f \
322 and f not in ignore_files \
323 and not any(folder in f for folder in ignore_folders):
324 libvpx_files.append(f)
325 return libvpx_files
326
327 def get_sources(prefix):
328 source = {}
329 unknown = {}
330 disabled = {}
331
332 for mk in mk_files:
333 with open(os.path.join(prefix, mk)) as f:
334 base = os.path.dirname(mk)
335 for l in f:
336 if '+=' in l:
337 l = l.split('+=')
338 key = l[0].strip()
339 value = l[1].strip().replace('$(ASM)', '.asm')
340 value = os.path.join(base, value)
341 if not key.startswith('#') and os.path.splitext(value)[-1] in extensions:
342 if key not in source:
343 source[key] = []
344 source[key].append(value)
345
346 for key in source:
347 for f in source[key]:
348 if key.endswith('EXPORTS') and f.endswith('.h'):
349 files['EXPORTS'].append(f)
350 if os.path.splitext(f)[-1] in ('.c', '.asm') and not f in manual:
351 module = get_module(key)
352 if module:
353 if not module in files:
354 files[module] = []
355 t = files[module]
356 elif key in DISABLED_MODULES:
357 if not key in disabled:
358 disabled[key] = []
359 t = disabled[key]
360 else:
361 if not key in unknown:
362 unknown[key] = []
363 t = unknown[key]
364 t.append(f)
365
366 files['UNIFIED_SOURCES'] = [f for f in files['UNIFIED_SOURCES'] if f not in files['SOURCES']]
367
368 for key in files:
369 files[key] = list(sorted(set(files[key])))
370
371 return source, files, disabled, unknown
372
373 def update_sources_mozbuild(files, sources_mozbuild):
374 f = StringIO()
375 pprint(files, stream=f)
376 sources_mozbuild_new = "files = {\n %s\n}\n" % f.getvalue().strip()[1:-1]
377 if sources_mozbuild != sources_mozbuild_new:
378 print 'updating sources.mozbuild'
379 with open('sources.mozbuild', 'w') as f:
380 f.write(sources_mozbuild_new)
381
382 def get_current_files():
383 current_files = []
384 for root, folders, files in os.walk('.'):
385 for f in files:
386 f = os.path.join(root, f)[len('.%s'%os.sep):]
387 if 'upstream%s'%os.sep in f or not os.sep in f:
388 continue
389 if os.path.splitext(f)[-1] in extensions:
390 current_files.append(f)
391 return current_files
392
393 def is_new(a, b):
394 return not os.path.exists(a) \
395 or not os.path.exists(b) \
396 or open(a).read() != open(b).read()
397
398 def get_sources_mozbuild():
399 with open('sources.mozbuild') as f:
400 sources_mozbuild = f.read()
401 exec(sources_mozbuild)
402 return sources_mozbuild, files
403
404 def update_and_remove_files(prefix, libvpx_files, files):
405 current_files = get_current_files()
406
407 def copy(src, dst):
408 print ' ', dst
409 shutil.copy(src, dst)
410
411 # Update files
412 first = True
413 for f in libvpx_files:
414 fdir = os.path.dirname(f)
415 if fdir and not os.path.exists(fdir):
416 os.makedirs(fdir)
417 s = os.path.join(prefix, f)
418 if is_new(f, s):
419 if first:
420 print "Copy files:"
421 first = False
422 copy(s, f)
423
424 # Copy configuration files for each platform
425 for target in PLATFORMS:
426 first = True
427 for f in platform_files:
428 t = os.path.splitext(f)
429 t = '%s_%s%s' % (t[0], target, t[1])
430 f = os.path.join(prefix, 'objdir', target, f)
431 if is_new(f, t):
432 if first:
433 print "Copy files for %s:" % target
434 first = False
435 copy(f, t)
436
437 # Copy vpx_version.h from one of the build targets
438 s = os.path.join(prefix, 'objdir/x86-linux-gcc/vpx_version.h')
439 f = 'vpx_version.h'
440 if is_new(s, f):
441 copy(s, f)
442
443 # Remove unknown files from tree
444 removed_files = [f for f in current_files if f not in libvpx_files]
445 if removed_files:
446 print "Remove files:"
447 for f in removed_files:
448 os.unlink(f)
449 print ' ', f
450
451 def apply_patches():
452 # Patch to permit vpx users to specify their own <stdint.h> types.
453 os.system("patch -p3 < stdint.patch")
454 os.system("patch -p3 < unified.patch")
455 os.system("patch -p3 < mingw.patch")
456
457 def update_readme(commit):
458 with open('README_MOZILLA') as f:
459 readme = f.read()
460
461 if 'The git commit ID used was' in readme:
462 new_readme = re.sub('The git commit ID used was [a-f0-9]+',
463 'The git commit ID used was %s' % commit, readme)
464 else:
465 new_readme = "%s\n\nThe git commit ID used was %s\n" % (readme, commit)
466
467 if readme != new_readme:
468 with open('README_MOZILLA', 'w') as f:
469 f.write(new_readme)
470
471 def print_info(source, files, disabled, unknown, moz_build_files):
472 for key in moz_build_files:
473 if key not in files:
474 print key, 'MISSING'
475 else:
476 gone = set(moz_build_files[key]) - set(files[key])
477 new = set(files[key]) - set(moz_build_files[key])
478 if gone:
479 print key, 'GONE:'
480 print ' '+ '\n '.join(gone)
481 if new:
482 print key, 'NEW:'
483 print ' '+ '\n '.join(new)
484
485 if unknown:
486 print "Please update this script, the following modules are unknown"
487 pprint(unknown)
488
489 if DEBUG:
490 print "===== SOURCE"
491 pprint(source)
492 print "===== FILES"
493 pprint(files)
494 print "===== DISABLED"
495 pprint(disabled)
496 print "===== UNKNOWN"
497 pprint(unknown)
498
499
500 if __name__ == '__main__':
501 parser = argparse.ArgumentParser(description='''This script only works on Mac OS X since the OS X Toolchain is not available on other platforms.
502 In addition you need XCode and the Android NDK installed.
503 If commit hash is not provided, current git master is used.''')
504 parser.add_argument('--debug', dest='debug', action="store_true")
505 parser.add_argument('--ndk', dest='ndk', type=str)
506 parser.add_argument('--commit', dest='commit', type=str, default=None)
507
508 args = parser.parse_args()
509
510 if sys.platform != 'darwin' or not args.ndk:
511 parser.print_help()
512 sys.exit(1)
513
514 ndk_path = args.ndk
515 commit = args.commit
516 DEBUG = args.debug
517
518 base = os.path.abspath(os.curdir)
519 prefix = os.path.join(base, 'upstream/')
520
521 commit = prepare_upstream(prefix, commit)
522
523 libvpx_files = get_libvpx_files(prefix)
524 source, files, disabled, unknown = get_sources(prefix)
525
526 sources_mozbuild, moz_build_files = get_sources_mozbuild()
527
528 print_info(source, files, disabled, unknown, moz_build_files)
529 update_sources_mozbuild(files, sources_mozbuild)
530 update_and_remove_files(prefix, libvpx_files, files)
531 apply_patches()
532 update_readme(commit)
533
534 cleanup_upstream()

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial