The Tor Browser / file comparison

comparison: layout/mathml/updateOperatorDictionary.pl

layout/mathml/updateOperatorDictionary.pl

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:a98bdbadec68
1 #!/usr/bin/perl
2 # -*- Mode: Perl; tab-width: 2; indent-tabs-mode: nil; -*-
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7 use XML::LibXSLT;
8 use XML::LibXML;
9 use LWP::Simple;
10
11 # output files
12 $FILE_UNICODE = "unicode.xml";
13 $FILE_DICTIONARY = "dictionary.xml";
14 $FILE_DIFFERENCES = "differences.txt";
15 $FILE_NEW_DICTIONARY = "new_dictionary.txt";
16 $FILE_SYNTAX_ERRORS = "syntax_errors.txt";
17 $FILE_JS = "tests/stretchy-and-large-operators.js";
18
19 # our dictionary (property file)
20 $MOZ_DICTIONARY = "mathfont.properties";
21
22 # dictionary provided by the W3C in "XML Entity Definitions for Characters"
23 $WG_DICTIONARY_URL = "http://www.w3.org/2003/entities/2007xml/unicode.xml";
24
25 # XSL stylesheet to extract relevant data from the dictionary
26 $DICTIONARY_XSL = "operatorDictionary.xsl";
27
28 # dictionary provided by the W3C transformed with operatorDictionary.xsl
29 $WG_DICTIONARY = $FILE_DICTIONARY;
30
31 if (!($#ARGV >= 0 &&
32 ((($ARGV[0] eq "download") && $#ARGV <= 1) ||
33 (($ARGV[0] eq "compare") && $#ARGV <= 1) ||
34 (($ARGV[0] eq "check") && $#ARGV <= 0) ||
35 (($ARGV[0] eq "make-js") && $#ARGV <= 0) ||
36 (($ARGV[0] eq "clean") && $#ARGV <= 0)))) {
37 &usage;
38 }
39
40 if ($ARGV[0] eq "download") {
41 if ($#ARGV == 1) {
42 $WG_DICTIONARY_URL = $ARGV[1];
43 }
44 print "Downloading $WG_DICTIONARY_URL...\n";
45 getstore($WG_DICTIONARY_URL, $FILE_UNICODE);
46
47 print "Converting $FILE_UNICODE into $FILE_DICTIONARY...\n";
48 my $xslt = XML::LibXSLT->new();
49 my $source = XML::LibXML->load_xml(location => $FILE_UNICODE);
50 my $style_doc = XML::LibXML->load_xml(location => $DICTIONARY_XSL,
51 no_cdata=>1);
52 my $stylesheet = $xslt->parse_stylesheet($style_doc);
53 my $results = $stylesheet->transform($source);
54 open($file, ">$FILE_DICTIONARY") || die ("Couldn't open $FILE_DICTIONARY!");
55 print $file $stylesheet->output_as_bytes($results);
56 close($file);
57 exit 0;
58 }
59
60 if ($ARGV[0] eq "clean") {
61 unlink($FILE_UNICODE,
62 $FILE_DICTIONARY,
63 $FILE_DIFFERENCES,
64 $FILE_NEW_DICTIONARY,
65 $FILE_SYNTAX_ERRORS);
66 exit 0;
67 }
68
69 if ($ARGV[0] eq "compare" && $#ARGV == 1) {
70 $WG_DICTIONARY = $ARGV[1];
71 }
72
73 ################################################################################
74 # structure of the dictionary used by this script:
75 # - key: same as in mathfont.properties
76 # - table:
77 # index | value
78 # 0 | description
79 # 1 | lspace
80 # 2 | rspace
81 # 3 | minsize
82 # 4 | largeop
83 # 5 | movablelimits
84 # 6 | stretchy
85 # 7 | separator
86 # 8 | accent
87 # 9 | fence
88 # 10 | symmetric
89 # 11 | priority
90 # 12 | linebreakstyle
91 # 13 | direction
92 # 14 | integral
93 # 15 | mirrorable
94
95 # 1) build %moz_hash from $MOZ_DICTIONARY
96
97 print "loading $MOZ_DICTIONARY...\n";
98 open($file, $MOZ_DICTIONARY) || die ("Couldn't open $MOZ_DICTIONARY!");
99
100 print "building dictionary...\n";
101 while (<$file>) {
102 next unless (m/^operator\.(.*)$/);
103 (m/^([\w|\.|\\]*)\s=\s(.*)\s#\s(.*)$/);
104
105 # 1.1) build the key
106 $key = $1;
107
108 # 1.2) build the array
109 $_ = $2;
110 @value = ();
111 $value[0] = $3;
112 if (m/^(.*)lspace:(\d)(.*)$/) { $value[1] = $2; } else { $value[1] = "5"; }
113 if (m/^(.*)rspace:(\d)(.*)$/) { $value[2] = $2; } else { $value[2] = "5"; }
114 if (m/^(.*)minsize:(\d)(.*)$/) { $value[3] = $2; } else { $value[3] = "1"; }
115 $value[4] = (m/^(.*)largeop(.*)$/);
116 $value[5] = (m/^(.*)movablelimits(.*)$/);
117 $value[6] = (m/^(.*)stretchy(.*)$/);
118 $value[7] = (m/^(.*)separator(.*)$/);
119 $value[8] = (m/^(.*)accent(.*)$/);
120 $value[9] = (m/^(.*)fence(.*)$/);
121 $value[10] = (m/^(.*)symmetric(.*)$/);
122 $value[11] = ""; # we don't store "priority" in our dictionary
123 $value[12] = ""; # we don't store "linebreakstyle" in our dictionary
124 if (m/^(.*)direction:([a-z]*)(.*)$/) { $value[13] = $2; }
125 else { $value[13] = ""; }
126 $value[14] = (m/^(.*)integral(.*)$/);
127 $value[15] = (m/^(.*)mirrorable(.*)$/);
128
129 # 1.3) save the key and value
130 $moz_hash{$key} = [ @value ];
131 }
132
133 close($file);
134
135 ################################################################################
136 # 2) If mode "make-js", generate tests/stretchy-and-large-operators.js and quit.
137 # If mode "check", verify validity of our operator dictionary and quit.
138 # If mode "compare", go to step 3)
139
140 if ($ARGV[0] eq "make-js") {
141 print "generating file $FILE_JS...\n";
142 open($file_js, ">$FILE_JS") ||
143 die ("Couldn't open $FILE_JS!");
144 print $file_js "// This file is automatically generated. Do not edit.\n";
145 print $file_js "var stretchy_and_large_operators = [";
146 @moz_keys = (keys %moz_hash);
147 while ($key = pop(@moz_keys)) {
148 @moz = @{ $moz_hash{$key} };
149
150 $_ = $key;
151 (m/^operator\.([\w|\.|\\]*)\.(prefix|infix|postfix)$/);
152 $opname = "\\$1.$2: ";
153
154 if (@moz[4]) {
155 print $file_js "['$opname', '$1','l','$2'],";
156 }
157
158 if (@moz[6]) {
159 $_ = substr(@moz[13], 0, 1);
160 print $file_js "['$opname', '$1','$_','$2'],";
161 }
162 }
163 print $file_js "];\n";
164 close($file_js);
165 exit 0;
166 }
167
168 if ($ARGV[0] eq "check") {
169 print "checking operator dictionary...\n";
170 open($file_syntax_errors, ">$FILE_SYNTAX_ERRORS") ||
171 die ("Couldn't open $FILE_SYNTAX_ERRORS!");
172
173 $nb_errors = 0;
174 $nb_warnings = 0;
175 @moz_keys = (keys %moz_hash);
176 # check the validity of our private data
177 while ($key = pop(@moz_keys)) {
178 @moz = @{ $moz_hash{$key} };
179 $entry = &generateEntry($key, @moz);
180 $valid = 1;
181
182 if (!(@moz[13] eq "" ||
183 @moz[13] eq "horizontal" ||
184 @moz[13] eq "vertical")) {
185 $valid = 0;
186 $nb_errors++;
187 print $file_syntax_errors "error: invalid direction \"$moz[13]\"\n";
188 }
189
190 if (!@moz[4] && @moz[14]) {
191 $valid = 0;
192 $nb_warnings++;
193 print $file_syntax_errors "warning: operator is integral but not largeop\n";
194 }
195
196 $_ = @moz[0];
197 if ((m/^(.*)[iI]ntegral(.*)$/) && !@moz[14]) {
198 $valid = 0;
199 $nb_warnings++;
200 print $file_syntax_errors "warning: operator contains the term \"integral\" in its comment, but is not integral\n";
201 }
202
203 if (!$valid) {
204 print $file_syntax_errors $entry;
205 print $file_syntax_errors "\n";
206 }
207 }
208
209 # check that all forms have the same direction.
210 @moz_keys = (keys %moz_hash);
211 while ($key = pop(@moz_keys)) {
212
213 if (@{ $moz_hash{$key} }) {
214 # the operator has not been removed from the hash table yet.
215
216 $_ = $key;
217 (m/^([\w|\.|\\]*)\.(prefix|infix|postfix)$/);
218 $key_prefix = "$1.prefix";
219 $key_infix = "$1.infix";
220 $key_postfix = "$1.postfix";
221 @moz_prefix = @{ $moz_hash{$key_prefix} };
222 @moz_infix = @{ $moz_hash{$key_infix} };
223 @moz_postfix = @{ $moz_hash{$key_postfix} };
224
225 $same_direction = 1;
226
227 if (@moz_prefix) {
228 if (@moz_infix &&
229 !($moz_infix[13] eq $moz_prefix[13])) {
230 $same_direction = 0;
231 }
232 if (@moz_postfix &&
233 !($moz_postfix[13] eq $moz_prefix[13])) {
234 $same_direction = 0;
235 }
236 }
237 if (@moz_infix) {
238 if (@moz_postfix &&
239 !($moz_postfix[13] eq $moz_infix[13])) {
240 $same_direction = 0;
241 }
242 }
243
244 if (!$same_direction) {
245 $nb_errors++;
246 print $file_syntax_errors
247 "error: operator has a stretchy form, but all forms";
248 print $file_syntax_errors
249 " have not the same direction\n";
250 if (@moz_prefix) {
251 $_ = &generateEntry($key_prefix, @moz_prefix);
252 print $file_syntax_errors $_;
253 }
254 if (@moz_infix) {
255 $_ = &generateEntry($key_infix, @moz_infix);
256 print $file_syntax_errors $_;
257 }
258 if (@moz_postfix) {
259 $_ = &generateEntry($key_postfix, @moz_postfix);
260 print $file_syntax_errors $_;
261 }
262 print $file_syntax_errors "\n";
263 }
264
265 if (@moz_prefix) {
266 delete $moz_hash{$key.prefix};
267 }
268 if (@moz_infix) {
269 delete $moz_hash{$key_infix};
270 }
271 if (@moz_postfix) {
272 delete $moz_hash{$key_postfix};
273 }
274 }
275 }
276
277 close($file_syntax_errors);
278 print "\n";
279 if ($nb_errors > 0 || $nb_warnings > 0) {
280 print "$nb_errors error(s) found\n";
281 print "$nb_warnings warning(s) found\n";
282 print "See output file $FILE_SYNTAX_ERRORS.\n\n";
283 } else {
284 print "No error found.\n\n";
285 }
286
287 exit 0;
288 }
289
290 ################################################################################
291 # 3) build %wg_hash and @wg_keys from the page $WG_DICTIONARY
292
293 print "loading $WG_DICTIONARY...\n";
294 my $parser = XML::LibXML->new();
295 my $doc = $parser->parse_file($WG_DICTIONARY);
296
297 print "building dictionary...\n";
298 @wg_keys = ();
299
300 foreach my $entry ($doc->findnodes('/root/entry')) {
301 # 3.1) build the key
302 $key = "operator.";
303
304 $_ = $entry->getAttribute("unicode");
305 $_ = "$_-";
306 while (m/^U?0(\w*)-(.*)$/) {
307 # Concatenate .\uNNNN
308 $key = "$key\\u$1";
309 $_ = $2;
310 }
311
312 $_ = $entry->getAttribute("form"); # "Form"
313 $key = "$key.$_";
314
315 # 3.2) build the array
316 @value = ();
317 $value[0] = lc($entry->getAttribute("description"));
318 $value[1] = $entry->getAttribute("lspace");
319 if ($value[1] eq "") { $value[1] = "5"; }
320 $value[2] = $entry->getAttribute("rspace");
321 if ($value[2] eq "") { $value[2] = "5"; }
322 $value[3] = $entry->getAttribute("minsize");
323 if ($value[3] eq "") { $value[3] = "1"; }
324
325 $_ = $entry->getAttribute("properties");
326 $value[4] = (m/^(.*)largeop(.*)$/);
327 $value[5] = (m/^(.*)movablelimits(.*)$/);
328 $value[6] = (m/^(.*)stretchy(.*)$/);
329 $value[7] = (m/^(.*)separator(.*)$/);
330 $value[8] = (m/^(.*)accent(.*)$/);
331 $value[9] = (m/^(.*)fence(.*)$/);
332 $value[10] = (m/^(.*)symmetric(.*)$/);
333 $value[15] = (m/^(.*)mirrorable(.*)$/);
334 $value[11] = $entry->getAttribute("priority");
335 $value[12] = $entry->getAttribute("linebreakstyle");
336
337 # not stored in the WG dictionary
338 $value[13] = ""; # direction
339 $value[14] = ""; # integral
340
341 # 3.3) save the key and value
342 push(@wg_keys, $key);
343 $wg_hash{$key} = [ @value ];
344 }
345 @wg_keys = reverse(@wg_keys);
346
347 ################################################################################
348 # 4) Compare the two dictionaries and output the result
349
350 print "comparing dictionaries...\n";
351 open($file_differences, ">$FILE_DIFFERENCES") ||
352 die ("Couldn't open $FILE_DIFFERENCES!");
353 open($file_new_dictionary, ">$FILE_NEW_DICTIONARY") ||
354 die ("Couldn't open $FILE_NEW_DICTIONARY!");
355
356 $conflicting = 0; $conflicting_stretching = 0;
357 $new = 0; $new_stretching = 0;
358 $obsolete = 0; $obsolete_stretching = 0;
359 $unchanged = 0;
360
361 # 4.1) look to the entries of the WG dictionary
362 while ($key = pop(@wg_keys)) {
363
364 @wg = @{ $wg_hash{$key} };
365 delete $wg_hash{$key};
366 $wg_value = &generateCommon(@wg);
367
368 if (exists($moz_hash{$key})) {
369 # entry is in both dictionary
370 @moz = @{ $moz_hash{$key} };
371 delete $moz_hash{$key};
372 $moz_value = &generateCommon(@moz);
373 if ($moz_value ne $wg_value) {
374 # conflicting entry
375 print $file_differences "[conflict]";
376 $conflicting++;
377 if ($moz[6] != $wg[6]) {
378 print $file_differences "[stretching]";
379 $conflicting_stretching++;
380 }
381 print $file_differences " - $key ($wg[0])\n";
382 print $file_differences "-$moz_value\n+$wg_value\n\n";
383 $_ = &completeCommon($wg_value, $key, @moz, @wg);
384 print $file_new_dictionary $_;
385 } else {
386 # unchanged entry
387 $unchanged++;
388 $_ = &completeCommon($wg_value, $key, @moz, @wg);
389 print $file_new_dictionary $_;
390 }
391 } else {
392 # we don't have this entry in our dictionary yet
393 print $file_differences "[new entry]";
394 $new++;
395 if ($wg[6]) {
396 print $file_differences "[stretching]";
397 $new_stretching++;
398 }
399 print $file_differences " - $key ($wg[0])\n";
400 print $file_differences "-\n+$wg_value\n\n";
401 $_ = &completeCommon($wg_value, $key, (), @wg);
402 print $file_new_dictionary $_;
403 }
404 }
405
406 print $file_new_dictionary
407 "\n# Entries below are not part of the official MathML dictionary\n\n";
408 # 4.2) look in our dictionary the remaining entries
409 @moz_keys = (keys %moz_hash);
410 @moz_keys = reverse(sort(@moz_keys));
411
412 while ($key = pop(@moz_keys)) {
413 @moz = @{ $moz_hash{$key} };
414 $moz_value = &generateCommon(@moz);
415 print $file_differences "[obsolete entry]";
416 $obsolete++;
417 if ($moz[6]) {
418 print $file_differences "[stretching]";
419 $obsolete_stretching++;
420 }
421 print $file_differences " - $key ($moz[0])\n";
422 print $file_differences "-$moz_value\n+\n\n";
423 $_ = &completeCommon($moz_value, $key, (), @moz);
424 print $file_new_dictionary $_;
425 }
426
427 close($file_differences);
428 close($file_new_dictionary);
429
430 print "\n";
431 print "- $obsolete obsolete entries ";
432 print "($obsolete_stretching of them are related to stretching)\n";
433 print "- $unchanged unchanged entries\n";
434 print "- $conflicting conflicting entries ";
435 print "($conflicting_stretching of them are related to stretching)\n";
436 print "- $new new entries ";
437 print "($new_stretching of them are related to stretching)\n";
438 print "\nSee output files $FILE_DIFFERENCES and $FILE_NEW_DICTIONARY.\n\n";
439 print "After having modified the dictionary, please run";
440 print "./updateOperatorDictionary check\n\n";
441 exit 0;
442
443 ################################################################################
444 sub usage {
445 # display the accepted command syntax and quit
446 print "usage:\n";
447 print " ./updateOperatorDictionary.pl download [unicode.xml]\n";
448 print " ./updateOperatorDictionary.pl compare [dictionary.xml]\n";
449 print " ./updateOperatorDictionary.pl check\n";
450 print " ./updateOperatorDictionary.pl make-js\n";
451 print " ./updateOperatorDictionary.pl clean\n";
452 exit 0;
453 }
454
455 sub generateCommon {
456 # helper function to generate the string of data shared by both dictionaries
457 my(@v) = @_;
458 $entry = "lspace:$v[1] rspace:$v[2]";
459 if ($v[3] ne "1") { $entry = "$entry minsize:$v[3]"; }
460 if ($v[4]) { $entry = "$entry largeop"; }
461 if ($v[5]) { $entry = "$entry movablelimits"; }
462 if ($v[6]) { $entry = "$entry stretchy"; }
463 if ($v[7]) { $entry = "$entry separator"; }
464 if ($v[8]) { $entry = "$entry accent"; }
465 if ($v[9]) { $entry = "$entry fence"; }
466 if ($v[10]) { $entry = "$entry symmetric"; }
467 if ($v[15]) { $entry = "$entry mirrorable"; }
468 return $entry;
469 }
470
471 sub completeCommon {
472 # helper to add key and private data to generateCommon
473 my($entry, $key, @v_moz, @v_wg) = @_;
474
475 $entry = "$key = $entry";
476
477 if ($v_moz[13]) { $entry = "$entry direction:$v_moz[13]"; }
478 if ($v_moz[14]) { $entry = "$entry integral"; }
479 if ($v_moz[15]) { $entry = "$entry mirrorable"; }
480
481 if ($v_moz[0]) {
482 # keep our previous comment
483 $entry = "$entry # $v_moz[0]";
484 } else {
485 # otherwise use the description given by the WG
486 $entry = "$entry # $v_wg[0]";
487 }
488
489 $entry = "$entry\n";
490 return $entry;
491 }
492
493 sub generateEntry {
494 # helper function to generate an entry of our operator dictionary
495 my($key, @moz) = @_;
496 $entry = &generateCommon(@moz);
497 $entry = &completeCommon($entry, $key, @moz, @moz);
498 return $entry;
499 }

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial