The Tor Browser: intl/uconv/tools/gen-big5hkscs-2001-mozilla.pl@97036ab72558 (annotated)

intl/uconv/tools/gen-big5hkscs-2001-mozilla.pl@97036ab72558 (annotated)

intl/uconv/tools/gen-big5hkscs-2001-mozilla.pl

Tue, 06 Jan 2015 21:39:09 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Tue, 06 Jan 2015 21:39:09 +0100
branch: TOR_BUG_9701
changeset 8: 97036ab72558
permissions: -rw-r--r--

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

 #!/usr/bin/perl -w
 #
 #  gen-big5hkscs-2001-mozilla.pl
 #      a Perl script that generates Big5-HKSCS <-> Unicode
 #      conversion tables for Mozilla
 #
 #  Author (of the original Perl script):
 #      Anthony Fok <anthony@thizlinux.com> <foka@debian.org>
 #  Copyright (C) 2001, 2002 ThizLinux Laboratory Ltd.
 #  License: GNU General Public License, v2 or later.
 #
 #  This version includes original C source code from
 #  glibc-2.2.5/iconvdata/big5hkscs.c by Ulrich Drepper <drepper@redhat.com>
 #  Roger So <roger.so@sw-linux.com>
 #
 #                         First attempt for Qt-2.3.x: 2001-09-21
 #                     A working version for Qt-2.3.x: 2001-10-30
 #              Ported to glibc-2.2.5 with HKSCS-2001: 2002-03-21
 #  Adapted to generate conversion tables for Mozilla: 2002-11-26
 #  Adapted to generate conversion tables for Mozilla: 2002-11-30
 #                     Cleaned up the script somewhat: 2002-12-04
 # Minor revisions for submitting to Mozilla Bugzilla: 2002-12-10
 #
 #  Notes:
 #
 #   1. The latest version of this script may be found in:
 #          http://www.thizlinux.com/~anthony/hkscs/gen-glibc-big5hkscs.pl
 #          http://people.debian.org/~foka/hkscs/gen-glibc-big5hkscs.pl
 #      Or, better yet, e-mail me and ask for the latest version.
 #
 #   2. This script generates data from 3 tables:
 #       a. http://www.microsoft.com/typography/unicode/950.txt
 #       b. http://www.info.gov.hk/digital21/chi/hkscs/download/big5-iso.txt
 #       c. http://www.info.gov.hk/digital21/chi/hkscs/download/big5cmp.txt
 #
 #      Make sure your big5-iso.txt is the latest HKSCS-2001 version.
 #
 #   3. [glibc]: I have currently split the ucs_to_big5_hkscs_?[] tables into
 #       different areas similar to the way Ulrich and Roger did it,
 #       but extended for HKSCS-2001.
 #
 #   4. [Mozilla]: This script is very quick-and-dirty in some places.
 #       Call either gen_mozilla_uf() or gen_mozilla_ut() to generate
 #       the appropriate tables for feeding into "fromu" or "tou".
 #
 #   5. [CharMapML]: The comments regarding TW-BIG5 herein need to be organized.
 #       Also, please make sure "$hkscs_mode = 0;" for TW-BIG5 mode.
 #       Otherwise, this script would generate a HKSCS table.
 #       (Yes, I know, I should clean up this script and make it more modular,
 #       and with command-line options or whatnot.  I'll do that later.  :-)
 #
 #  If you have any questions or concerns, please feel free to contact me
 #  at Anthony Fok <anthony@thizlinux.com> or <foka@debian.org>  :-)
 #
 #  Last but not least, special thanks to ThizLinux Laboratory Ltd. (HK)
 #  for their generous support in this work.
 #
 # 1. UDA3, 0x8840 - 0x8dfe
 # 2. UDA2, 0x8e40 - 0xa0fe
 # 3. VDA,  0xc6a1 - 0xc8fe
 #use Getopt::Std;
 my ( %b2u, %u2b, $unicode, $big5, $high, $low, $i, $count );
 my $debug = 0;
 my $hkscs_mode = 1;
 my $kangxi = 0;
 my $use_range  = 0;
 my $bmp_only  = 1;
 #
 # Subroutine Declaration
 #
 sub read_cp950();
 sub adjust_radicals();
 sub read_hkscs_main();
 sub read_hkscs_cmp();
 sub post_tuning();
 sub gen_charmapml();
 sub gen_check_b2u();
 sub gen_check_u2b();
 sub gen_mozilla_uf();
 sub gen_mozilla_ut();
 sub gen_glibc();
 ###########################################################################
 #
 # Main program
 #
 # First, read Microsoft's CP950 as base Big5.
 read_cp950 ();
 # Add mappings to Kangxi Radicals.
 # The b2u direction is added only if $kangxi is not null.
 adjust_radicals ();
 # Then, read the HKSCS table.
 # Again, see the $hkscs_mode variable.
 read_hkscs_main ();
 read_hkscs_cmp () if $hkscs_mode;
 post_tuning ();
 # Then, choose one of the following:
 #gen_charmapml();
 gen_mozilla_uf();
 #gen_mozilla_ut();
 #gen_check_u2b();
 #gen_glibc();
 # End of program
 exit 0;
 #############################################################################
 #
 #  Subroutines
 #
 sub read_cp950() {
     open( CP950, "950.txt" ) or die;
     my $mode = 0;
     while (<CP950>) {
         s/\r//;
         chomp;
         next if /^$/;
         last if /^ENDCODEPAGE/;
         if (/^DBCSTABLE (\d+)\s+;LeadByte = 0x([0-9a-f]{2})/) {
             $mode = 1;
             ( $count, $high ) = ( $1, $2 );
             $i = 0;
             next;
         }
         if (/^WCTABLE (\d+)/) {
             $mode  = 2;
             $count = $1;
             $i     = 0;
             next;
         }
         next if $mode == 0;
         if ( $mode == 1 ) {
             ( $low, $unicode, $comment ) = split "\t";
             $low     =~ s/^0x//;
             $unicode =~ s/^0x//;
             $big5 = $high . $low;
             $b2u{ uc($big5) } = uc($unicode);
             if ( ++$i == $count ) { $mode = 0; $count = 0; next; }
         }
         if ( $mode == 2 ) {
             ( $unicode, $big5, $comment ) = split "\t";
             $unicode =~ s/^0x//;
             $big5    =~ s/^0x//;
             my $u = hex($unicode);
             my $b = hex($big5);
             $u2b{ uc($unicode) } = uc($big5) unless
               # Skip Microsoft's over-generous (or over-zealous?) mappings
               # "Faked" accented latin characters
               ( $b <= 0xFF and $b != $u )
               # "Faked" Ideographic Annotation ___ Mark
               or ( $u >= 0x3192 and $u <= 0x319F )
               # "Faked" Parenthesized Ideograph ___
               or ( $u >= 0x3220 and $u <= 0x3243 )
               # "Faked" Circled Ideograph ___ except Circled Ideograph Correct
               or ( $u >= 0x3280 and $u <= 0x32B0 and $u != 0x32A3 )
               # �F�G�D���g�M
               or ( $u == 0xA2
                 or $u == 0xA3
                 or $u == 0xA5
                 or $u == 0xB4
                 or $u == 0xB5
                 or $u == 0xB8 )
               # �¢w���E�����âB�X�ݡ[�����
               or ( $u == 0x0305		# ???
                 or $u == 0x2015
                 or $u == 0x2016
                 or $u == 0x2022
                 or $u == 0x2024
                 or $u == 0x2033
                 or $u == 0x203E		# ???
                 or $u == 0x2216
                 or $u == 0x2218
                 or $u == 0x2263
                 or $u == 0x2307
                 or $u == 0x2609
                 or $u == 0x2641
                 or $u == 0x301C
                 or $u == 0x3030 )
               # �s���N
               or ( $u == 0xFF3E or $u == 0xFF40 or $u == 0xFF64 );
             if ( ++$i == $count ) { $mode = 0; $count = 0; next; }
         }
     }
 }
 sub adjust_radicals() {
     # B5+C6BF - B5+C6D7: Radicals (?)
     # TW-BIG5 drafted by Autrijus uses Kangxi Radicals whenever possible.
     #
     #   Big5-HKSCS tends towards using the character in Unicode CJK Ideographs
     #   Note that HKSCS does not explicitly define
     #       B5+C6CF, B5+C6D3, B5+C6D5, B5+C6D7 (�ϡB�ӡB�աB��),
     #   but do have these characters at B5+FBFD, B5+FCD3, B5+FEC1, B5+90C4,
     #   mapped to U+5EF4, U+65E0, U+7676, U+96B6 respectively.
     #
     #   As for B5+C6CD (��), HKSCS maps it to U+2F33 just like TW-BIG5.
     #   However, it also maps B5+FBF4 (��) to U+5E7A.
     $b2u{"C6BF"} = "2F02" if $kangxi;
     $u2b{"2F02"} = "C6BF";              # ƿ
     $b2u{"C6C0"} = "2F03" if $kangxi;
     $u2b{"2F03"} = "C6C0";              # ��
     $b2u{"C6C1"} = "2F05" if $kangxi;
     $u2b{"2F05"} = "C6C1";              # ��
     $b2u{"C6C2"} = "2F07" if $kangxi;
     $u2b{"2F07"} = "C6C2";              # ��
     $b2u{"C6C3"} = "2F0C" if $kangxi;
     $u2b{"2F0C"} = "C6C3";              # ��
     $b2u{"C6C4"} = "2F0D" if $kangxi;
     $u2b{"2F0D"} = "C6C4";              # ��
     $b2u{"C6C5"} = "2F0E" if $kangxi;
     $u2b{"2F0E"} = "C6C5";              # ��
     $b2u{"C6C6"} = "2F13" if $kangxi;
     $u2b{"2F13"} = "C6C6";              # ��
     $b2u{"C6C7"} = "2F16" if $kangxi;
     $u2b{"2F16"} = "C6C7";              # ��
     $b2u{"C6C8"} = "2F19" if $kangxi;
     $u2b{"2F19"} = "C6C8";              # ��
     $b2u{"C6C9"} = "2F1B" if $kangxi;
     $u2b{"2F1B"} = "C6C9";              # ��
     $b2u{"C6CA"} = "2F22" if $kangxi;
     $u2b{"2F22"} = "C6CA";              # ��
     $b2u{"C6CB"} = "2F27" if $kangxi;
     $u2b{"2F27"} = "C6CB";              # ��
     $b2u{"C6CC"} = "2F2E" if $kangxi;
     $u2b{"2F2E"} = "C6CC";              # ��
     $b2u{"C6CD"} = "2F33" if $kangxi;
     $u2b{"2F33"} = "C6CD";              # ��
     $b2u{"C6CE"} = "2F34" if $kangxi;
     $u2b{"2F34"} = "C6CE";              # ��
     $b2u{"C6CF"} = "2F35" if $kangxi;
     $u2b{"2F35"} = "C6CF";              # ��
     $b2u{"C6D0"} = "2F39" if $kangxi;
     $u2b{"2F39"} = "C6D0";              # ��
     $b2u{"C6D1"} = "2F3A" if $kangxi;
     $u2b{"2F3A"} = "C6D1";              # ��
     $b2u{"C6D2"} = "2F41" if $kangxi;
     $u2b{"2F41"} = "C6D2";              # ��
     $b2u{"C6D3"} = "2F46" if $kangxi;
     $u2b{"2F46"} = "C6D3";              # ��
     $b2u{"C6D4"} = "2F67" if $kangxi;
     $u2b{"2F67"} = "C6D4";              # ��
     $b2u{"C6D5"} = "2F68" if $kangxi;
     $u2b{"2F68"} = "C6D5";              # ��
     $b2u{"C6D6"} = "2FA1" if $kangxi;
     $u2b{"2FA1"} = "C6D6";              # ��
     $b2u{"C6D7"} = "2FAA" if $kangxi;
     $u2b{"2FAA"} = "C6D7";              # ��
 }
 sub read_hkscs_main() {
     open( B2U, "<big5-iso.txt" ) or die;
     while (<B2U>) {
         next
           unless
 /([[:xdigit:]]{4})\s+([[:xdigit:]]{4})\s+([[:xdigit:]]{4})\s+([[:xdigit:]]{4,5})/;
         ( $big5, $iso1993, $iso2000, $iso2001 ) = ( $1, $2, $3, $4 );
         my $b = hex($big5);
         # For non-HKSCS mode, only take data in the VDA range (?)
         next unless $hkscs_mode
           # Note that we don't go from B5+C6A1-B5+C6FE, but rather only
           # C6A1-C8D3 excluding C6BF-C6D7 (Kangxi Radicals)
           # because C8D4-C8FE are not assigned in TW-BIG5
           # if we are to follow Arphic PL Big-5 fonts.  (To be discussed)
           or
           ( $b >= 0xC6A1 && $b <= 0xC8D3 and !( $b >= 0xC6BF && $b <= 0xC6D7 ) )
           or ( $b >= 0xF9D6 && $b <= 0xF9FE );
         print STDERR
           "B2U, 2000: $big5 redefined from U+$b2u{$big5} to U+$iso2000.\n"
           if $debug
           and defined( $b2u{$big5} )
           and $b2u{$big5} ne $iso2000;
         $b2u{$big5} = $bmp_only ? $iso2000 : $iso2001
           unless !$hkscs_mode
           and $b == 0xF9FE;
         # B5+F9FE is mapped differently in TW-BIG5 and HKSCS, to
         # U+2593 (Dark Shade) and U+FFED (Halfwidth Black Square) respectively.
         # Which is more correct?  I don't know!  (To be discussed)
         print STDERR
           "1993: U+$iso1993 redefined from $u2b{$iso1993} to $big5.\n"
           if $debug
           and defined( $u2b{$iso1993} )
           and $u2b{$iso1993} ne $big5;
         $u2b{$iso1993} = $big5;
         print STDERR
           "2000: U+$iso2000 redefined from $u2b{$iso2000} to $big5.\n"
           if $debug
           and defined( $u2b{$iso2000} )
           and $u2b{$iso2000} ne $big5;
         $u2b{$iso2000} = $big5;
         print STDERR
           "2001: U+$iso2001 redefined from $u2b{$iso2001} to $big5.\n"
           if $debug
           and defined( $u2b{$iso2001} )
           and $u2b{$iso2001} ne $big5;
         $u2b{$iso2001} = $big5;
     }
     close B2U;
 }    # read_hkscs_main()
 sub read_hkscs_cmp() {
     ###########################################################################
     # Add Big5 compatibility coding...
     #
     # Stephan, here is the code segment that you may want to implement
     # in your convertbig5hkscs2001.pl
     #
     open( B5CMP, "<big5cmp.txt" ) or die;
     $mode = 0;
     while (<B5CMP>) {
         if (/^=====/) { $mode = 1; next; }
         next if $mode == 0;
         last if $mode == 1 and /^\s+/;
         chomp;
         my ( $big5cmp, $big5 ) = split " ";
         $big5cmp = uc($big5cmp);
         $big5    = uc($big5);
         my $uni    = $b2u{$big5};
         my $unicmp = $b2u{$big5cmp};
         print STDERR
           "Was: U+$unicmp -> $u2b{$unicmp}, $big5cmp -> U+$b2u{$big5cmp}\t"
           if $debug;
         $b2u{$big5cmp} = $uni;
         $u2b{$unicmp}  = $big5;
         print STDERR
           "Now:  U+$unicmp -> $u2b{$unicmp}, $big5cmp -> U+$b2u{$big5cmp}\n"
           if $debug;
     }
     close B5CMP;
 }    # read_hkscs_cmp();
 sub post_tuning() {
     # And finally, fine-tuning...
     for $i ( 0x00 .. 0x80 ) {
         $big5 = $unicode = sprintf( "%04X", $i );
         $b2u{$big5} = $unicode;
     }
     # Add Euro '��' (I wonder why this 950.txt doesn't have it.)
     $b2u{"A3E1"} = "20AC";
     $u2b{"20AC"} = "A3E1";
     # Box drawing characters:
     # Align with Big-5E (To be discussed, as it differs from CP950 and HKSCS)
     # (To be discussed)
     if ( !$hkscs_mode ) {
         $u2b{"2550"} = "A2A4";    # Big5: ��	(also B5-F9F9)
         $u2b{"255E"} = "A2A5";    # Big5: ��	(also B5-F9E9)
         $u2b{"2561"} = "A2A7";    # Big5: ��	(also B5-F9EB)
         $u2b{"256A"} = "A2A6";    # Big5: ��	(also B5-F9EA)
         $u2b{"256D"} = "A27E";    # Big5: �~	(also B5-F9FA)
         $u2b{"256E"} = "A2A1";    # Big5: ��	(also B5-F9FB)
         $u2b{"256F"} = "A2A3";    # Big5: ��	(also B5-F9FD)
         $u2b{"2570"} = "A2A2";    # Big5: ��	(also B5-F9FC)
     }
     # "Hangzhou" or "Suzhou" Chinese numerals 10, 20, 30 (�̢͢�)
     # (To be discussed)
     if ( !$hkscs_mode ) {
         $b2u{"A2CC"} = "3038";
         $u2b{"3038"} = "A2CC";
         $b2u{"A2CD"} = "3039";
         $u2b{"3039"} = "A2CD";
         $b2u{"A2CE"} = "303A";
         $u2b{"303A"} = "A2CE";
     }
     # The character for ethnic group "Yi" (�U):
     # (To be discussed)
     $u2b{"5F5E"} = "C255";    # Always add this.
     if ( !$hkscs_mode ) {
         $b2u{"C255"} = "5F5E";
     }
 }    # post_tuning()
 sub gen_charmapml() {
     ###########################################################################
     #
     #  Codes for generating CharMapML XML file
     print <<EOT;
 <?xml version="1.0" encoding="UTF-8" ?>
 <!DOCTYPE characterMapping SYSTEM "http://www.unicode.org/unicode/reports/tr22/CharacterMapping.dtd">
 EOT
     if ($hkscs_mode) {
         print <<EOT;
 <characterMapping id="big5-hkscs-2001" version="1">
  <history>
   <modified version="1" date="2002-11-30">
    Trial version generated from 950.txt + part of big5-iso.txt (HKSCS-2001)
    with Euro added, with CP950's excessive fub (fallbacks uni->big5) removed,
    and with some other manual tweaking.
   </modified>
  </history>
 EOT
     }
     else {
         print <<EOT;
 <characterMapping id="tw-big5-2002" version="1">
  <history>
   <modified version="1" date="2002-11-30">
    Trial version generated from 950.txt + part of big5-iso.txt (HKSCS-2001)
    with Euro added, with CP950's excessive fub (fallbacks uni->big5) removed,
    and with some other manual tweaking.
   </modified>
  </history>
 EOT
     }
     print <<EOT;
  <validity>
   <state type="FIRST" next="VALID" s="0" e="80" max="FFFF"/>
   <state type="FIRST" next="SECOND" s="81" e="FE" max="FFFF"/>
   <state type="SECOND" next="VALID" s="40" e="7E" max="FFFF"/>
   <state type="SECOND" next="VALID" s="A1" e="FE" max="FFFF"/>
  </validity>
  <assignments sub="3F">
 EOT
     print "  <!-- One to one mappings -->\n";
     for $unicode ( sort { hex($a) <=> hex($b) } keys %u2b ) {
         $big5 = $u2b{$unicode};
         $u    = hex($unicode);
         next
           unless defined( $b2u{$big5} )
           and $unicode eq $b2u{$big5}
           and
           not( $use_range and !$hkscs_mode and $u >= 0xE000 && $u <= 0xF6B0 );
         printf "  <a u=\"%04X\" ", $u;
         if ( hex($big5) <= 0xFF ) {
             printf "b=\"%02X\"/>\n", hex($big5);
         }
         else {
             printf "b=\"%s %s\"/>\n", substr( $big5, 0, 2 ),
               substr( $big5, 2, 2 );
         }
     }
     print "  <!-- Fallback mappings from Unicode to bytes -->\n";
     for $unicode ( sort { hex($a) <=> hex($b) } keys %u2b ) {
         $big5 = $u2b{$unicode};
         next if defined( $b2u{$big5} ) and hex($unicode) == hex( $b2u{$big5} );
         if ( $unicode eq "F900" ) {
             print "  <!-- CJK Compatibility Ideographs: U+F900 - U+FA6A.\n";
             print
 "       These are included in CP950 (Unicode->Big5 direction only).\n";
             print "       Should we include this area in TW-BIG5 or not? -->\n";
         }
         printf "  <fub u=\"%04X\" b=\"%s %s\"/>\n", hex($unicode),
           substr( $big5, 0, 2 ), substr( $big5, 2, 2 );
     }
     my %fbu;
     print "  <!-- Fallback mappings from bytes to Unicode -->\n";
     for $big5 ( sort { hex($a) <=> hex($b) } keys %b2u ) {
         $unicode = $b2u{$big5};
         if ( !defined( $u2b{$unicode} ) or hex($big5) != hex( $u2b{$unicode} ) )
         {
             $fbu{$unicode} = $big5;
         }
     }
     for $unicode ( sort { hex($a) <=> hex($b) } keys %fbu ) {
         $big5 = $fbu{$unicode};
         printf "  <fbu u=\"%04X\" b=\"%s %s\"/>\n", hex($unicode),
           substr( $big5, 0, 2 ), substr( $big5, 2, 2 );
     }
     if ( $use_range and !$hkscs_mode ) {
         print <<EOT;
   <!-- Roundtrip-mappings that can be enumerated
        Note: We can only use the <range> tag for TW-BIG5.
              Big-5E and Big5-HKSCS have assigned characters in these areas,
 	     and we will have to use the <a> and <fub> tags instead.
     -->
   <!-- User-Defined Area 1 (UDA1) -->
   <range uFirst="E000" uLast="E310"  bFirst="FA 40" bLast="FE FE" bMin="81 40" bMax="FE FE"/>
   <!-- User-Defined Area 2 (UDA2) -->
   <range uFirst="E311" uLast="EEB7"  bFirst="8E 40" bLast="A0 FE" bMin="81 40" bMax="FE FE"/>
   <!-- User-Defined Area 3 (UDA3) -->
   <range uFirst="EEB8" uLast="F6B0"  bFirst="81 40" bLast="8D FE" bMin="81 40" bMax="FE FE"/>
 EOT
     }
     print <<EOT;
  </assignments>
 </characterMapping>
 EOT
 }    # gen_charmapml()
 sub gen_check_b2u() {
     ###########################################################################
     #
     #  Codes for generating a raw table for verification and testing
     #
     # #print $u2b{"F7D1"}, "\n";
     # print $b2u{$u2b{"F7D1"}}, "\n";
     # print "FA59 -> U+", $b2u{"FA59"}, "\n";
     foreach $big5 ( sort { hex($a) <=> hex($b) } keys %b2u ) {
         $unicode = $b2u{$big5};
         $big5 =~ s/^00//;
         print "U+", $unicode, ": ", $big5, "\n";
     }
 }
 sub gen_check_u2b() {
     foreach $unicode ( sort { hex($a) <=> hex($b) } keys %u2b ) {
         $big5 = $u2b{$unicode};
         $big5 =~ s/^00//;
         print "U+", $unicode, ": ", $big5, "\n";
     }
 }
 ###########################################################################
 #
 #  Codes for generating hkscs.ut and hkscs.uf files for Mozilla
 #
 sub gen_mozilla_uf() {
     # hkscs.uf
     foreach $unicode ( sort keys %u2b ) {
         $big5 = $u2b{$unicode};
 	my $b = hex($big5);
         print "0x", uc($big5), "\t0x", uc($unicode), "\n"
           unless ( $b >= 0xA140 and $b <= 0xC6A0 )
           or ( $b >= 0xC940 and $b <= 0xF9D5 )
           or ( $b < 0x8140 )
           or ( hex($unicode) > 0xFFFF );
     }
 }
 sub gen_mozilla_ut() {
     # hkscs.ut
     foreach $big5 ( sort keys %b2u ) {
         my $b = hex($big5);
         print "0x", uc($big5), "\t0x", uc( $b2u{$big5} ), "\n"
           unless ( $b >= 0xA140 and $b <= 0xC6A0 )
 	  or ( $b < 0x8140 )
           or ( $b >= 0xC940 and $b <= 0xF9D5 );
     }
 }
 ###########################################################################
 sub gen_glibc() {
     ##########################################################################
     #
     #   Generate index for UCS4 to Big5-HKSCS conversion table
     #
     @index_array = ();
     $mode  = 0;
     $count = 0;
     for ( $uni = 0x81 ; $uni <= 0x2FFFF ; $uni++ ) {
         $unicode = sprintf( "%04X", $uni );
         # print "  /* U+$unicode */\t" if $low % 4 == 0;
         if ( defined( $u2b{$unicode} ) ) {
             if ( $mode == 0 ) {
                 $range_start = $range_end = $uni;
                 # printf "  { %7s, ", sprintf("0x%04X", $range_start);
                 $mode = 1;
             }
             else {
                 $range_end = $uni;
             }
         }
         elsif ( $mode == 1 and ( $uni - $range_end ) >= 0x80 ) {
             # Start a new range if the gap is 0x80 or larger
             # printf "%7s, %5d },\n", sprintf("0x%04X", $range_end), $count;
             push @index_array, [ ( $range_start, $range_end, $count ) ];
             $count += $range_end - $range_start + 1;
             $mode = 0;
         }
     }
     #
     #  Note that $count and $range_end are used again as global variables
     #  below
     #
     ###########################################################################
     #
     #  Start generating real C code...
     #
     print <<'EOT';
 /* Mapping tables for Big5-HKSCS handling.
    Copyright (C) 1997, 1998, 2000, 2001, 2002 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
    Modified for Big5-HKSCS by Roger So <roger.so@sw-linux.com>, 2000.
    Updated for HKSCS-2001 by James Su <suzhe@turbolinux.com.cn>
                          and Anthony Fok <anthony@thizlinux.com>, 2002
    The GNU C Library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.
    The GNU C Library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.
    You should have received a copy of the GNU Lesser General Public
    License along with the GNU C Library; if not, write to the Free
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
 -1307 USA.  */
 #include <dlfcn.h>
 #include <gconv.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <wchar.h>
 /* Table for Big5-HKSCS to UCS conversion.
    Original comments by Roger So when he updated the tables for HKSCS-1999:
      With HKSCS mappings 0x8140-0xA0FE and 0xFA40-0xFEFE added; more info:
      http://www.digital21.gov.hk/eng/hkscs/index.html
        - spacehunt 07/01/2000
    The BIG5-HKSCS mapping tables are generated from 950.txt, big5-iso.txt
    and big5cmp.txt using a Perl script while merging C source code from
    other developers.  A copy of the source Perl script is available at:
       http://www.thizlinux.com/~anthony/hkscs/gen-glibc-big5hkscs.pl
       http://people.debian.org/~foka/hkscs/gen-glibc-big5hkscs.pl
   Revisions:
 -10-30  made codec for Qt
 -03-21  ported to glibc-2.2.5 and added HKSCS-2001
   Todo:
     Use a hash for characters beyond BMP to save space and make it
     more efficient
    - Anthony Fok <anthony@thizlinux.com>  21 Mar 2002
      On behalf of ThizLinux Laboratory Ltd., Hong Kong SAR, China
 */
 EOT
     ##########################################################################
     #
     # Generate Big5-HKSCS to Unicode conversion table
     #
     ## print "Big5HKSCS to Unicode\n";
     # for $high (0x81..0x8d, 0x8e..0xa0, 0xc6..0xc8, 0xf9, 0xfa..0xfe) {
     $high_start = 0x88;
     $high_end   = 0xfe;
     print "static const uint16_t big5_hkscs_to_ucs[";
     print( ( $high_end - $high_start + 1 ) * 157 );
     print "] =\n{\n";
     for $high ( 0x88 .. 0xfe ) {
         for $low ( 0x40 .. 0x7e, 0xa1 .. 0xfe ) {
             if ( $low == 0x40 ) {
                 print "\n" unless $high == $high_start;
                 printf
                   "\t/* Big5-HKSCS 0x%02X40..0x%02X7E, 0x%02XA1..0x%02XFE */\n",
                   $high, $high, $high, $high;
             }
             elsif ( $low == 0xa1 ) {
                 print "\t\t";
             }
             $big5 = sprintf( "%02X%02X", $high, $low );
             print "\t" if $low % 8 == 0;
             if ( defined( $b2u{$big5} ) ) {
                 $unicode = $b2u{$big5};
                 print "0x", $unicode, ",";
             }
             else {
                 print "0x0000,";    # for glibc
             }
             print( ( $low % 8 == 7 or $low == 0x7e or $low == 0xfe )
                 ? "\n"
                 : "\t" );
         }
     }
     print "};\n\n";
     ##########################################################################
     #
     # Generate Unicode to Big5-HKSCS conversion table
     #
     print "static const unsigned char ucs4_to_big5_hkscs[$count][2] =\n{\n";
     foreach $index (@index_array) {
         ( $start, $end ) = ( @$index[0], @$index[1] );
         printf( "  /* U+%04X */\t", $start ) if ( $start % 4 != 0 );
         print "\t" x ( ( $start % 4 ) * 1.5 ) . "    " x ( $start % 2 );
         for ( $i = $start ; $i <= $end ; $i++ ) {
             printf( "  /* U+%04X */\t", $i ) if ( $i % 4 == 0 );
             $unicode = sprintf( "%04X", $i );
             if ( defined( $big5 = $u2b{$unicode} ) ) {
                 if ( $big5 =~ /^00/ ) {
                     print '"\x', substr( $big5, 2, 2 ), '\x00",';
                 }
                 else {
                     print '"\x', substr( $big5, 0, 2 ), '\x',
                       substr( $big5, 2, 2 ), '",';
                 }
             }
             else {
                 print '"\x00\x00",';
             }
             print( ( $i % 4 == 3 ) ? "\n" : " " ) unless $i == $end;
         }
         print $end == $range_end ? "\n" : "\n\n";
     }
     print "};\n\n";
     ###########################################################################
     print <<EOT;
 static struct
 {
     /* Note: We are going to split this table so that we can use
        uint16_t for "from" and "to" again.  Anthony Fok, 2002-03-21 */
     uint32_t from;
     uint32_t to;
     uint32_t offset;
 } from_ucs4_idx[] =
 {
 EOT
     foreach $index (@index_array) {
         printf "    { %7s, %7s, %5d },\n", sprintf( "0x%04X", @$index[0] ),
           sprintf( "0x%04X", @$index[1] ), @$index[2];
     }
     print "};\n\n";
     #foreach $i (sort keys %b2u) {
     #    print $b2u{$i} . ' ';
     #}
     print <<'EOT';
 /* Definitions used in the body of the `gconv' function.  */
 #define CHARSET_NAME		"BIG5HKSCS//"
 #define FROM_LOOP		from_big5
 #define TO_LOOP			to_big5
 #define DEFINE_INIT		1
 #define DEFINE_FINI		1
 #define MIN_NEEDED_FROM		1
 #define MAX_NEEDED_FROM		2
 #define MIN_NEEDED_TO		4
 /* First define the conversion function from Big5-HKSCS to UCS4.  */
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MAX_NEEDED_INPUT	MAX_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
 #define LOOPFCT			FROM_LOOP
 #define BODY \
   {									      \
     uint32_t ch = *inptr;						      \
 									      \
     if (ch >= 0x81 && ch <= 0xfe)					      \
       {									      \
 	/* Two-byte character.  First test whether the next character	      \
 	   is also available.  */					      \
 	uint32_t ch2;							      \
 	int idx;							      \
 									      \
 	if (__builtin_expect (inptr + 1 >= inend, 0))			      \
 	  {								      \
 	    /* The second character is not available.  */		      \
 	    result = __GCONV_INCOMPLETE_INPUT;				      \
 	    break;							      \
 	  }								      \
 									      \
 	ch2 = inptr[1];							      \
 	/* See whether the second byte is in the correct range.  */	      \
 	if ((ch2 >= 0x40 && ch2 <= 0x7e) || (ch2 >= 0xa1 && ch2 <= 0xfe))     \
 	  {								      \
 	    if (ch >= 0x88)						      \
 	      {								      \
 		/* Look up the table */					      \
 		idx = (ch - 0x88) * 157 + ch2 - (ch2 <= 0x7e ? 0x40 : 0x62);  \
 		if ((ch = big5_hkscs_to_ucs[idx]) == 0)			      \
 		  {							      \
 		    /* This is illegal.  */				      \
 		    if (! ignore_errors_p ())				      \
 		      {							      \
 			result = __GCONV_ILLEGAL_INPUT;			      \
 			break;						      \
 		      }							      \
 									      \
 		    ++inptr;						      \
 		    ++*irreversible;					      \
 		    continue;						      \
 		  }							      \
 	      }								      \
 	    else							      \
 	      {								      \
 		/* 0x81..0x87 in UDA3, currently maps linearly to PUA */      \
 		ch = (ch - 0x81) * 157 + ch2 - (ch2 <= 0x7e ? 0x40 : 0x62)    \
 		      + 0xeeb8;						      \
 	      }								      \
 	  }								      \
 	else								      \
 	  {								      \
 	    /* This is illegal.  */					      \
 	    if (! ignore_errors_p ())					      \
 	      {								      \
 		result = __GCONV_ILLEGAL_INPUT;				      \
 		break;							      \
 	      }								      \
 									      \
 	    ++inptr;							      \
 	    ++*irreversible;						      \
 	    continue;							      \
 	  }								      \
 									      \
 	inptr += 2;							      \
       }									      \
     else if (__builtin_expect (ch, 0) == 0xff)				      \
       {									      \
 	result = __GCONV_ILLEGAL_INPUT;					      \
 	break;								      \
       }									      \
     else  /* 0x00 to 0x80 */						      \
       ++inptr;								      \
 									      \
     put32 (outptr, ch);							      \
     outptr += 4;							      \
   }
 #define LOOP_NEED_FLAGS
 #include <iconv/loop.c>
 /* Next, define the other direction.  */
 #define MIN_NEEDED_INPUT	MIN_NEEDED_TO
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_FROM
 #define MAX_NEEDED_OUTPUT	MAX_NEEDED_FROM
 #define LOOPFCT			TO_LOOP
 #define BODY \
   {									      \
     uint32_t ch = get32 (inptr);					      \
     const unsigned char *cp = "";						      \
     unsigned char b5ch[2] = "\0\0";					      \
     int i;								      \
     									      \
     for (i = 0;								      \
 	 i < (int) (sizeof (from_ucs4_idx) / sizeof (from_ucs4_idx[0]));      \
 	 ++i)								      \
       {									      \
 	if (ch < from_ucs4_idx[i].from)					      \
 	  break;							      \
 	if (from_ucs4_idx[i].to >= ch)					      \
 	  {								      \
 	    cp = ucs4_to_big5_hkscs[from_ucs4_idx[i].offset		      \
 			  + ch - from_ucs4_idx[i].from];		      \
 	    break;							      \
 	  }								      \
       }									      \
 									      \
     if (ch <= 0x80)							      \
       {									      \
 	b5ch[0] = ch;							      \
 	cp = b5ch;							      \
       }									      \
 									      \
     if (cp[0] == '\0' && ch != 0)					      \
       {									      \
 	UNICODE_TAG_HANDLER (ch, 4);					      \
 									      \
 	/* Illegal character.  */					      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
     else								      \
       {									      \
 	/* See whether there is enough room for the second byte we write.  */ \
 	if (__builtin_expect (cp[1], '\1') != '\0'			      \
 	    && __builtin_expect (outptr + 1 >= outend, 0))		      \
 	  {								      \
 	    /* We have not enough room.  */				      \
 	    result = __GCONV_FULL_OUTPUT;				      \
 	    break;							      \
 	  }								      \
 									      \
 	*outptr++ = cp[0];						      \
 	if (cp[1] != '\0')						      \
 	  *outptr++ = cp[1];						      \
       }									      \
 									      \
     inptr += 4;								      \
   }
 #define LOOP_NEED_FLAGS
 #include <iconv/loop.c>
 /* Now define the toplevel functions.  */
 #include <iconv/skeleton.c>
 EOT
 }

The Tor Browser / annotate

intl/uconv/tools/gen-big5hkscs-2001-mozilla.pl@97036ab72558 (annotated)

intl/uconv/tools/gen-big5hkscs-2001-mozilla.pl