michael@0: #!/usr/bin/perl 
michael@0: #
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: $header = <<END_OF_HEADER;
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: # 
michael@0: # THIS FILE IS GENERATED BY mozilla/intl/unicharutil/tools/gentransliterate.pl
michael@0: # PLEASE DO NOT MODIFY THIS FILE BY HAND
michael@0: #
michael@0: entity.list.name=transliterate
michael@0: entity.169=(c)
michael@0: #
michael@0: #
michael@0: # Here are the windows-1252 characters from the range 0x80 - 0x9F
michael@0: #
michael@0: END_OF_HEADER
michael@0: 
michael@0: $handcoded = <<END_OF_HANDCODED;
michael@0: # EURO SIGN
michael@0: entity.8364=EUR
michael@0: # SINGLE LOW-9 QUOTATION MARK
michael@0: entity.8218=,
michael@0: # LATIN SMALL LETTER F WITH HOOK
michael@0: entity.402=f
michael@0: # DOUBLE LOW-9 QUOTATION MARK
michael@0: entity.8222="
michael@0: # DAGGER
michael@0: entity.8224=+
michael@0: # DOUBLE DAGGER
michael@0: entity.8225=++
michael@0: # MODIFIER LETTER CIRCUMFLEX ACCENT
michael@0: entity.710=^
michael@0: # PER MILLE SIGN
michael@0: entity.8240=0/00
michael@0: # SINGLE LEFT-POINTING ANGLE QUOTATION MARK
michael@0: entity.8249=<
michael@0: # LATIN CAPITAL LIGATURE OE
michael@0: entity.338=OE
michael@0: # LEFT SINGLE QUOTATION MARK
michael@0: entity.8216='
michael@0: # RIGHT SINGLE QUOTATION MARK
michael@0: entity.8217='
michael@0: # LEFT DOUBLE QUOTATION MARK
michael@0: entity.8220="
michael@0: # RIGHT DOUBLE QUOTATION MARK
michael@0: entity.8221="
michael@0: # BULLET
michael@0: entity.8226=.
michael@0: # EN DASH
michael@0: entity.8211=--
michael@0: # EM DASH
michael@0: entity.8212=---
michael@0: # SMALL TILDE
michael@0: entity.732=~
michael@0: # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
michael@0: entity.8250=>
michael@0: # LATIN SMALL LIGATURE OE
michael@0: entity.339=oe
michael@0: # U+2000 EN QUAD
michael@0: entity.8192=\\u0020
michael@0: # U+2001 EM QUAD
michael@0: entity.8193=\\u0020
michael@0: # U+2010 HYPHEN
michael@0: entity.8208=-
michael@0: # U+2011 NON-BREAKING HYPHEN
michael@0: entity.8209=-
michael@0: # U+2012 FIGURE DASH
michael@0: entity.8210=-
michael@0: # U+2015 HORIZONTAL BAR
michael@0: entity.8213=--
michael@0: # U+200B, ZERO WIDTH SPACE (a.k.a. InvisibleComma)
michael@0: entity.8203=
michael@0: # U+2061, ApplyFunction, character showing function application in presentation tagging
michael@0: entity.8289=
michael@0: # U+2062, InvisibleTimes, marks multiplication when it is understood without a mark
michael@0: entity.8290=
michael@0: # U+2146, DifferentialD, d for use in differentials, e.g., within integrals
michael@0: entity.8518=d
michael@0: # U+2212, MINUS SIGN, official Unicode minus sign
michael@0: entity.8722=-
michael@0: # Hebrew punctuation
michael@0: # U+05BE HEBREW PUNCTUATION MAQAF
michael@0: entity.1470=-
michael@0: # U+05C0 HEBREW PUNCTUATION PASEQ
michael@0: entity.1472=|
michael@0: # U+05C3 HEBREW PUNCTUATION SOF PASUQ
michael@0: entity.1475=:
michael@0: # U+05F3 HEBREW PUNCTUATION GERESH
michael@0: entity.1523='
michael@0: # U+05F4 HEBREW PUNCTUATION GERSHAYIM
michael@0: entity.1524="
michael@0: ##
michael@0: ## End of hand coded section
michael@0: ## Below are generated from the unicode character database
michael@0: ##
michael@0: END_OF_HANDCODED
michael@0: 
michael@0: @table = ();
michael@0: sub FromLatinComment
michael@0: {
michael@0:   my ($cmt) = (@_);
michael@0:   $char = "";
michael@0:   if($cmt =~ /PRECEDED BY APOSTROPHE/) {
michael@0:       $char = "\'";
michael@0:   }
michael@0:   if($cmt =~ /CAPITAL LETTER ([A-Z]*)/) {
michael@0:       $char = $char . $1;
michael@0:   }
michael@0:   if($cmt =~ /SMALL LETTER ([A-Z]*)/) {
michael@0:       $char = $char . lc($1);
michael@0:   }
michael@0:   @f = split(/ / , $cmt); 
michael@0:   while($item = shift @f) {
michael@0:      if($item eq "DOT") {
michael@0:        $char .= ".";
michael@0:      } elsif ($item eq "DIAERESIS") {
michael@0:        $char .= "\"";
michael@0:      } elsif ($item eq "BREVE") {
michael@0:        $char .= "(";
michael@0:      } elsif ($item eq "ACUTE") {
michael@0:        $char .= "\'";
michael@0:      } elsif ($item eq "GRAVE") {
michael@0:        $char .= "`";
michael@0:      } elsif ($item eq "TILDE") {
michael@0:        $char .= "~";
michael@0:      } elsif ($item eq "CARON") {
michael@0:        $char .= "(";
michael@0:      } elsif ($item eq "HOOK") {
michael@0:        $char .= "?";
michael@0:      } elsif ($item eq "CEDILLA") {
michael@0:        $char .= ",";
michael@0:      } elsif ($item eq "MACRON") {
michael@0:        $char .= "-";
michael@0:      } elsif ($item eq "CIRCUMFLEX") {
michael@0:        $char .= "^";
michael@0:      } elsif ($item eq "RING") {
michael@0:        $char .= "*";
michael@0:      } elsif ($item eq "OGONEK") {
michael@0:        $char .= ";";
michael@0:      } elsif ($item eq "LINE") {
michael@0:        $char .= "_";
michael@0:      } elsif ($item eq "COMMA") {
michael@0:        $char .= ",";
michael@0:      } elsif ($item eq "STROKE") {
michael@0:        $char .= "/";
michael@0:      } elsif ($item eq "HORN") {
michael@0:        $char .= "+";
michael@0:      } elsif ($item =~ /^(LATIN|CAPITAL|SMALL|LETTER|WITH|ABOVE|BELOW|INVERTED|MIDDLE|AND|BY|APOSTROPHE|[A-Z])$/) {
michael@0:        # ignore
michael@0:      } else {
michael@0:        #print "AAAA $item\n";
michael@0:      }
michael@0:   }
michael@0:   
michael@0:   return $char;
michael@0: }
michael@0: sub warning
michael@0: {
michael@0:   my ($warning) = (@_);
michael@0:   print "WARNING: $warning \n";
michael@0: }
michael@0: sub doutput
michael@0: {
michael@0:   my ($u, $cmt, $udec, $str) = (@_);
michael@0:   # don't print out comments - for debugging purposes only
michael@0:   # print "# U+$u $cmt\n";
michael@0:   print "entity.$udec=$str\n";
michael@0: }
michael@0: sub output
michael@0: {
michael@0:   my ($u, $cmt, $udec, $str) = (@_);
michael@0:   if(decomposeIntoNonASCII($str)) {
michael@0:     if(($cmt =~ "LATIN")  && ($cmt =~ "LETTER") && !($cmt =~ "LONG")) {
michael@0:        $str = FromLatinComment($cmt);
michael@0:        output($u,$cmt,$udec,$str);
michael@0:     }
michael@0:   } else {
michael@0:     # don't print out comments - for debugging purposes only
michael@0:     # print OUT "# U+$u $cmt\n";
michael@0:     print OUT "entity.$udec=$str\n";
michael@0:   }
michael@0: }
michael@0: 
michael@0: sub decomposeIntoNonASCII
michael@0: {
michael@0:   my ($dec) = (@_);
michael@0:   return $dec =~ /\\u([1-9A-F][0-9A-F][0-9A-F]|[0-9A-F][1-9A-F][0-9A-F]|00[8-9A-F])[0-9A-F]/;
michael@0: }
michael@0: 
michael@0: sub foldcombining
michael@0: {
michael@0:   my ($dec) = (@_);
michael@0:   $grave = "0060";
michael@0:   $acute = "0027";
michael@0:   $hat = "005E";
michael@0:   $hat = "005E";
michael@0:   $tilde = "007E";
michael@0:   $overscore = "002D"; ## should be 00AF but we can only handle ASCII now
michael@0:   $umlaut = "0022"; ## should be 00A8 but we can only handle ASCII now
michael@0:   $doubleacute = "0022";
michael@0:   $dot = "002E";
michael@0:   $doublegrave = "0060 0060";
michael@0: 
michael@0: 
michael@0:   $dec =~ s/00A8/$umlaut/eg;
michael@0:   $dec =~ s/00AF/$overscore/eg;
michael@0:  # $dec =~ s/00B0//eg;
michael@0:   $dec =~ s/00B4/$acute/eg;
michael@0:   $dec =~ s/00B7/$dot/eg;
michael@0:  # $dec =~ s/00B8//eg;
michael@0:   $dec =~ s/0300/$grave/eg;
michael@0:   $dec =~ s/0301/$acute/eg;
michael@0:   $dec =~ s/0302/$hat/eg;
michael@0:   $dec =~ s/0303/$tilde/eg;
michael@0:   $dec =~ s/0304/$overscore/eg;
michael@0:   $dec =~ s/0305/$overscore/eg;
michael@0:  #$dec =~ s/0306/?/eg;
michael@0:   $dec =~ s/0307/$dot/eg;
michael@0:   $dec =~ s/0308/$umlaut/eg;
michael@0:  #$dec =~ s/0309/?/eg;
michael@0:  #$dec =~ s/030A/?/eg;
michael@0:   $dec =~ s/030B/$doubleacute/eg;
michael@0:  #$dec =~ s/030C/?/eg;
michael@0:   $dec =~ s/030D/$acute/eg;
michael@0:   $dec =~ s/030E/$doubleacute/eg;
michael@0:   $dec =~ s/030F/$doublegrave/eg;
michael@0: 
michael@0:  # $dec =~ s/03[0-9A-F][0-9A-F]//eg; ## drop others
michael@0:   return $dec;
michael@0: }
michael@0: sub rdecompose
michael@0: {
michael@0:   my ($dec) = (@_);
michael@0:   if(exists $table{$dec}) {
michael@0:     $t = $table{$dec};
michael@0:     $t =~ s/<[a-zA-Z]*>//eg;
michael@0:     $t = foldcombining($t);
michael@0:     return rdecompose( $table{$t});
michael@0:   }
michael@0:   return $dec;
michael@0: }
michael@0: sub decompose
michael@0: {
michael@0:   my ($removeprefix, $dec) = (@_);
michael@0:   $removeprefix .= " ";
michael@0:   
michael@0:   $dec =~ s/$removeprefix//eg;
michael@0:   if($dec eq "0020") {
michael@0:    $dec = "\\u0020";
michael@0:   } elsif($dec eq "005C") {
michael@0:    $dec = "\\u005C";
michael@0:   } else {
michael@0:    $k = "\/";
michael@0:    $dec =~ s/2044/$k/eg;
michael@0:    $dec =~ s/([0-9A-F][0-9A-F][0-9A-F][0-9A-F])/rdecompose($1)/eg;
michael@0:    $dec =~ s/([0-9A-F][0-9A-F][0-9A-F][0-9A-F])/\\u$1/g;
michael@0:    $dec =~ s/\\u00([0-7][0-9A-F])/pack("C",hex($1))/eg;
michael@0:    $dec =~ s/ //eg;
michael@0:   } 
michael@0:   return $dec;
michael@0: }
michael@0: 
michael@0: ######################################################################
michael@0: #
michael@0: # Open the unicode database file
michael@0: #
michael@0: ######################################################################
michael@0: open ( UNICODATA , "< UnicodeData-Latest.txt") 
michael@0:    || die "cannot find UnicodeData-Latest.txt";
michael@0: 
michael@0: open ( UNICODATA2 , "< UnicodeData-Latest.txt") 
michael@0:    || die "cannot find UnicodeData-Latest.txt";
michael@0: ######################################################################
michael@0: #
michael@0: # Open the output file
michael@0: #
michael@0: ######################################################################
michael@0: open ( OUT , "> ../tables/transliterate.properties") 
michael@0:   || die "cannot open output ../tables/transliterate.properties file";
michael@0: 
michael@0: print OUT $header;
michael@0: 
michael@0: # remove comments from $handcoded
michael@0: $handcoded =~ s/^#[^#].*\n//mg;
michael@0: print OUT $handcoded;
michael@0: 
michael@0: ######################################################################
michael@0: #
michael@0: # Process the file line by line
michael@0: #
michael@0: ######################################################################
michael@0: while(<UNICODATA2>) {
michael@0:    chop;
michael@0:    @f = split(/;/ , $_); 
michael@0:    $udec = hex($u);
michael@0:    if(($udec > 256 ) && ($f[5] ne "")) {
michael@0:      $table{$f[0]}=$f[5];
michael@0:    }
michael@0: }
michael@0: while(<UNICODATA>) {
michael@0:    chop;
michael@0:    ######################################################################
michael@0:    #
michael@0:    # Get value from fields
michael@0:    #
michael@0:    ######################################################################
michael@0:    @f = split(/;/ , $_); 
michael@0:    $u = $f[0];    # The unicode value
michael@0:    $cmt = $f[1];  # The comment
michael@0:    $dec = $f[5];  # The decomposed value
michael@0:    $d1 = $f[6];  
michael@0:    $d2 = $f[7];  
michael@0:    $d3 = $f[8];  
michael@0:    $udec = hex($u);
michael@0: 
michael@0:    if($udec > 128) 
michael@0:    {
michael@0:      # not ASCII
michael@0:      if($dec ne "") 
michael@0:      {
michael@0:        # have decomposition
michael@0:        if($dec =~ /</)  {
michael@0:            # formated decomposition
michael@0:            if($dec =~ /<wide>/)  {
michael@0:               output($u,$cmt,$udec,&decompose("<wide>", $dec));
michael@0:            } elsif($dec =~ /<narrow>/)  {
michael@0:               # ignore non ASCII decomposition
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<circle>/)  {
michael@0:               output($u,$cmt,$udec,&decompose("<circle>", "(".$dec.")"));
michael@0:            } elsif($dec =~ /<fraction>/)  {
michael@0:               output($u,$cmt,$udec,&decompose("<fraction>", $dec));
michael@0:            } elsif($dec =~ /<small>/)  {
michael@0:               output($u,$cmt,$udec,&decompose("<small>", $dec));
michael@0:            } elsif($dec =~ /<vertical>/)  {
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<super>/)  {
michael@0:               output($u,$cmt,$udec,"^(".&decompose("<super>", $dec).")");
michael@0:            } elsif($dec =~ /<sub>/)  {
michael@0:               output($u,$cmt,$udec,"v(".&decompose("<sub>", $dec).")");
michael@0:            } elsif($dec =~ /<font>/)  {
michael@0:                output($u,$cmt,$udec,&decompose("<font>", $dec));
michael@0:            } elsif($dec =~ /<square>/)  {
michael@0:               # ignore <square>
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<compat>/)  {
michael@0:                output($u,$cmt,$udec,&decompose("<compat>", $dec));
michael@0:            } elsif($dec =~ /<isolated>/)  {
michael@0:               # ignore <isolated>
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<medial>/)  {
michael@0:               # ignore <medial>
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<final>/)  {
michael@0:               # ignore <final>
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<initial>/)  {
michael@0:               # ignore <initial>
michael@0:               # warning($_);
michael@0:            } elsif($dec =~ /<noBreak>/)  {
michael@0:              if($dec eq "<noBreak> 0020")
michael@0:              {
michael@0:                output($u,$cmt,$udec,"\\u0020");
michael@0:              } else {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:              }
michael@0:            } else {
michael@0:              warning($_);
michael@0:            }
michael@0:        } else {
michael@0:          # decomposition without format code
michael@0:          if($cmt =~ /LATIN/) {
michael@0:            $dec = foldcombining($dec);
michael@0:               output($u,$cmt,$udec,&decompose("", $dec));
michael@0:          } elsif($cmt =~ /CYRILLIC/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /GREEK/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /ARABIC/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /CJK/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /HEBREW/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /DEVANAGARI/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /BENGALI/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /GURMUKHI/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /ORIYA/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /TAMIL/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /TELUGU/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /KANNADA/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /MALAYALAM/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /SINHALA/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /TIBETAN/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /MYANMAR/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /KATAKANA/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } elsif($cmt =~ /HIRAGANA/) {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          } else {
michael@0:               # ignore 
michael@0:               # warning($_);
michael@0:          }
michael@0:        }
michael@0:      } else {
michael@0:        # do not have decomposition
michael@0:        if ($d1 ne "") 
michael@0:        {
michael@0:          # are numeric characters
michael@0:          output($u,$cmt,$udec,$d1);
michael@0:        } elsif ($d2 ne "") {
michael@0:          if($cmt =~ /CIRCLED/) {
michael@0:            # circled
michael@0:            output($u,$cmt,$udec,"(".$d2.")");
michael@0:          } else {
michael@0:            # others, use [ ]
michael@0:            output($u,$cmt,$udec,"[".$d2."]");
michael@0:          }
michael@0:        } elsif ($d3 ne "") {
michael@0:          if($cmt =~ /CIRCLED/) {
michael@0:            # circled
michael@0:            output($u,$cmt,$udec,"(".$d3.")");
michael@0:          } else {
michael@0:            # others, use [ ]
michael@0:            output($u,$cmt,$udec,"[".$d3."]");
michael@0:          }
michael@0:        } else {
michael@0:          # not numeric characters
michael@0: 
michael@0:        } # end of no decomposition
michael@0:      } # end of have/not decomposition
michael@0:    }
michael@0: }
michael@0: ######################################################################
michael@0: #
michael@0: # Close files
michael@0: #
michael@0: ######################################################################
michael@0: close(UNIDATA);
michael@0: close(OUT);
michael@0: