The Tor Browser / file revision

 #!/usr/bin/perl

 #  ********************************************************************

 #  * COPYRIGHT:

 #  * Copyright (c) 2002-2007, International Business Machines Corporation and

 #  * others. All Rights Reserved.

 #  ********************************************************************

 #

 #  regexcst.pl

 #            Compile the regular expression paser state table data into initialized C data.

 #            Usage:

 #                   cd icu/source/i18n

 #                   perl regexcst.pl < regexcst.txt > regexcst.h

 #

 #             The output file, regexcst.h, is included by some of the .cpp regex

 #             implementation files.   This perl script is NOT run as part

 #             of a normal ICU build.  It is run by hand when needed, and the

 #             regexcst.h generated file is put back into cvs.

 #

 #             See regexcst.txt for a description of the input format for this script.

 #

 #             This script is derived from rbbicst.pl, which peforms the same function

 #             for the Rule Based Break Iterator Rule Parser.  Perhaps they could be

 #             merged?

 #

 $num_states = 1;         # Always the state number for the line being compiled.

 $line_num  = 0;          # The line number in the input file.

 $states{"pop"} = 255;    # Add the "pop"  to the list of defined state names.

                          # This prevents any state from being labelled with "pop",

                          #  and resolves references to "pop" in the next state field.

 line_loop: while (<>) {

     chomp();

     $line = $_;

     @fields = split();

     $line_num++;

     # Remove # comments, which are any fields beginning with a #, plus all

     #  that follow on the line.

     for ($i=0; $i<@fields; $i++) {

         if ($fields[$i] =~ /^#/) {

             @fields = @fields[0 .. $i-1];

             last;

         }

     }

     # ignore blank lines, and those with no fields left after stripping comments..

     if (@fields == 0) {

         next;

     }

     #

     # State Label:  handling.

     #    Does the first token end with a ":"?  If so, it's the name  of a state.

     #    Put in a hash, together with the current state number,

     #        so that we can later look up the number from the name.

     #

     if (@fields[0] =~ /.*:$/) {

         $state_name = @fields[0];

         $state_name =~ s/://;        # strip off the colon from the state name.

         if ($states{$state_name} != 0) {

             print "  rbbicst: at line $line-num duplicate definition of state $state_name\n";

         }

         $states{$state_name} = $num_states;

         $stateNames[$num_states] = $state_name;

         # if the label was the only thing on this line, go on to the next line,

         # otherwise assume that a state definition is on the same line and fall through.

         if (@fields == 1) {

             next line_loop;

         }

         shift @fields;                       # shift off label field in preparation

                                              #  for handling the rest of the line.

     }

     #

     # State Transition line.

     #   syntax is this,

     #       character   [n]  target-state  [^push-state]  [function-name]

     #   where

     #      [something]   is an optional something

     #      character     is either a single quoted character e.g. '['

     #                       or a name of a character class, e.g. white_space

     #

     $state_line_num[$num_states] = $line_num;   # remember line number with each state

                                                 #  so we can make better error messages later.

     #

     # First field, character class or literal character for this transition.

     #

     if ($fields[0] =~ /^'.'$/) {

         # We've got a quoted literal character.

         $state_literal_chars[$num_states] = $fields[0];

         $state_literal_chars[$num_states] =~ s/'//g;

     } else {

         # We've got the name of a character class.

         $state_char_class[$num_states] = $fields[0];

         if ($fields[0] =~ /[\W]/) {

             print "  rbbicsts:  at line $line_num, bad character literal or character class name.\n";

             print "     scanning $fields[0]\n";

             exit(-1);

         }

     }

     shift @fields;

     #

     # do the 'n' flag

     #

     $state_flag[$num_states] = "FALSE";

     if ($fields[0] eq "n") {

         $state_flag[$num_states] = "TRUE";

         shift @fields;

     }

     #

     # do the destination state.

     #

     $state_dest_state[$num_states] = $fields[0];

     if ($fields[0] eq "") {

         print "  rbbicsts:  at line $line_num, destination state missing.\n";

         exit(-1);

     }

     shift @fields;

     #

     # do the push state, if present.

     #

     if ($fields[0] =~ /^\^/) {

         $fields[0] =~ s/^\^//;

         $state_push_state[$num_states] = $fields[0];

         if ($fields[0] eq "" ) {

             print "  rbbicsts:  at line $line_num, expected state after ^ (no spaces).\n";

             exit(-1);

         }

         shift @fields;

     }

     #

     # Lastly, do the optional action name.

     #

     if ($fields[0] ne "") {

         $state_func_name[$num_states] = $fields[0];

         shift @fields;

     }

     #

     #  There should be no fields left on the line at this point.

     #

     if (@fields > 0) {

        print "  rbbicsts:  at line $line_num, unexpected extra stuff on input line.\n";

        print "     scanning $fields[0]\n";

    }

    $num_states++;

 }

 #

 # We've read in the whole file, now go back and output the

 #   C source code for the state transition table.

 #

 # We read all states first, before writing anything,  so that the state numbers

 # for the destination states are all available to be written.

 #

 #

 # Make hashes for the names of the character classes and

 #      for the names of the actions that appeared.

 #

 for ($state=1; $state < $num_states; $state++) {

     if ($state_char_class[$state] ne "") {

         if ($charClasses{$state_char_class[$state]} == 0) {

             $charClasses{$state_char_class[$state]} = 1;

         }

     }

     if ($state_func_name[$state] eq "") {

         $state_func_name[$state] = "doNOP";

     }

     if ($actions{$state_action_name[$state]} == 0) {

         $actions{$state_func_name[$state]} = 1;

     }

 }

 #

 # Check that all of the destination states have been defined

 #

 #

 $states{"exit"} = 0;              # Predefined state name, terminates state machine.

 for ($state=1; $state<$num_states; $state++) {

    if ($states{$state_dest_state[$state]} == 0 && $state_dest_state[$state] ne "exit") {

        print "Error at line $state_line_num[$state]: target state \"$state_dest_state[$state]\" is not defined.\n";

        $errors++;

    }

    if ($state_push_state[$state] ne "" && $states{$state_push_state[$state]} == 0) {

        print "Error at line $state_line_num[$state]: target state \"$state_push_state[$state]\" is not defined.\n";

        $errors++;

    }

 }

 die if ($errors>0);

 print "//---------------------------------------------------------------------------------\n";

 print "//\n";

 print "// Generated Header File.  Do not edit by hand.\n";

 print "//    This file contains the state table for the ICU Regular Expression Pattern Parser\n";

 print "//    It is generated by the Perl script \"regexcst.pl\" from\n";

 print "//    the rule parser state definitions file \"regexcst.txt\".\n";

 print "//\n";

 print "//   Copyright (C) 2002-2007 International Business Machines Corporation \n";

 print "//   and others. All rights reserved.  \n";

 print "//\n";

 print "//---------------------------------------------------------------------------------\n";

 print "#ifndef RBBIRPT_H\n";

 print "#define RBBIRPT_H\n";

 print "\n";

 print "U_NAMESPACE_BEGIN\n";

 #

 # Emit the constants for indicies of Unicode Sets

 #   Define one constant for each of the character classes encountered.

 #   At the same time, store the index corresponding to the set name back into hash.

 #

 print "//\n";

 print "// Character classes for regex pattern scanning.\n";

 print "//\n";

 $i = 128;                   # State Table values for Unicode char sets range from 128-250.

                             # Sets "default", "quoted", etc. get special handling.

                             #  They have no corresponding UnicodeSet object in the state machine,

                             #    but are handled by special case code.  So we emit no reference

                             #    to a UnicodeSet object to them here.

 foreach $setName (keys %charClasses) {

     if ($setName eq "default") {

         $charClasses{$setName} = 255;}

     elsif ($setName eq "quoted") {

         $charClasses{$setName} = 254;}

     elsif ($setName eq "eof") {

         $charClasses{$setName} = 253;}

     else {

         # Normal character class.  Fill in array with a ptr to the corresponding UnicodeSet in the state machine.

        print "    static const uint8_t kRuleSet_$setName = $i;\n";

         $charClasses{$setName} = $i;

         $i++;

     }

 }

 print "\n\n";

 #

 # Emit the enum for the actions to be performed.

 #

 print "enum Regex_PatternParseAction {\n";

 foreach $act (keys %actions) {

     print "    $act,\n";

 }

 print "    rbbiLastAction};\n\n";

 #

 # Emit the struct definition for transtion table elements.

 #

 print "//-------------------------------------------------------------------------------\n";

 print "//\n";

 print "//  RegexTableEl       represents the structure of a row in the transition table\n";

 print "//                     for the pattern parser state machine.\n";

 print "//-------------------------------------------------------------------------------\n";

 print "struct RegexTableEl {\n";

 print "    Regex_PatternParseAction      fAction;\n";

 print "    uint8_t                       fCharClass;       // 0-127:    an individual ASCII character\n";

 print "                                                    // 128-255:  character class index\n";

 print "    uint8_t                       fNextState;       // 0-250:    normal next-state numbers\n";

 print "                                                    // 255:      pop next-state from stack.\n";

 print "    uint8_t                       fPushState;\n";

 print "    UBool                         fNextChar;\n";

 print "};\n\n";

 #

 # emit the state transition table

 #

 print "static const struct RegexTableEl gRuleParseStateTable[] = {\n";

 print "    {doNOP, 0, 0, 0, TRUE}\n";    # State 0 is a dummy.  Real states start with index = 1.

 for ($state=1; $state < $num_states; $state++) {

     print "    , {$state_func_name[$state],";

     if ($state_literal_chars[$state] ne "") {

         $c = $state_literal_chars[$state];

         printf(" %d /* $c */,", ord($c));   #  use numeric value, so EBCDIC machines are ok.

     }else {

         print " $charClasses{$state_char_class[$state]},";

     }

     print " $states{$state_dest_state[$state]},";

     # The push-state field is optional.  If omitted, fill field with a zero, which flags

     #   the state machine that there is no push state.

     if ($state_push_state[$state] eq "") {

         print "0, ";

     } else {

         print " $states{$state_push_state[$state]},";

     }

     print " $state_flag[$state]} ";

     # Put out a C++ comment showing the number (index) of this state row,

     #   and, if this is the first row of the table for this state, the state name.

     print "    //  $state ";

     if ($stateNames[$state] ne "") {

         print "     $stateNames[$state]";

     }

     print "\n";

 };

 print " };\n";

 #

 # emit a mapping array from state numbers to state names.

 #

 #    This array is used for producing debugging output from the pattern parser.

 #

 print "static const char * const RegexStateNames[] = {";

 for ($state=0; $state<$num_states; $state++) {

     if ($stateNames[$state] ne "") {

         print "     \"$stateNames[$state]\",\n";

     } else {

         print "    0,\n";

     }

 }

 print "    0};\n\n";

 print "U_NAMESPACE_END\n";

 print "#endif\n";