michael@0: 
michael@0: #*****************************************************************************
michael@0: #
michael@0: #   Copyright (C) 2002-2007, International Business Machines Corporation and others.
michael@0: #   All Rights Reserved.
michael@0: #
michael@0: #*****************************************************************************
michael@0: #
michael@0: #  file:  regexcst.txt
michael@0: #  ICU Regular Expression Parser State Table
michael@0: #
michael@0: #     This state table is used when reading and parsing a regular expression pattern
michael@0: #     The pattern parser uses a state machine; the data in this file define the
michael@0: #     state transitions that occur for each input character.
michael@0: #
michael@0: #     *** This file defines the regex pattern grammar.   This is it.
michael@0: #     *** The determination of what is accepted is here.
michael@0: #
michael@0: #     This file is processed by a perl script "regexcst.pl" to produce initialized C arrays
michael@0: #     that are then built with the rule parser.
michael@0: #
michael@0: 
michael@0: #
michael@0: # Here is the syntax of the state definitions in this file:
michael@0: #
michael@0: #
michael@0: #StateName:
michael@0: #   input-char           n next-state           ^push-state     action
michael@0: #   input-char           n next-state           ^push-state     action
michael@0: #       |                |   |                      |             |
michael@0: #       |                |   |                      |             |--- action to be performed by state machine
michael@0: #       |                |   |                      |                  See function RBBIRuleScanner::doParseActions()
michael@0: #       |                |   |                      |
michael@0: #       |                |   |                      |--- Push this named state onto the state stack.
michael@0: #       |                |   |                           Later, when next state is specified as "pop",
michael@0: #       |                |   |                           the pushed state will become the current state.
michael@0: #       |                |   |
michael@0: #       |                |   |--- Transition to this state if the current input character matches the input
michael@0: #       |                |        character or char class in the left hand column.  "pop" causes the next
michael@0: #       |                |        state to be popped from the state stack.
michael@0: #       |                |
michael@0: #       |                |--- When making the state transition specified on this line, advance to the next
michael@0: #       |                     character from the input only if 'n' appears here.
michael@0: #       |
michael@0: #       |--- Character or named character classes to test for.  If the current character being scanned
michael@0: #            matches, peform the actions and go to the state specified on this line.
michael@0: #            The input character is tested sequentally, in the order written.  The characters and
michael@0: #            character classes tested for do not need to be mutually exclusive.  The first match wins.
michael@0: #
michael@0: 
michael@0: 
michael@0: 
michael@0: 
michael@0: #
michael@0: #  start state, scan position is at the beginning of the pattern.
michael@0: #
michael@0: start:
michael@0:    default                 term                                     doPatStart
michael@0: 
michael@0: 
michael@0: 
michael@0: 
michael@0: #
michael@0: #  term.  At a position where we can accept the start most items in a pattern.
michael@0: #
michael@0: term:
michael@0:     quoted               n expr-quant                               doLiteralChar
michael@0:     rule_char            n expr-quant                               doLiteralChar
michael@0:     '['                  n set-open       ^set-finish               doSetBegin
michael@0:     '('                  n open-paren
michael@0:     '.'                  n expr-quant                               doDotAny
michael@0:     '^'                  n expr-quant                               doCaret
michael@0:     '$'                  n expr-quant                               doDollar
michael@0:     '\'                  n backslash
michael@0:     '|'                  n  term                                    doOrOperator
michael@0:     ')'                  n  pop                                     doCloseParen
michael@0:     eof	                   term                                     doPatFinish
michael@0:     default                errorDeath                               doRuleError
michael@0: 
michael@0: 
michael@0: 
michael@0: #
michael@0: #   expr-quant    We've just finished scanning a term, now look for the optional
michael@0: #                 trailing quantifier - *, +, ?, *?,  etc.
michael@0: #
michael@0: expr-quant:
michael@0:     '*'                  n  quant-star
michael@0:     '+'                  n  quant-plus
michael@0:     '?'                  n  quant-opt
michael@0:     '{'                  n  interval-open                          doIntervalInit
michael@0:     '('                  n  open-paren-quant
michael@0:     default                 expr-cont
michael@0: 
michael@0: 
michael@0: #
michael@0: #  expr-cont      Expression, continuation.  At a point where additional terms are
michael@0: #                                            allowed, but not required.  No Quantifiers
michael@0: #
michael@0: expr-cont:
michael@0:     '|'                  n  term                                    doOrOperator
michael@0:     ')'                  n  pop                                     doCloseParen
michael@0:     default                 term
michael@0: 
michael@0: 
michael@0: #
michael@0: #   open-paren-quant   Special case handling for comments appearing before a quantifier,
michael@0: #                        e.g.   x(?#comment )*
michael@0: #                      Open parens from expr-quant come here; anything but a (?# comment
michael@0: #                      branches into the normal parenthesis sequence as quickly as possible.
michael@0: #
michael@0: open-paren-quant:
michael@0:     '?'                  n  open-paren-quant2                      doSuppressComments
michael@0:     default                 open-paren
michael@0: 
michael@0: open-paren-quant2:
michael@0:     '#'                  n  paren-comment   ^expr-quant
michael@0:     default                 open-paren-extended
michael@0: 
michael@0: 
michael@0: #
michael@0: #   open-paren    We've got an open paren.  We need to scan further to
michael@0: #                 determine what kind of quantifier it is - plain (, (?:, (?>, or whatever.
michael@0: #
michael@0: open-paren:
michael@0:     '?'                  n  open-paren-extended                     doSuppressComments
michael@0:     default                 term            ^expr-quant             doOpenCaptureParen
michael@0: 
michael@0: open-paren-extended:
michael@0:     ':'                  n  term            ^expr-quant             doOpenNonCaptureParen  #  (?:
michael@0:     '>'                  n  term            ^expr-quant             doOpenAtomicParen      #  (?>
michael@0:     '='                  n  term            ^expr-cont              doOpenLookAhead        #  (?=
michael@0:     '!'                  n  term            ^expr-cont              doOpenLookAheadNeg     #  (?!
michael@0:     '<'                  n  open-paren-lookbehind
michael@0:     '#'                  n  paren-comment   ^term
michael@0:     'i'                     paren-flag                              doBeginMatchMode
michael@0:     'd'                     paren-flag                              doBeginMatchMode
michael@0:     'm'                     paren-flag                              doBeginMatchMode
michael@0:     's'                     paren-flag                              doBeginMatchMode
michael@0:     'u'                     paren-flag                              doBeginMatchMode
michael@0:     'w'                     paren-flag                              doBeginMatchMode
michael@0:     'x'                     paren-flag                              doBeginMatchMode
michael@0:     '-'                     paren-flag                              doBeginMatchMode
michael@0:     '('                  n  errorDeath                              doConditionalExpr
michael@0:     '{'                  n  errorDeath                              doPerlInline
michael@0:     default                 errorDeath                              doBadOpenParenType
michael@0: 
michael@0: open-paren-lookbehind:
michael@0:     '='                  n  term            ^expr-cont              doOpenLookBehind       #  (?<=
michael@0:     '!'                  n  term            ^expr-cont              doOpenLookBehindNeg    #  (?<!
michael@0:     default                 errorDeath                              doBadOpenParenType
michael@0: 
michael@0: 
michael@0: #
michael@0: #   paren-comment    We've got a (?# ... )  style comment.  Eat pattern text till we get to the ')'
michael@0: #
michael@0: paren-comment:
michael@0:     ')'                  n  pop
michael@0:     eof		                errorDeath                              doMismatchedParenErr
michael@0:     default              n  paren-comment
michael@0: 
michael@0: #
michael@0: #  paren-flag    Scanned a (?ismx-ismx  flag setting
michael@0: #
michael@0: paren-flag:
michael@0:     'i'                  n  paren-flag                              doMatchMode
michael@0:     'd'                  n  paren-flag                              doMatchMode
michael@0:     'm'                  n  paren-flag                              doMatchMode
michael@0:     's'                  n  paren-flag                              doMatchMode
michael@0:     'u'                  n  paren-flag                              doMatchMode
michael@0:     'w'                  n  paren-flag                              doMatchMode
michael@0:     'x'                  n  paren-flag                              doMatchMode
michael@0:     '-'                  n  paren-flag                              doMatchMode
michael@0:     ')'                  n  term                                    doSetMatchMode
michael@0:     ':'                  n  term              ^expr-quant           doMatchModeParen
michael@0:     default                 errorDeath                              doBadModeFlag
michael@0: 
michael@0: 
michael@0: #
michael@0: #  quant-star     Scanning a '*' quantifier.  Need to look ahead to decide
michael@0: #                 between plain '*', '*?', '*+'
michael@0: #
michael@0: quant-star:
michael@0:      '?'                 n  expr-cont                               doNGStar               #  *?
michael@0:      '+'                 n  expr-cont                               doPossessiveStar       #  *+
michael@0:      default                expr-cont                               doStar
michael@0: 
michael@0: 
michael@0: #
michael@0: #  quant-plus     Scanning a '+' quantifier.  Need to look ahead to decide
michael@0: #                 between plain '+', '+?', '++'
michael@0: #
michael@0: quant-plus:
michael@0:      '?'                 n  expr-cont                               doNGPlus               #  *?
michael@0:      '+'                 n  expr-cont                               doPossessivePlus       #  *+
michael@0:      default                expr-cont                               doPlus
michael@0: 
michael@0: 
michael@0: #
michael@0: #  quant-opt  Scanning a '?' quantifier.  Need to look ahead to decide
michael@0: #                  between plain '?', '??', '?+'
michael@0: #
michael@0: quant-opt:
michael@0:      '?'                 n  expr-cont                               doNGOpt                 #  ??
michael@0:      '+'                 n  expr-cont                               doPossessiveOpt         #  ?+
michael@0:      default                expr-cont                               doOpt                   #  ?
michael@0: 
michael@0: 
michael@0: #
michael@0: #   Interval         scanning a '{', the opening delimiter for an interval specification
michael@0: #                                   {number} or {min, max} or {min,}
michael@0: #
michael@0: interval-open:
michael@0:     digit_char              interval-lower
michael@0:     default                 errorDeath                              doIntervalError
michael@0: 
michael@0: interval-lower:
michael@0:     digit_char           n  interval-lower                          doIntevalLowerDigit
michael@0:     ','			         n  interval-upper
michael@0:     '}'                  n  interval-type                           doIntervalSame             # {n}
michael@0:     default                 errorDeath                              doIntervalError
michael@0: 
michael@0: interval-upper:
michael@0:     digit_char           n  interval-upper                          doIntervalUpperDigit
michael@0:     '}'                  n  interval-type
michael@0:     default                 errorDeath                              doIntervalError
michael@0: 
michael@0: interval-type:
michael@0:     '?'                  n  expr-cont                               doNGInterval                # {n,m}?
michael@0:     '+'                  n  expr-cont                               doPossessiveInterval        # {n,m}+
michael@0:     default                 expr-cont                               doInterval                  # {m,n}
michael@0: 
michael@0: 
michael@0: #
michael@0: #  backslash        #  Backslash.  Figure out which of the \thingies we have encountered.
michael@0: #                                  The low level next-char function will have preprocessed
michael@0: #                                  some of them already; those won't come here.
michael@0: backslash:
michael@0:    'A'                   n  term                                    doBackslashA
michael@0:    'B'                   n  term                                    doBackslashB
michael@0:    'b'                   n  term                                    doBackslashb
michael@0:    'd'                   n  expr-quant                              doBackslashd
michael@0:    'D'                   n  expr-quant                              doBackslashD
michael@0:    'G'                   n  term                                    doBackslashG
michael@0:    'N'                      expr-quant                              doNamedChar      #   \N{NAME}  named char
michael@0:    'p'                      expr-quant                              doProperty       #   \p{Lu}  style property
michael@0:    'P'                      expr-quant                              doProperty
michael@0:    'Q'                   n  term                                    doEnterQuoteMode
michael@0:    'S'                   n  expr-quant                              doBackslashS
michael@0:    's'                   n  expr-quant                              doBackslashs
michael@0:    'W'                   n  expr-quant                              doBackslashW
michael@0:    'w'                   n  expr-quant                              doBackslashw
michael@0:    'X'                   n  expr-quant                              doBackslashX
michael@0:    'Z'                   n  term                                    doBackslashZ
michael@0:    'z'                   n  term                                    doBackslashz
michael@0:    digit_char            n  expr-quant                              doBackRef         #  Will scan multiple digits
michael@0:    eof                      errorDeath                              doEscapeError
michael@0:    default               n  expr-quant                              doEscapedLiteralChar
michael@0: 
michael@0: 
michael@0: 
michael@0: #
michael@0: # [set expression] parsing,
michael@0: #    All states involved in parsing set expressions have names beginning with "set-"
michael@0: #
michael@0: 
michael@0: set-open:
michael@0:    '^'                   n  set-open2                               doSetNegate
michael@0:    ':'                      set-posix                               doSetPosixProp
michael@0:    default                  set-open2
michael@0: 
michael@0: set-open2:
michael@0:    ']'                   n  set-after-lit                           doSetLiteral
michael@0:    default                  set-start
michael@0: 
michael@0: #  set-posix:
michael@0: #                  scanned a '[:'  If it really is a [:property:], doSetPosixProp will have
michael@0: #                  moved the scan to the closing ']'.  If it wasn't a property
michael@0: #                  expression, the scan will still be at the opening ':', which should
michael@0: #                  be interpreted as a normal set expression.
michael@0: set-posix:
michael@0:     ']'                  n   pop                                    doSetEnd
michael@0:     ':'                      set-start
michael@0:     default                  errorDeath                             doRuleError  # should not be possible.
michael@0: 
michael@0: #
michael@0: #   set-start   after the [ and special case leading characters (^ and/or ]) but before
michael@0: #               everything else.   A '-' is literal at this point.
michael@0: #
michael@0: set-start:
michael@0:     ']'                  n  pop                                     doSetEnd
michael@0:     '['                  n  set-open      ^set-after-set            doSetBeginUnion
michael@0:     '\'                  n  set-escape
michael@0:     '-'                  n  set-start-dash
michael@0:     '&'                  n  set-start-amp
michael@0:     default              n  set-after-lit                           doSetLiteral
michael@0: 
michael@0: #    set-start-dash    Turn "[--" into a syntax error.
michael@0: #                           "[-x" is good, - and x are literals.
michael@0: #
michael@0: set-start-dash:
michael@0:     '-'                     errorDeath                              doRuleError
michael@0:     default                 set-after-lit                           doSetAddDash
michael@0: 
michael@0: #    set-start-amp     Turn "[&&" into a syntax error.
michael@0: #                           "[&x" is good, & and x are literals.
michael@0: #
michael@0: set-start-amp:
michael@0:     '&'                     errorDeath                              doRuleError
michael@0:     default                 set-after-lit                           doSetAddAmp
michael@0: 
michael@0: #
michael@0: #   set-after-lit    The last thing scanned was a literal character within a set.
michael@0: #                    Can be followed by anything.  Single '-' or '&' are
michael@0: #                    literals in this context, not operators.
michael@0: set-after-lit:
michael@0:     ']'                  n  pop                                     doSetEnd
michael@0:     '['                  n  set-open      ^set-after-set            doSetBeginUnion
michael@0:     '-'                  n  set-lit-dash
michael@0:     '&'                  n  set-lit-amp
michael@0:     '\'                  n  set-escape
michael@0:     eof                     errorDeath                              doSetNoCloseError
michael@0:     default              n  set-after-lit                           doSetLiteral
michael@0: 
michael@0: set-after-set:
michael@0:     ']'                  n  pop                                     doSetEnd
michael@0:     '['                  n  set-open      ^set-after-set            doSetBeginUnion
michael@0:     '-'                  n  set-set-dash
michael@0:     '&'                  n  set-set-amp
michael@0:     '\'                  n  set-escape
michael@0:     eof                     errorDeath                              doSetNoCloseError
michael@0:     default              n  set-after-lit                           doSetLiteral
michael@0: 
michael@0: set-after-range:
michael@0:     ']'                  n  pop                                     doSetEnd
michael@0:     '['                  n  set-open      ^set-after-set            doSetBeginUnion
michael@0:     '-'                  n  set-range-dash
michael@0:     '&'                  n  set-range-amp
michael@0:     '\'                  n  set-escape
michael@0:     eof                     errorDeath                              doSetNoCloseError
michael@0:     default              n  set-after-lit                           doSetLiteral
michael@0:     
michael@0: 
michael@0: # set-after-op
michael@0: #     After a --  or &&
michael@0: #     It is an error to close a set at this point.
michael@0: #
michael@0: set-after-op:
michael@0:     '['                  n  set-open         ^set-after-set         doSetBeginUnion
michael@0:     ']'                     errorDeath                              doSetOpError
michael@0:     '\'                  n  set-escape
michael@0:     default              n  set-after-lit                           doSetLiteral
michael@0: 
michael@0: #
michael@0: #   set-set-amp
michael@0: #      Have scanned [[set]&
michael@0: #      Could be a '&' intersection operator, if a set follows.
michael@0: #      Could be the start of a '&&' operator.
michael@0: #      Otherewise is a literal.
michael@0: set-set-amp:
michael@0:     '['                  n  set-open      ^set-after-set           doSetBeginIntersection1
michael@0:     '&'                  n  set-after-op                           doSetIntersection2
michael@0:     default                 set-after-lit                          doSetAddAmp
michael@0: 
michael@0: 
michael@0: # set-lit-amp   Have scanned "[literals&"
michael@0: #               Could be a start of "&&" operator or a literal
michael@0: #               In [abc&[def]],   the '&' is a literal
michael@0: #
michael@0: set-lit-amp:
michael@0:     '&'                  n  set-after-op                            doSetIntersection2
michael@0:     default                 set-after-lit                           doSetAddAmp
michael@0: 
michael@0: 
michael@0: #
michael@0: #  set-set-dash
michael@0: #      Have scanned [set]-
michael@0: #      Could be a '-' difference operator, if a [set] follows.
michael@0: #      Could be the start of a '--' operator.
michael@0: #      Otherewise is a literal.
michael@0: set-set-dash:
michael@0:     '['                  n  set-open      ^set-after-set           doSetBeginDifference1
michael@0:     '-'                  n  set-after-op                           doSetDifference2
michael@0:     default                 set-after-lit                          doSetAddDash
michael@0: 
michael@0: 
michael@0: #
michael@0: #  set-range-dash
michael@0: #      scanned  a-b-  or \w-
michael@0: #         any set or range like item where the trailing single '-' should
michael@0: #         be literal, not a set difference operation.
michael@0: #         A trailing "--" is still a difference operator.
michael@0: set-range-dash:
michael@0:     '-'                  n  set-after-op                           doSetDifference2
michael@0:     default                 set-after-lit                          doSetAddDash
michael@0: 
michael@0: 
michael@0: set-range-amp:
michael@0:     '&'                  n  set-after-op                           doSetIntersection2
michael@0:     default                 set-after-lit                          doSetAddAmp
michael@0: 
michael@0: 
michael@0: #  set-lit-dash
michael@0: #     Have scanned "[literals-" Could be a range or a -- operator or a literal
michael@0: #     In [abc-[def]], the '-' is a literal (confirmed with a Java test)
michael@0: #        [abc-\p{xx}  the '-' is an error
michael@0: #        [abc-]       the '-' is a literal
michael@0: #        [ab-xy]      the '-' is a range
michael@0: #
michael@0: set-lit-dash:
michael@0:     '-'                  n  set-after-op                            doSetDifference2
michael@0:     '['                     set-after-lit                           doSetAddDash
michael@0:     ']'                     set-after-lit                           doSetAddDash
michael@0:     '\'                  n  set-lit-dash-escape
michael@0:     default              n  set-after-range                         doSetRange
michael@0: 
michael@0: # set-lit-dash-escape
michael@0: #
michael@0: #    scanned "[literal-\"
michael@0: #    Could be a range, if the \ introduces an escaped literal char or a named char.
michael@0: #    Otherwise it is an error.
michael@0: #
michael@0: set-lit-dash-escape:
michael@0:    's'                      errorDeath                             doSetOpError
michael@0:    'S'                      errorDeath                             doSetOpError
michael@0:    'w'                      errorDeath                             doSetOpError
michael@0:    'W'                      errorDeath                             doSetOpError
michael@0:    'd'                      errorDeath                             doSetOpError
michael@0:    'D'                      errorDeath                             doSetOpError
michael@0:    'N'                      set-after-range                        doSetNamedRange
michael@0:    default               n  set-after-range                        doSetRange
michael@0: 
michael@0:    
michael@0: #
michael@0: #  set-escape
michael@0: #       Common back-slash escape processing within set expressions
michael@0: #
michael@0: set-escape:
michael@0:    'p'                      set-after-set                           doSetProp
michael@0:    'P'                      set-after-set                           doSetProp
michael@0:    'N'                      set-after-lit                           doSetNamedChar
michael@0:    's'                   n  set-after-range                         doSetBackslash_s
michael@0:    'S'                   n  set-after-range                         doSetBackslash_S
michael@0:    'w'                   n  set-after-range                         doSetBackslash_w
michael@0:    'W'                   n  set-after-range                         doSetBackslash_W
michael@0:    'd'                   n  set-after-range                         doSetBackslash_d
michael@0:    'D'                   n  set-after-range                         doSetBackslash_D
michael@0:    default               n  set-after-lit                           doSetLiteralEscaped 
michael@0: 
michael@0: #
michael@0: # set-finish
michael@0: #     Have just encountered the final ']' that completes a [set], and
michael@0: #     arrived here via a pop.  From here, we exit the set parsing world, and go
michael@0: #     back to generic regular expression parsing.
michael@0: #
michael@0: set-finish:
michael@0:     default                 expr-quant                              doSetFinish
michael@0: 
michael@0: 
michael@0: #
michael@0: # errorDeath.   This state is specified as the next state whenever a syntax error
michael@0: #               in the source rules is detected.  Barring bugs, the state machine will never
michael@0: #               actually get here, but will stop because of the action associated with the error.
michael@0: #               But, just in case, this state asks the state machine to exit.
michael@0: errorDeath:
michael@0:     default              n errorDeath                               doExit
michael@0: 
michael@0: