michael@0: michael@0: // michael@0: // file: rbbiscan.cpp michael@0: // michael@0: // Copyright (C) 2002-2012, International Business Machines Corporation and others. michael@0: // All Rights Reserved. michael@0: // michael@0: // This file contains the Rule Based Break Iterator Rule Builder functions for michael@0: // scanning the rules and assembling a parse tree. This is the first phase michael@0: // of compiling the rules. michael@0: // michael@0: // The overall of the rules is managed by class RBBIRuleBuilder, which will michael@0: // create and use an instance of this class as part of the process. michael@0: // michael@0: michael@0: #include "unicode/utypes.h" michael@0: michael@0: #if !UCONFIG_NO_BREAK_ITERATION michael@0: michael@0: #include "unicode/unistr.h" michael@0: #include "unicode/uniset.h" michael@0: #include "unicode/uchar.h" michael@0: #include "unicode/uchriter.h" michael@0: #include "unicode/parsepos.h" michael@0: #include "unicode/parseerr.h" michael@0: #include "cmemory.h" michael@0: #include "cstring.h" michael@0: michael@0: #include "rbbirpt.h" // Contains state table for the rbbi rules parser. michael@0: // generated by a Perl script. michael@0: #include "rbbirb.h" michael@0: #include "rbbinode.h" michael@0: #include "rbbiscan.h" michael@0: #include "rbbitblb.h" michael@0: michael@0: #include "uassert.h" michael@0: michael@0: #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // Unicode Set init strings for each of the character classes needed for parsing a rule file. michael@0: // (Initialized with hex values for portability to EBCDIC based machines. michael@0: // Really ugly, but there's no good way to avoid it.) michael@0: // michael@0: // The sets are referred to by name in the rbbirpt.txt, which is the michael@0: // source form of the state transition table for the RBBI rule parser. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: static const UChar gRuleSet_rule_char_pattern[] = { michael@0: // [ ^ [ \ p { Z } \ u 0 0 2 0 michael@0: 0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30, michael@0: // - \ u 0 0 7 f ] - [ \ p michael@0: 0x2d, 0x5c, 0x75, 0x30, 0x30, 0x37, 0x66, 0x5d, 0x2d, 0x5b, 0x5c, 0x70, michael@0: // { L } ] - [ \ p { N } ] ] michael@0: 0x7b, 0x4c, 0x7d, 0x5d, 0x2d, 0x5b, 0x5c, 0x70, 0x7b, 0x4e, 0x7d, 0x5d, 0x5d, 0}; michael@0: michael@0: static const UChar gRuleSet_name_char_pattern[] = { michael@0: // [ _ \ p { L } \ p { N } ] michael@0: 0x5b, 0x5f, 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x5c, 0x70, 0x7b, 0x4e, 0x7d, 0x5d, 0}; michael@0: michael@0: static const UChar gRuleSet_digit_char_pattern[] = { michael@0: // [ 0 - 9 ] michael@0: 0x5b, 0x30, 0x2d, 0x39, 0x5d, 0}; michael@0: michael@0: static const UChar gRuleSet_name_start_char_pattern[] = { michael@0: // [ _ \ p { L } ] michael@0: 0x5b, 0x5f, 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x5d, 0 }; michael@0: michael@0: static const UChar kAny[] = {0x61, 0x6e, 0x79, 0x00}; // "any" michael@0: michael@0: michael@0: U_CDECL_BEGIN michael@0: static void U_CALLCONV RBBISetTable_deleter(void *p) { michael@0: icu::RBBISetTableEl *px = (icu::RBBISetTableEl *)p; michael@0: delete px->key; michael@0: // Note: px->val is owned by the linked list "fSetsListHead" in scanner. michael@0: // Don't delete the value nodes here. michael@0: uprv_free(px); michael@0: } michael@0: U_CDECL_END michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // Constructor. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: RBBIRuleScanner::RBBIRuleScanner(RBBIRuleBuilder *rb) michael@0: { michael@0: fRB = rb; michael@0: fStackPtr = 0; michael@0: fStack[fStackPtr] = 0; michael@0: fNodeStackPtr = 0; michael@0: fRuleNum = 0; michael@0: fNodeStack[0] = NULL; michael@0: michael@0: fSymbolTable = NULL; michael@0: fSetTable = NULL; michael@0: michael@0: fScanIndex = 0; michael@0: fNextIndex = 0; michael@0: michael@0: fReverseRule = FALSE; michael@0: fLookAheadRule = FALSE; michael@0: michael@0: fLineNum = 1; michael@0: fCharNum = 0; michael@0: fQuoteMode = FALSE; michael@0: michael@0: // Do not check status until after all critical fields are sufficiently initialized michael@0: // that the destructor can run cleanly. michael@0: if (U_FAILURE(*rb->fStatus)) { michael@0: return; michael@0: } michael@0: michael@0: // michael@0: // Set up the constant Unicode Sets. michael@0: // Note: These could be made static, lazily initialized, and shared among michael@0: // all instances of RBBIRuleScanners. BUT this is quite a bit simpler, michael@0: // and the time to build these few sets should be small compared to a michael@0: // full break iterator build. michael@0: fRuleSets[kRuleSet_rule_char-128] michael@0: = UnicodeSet(UnicodeString(gRuleSet_rule_char_pattern), *rb->fStatus); michael@0: // fRuleSets[kRuleSet_white_space-128] = [:Pattern_White_Space:] michael@0: fRuleSets[kRuleSet_white_space-128]. michael@0: add(9, 0xd).add(0x20).add(0x85).add(0x200e, 0x200f).add(0x2028, 0x2029); michael@0: fRuleSets[kRuleSet_name_char-128] michael@0: = UnicodeSet(UnicodeString(gRuleSet_name_char_pattern), *rb->fStatus); michael@0: fRuleSets[kRuleSet_name_start_char-128] michael@0: = UnicodeSet(UnicodeString(gRuleSet_name_start_char_pattern), *rb->fStatus); michael@0: fRuleSets[kRuleSet_digit_char-128] michael@0: = UnicodeSet(UnicodeString(gRuleSet_digit_char_pattern), *rb->fStatus); michael@0: if (*rb->fStatus == U_ILLEGAL_ARGUMENT_ERROR) { michael@0: // This case happens if ICU's data is missing. UnicodeSet tries to look up property michael@0: // names from the init string, can't find them, and claims an illegal argument. michael@0: // Change the error so that the actual problem will be clearer to users. michael@0: *rb->fStatus = U_BRK_INIT_ERROR; michael@0: } michael@0: if (U_FAILURE(*rb->fStatus)) { michael@0: return; michael@0: } michael@0: michael@0: fSymbolTable = new RBBISymbolTable(this, rb->fRules, *rb->fStatus); michael@0: if (fSymbolTable == NULL) { michael@0: *rb->fStatus = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: fSetTable = uhash_open(uhash_hashUnicodeString, uhash_compareUnicodeString, NULL, rb->fStatus); michael@0: if (U_FAILURE(*rb->fStatus)) { michael@0: return; michael@0: } michael@0: uhash_setValueDeleter(fSetTable, RBBISetTable_deleter); michael@0: } michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // Destructor michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: RBBIRuleScanner::~RBBIRuleScanner() { michael@0: delete fSymbolTable; michael@0: if (fSetTable != NULL) { michael@0: uhash_close(fSetTable); michael@0: fSetTable = NULL; michael@0: michael@0: } michael@0: michael@0: michael@0: // Node Stack. michael@0: // Normally has one entry, which is the entire parse tree for the rules. michael@0: // If errors occured, there may be additional subtrees left on the stack. michael@0: while (fNodeStackPtr > 0) { michael@0: delete fNodeStack[fNodeStackPtr]; michael@0: fNodeStackPtr--; michael@0: } michael@0: michael@0: } michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // doParseAction Do some action during rule parsing. michael@0: // Called by the parse state machine. michael@0: // Actions build the parse tree and Unicode Sets, michael@0: // and maintain the parse stack for nested expressions. michael@0: // michael@0: // TODO: unify EParseAction and RBBI_RuleParseAction enum types. michael@0: // They represent exactly the same thing. They're separate michael@0: // only to work around enum forward declaration restrictions michael@0: // in some compilers, while at the same time avoiding multiple michael@0: // definitions problems. I'm sure that there's a better way. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: UBool RBBIRuleScanner::doParseActions(int32_t action) michael@0: { michael@0: RBBINode *n = NULL; michael@0: michael@0: UBool returnVal = TRUE; michael@0: michael@0: switch (action) { michael@0: michael@0: case doExprStart: michael@0: pushNewNode(RBBINode::opStart); michael@0: fRuleNum++; michael@0: break; michael@0: michael@0: michael@0: case doExprOrOperator: michael@0: { michael@0: fixOpStack(RBBINode::precOpCat); michael@0: RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; michael@0: RBBINode *orNode = pushNewNode(RBBINode::opOr); michael@0: orNode->fLeftChild = operandNode; michael@0: operandNode->fParent = orNode; michael@0: } michael@0: break; michael@0: michael@0: case doExprCatOperator: michael@0: // concatenation operator. michael@0: // For the implicit concatenation of adjacent terms in an expression that are michael@0: // not separated by any other operator. Action is invoked between the michael@0: // actions for the two terms. michael@0: { michael@0: fixOpStack(RBBINode::precOpCat); michael@0: RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; michael@0: RBBINode *catNode = pushNewNode(RBBINode::opCat); michael@0: catNode->fLeftChild = operandNode; michael@0: operandNode->fParent = catNode; michael@0: } michael@0: break; michael@0: michael@0: case doLParen: michael@0: // Open Paren. michael@0: // The openParen node is a dummy operation type with a low precedence, michael@0: // which has the affect of ensuring that any real binary op that michael@0: // follows within the parens binds more tightly to the operands than michael@0: // stuff outside of the parens. michael@0: pushNewNode(RBBINode::opLParen); michael@0: break; michael@0: michael@0: case doExprRParen: michael@0: fixOpStack(RBBINode::precLParen); michael@0: break; michael@0: michael@0: case doNOP: michael@0: break; michael@0: michael@0: case doStartAssign: michael@0: // We've just scanned "$variable = " michael@0: // The top of the node stack has the $variable ref node. michael@0: michael@0: // Save the start position of the RHS text in the StartExpression node michael@0: // that precedes the $variableReference node on the stack. michael@0: // This will eventually be used when saving the full $variable replacement michael@0: // text as a string. michael@0: n = fNodeStack[fNodeStackPtr-1]; michael@0: n->fFirstPos = fNextIndex; // move past the '=' michael@0: michael@0: // Push a new start-of-expression node; needed to keep parse of the michael@0: // RHS expression happy. michael@0: pushNewNode(RBBINode::opStart); michael@0: break; michael@0: michael@0: michael@0: michael@0: michael@0: case doEndAssign: michael@0: { michael@0: // We have reached the end of an assignement statement. michael@0: // Current scan char is the ';' that terminates the assignment. michael@0: michael@0: // Terminate expression, leaves expression parse tree rooted in TOS node. michael@0: fixOpStack(RBBINode::precStart); michael@0: michael@0: RBBINode *startExprNode = fNodeStack[fNodeStackPtr-2]; michael@0: RBBINode *varRefNode = fNodeStack[fNodeStackPtr-1]; michael@0: RBBINode *RHSExprNode = fNodeStack[fNodeStackPtr]; michael@0: michael@0: // Save original text of right side of assignment, excluding the terminating ';' michael@0: // in the root of the node for the right-hand-side expression. michael@0: RHSExprNode->fFirstPos = startExprNode->fFirstPos; michael@0: RHSExprNode->fLastPos = fScanIndex; michael@0: fRB->fRules.extractBetween(RHSExprNode->fFirstPos, RHSExprNode->fLastPos, RHSExprNode->fText); michael@0: michael@0: // Expression parse tree becomes l. child of the $variable reference node. michael@0: varRefNode->fLeftChild = RHSExprNode; michael@0: RHSExprNode->fParent = varRefNode; michael@0: michael@0: // Make a symbol table entry for the $variableRef node. michael@0: fSymbolTable->addEntry(varRefNode->fText, varRefNode, *fRB->fStatus); michael@0: if (U_FAILURE(*fRB->fStatus)) { michael@0: // This is a round-about way to get the parse position set michael@0: // so that duplicate symbols error messages include a line number. michael@0: UErrorCode t = *fRB->fStatus; michael@0: *fRB->fStatus = U_ZERO_ERROR; michael@0: error(t); michael@0: } michael@0: michael@0: // Clean up the stack. michael@0: delete startExprNode; michael@0: fNodeStackPtr-=3; michael@0: break; michael@0: } michael@0: michael@0: case doEndOfRule: michael@0: { michael@0: fixOpStack(RBBINode::precStart); // Terminate expression, leaves expression michael@0: if (U_FAILURE(*fRB->fStatus)) { // parse tree rooted in TOS node. michael@0: break; michael@0: } michael@0: #ifdef RBBI_DEBUG michael@0: if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "rtree")) {printNodeStack("end of rule");} michael@0: #endif michael@0: U_ASSERT(fNodeStackPtr == 1); michael@0: michael@0: // If this rule includes a look-ahead '/', add a endMark node to the michael@0: // expression tree. michael@0: if (fLookAheadRule) { michael@0: RBBINode *thisRule = fNodeStack[fNodeStackPtr]; michael@0: RBBINode *endNode = pushNewNode(RBBINode::endMark); michael@0: RBBINode *catNode = pushNewNode(RBBINode::opCat); michael@0: fNodeStackPtr -= 2; michael@0: catNode->fLeftChild = thisRule; michael@0: catNode->fRightChild = endNode; michael@0: fNodeStack[fNodeStackPtr] = catNode; michael@0: endNode->fVal = fRuleNum; michael@0: endNode->fLookAheadEnd = TRUE; michael@0: } michael@0: michael@0: // All rule expressions are ORed together. michael@0: // The ';' that terminates an expression really just functions as a '|' with michael@0: // a low operator prededence. michael@0: // michael@0: // Each of the four sets of rules are collected separately. michael@0: // (forward, reverse, safe_forward, safe_reverse) michael@0: // OR this rule into the appropriate group of them. michael@0: // michael@0: RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : fRB->fDefaultTree); michael@0: michael@0: if (*destRules != NULL) { michael@0: // This is not the first rule encounted. michael@0: // OR previous stuff (from *destRules) michael@0: // with the current rule expression (on the Node Stack) michael@0: // with the resulting OR expression going to *destRules michael@0: // michael@0: RBBINode *thisRule = fNodeStack[fNodeStackPtr]; michael@0: RBBINode *prevRules = *destRules; michael@0: RBBINode *orNode = pushNewNode(RBBINode::opOr); michael@0: orNode->fLeftChild = prevRules; michael@0: prevRules->fParent = orNode; michael@0: orNode->fRightChild = thisRule; michael@0: thisRule->fParent = orNode; michael@0: *destRules = orNode; michael@0: } michael@0: else michael@0: { michael@0: // This is the first rule encountered (for this direction). michael@0: // Just move its parse tree from the stack to *destRules. michael@0: *destRules = fNodeStack[fNodeStackPtr]; michael@0: } michael@0: fReverseRule = FALSE; // in preparation for the next rule. michael@0: fLookAheadRule = FALSE; michael@0: fNodeStackPtr = 0; michael@0: } michael@0: break; michael@0: michael@0: michael@0: case doRuleError: michael@0: error(U_BRK_RULE_SYNTAX); michael@0: returnVal = FALSE; michael@0: break; michael@0: michael@0: michael@0: case doVariableNameExpectedErr: michael@0: error(U_BRK_RULE_SYNTAX); michael@0: break; michael@0: michael@0: michael@0: // michael@0: // Unary operands + ? * michael@0: // These all appear after the operand to which they apply. michael@0: // When we hit one, the operand (may be a whole sub expression) michael@0: // will be on the top of the stack. michael@0: // Unary Operator becomes TOS, with the old TOS as its one child. michael@0: case doUnaryOpPlus: michael@0: { michael@0: RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; michael@0: RBBINode *plusNode = pushNewNode(RBBINode::opPlus); michael@0: plusNode->fLeftChild = operandNode; michael@0: operandNode->fParent = plusNode; michael@0: } michael@0: break; michael@0: michael@0: case doUnaryOpQuestion: michael@0: { michael@0: RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; michael@0: RBBINode *qNode = pushNewNode(RBBINode::opQuestion); michael@0: qNode->fLeftChild = operandNode; michael@0: operandNode->fParent = qNode; michael@0: } michael@0: break; michael@0: michael@0: case doUnaryOpStar: michael@0: { michael@0: RBBINode *operandNode = fNodeStack[fNodeStackPtr--]; michael@0: RBBINode *starNode = pushNewNode(RBBINode::opStar); michael@0: starNode->fLeftChild = operandNode; michael@0: operandNode->fParent = starNode; michael@0: } michael@0: break; michael@0: michael@0: case doRuleChar: michael@0: // A "Rule Character" is any single character that is a literal part michael@0: // of the regular expression. Like a, b and c in the expression "(abc*) | [:L:]" michael@0: // These are pretty uncommon in break rules; the terms are more commonly michael@0: // sets. To keep things uniform, treat these characters like as michael@0: // sets that just happen to contain only one character. michael@0: { michael@0: n = pushNewNode(RBBINode::setRef); michael@0: findSetFor(UnicodeString(fC.fChar), n); michael@0: n->fFirstPos = fScanIndex; michael@0: n->fLastPos = fNextIndex; michael@0: fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); michael@0: break; michael@0: } michael@0: michael@0: case doDotAny: michael@0: // scanned a ".", meaning match any single character. michael@0: { michael@0: n = pushNewNode(RBBINode::setRef); michael@0: findSetFor(UnicodeString(TRUE, kAny, 3), n); michael@0: n->fFirstPos = fScanIndex; michael@0: n->fLastPos = fNextIndex; michael@0: fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); michael@0: break; michael@0: } michael@0: michael@0: case doSlash: michael@0: // Scanned a '/', which identifies a look-ahead break position in a rule. michael@0: n = pushNewNode(RBBINode::lookAhead); michael@0: n->fVal = fRuleNum; michael@0: n->fFirstPos = fScanIndex; michael@0: n->fLastPos = fNextIndex; michael@0: fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); michael@0: fLookAheadRule = TRUE; michael@0: break; michael@0: michael@0: michael@0: case doStartTagValue: michael@0: // Scanned a '{', the opening delimiter for a tag value within a rule. michael@0: n = pushNewNode(RBBINode::tag); michael@0: n->fVal = 0; michael@0: n->fFirstPos = fScanIndex; michael@0: n->fLastPos = fNextIndex; michael@0: break; michael@0: michael@0: case doTagDigit: michael@0: // Just scanned a decimal digit that's part of a tag value michael@0: { michael@0: n = fNodeStack[fNodeStackPtr]; michael@0: uint32_t v = u_charDigitValue(fC.fChar); michael@0: U_ASSERT(v < 10); michael@0: n->fVal = n->fVal*10 + v; michael@0: break; michael@0: } michael@0: michael@0: case doTagValue: michael@0: n = fNodeStack[fNodeStackPtr]; michael@0: n->fLastPos = fNextIndex; michael@0: fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); michael@0: break; michael@0: michael@0: case doTagExpectedError: michael@0: error(U_BRK_MALFORMED_RULE_TAG); michael@0: returnVal = FALSE; michael@0: break; michael@0: michael@0: case doOptionStart: michael@0: // Scanning a !!option. At the start of string. michael@0: fOptionStart = fScanIndex; michael@0: break; michael@0: michael@0: case doOptionEnd: michael@0: { michael@0: UnicodeString opt(fRB->fRules, fOptionStart, fScanIndex-fOptionStart); michael@0: if (opt == UNICODE_STRING("chain", 5)) { michael@0: fRB->fChainRules = TRUE; michael@0: } else if (opt == UNICODE_STRING("LBCMNoChain", 11)) { michael@0: fRB->fLBCMNoChain = TRUE; michael@0: } else if (opt == UNICODE_STRING("forward", 7)) { michael@0: fRB->fDefaultTree = &fRB->fForwardTree; michael@0: } else if (opt == UNICODE_STRING("reverse", 7)) { michael@0: fRB->fDefaultTree = &fRB->fReverseTree; michael@0: } else if (opt == UNICODE_STRING("safe_forward", 12)) { michael@0: fRB->fDefaultTree = &fRB->fSafeFwdTree; michael@0: } else if (opt == UNICODE_STRING("safe_reverse", 12)) { michael@0: fRB->fDefaultTree = &fRB->fSafeRevTree; michael@0: } else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) { michael@0: fRB->fLookAheadHardBreak = TRUE; michael@0: } else { michael@0: error(U_BRK_UNRECOGNIZED_OPTION); michael@0: } michael@0: } michael@0: break; michael@0: michael@0: case doReverseDir: michael@0: fReverseRule = TRUE; michael@0: break; michael@0: michael@0: case doStartVariableName: michael@0: n = pushNewNode(RBBINode::varRef); michael@0: if (U_FAILURE(*fRB->fStatus)) { michael@0: break; michael@0: } michael@0: n->fFirstPos = fScanIndex; michael@0: break; michael@0: michael@0: case doEndVariableName: michael@0: n = fNodeStack[fNodeStackPtr]; michael@0: if (n==NULL || n->fType != RBBINode::varRef) { michael@0: error(U_BRK_INTERNAL_ERROR); michael@0: break; michael@0: } michael@0: n->fLastPos = fScanIndex; michael@0: fRB->fRules.extractBetween(n->fFirstPos+1, n->fLastPos, n->fText); michael@0: // Look the newly scanned name up in the symbol table michael@0: // If there's an entry, set the l. child of the var ref to the replacement expression. michael@0: // (We also pass through here when scanning assignments, but no harm is done, other michael@0: // than a slight wasted effort that seems hard to avoid. Lookup will be null) michael@0: n->fLeftChild = fSymbolTable->lookupNode(n->fText); michael@0: break; michael@0: michael@0: case doCheckVarDef: michael@0: n = fNodeStack[fNodeStackPtr]; michael@0: if (n->fLeftChild == NULL) { michael@0: error(U_BRK_UNDEFINED_VARIABLE); michael@0: returnVal = FALSE; michael@0: } michael@0: break; michael@0: michael@0: case doExprFinished: michael@0: break; michael@0: michael@0: case doRuleErrorAssignExpr: michael@0: error(U_BRK_ASSIGN_ERROR); michael@0: returnVal = FALSE; michael@0: break; michael@0: michael@0: case doExit: michael@0: returnVal = FALSE; michael@0: break; michael@0: michael@0: case doScanUnicodeSet: michael@0: scanSet(); michael@0: break; michael@0: michael@0: default: michael@0: error(U_BRK_INTERNAL_ERROR); michael@0: returnVal = FALSE; michael@0: break; michael@0: } michael@0: return returnVal; michael@0: } michael@0: michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // Error Report a rule parse error. michael@0: // Only report it if no previous error has been recorded. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: void RBBIRuleScanner::error(UErrorCode e) { michael@0: if (U_SUCCESS(*fRB->fStatus)) { michael@0: *fRB->fStatus = e; michael@0: if (fRB->fParseError) { michael@0: fRB->fParseError->line = fLineNum; michael@0: fRB->fParseError->offset = fCharNum; michael@0: fRB->fParseError->preContext[0] = 0; michael@0: fRB->fParseError->preContext[0] = 0; michael@0: } michael@0: } michael@0: } michael@0: michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // fixOpStack The parse stack holds partially assembled chunks of the parse tree. michael@0: // An entry on the stack may be as small as a single setRef node, michael@0: // or as large as the parse tree michael@0: // for an entire expression (this will be the one item left on the stack michael@0: // when the parsing of an RBBI rule completes. michael@0: // michael@0: // This function is called when a binary operator is encountered. michael@0: // It looks back up the stack for operators that are not yet associated michael@0: // with a right operand, and if the precedence of the stacked operator >= michael@0: // the precedence of the current operator, binds the operand left, michael@0: // to the previously encountered operator. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: void RBBIRuleScanner::fixOpStack(RBBINode::OpPrecedence p) { michael@0: RBBINode *n; michael@0: // printNodeStack("entering fixOpStack()"); michael@0: for (;;) { michael@0: n = fNodeStack[fNodeStackPtr-1]; // an operator node michael@0: if (n->fPrecedence == 0) { michael@0: RBBIDebugPuts("RBBIRuleScanner::fixOpStack, bad operator node"); michael@0: error(U_BRK_INTERNAL_ERROR); michael@0: return; michael@0: } michael@0: michael@0: if (n->fPrecedence < p || n->fPrecedence <= RBBINode::precLParen) { michael@0: // The most recent operand goes with the current operator, michael@0: // not with the previously stacked one. michael@0: break; michael@0: } michael@0: // Stack operator is a binary op ( '|' or concatenation) michael@0: // TOS operand becomes right child of this operator. michael@0: // Resulting subexpression becomes the TOS operand. michael@0: n->fRightChild = fNodeStack[fNodeStackPtr]; michael@0: fNodeStack[fNodeStackPtr]->fParent = n; michael@0: fNodeStackPtr--; michael@0: // printNodeStack("looping in fixOpStack() "); michael@0: } michael@0: michael@0: if (p <= RBBINode::precLParen) { michael@0: // Scan is at a right paren or end of expression. michael@0: // The scanned item must match the stack, or else there was an error. michael@0: // Discard the left paren (or start expr) node from the stack, michael@0: // leaving the completed (sub)expression as TOS. michael@0: if (n->fPrecedence != p) { michael@0: // Right paren encountered matched start of expression node, or michael@0: // end of expression matched with a left paren node. michael@0: error(U_BRK_MISMATCHED_PAREN); michael@0: } michael@0: fNodeStack[fNodeStackPtr-1] = fNodeStack[fNodeStackPtr]; michael@0: fNodeStackPtr--; michael@0: // Delete the now-discarded LParen or Start node. michael@0: delete n; michael@0: } michael@0: // printNodeStack("leaving fixOpStack()"); michael@0: } michael@0: michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // findSetFor given a UnicodeString, michael@0: // - find the corresponding Unicode Set (uset node) michael@0: // (create one if necessary) michael@0: // - Set fLeftChild of the caller's node (should be a setRef node) michael@0: // to the uset node michael@0: // Maintain a hash table of uset nodes, so the same one is always used michael@0: // for the same string. michael@0: // If a "to adopt" set is provided and we haven't seen this key before, michael@0: // add the provided set to the hash table. michael@0: // If the string is one (32 bit) char in length, the set contains michael@0: // just one element which is the char in question. michael@0: // If the string is "any", return a set containing all chars. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: void RBBIRuleScanner::findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt) { michael@0: michael@0: RBBISetTableEl *el; michael@0: michael@0: // First check whether we've already cached a set for this string. michael@0: // If so, just use the cached set in the new node. michael@0: // delete any set provided by the caller, since we own it. michael@0: el = (RBBISetTableEl *)uhash_get(fSetTable, &s); michael@0: if (el != NULL) { michael@0: delete setToAdopt; michael@0: node->fLeftChild = el->val; michael@0: U_ASSERT(node->fLeftChild->fType == RBBINode::uset); michael@0: return; michael@0: } michael@0: michael@0: // Haven't seen this set before. michael@0: // If the caller didn't provide us with a prebuilt set, michael@0: // create a new UnicodeSet now. michael@0: if (setToAdopt == NULL) { michael@0: if (s.compare(kAny, -1) == 0) { michael@0: setToAdopt = new UnicodeSet(0x000000, 0x10ffff); michael@0: } else { michael@0: UChar32 c; michael@0: c = s.char32At(0); michael@0: setToAdopt = new UnicodeSet(c, c); michael@0: } michael@0: } michael@0: michael@0: // michael@0: // Make a new uset node to refer to this UnicodeSet michael@0: // This new uset node becomes the child of the caller's setReference node. michael@0: // michael@0: RBBINode *usetNode = new RBBINode(RBBINode::uset); michael@0: if (usetNode == NULL) { michael@0: error(U_MEMORY_ALLOCATION_ERROR); michael@0: return; michael@0: } michael@0: usetNode->fInputSet = setToAdopt; michael@0: usetNode->fParent = node; michael@0: node->fLeftChild = usetNode; michael@0: usetNode->fText = s; michael@0: michael@0: michael@0: // michael@0: // Add the new uset node to the list of all uset nodes. michael@0: // michael@0: fRB->fUSetNodes->addElement(usetNode, *fRB->fStatus); michael@0: michael@0: michael@0: // michael@0: // Add the new set to the set hash table. michael@0: // michael@0: el = (RBBISetTableEl *)uprv_malloc(sizeof(RBBISetTableEl)); michael@0: UnicodeString *tkey = new UnicodeString(s); michael@0: if (tkey == NULL || el == NULL || setToAdopt == NULL) { michael@0: // Delete to avoid memory leak michael@0: delete tkey; michael@0: tkey = NULL; michael@0: uprv_free(el); michael@0: el = NULL; michael@0: delete setToAdopt; michael@0: setToAdopt = NULL; michael@0: michael@0: error(U_MEMORY_ALLOCATION_ERROR); michael@0: return; michael@0: } michael@0: el->key = tkey; michael@0: el->val = usetNode; michael@0: uhash_put(fSetTable, el->key, el, fRB->fStatus); michael@0: michael@0: return; michael@0: } michael@0: michael@0: michael@0: michael@0: // michael@0: // Assorted Unicode character constants. michael@0: // Numeric because there is no portable way to enter them as literals. michael@0: // (Think EBCDIC). michael@0: // michael@0: static const UChar chCR = 0x0d; // New lines, for terminating comments. michael@0: static const UChar chLF = 0x0a; michael@0: static const UChar chNEL = 0x85; // NEL newline variant michael@0: static const UChar chLS = 0x2028; // Unicode Line Separator michael@0: static const UChar chApos = 0x27; // single quote, for quoted chars. michael@0: static const UChar chPound = 0x23; // '#', introduces a comment. michael@0: static const UChar chBackSlash = 0x5c; // '\' introduces a char escape michael@0: static const UChar chLParen = 0x28; michael@0: static const UChar chRParen = 0x29; michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // stripRules Return a rules string without unnecessary michael@0: // characters. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) { michael@0: UnicodeString strippedRules; michael@0: int rulesLength = rules.length(); michael@0: for (int idx = 0; idx < rulesLength; ) { michael@0: UChar ch = rules[idx++]; michael@0: if (ch == chPound) { michael@0: while (idx < rulesLength michael@0: && ch != chCR && ch != chLF && ch != chNEL) michael@0: { michael@0: ch = rules[idx++]; michael@0: } michael@0: } michael@0: if (!u_isISOControl(ch)) { michael@0: strippedRules.append(ch); michael@0: } michael@0: } michael@0: // strippedRules = strippedRules.unescape(); michael@0: return strippedRules; michael@0: } michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // nextCharLL Low Level Next Char from rule input source. michael@0: // Get a char from the input character iterator, michael@0: // keep track of input position for error reporting. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: UChar32 RBBIRuleScanner::nextCharLL() { michael@0: UChar32 ch; michael@0: michael@0: if (fNextIndex >= fRB->fRules.length()) { michael@0: return (UChar32)-1; michael@0: } michael@0: ch = fRB->fRules.char32At(fNextIndex); michael@0: fNextIndex = fRB->fRules.moveIndex32(fNextIndex, 1); michael@0: michael@0: if (ch == chCR || michael@0: ch == chNEL || michael@0: ch == chLS || michael@0: (ch == chLF && fLastChar != chCR)) { michael@0: // Character is starting a new line. Bump up the line number, and michael@0: // reset the column to 0. michael@0: fLineNum++; michael@0: fCharNum=0; michael@0: if (fQuoteMode) { michael@0: error(U_BRK_NEW_LINE_IN_QUOTED_STRING); michael@0: fQuoteMode = FALSE; michael@0: } michael@0: } michael@0: else { michael@0: // Character is not starting a new line. Except in the case of a michael@0: // LF following a CR, increment the column position. michael@0: if (ch != chLF) { michael@0: fCharNum++; michael@0: } michael@0: } michael@0: fLastChar = ch; michael@0: return ch; michael@0: } michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // nextChar for rules scanning. At this level, we handle stripping michael@0: // out comments and processing backslash character escapes. michael@0: // The rest of the rules grammar is handled at the next level up. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: void RBBIRuleScanner::nextChar(RBBIRuleChar &c) { michael@0: michael@0: // Unicode Character constants needed for the processing done by nextChar(), michael@0: // in hex because literals wont work on EBCDIC machines. michael@0: michael@0: fScanIndex = fNextIndex; michael@0: c.fChar = nextCharLL(); michael@0: c.fEscaped = FALSE; michael@0: michael@0: // michael@0: // check for '' sequence. michael@0: // These are recognized in all contexts, whether in quoted text or not. michael@0: // michael@0: if (c.fChar == chApos) { michael@0: if (fRB->fRules.char32At(fNextIndex) == chApos) { michael@0: c.fChar = nextCharLL(); // get nextChar officially so character counts michael@0: c.fEscaped = TRUE; // stay correct. michael@0: } michael@0: else michael@0: { michael@0: // Single quote, by itself. michael@0: // Toggle quoting mode. michael@0: // Return either '(' or ')', because quotes cause a grouping of the quoted text. michael@0: fQuoteMode = !fQuoteMode; michael@0: if (fQuoteMode == TRUE) { michael@0: c.fChar = chLParen; michael@0: } else { michael@0: c.fChar = chRParen; michael@0: } michael@0: c.fEscaped = FALSE; // The paren that we return is not escaped. michael@0: return; michael@0: } michael@0: } michael@0: michael@0: if (fQuoteMode) { michael@0: c.fEscaped = TRUE; michael@0: } michael@0: else michael@0: { michael@0: // We are not in a 'quoted region' of the source. michael@0: // michael@0: if (c.fChar == chPound) { michael@0: // Start of a comment. Consume the rest of it. michael@0: // The new-line char that terminates the comment is always returned. michael@0: // It will be treated as white-space, and serves to break up anything michael@0: // that might otherwise incorrectly clump together with a comment in michael@0: // the middle (a variable name, for example.) michael@0: for (;;) { michael@0: c.fChar = nextCharLL(); michael@0: if (c.fChar == (UChar32)-1 || // EOF michael@0: c.fChar == chCR || michael@0: c.fChar == chLF || michael@0: c.fChar == chNEL || michael@0: c.fChar == chLS) {break;} michael@0: } michael@0: } michael@0: if (c.fChar == (UChar32)-1) { michael@0: return; michael@0: } michael@0: michael@0: // michael@0: // check for backslash escaped characters. michael@0: // Use UnicodeString::unescapeAt() to handle them. michael@0: // michael@0: if (c.fChar == chBackSlash) { michael@0: c.fEscaped = TRUE; michael@0: int32_t startX = fNextIndex; michael@0: c.fChar = fRB->fRules.unescapeAt(fNextIndex); michael@0: if (fNextIndex == startX) { michael@0: error(U_BRK_HEX_DIGITS_EXPECTED); michael@0: } michael@0: fCharNum += fNextIndex-startX; michael@0: } michael@0: } michael@0: // putc(c.fChar, stdout); michael@0: } michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // Parse RBBI rules. The state machine for rules parsing is here. michael@0: // The state tables are hand-written in the file rbbirpt.txt, michael@0: // and converted to the form used here by a perl michael@0: // script rbbicst.pl michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: void RBBIRuleScanner::parse() { michael@0: uint16_t state; michael@0: const RBBIRuleTableEl *tableEl; michael@0: michael@0: if (U_FAILURE(*fRB->fStatus)) { michael@0: return; michael@0: } michael@0: michael@0: state = 1; michael@0: nextChar(fC); michael@0: // michael@0: // Main loop for the rule parsing state machine. michael@0: // Runs once per state transition. michael@0: // Each time through optionally performs, depending on the state table, michael@0: // - an advance to the the next input char michael@0: // - an action to be performed. michael@0: // - pushing or popping a state to/from the local state return stack. michael@0: // michael@0: for (;;) { michael@0: // Bail out if anything has gone wrong. michael@0: // RBBI rule file parsing stops on the first error encountered. michael@0: if (U_FAILURE(*fRB->fStatus)) { michael@0: break; michael@0: } michael@0: michael@0: // Quit if state == 0. This is the normal way to exit the state machine. michael@0: // michael@0: if (state == 0) { michael@0: break; michael@0: } michael@0: michael@0: // Find the state table element that matches the input char from the rule, or the michael@0: // class of the input character. Start with the first table row for this michael@0: // state, then linearly scan forward until we find a row that matches the michael@0: // character. The last row for each state always matches all characters, so michael@0: // the search will stop there, if not before. michael@0: // michael@0: tableEl = &gRuleParseStateTable[state]; michael@0: #ifdef RBBI_DEBUG michael@0: if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { michael@0: RBBIDebugPrintf("char, line, col = (\'%c\', %d, %d) state=%s ", michael@0: fC.fChar, fLineNum, fCharNum, RBBIRuleStateNames[state]); michael@0: } michael@0: #endif michael@0: michael@0: for (;;) { michael@0: #ifdef RBBI_DEBUG michael@0: if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPrintf(".");} michael@0: #endif michael@0: if (tableEl->fCharClass < 127 && fC.fEscaped == FALSE && tableEl->fCharClass == fC.fChar) { michael@0: // Table row specified an individual character, not a set, and michael@0: // the input character is not escaped, and michael@0: // the input character matched it. michael@0: break; michael@0: } michael@0: if (tableEl->fCharClass == 255) { michael@0: // Table row specified default, match anything character class. michael@0: break; michael@0: } michael@0: if (tableEl->fCharClass == 254 && fC.fEscaped) { michael@0: // Table row specified "escaped" and the char was escaped. michael@0: break; michael@0: } michael@0: if (tableEl->fCharClass == 253 && fC.fEscaped && michael@0: (fC.fChar == 0x50 || fC.fChar == 0x70 )) { michael@0: // Table row specified "escaped P" and the char is either 'p' or 'P'. michael@0: break; michael@0: } michael@0: if (tableEl->fCharClass == 252 && fC.fChar == (UChar32)-1) { michael@0: // Table row specified eof and we hit eof on the input. michael@0: break; michael@0: } michael@0: michael@0: if (tableEl->fCharClass >= 128 && tableEl->fCharClass < 240 && // Table specs a char class && michael@0: fC.fEscaped == FALSE && // char is not escaped && michael@0: fC.fChar != (UChar32)-1) { // char is not EOF michael@0: U_ASSERT((tableEl->fCharClass-128) < LENGTHOF(fRuleSets)); michael@0: if (fRuleSets[tableEl->fCharClass-128].contains(fC.fChar)) { michael@0: // Table row specified a character class, or set of characters, michael@0: // and the current char matches it. michael@0: break; michael@0: } michael@0: } michael@0: michael@0: // No match on this row, advance to the next row for this state, michael@0: tableEl++; michael@0: } michael@0: if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "scan")) { RBBIDebugPuts("");} michael@0: michael@0: // michael@0: // We've found the row of the state table that matches the current input michael@0: // character from the rules string. michael@0: // Perform any action specified by this row in the state table. michael@0: if (doParseActions((int32_t)tableEl->fAction) == FALSE) { michael@0: // Break out of the state machine loop if the michael@0: // the action signalled some kind of error, or michael@0: // the action was to exit, occurs on normal end-of-rules-input. michael@0: break; michael@0: } michael@0: michael@0: if (tableEl->fPushState != 0) { michael@0: fStackPtr++; michael@0: if (fStackPtr >= kStackSize) { michael@0: error(U_BRK_INTERNAL_ERROR); michael@0: RBBIDebugPuts("RBBIRuleScanner::parse() - state stack overflow."); michael@0: fStackPtr--; michael@0: } michael@0: fStack[fStackPtr] = tableEl->fPushState; michael@0: } michael@0: michael@0: if (tableEl->fNextChar) { michael@0: nextChar(fC); michael@0: } michael@0: michael@0: // Get the next state from the table entry, or from the michael@0: // state stack if the next state was specified as "pop". michael@0: if (tableEl->fNextState != 255) { michael@0: state = tableEl->fNextState; michael@0: } else { michael@0: state = fStack[fStackPtr]; michael@0: fStackPtr--; michael@0: if (fStackPtr < 0) { michael@0: error(U_BRK_INTERNAL_ERROR); michael@0: RBBIDebugPuts("RBBIRuleScanner::parse() - state stack underflow."); michael@0: fStackPtr++; michael@0: } michael@0: } michael@0: michael@0: } michael@0: michael@0: // michael@0: // If there were NO user specified reverse rules, set up the equivalent of ".*;" michael@0: // michael@0: if (fRB->fReverseTree == NULL) { michael@0: fRB->fReverseTree = pushNewNode(RBBINode::opStar); michael@0: RBBINode *operand = pushNewNode(RBBINode::setRef); michael@0: findSetFor(UnicodeString(TRUE, kAny, 3), operand); michael@0: fRB->fReverseTree->fLeftChild = operand; michael@0: operand->fParent = fRB->fReverseTree; michael@0: fNodeStackPtr -= 2; michael@0: } michael@0: michael@0: michael@0: // michael@0: // Parsing of the input RBBI rules is complete. michael@0: // We now have a parse tree for the rule expressions michael@0: // and a list of all UnicodeSets that are referenced. michael@0: // michael@0: #ifdef RBBI_DEBUG michael@0: if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "symbols")) {fSymbolTable->rbbiSymtablePrint();} michael@0: if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "ptree")) michael@0: { michael@0: RBBIDebugPrintf("Completed Forward Rules Parse Tree...\n"); michael@0: fRB->fForwardTree->printTree(TRUE); michael@0: RBBIDebugPrintf("\nCompleted Reverse Rules Parse Tree...\n"); michael@0: fRB->fReverseTree->printTree(TRUE); michael@0: RBBIDebugPrintf("\nCompleted Safe Point Forward Rules Parse Tree...\n"); michael@0: fRB->fSafeFwdTree->printTree(TRUE); michael@0: RBBIDebugPrintf("\nCompleted Safe Point Reverse Rules Parse Tree...\n"); michael@0: fRB->fSafeRevTree->printTree(TRUE); michael@0: } michael@0: #endif michael@0: } michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // printNodeStack for debugging... michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: #ifdef RBBI_DEBUG michael@0: void RBBIRuleScanner::printNodeStack(const char *title) { michael@0: int i; michael@0: RBBIDebugPrintf("%s. Dumping node stack...\n", title); michael@0: for (i=fNodeStackPtr; i>0; i--) {fNodeStack[i]->printTree(TRUE);} michael@0: } michael@0: #endif michael@0: michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // pushNewNode create a new RBBINode of the specified type and push it michael@0: // onto the stack of nodes. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) { michael@0: fNodeStackPtr++; michael@0: if (fNodeStackPtr >= kStackSize) { michael@0: error(U_BRK_INTERNAL_ERROR); michael@0: RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow."); michael@0: *fRB->fStatus = U_BRK_INTERNAL_ERROR; michael@0: return NULL; michael@0: } michael@0: fNodeStack[fNodeStackPtr] = new RBBINode(t); michael@0: if (fNodeStack[fNodeStackPtr] == NULL) { michael@0: *fRB->fStatus = U_MEMORY_ALLOCATION_ERROR; michael@0: } michael@0: return fNodeStack[fNodeStackPtr]; michael@0: } michael@0: michael@0: michael@0: michael@0: //------------------------------------------------------------------------------ michael@0: // michael@0: // scanSet Construct a UnicodeSet from the text at the current scan michael@0: // position. Advance the scan position to the first character michael@0: // after the set. michael@0: // michael@0: // A new RBBI setref node referring to the set is pushed onto the node michael@0: // stack. michael@0: // michael@0: // The scan position is normally under the control of the state machine michael@0: // that controls rule parsing. UnicodeSets, however, are parsed by michael@0: // the UnicodeSet constructor, not by the RBBI rule parser. michael@0: // michael@0: //------------------------------------------------------------------------------ michael@0: void RBBIRuleScanner::scanSet() { michael@0: UnicodeSet *uset; michael@0: ParsePosition pos; michael@0: int startPos; michael@0: int i; michael@0: michael@0: if (U_FAILURE(*fRB->fStatus)) { michael@0: return; michael@0: } michael@0: michael@0: pos.setIndex(fScanIndex); michael@0: startPos = fScanIndex; michael@0: UErrorCode localStatus = U_ZERO_ERROR; michael@0: uset = new UnicodeSet(); michael@0: if (uset == NULL) { michael@0: localStatus = U_MEMORY_ALLOCATION_ERROR; michael@0: } else { michael@0: uset->applyPatternIgnoreSpace(fRB->fRules, pos, fSymbolTable, localStatus); michael@0: } michael@0: if (U_FAILURE(localStatus)) { michael@0: // TODO: Get more accurate position of the error from UnicodeSet's return info. michael@0: // UnicodeSet appears to not be reporting correctly at this time. michael@0: #ifdef RBBI_DEBUG michael@0: RBBIDebugPrintf("UnicodeSet parse postion.ErrorIndex = %d\n", pos.getIndex()); michael@0: #endif michael@0: error(localStatus); michael@0: delete uset; michael@0: return; michael@0: } michael@0: michael@0: // Verify that the set contains at least one code point. michael@0: // michael@0: U_ASSERT(uset!=NULL); michael@0: if (uset->isEmpty()) { michael@0: // This set is empty. michael@0: // Make it an error, because it almost certainly is not what the user wanted. michael@0: // Also, avoids having to think about corner cases in the tree manipulation code michael@0: // that occurs later on. michael@0: error(U_BRK_RULE_EMPTY_SET); michael@0: delete uset; michael@0: return; michael@0: } michael@0: michael@0: michael@0: // Advance the RBBI parse postion over the UnicodeSet pattern. michael@0: // Don't just set fScanIndex because the line/char positions maintained michael@0: // for error reporting would be thrown off. michael@0: i = pos.getIndex(); michael@0: for (;;) { michael@0: if (fNextIndex >= i) { michael@0: break; michael@0: } michael@0: nextCharLL(); michael@0: } michael@0: michael@0: if (U_SUCCESS(*fRB->fStatus)) { michael@0: RBBINode *n; michael@0: michael@0: n = pushNewNode(RBBINode::setRef); michael@0: n->fFirstPos = startPos; michael@0: n->fLastPos = fNextIndex; michael@0: fRB->fRules.extractBetween(n->fFirstPos, n->fLastPos, n->fText); michael@0: // findSetFor() serves several purposes here: michael@0: // - Adopts storage for the UnicodeSet, will be responsible for deleting. michael@0: // - Mantains collection of all sets in use, needed later for establishing michael@0: // character categories for run time engine. michael@0: // - Eliminates mulitiple instances of the same set. michael@0: // - Creates a new uset node if necessary (if this isn't a duplicate.) michael@0: findSetFor(n->fText, n, uset); michael@0: } michael@0: michael@0: } michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: #endif /* #if !UCONFIG_NO_BREAK_ITERATION */