The Tor Browser: intl/icu/source/i18n/rbt_pars.cpp@fc2d59ddac77 (annotated)

intl/icu/source/i18n/rbt_pars.cpp@fc2d59ddac77 (annotated)

intl/icu/source/i18n/rbt_pars.cpp

Wed, 31 Dec 2014 07:22:50 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 07:22:50 +0100
branch: TOR_BUG_3246
changeset 4: fc2d59ddac77
permissions: -rw-r--r--

Correct previous dual key logic pending first delivery installment.

 /*
  **********************************************************************
  *   Copyright (C) 1999-2011, International Business Machines
  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  *   Date        Name        Description
  *   11/17/99    aliu        Creation.
  **********************************************************************
  */
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_TRANSLITERATION
 #include "unicode/uobject.h"
 #include "unicode/parseerr.h"
 #include "unicode/parsepos.h"
 #include "unicode/putil.h"
 #include "unicode/uchar.h"
 #include "unicode/ustring.h"
 #include "unicode/uniset.h"
 #include "unicode/utf16.h"
 #include "cstring.h"
 #include "funcrepl.h"
 #include "hash.h"
 #include "quant.h"
 #include "rbt.h"
 #include "rbt_data.h"
 #include "rbt_pars.h"
 #include "rbt_rule.h"
 #include "strmatch.h"
 #include "strrepl.h"
 #include "unicode/symtable.h"
 #include "tridpars.h"
 #include "uvector.h"
 #include "hash.h"
 #include "patternprops.h"
 #include "util.h"
 #include "cmemory.h"
 #include "uprops.h"
 #include "putilimp.h"
 // Operators
 #define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/
 #define FORWARD_RULE_OP ((UChar)0x003E) /*>*/
 #define REVERSE_RULE_OP ((UChar)0x003C) /*<*/
 #define FWDREV_RULE_OP  ((UChar)0x007E) /*~*/ // internal rep of <> op
 // Other special characters
 #define QUOTE             ((UChar)0x0027) /*'*/
 #define ESCAPE            ((UChar)0x005C) /*\*/
 #define END_OF_RULE       ((UChar)0x003B) /*;*/
 #define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/
 #define SEGMENT_OPEN       ((UChar)0x0028) /*(*/
 #define SEGMENT_CLOSE      ((UChar)0x0029) /*)*/
 #define CONTEXT_ANTE       ((UChar)0x007B) /*{*/
 #define CONTEXT_POST       ((UChar)0x007D) /*}*/
 #define CURSOR_POS         ((UChar)0x007C) /*|*/
 #define CURSOR_OFFSET      ((UChar)0x0040) /*@*/
 #define ANCHOR_START       ((UChar)0x005E) /*^*/
 #define KLEENE_STAR        ((UChar)0x002A) /***/
 #define ONE_OR_MORE        ((UChar)0x002B) /*+*/
 #define ZERO_OR_ONE        ((UChar)0x003F) /*?*/
 #define DOT                ((UChar)46)     /*.*/
 static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]";
 , 94, 91, 58, 90, 112, 58, 93, 91, 58, 90,
 , 58, 93, 92, 114, 92, 110, 36, 93, 0
 };
 // A function is denoted &Source-Target/Variant(text)
 #define FUNCTION           ((UChar)38)     /*&*/
 // Aliases for some of the syntax characters. These are provided so
 // transliteration rules can be expressed in XML without clashing with
 // XML syntax characters '<', '>', and '&'.
 #define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow
 #define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow
 #define ALT_FWDREV_RULE_OP  ((UChar)0x2194) // Left Right Arrow
 #define ALT_FUNCTION        ((UChar)0x2206) // Increment (~Greek Capital Delta)
 // Special characters disallowed at the top level
 static const UChar ILLEGAL_TOP[] = {41,0}; // ")"
 // Special characters disallowed within a segment
 static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@"
 // Special characters disallowed within a function argument
 static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@"
 // By definition, the ANCHOR_END special character is a
 // trailing SymbolTable.SYMBOL_REF character.
 // private static final char ANCHOR_END       = '$';
 static const UChar gOPERATORS[] = { // "=><"
     VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
     ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
 
 };
 static const UChar HALF_ENDERS[] = { // "=><;"
     VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
     ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
     END_OF_RULE,
 
 };
 // These are also used in Transliterator::toRules()
 static const int32_t ID_TOKEN_LEN = 2;
 static const UChar   ID_TOKEN[]   = { 0x3A, 0x3A }; // ':', ':'
 /*
 commented out until we do real ::BEGIN/::END functionality
 static const int32_t BEGIN_TOKEN_LEN = 5;
 static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN'
 static const int32_t END_TOKEN_LEN = 3;
 static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END'
 */
 U_NAMESPACE_BEGIN
 //----------------------------------------------------------------------
 // BEGIN ParseData
 //----------------------------------------------------------------------
 /**
  * This class implements the SymbolTable interface.  It is used
  * during parsing to give UnicodeSet access to variables that
  * have been defined so far.  Note that it uses variablesVector,
  * _not_ data.setVariables.
  */
 class ParseData : public UMemory, public SymbolTable {
 public:
     const TransliterationRuleData* data; // alias
     const UVector* variablesVector; // alias
     const Hashtable* variableNames; // alias
     ParseData(const TransliterationRuleData* data = 0,
               const UVector* variablesVector = 0,
               const Hashtable* variableNames = 0);
     virtual ~ParseData();
     virtual const UnicodeString* lookup(const UnicodeString& s) const;
     virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const;
     virtual UnicodeString parseReference(const UnicodeString& text,
                                          ParsePosition& pos, int32_t limit) const;
     /**
      * Return true if the given character is a matcher standin or a plain
      * character (non standin).
      */
     UBool isMatcher(UChar32 ch);
     /**
      * Return true if the given character is a replacer standin or a plain
      * character (non standin).
      */
     UBool isReplacer(UChar32 ch);
 private:
     ParseData(const ParseData &other); // forbid copying of this class
     ParseData &operator=(const ParseData &other); // forbid copying of this class
 };
 ParseData::ParseData(const TransliterationRuleData* d,
                      const UVector* sets,
                      const Hashtable* vNames) :
     data(d), variablesVector(sets), variableNames(vNames) {}
 ParseData::~ParseData() {}
 /**
  * Implement SymbolTable API.
  */
 const UnicodeString* ParseData::lookup(const UnicodeString& name) const {
     return (const UnicodeString*) variableNames->get(name);
 }
 /**
  * Implement SymbolTable API.
  */
 const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const {
     // Note that we cannot use data.lookupSet() because the
     // set array has not been constructed yet.
     const UnicodeFunctor* set = NULL;
     int32_t i = ch - data->variablesBase;
     if (i >= 0 && i < variablesVector->size()) {
         int32_t i = ch - data->variablesBase;
         set = (i < variablesVector->size()) ?
             (UnicodeFunctor*) variablesVector->elementAt(i) : 0;
     }
     return set;
 }
 /**
  * Implement SymbolTable API.  Parse out a symbol reference
  * name.
  */
 UnicodeString ParseData::parseReference(const UnicodeString& text,
                                         ParsePosition& pos, int32_t limit) const {
     int32_t start = pos.getIndex();
     int32_t i = start;
     UnicodeString result;
     while (i < limit) {
         UChar c = text.charAt(i);
         if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) {
             break;
         }
         ++i;
     }
     if (i == start) { // No valid name chars
         return result; // Indicate failure with empty string
     }
     pos.setIndex(i);
     text.extractBetween(start, i, result);
     return result;
 }
 UBool ParseData::isMatcher(UChar32 ch) {
     // Note that we cannot use data.lookup() because the
     // set array has not been constructed yet.
     int32_t i = ch - data->variablesBase;
     if (i >= 0 && i < variablesVector->size()) {
         UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
         return f != NULL && f->toMatcher() != NULL;
     }
     return TRUE;
 }
 /**
  * Return true if the given character is a replacer standin or a plain
  * character (non standin).
  */
 UBool ParseData::isReplacer(UChar32 ch) {
     // Note that we cannot use data.lookup() because the
     // set array has not been constructed yet.
     int i = ch - data->variablesBase;
     if (i >= 0 && i < variablesVector->size()) {
         UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
         return f != NULL && f->toReplacer() != NULL;
     }
     return TRUE;
 }
 //----------------------------------------------------------------------
 // BEGIN RuleHalf
 //----------------------------------------------------------------------
 /**
  * A class representing one side of a rule.  This class knows how to
  * parse half of a rule.  It is tightly coupled to the method
  * RuleBasedTransliterator.Parser.parseRule().
  */
 class RuleHalf : public UMemory {
 public:
     UnicodeString text;
     int32_t cursor; // position of cursor in text
     int32_t ante;   // position of ante context marker '{' in text
     int32_t post;   // position of post context marker '}' in text
     // Record the offset to the cursor either to the left or to the
     // right of the key.  This is indicated by characters on the output
     // side that allow the cursor to be positioned arbitrarily within
     // the matching text.  For example, abc{def} > | @@@ xyz; changes
     // def to xyz and moves the cursor to before abc.  Offset characters
     // must be at the start or end, and they cannot move the cursor past
     // the ante- or postcontext text.  Placeholders are only valid in
     // output text.  The length of the ante and post context is
     // determined at runtime, because of supplementals and quantifiers.
     int32_t cursorOffset; // only nonzero on output side
     // Position of first CURSOR_OFFSET on _right_.  This will be -1
     // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc.
     int32_t cursorOffsetPos;
     UBool anchorStart;
     UBool anchorEnd;
     /**
      * The segment number from 1..n of the next '(' we see
      * during parsing; 1-based.
      */
     int32_t nextSegmentNumber;
     TransliteratorParser& parser;
     //--------------------------------------------------
     // Methods
     RuleHalf(TransliteratorParser& parser);
     ~RuleHalf();
     int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status);
     int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
                          UnicodeString& buf,
                          const UnicodeString& illegal,
                          UBool isSegment,
                          UErrorCode& status);
     /**
      * Remove context.
      */
     void removeContext();
     /**
      * Return true if this half looks like valid output, that is, does not
      * contain quantifiers or other special input-only elements.
      */
     UBool isValidOutput(TransliteratorParser& parser);
     /**
      * Return true if this half looks like valid input, that is, does not
      * contain functions or other special output-only elements.
      */
     UBool isValidInput(TransliteratorParser& parser);
     int syntaxError(UErrorCode code,
                     const UnicodeString& rule,
                     int32_t start,
                     UErrorCode& status) {
         return parser.syntaxError(code, rule, start, status);
     }
 private:
     // Disallowed methods; no impl.
     RuleHalf(const RuleHalf&);
     RuleHalf& operator=(const RuleHalf&);
 };
 RuleHalf::RuleHalf(TransliteratorParser& p) :
     parser(p)
 {
     cursor = -1;
     ante = -1;
     post = -1;
     cursorOffset = 0;
     cursorOffsetPos = 0;
     anchorStart = anchorEnd = FALSE;
     nextSegmentNumber = 1;
 }
 RuleHalf::~RuleHalf() {
 }
 /**
  * Parse one side of a rule, stopping at either the limit,
  * the END_OF_RULE character, or an operator.
  * @return the index after the terminating character, or
  * if limit was reached, limit
  */
 int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
     int32_t start = pos;
     text.truncate(0);
     pos = parseSection(rule, pos, limit, text, UnicodeString(TRUE, ILLEGAL_TOP, -1), FALSE, status);
     if (cursorOffset > 0 && cursor != cursorOffsetPos) {
         return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
     }
     return pos;
 }
 /**
  * Parse a section of one side of a rule, stopping at either
  * the limit, the END_OF_RULE character, an operator, or a
  * segment close character.  This method parses both a
  * top-level rule half and a segment within such a rule half.
  * It calls itself recursively to parse segments and nested
  * segments.
  * @param buf buffer into which to accumulate the rule pattern
  * characters, either literal characters from the rule or
  * standins for UnicodeMatcher objects including segments.
  * @param illegal the set of special characters that is illegal during
  * this parse.
  * @param isSegment if true, then we've already seen a '(' and
  * pos on entry points right after it.  Accumulate everything
  * up to the closing ')', put it in a segment matcher object,
  * generate a standin for it, and add the standin to buf.  As
  * a side effect, update the segments vector with a reference
  * to the segment matcher.  This works recursively for nested
  * segments.  If isSegment is false, just accumulate
  * characters into buf.
  * @return the index after the terminating character, or
  * if limit was reached, limit
  */
 int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
                                UnicodeString& buf,
                                const UnicodeString& illegal,
                                UBool isSegment, UErrorCode& status) {
     int32_t start = pos;
     ParsePosition pp;
     UnicodeString scratch;
     UBool done = FALSE;
     int32_t quoteStart = -1; // Most recent 'single quoted string'
     int32_t quoteLimit = -1;
     int32_t varStart = -1; // Most recent $variableReference
     int32_t varLimit = -1;
     int32_t bufStart = buf.length();
     while (pos < limit && !done) {
         // Since all syntax characters are in the BMP, fetching
         // 16-bit code units suffices here.
         UChar c = rule.charAt(pos++);
         if (PatternProps::isWhiteSpace(c)) {
             // Ignore whitespace.  Note that this is not Unicode
             // spaces, but Java spaces -- a subset, representing
             // whitespace likely to be seen in code.
             continue;
         }
         if (u_strchr(HALF_ENDERS, c) != NULL) {
             if (isSegment) {
                 // Unclosed segment
                 return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status);
             }
             break;
         }
         if (anchorEnd) {
             // Text after a presumed end anchor is a syntax err
             return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status);
         }
         if (UnicodeSet::resemblesPattern(rule, pos-1)) {
             pp.setIndex(pos-1); // Backup to opening '['
             buf.append(parser.parseSet(rule, pp, status));
             if (U_FAILURE(status)) {
                 return syntaxError(U_MALFORMED_SET, rule, start, status);
             }
             pos = pp.getIndex();
             continue;
         }
         // Handle escapes
         if (c == ESCAPE) {
             if (pos == limit) {
                 return syntaxError(U_TRAILING_BACKSLASH, rule, start, status);
             }
             UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\'
             if (escaped == (UChar32) -1) {
                 return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status);
             }
             if (!parser.checkVariableRange(escaped)) {
                 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
             }
             buf.append(escaped);
             continue;
         }
         // Handle quoted matter
         if (c == QUOTE) {
             int32_t iq = rule.indexOf(QUOTE, pos);
             if (iq == pos) {
                 buf.append(c); // Parse [''] outside quotes as [']
                 ++pos;
             } else {
                 /* This loop picks up a run of quoted text of the
                  * form 'aaaa' each time through.  If this run
                  * hasn't really ended ('aaaa''bbbb') then it keeps
                  * looping, each time adding on a new run.  When it
                  * reaches the final quote it breaks.
                  */
                 quoteStart = buf.length();
                 for (;;) {
                     if (iq < 0) {
                         return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status);
                     }
                     scratch.truncate(0);
                     rule.extractBetween(pos, iq, scratch);
                     buf.append(scratch);
                     pos = iq+1;
                     if (pos < limit && rule.charAt(pos) == QUOTE) {
                         // Parse [''] inside quotes as [']
                         iq = rule.indexOf(QUOTE, pos+1);
                         // Continue looping
                     } else {
                         break;
                     }
                 }
                 quoteLimit = buf.length();
                 for (iq=quoteStart; iq<quoteLimit; ++iq) {
                     if (!parser.checkVariableRange(buf.charAt(iq))) {
                         return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
                     }
                 }
             }
             continue;
         }
         if (!parser.checkVariableRange(c)) {
             return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
         }
         if (illegal.indexOf(c) >= 0) {
             syntaxError(U_ILLEGAL_CHARACTER, rule, start, status);
         }
         switch (c) {
         //------------------------------------------------------
         // Elements allowed within and out of segments
         //------------------------------------------------------
         case ANCHOR_START:
             if (buf.length() == 0 && !anchorStart) {
                 anchorStart = TRUE;
             } else {
               return syntaxError(U_MISPLACED_ANCHOR_START,
                                  rule, start, status);
             }
           break;
         case SEGMENT_OPEN:
             {
                 // bufSegStart is the offset in buf to the first
                 // character of the segment we are parsing.
                 int32_t bufSegStart = buf.length();
                 // Record segment number now, since nextSegmentNumber
                 // will be incremented during the call to parseSection
                 // if there are nested segments.
                 int32_t segmentNumber = nextSegmentNumber++; // 1-based
                 // Parse the segment
                 pos = parseSection(rule, pos, limit, buf, UnicodeString(TRUE, ILLEGAL_SEG, -1), TRUE, status);
                 // After parsing a segment, the relevant characters are
                 // in buf, starting at offset bufSegStart.  Extract them
                 // into a string matcher, and replace them with a
                 // standin for that matcher.
                 StringMatcher* m =
                     new StringMatcher(buf, bufSegStart, buf.length(),
                                       segmentNumber, *parser.curData);
                 if (m == NULL) {
                     return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
                 }
                 // Record and associate object and segment number
                 parser.setSegmentObject(segmentNumber, m, status);
                 buf.truncate(bufSegStart);
                 buf.append(parser.getSegmentStandin(segmentNumber, status));
             }
             break;
         case FUNCTION:
         case ALT_FUNCTION:
             {
                 int32_t iref = pos;
                 TransliteratorIDParser::SingleID* single =
                     TransliteratorIDParser::parseFilterID(rule, iref);
                 // The next character MUST be a segment open
                 if (single == NULL ||
                     !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) {
                     return syntaxError(U_INVALID_FUNCTION, rule, start, status);
                 }
                 Transliterator *t = single->createInstance();
                 delete single;
                 if (t == NULL) {
                     return syntaxError(U_INVALID_FUNCTION, rule, start, status);
                 }
                 // bufSegStart is the offset in buf to the first
                 // character of the segment we are parsing.
                 int32_t bufSegStart = buf.length();
                 // Parse the segment
                 pos = parseSection(rule, iref, limit, buf, UnicodeString(TRUE, ILLEGAL_FUNC, -1), TRUE, status);
                 // After parsing a segment, the relevant characters are
                 // in buf, starting at offset bufSegStart.
                 UnicodeString output;
                 buf.extractBetween(bufSegStart, buf.length(), output);
                 FunctionReplacer *r =
                     new FunctionReplacer(t, new StringReplacer(output, parser.curData));
                 if (r == NULL) {
                     return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
                 }
                 // Replace the buffer contents with a stand-in
                 buf.truncate(bufSegStart);
                 buf.append(parser.generateStandInFor(r, status));
             }
             break;
         case SymbolTable::SYMBOL_REF:
             // Handle variable references and segment references "$1" .. "$9"
             {
                 // A variable reference must be followed immediately
                 // by a Unicode identifier start and zero or more
                 // Unicode identifier part characters, or by a digit
                 // 1..9 if it is a segment reference.
                 if (pos == limit) {
                     // A variable ref character at the end acts as
                     // an anchor to the context limit, as in perl.
                     anchorEnd = TRUE;
                     break;
                 }
                 // Parse "$1" "$2" .. "$9" .. (no upper limit)
                 c = rule.charAt(pos);
                 int32_t r = u_digit(c, 10);
                 if (r >= 1 && r <= 9) {
                     r = ICU_Utility::parseNumber(rule, pos, 10);
                     if (r < 0) {
                         return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE,
                                            rule, start, status);
                     }
                     buf.append(parser.getSegmentStandin(r, status));
                 } else {
                     pp.setIndex(pos);
                     UnicodeString name = parser.parseData->
                                     parseReference(rule, pp, limit);
                     if (name.length() == 0) {
                         // This means the '$' was not followed by a
                         // valid name.  Try to interpret it as an
                         // end anchor then.  If this also doesn't work
                         // (if we see a following character) then signal
                         // an error.
                         anchorEnd = TRUE;
                         break;
                     }
                     pos = pp.getIndex();
                     // If this is a variable definition statement,
                     // then the LHS variable will be undefined.  In
                     // that case appendVariableDef() will append the
                     // special placeholder char variableLimit-1.
                     varStart = buf.length();
                     parser.appendVariableDef(name, buf, status);
                     varLimit = buf.length();
                 }
             }
             break;
         case DOT:
             buf.append(parser.getDotStandIn(status));
             break;
         case KLEENE_STAR:
         case ONE_OR_MORE:
         case ZERO_OR_ONE:
             // Quantifiers.  We handle single characters, quoted strings,
             // variable references, and segments.
             //  a+      matches  aaa
             //  'foo'+  matches  foofoofoo
             //  $v+     matches  xyxyxy if $v == xy
             //  (seg)+  matches  segsegseg
             {
                 if (isSegment && buf.length() == bufStart) {
                     // The */+ immediately follows '('
                     return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status);
                 }
                 int32_t qstart, qlimit;
                 // The */+ follows an isolated character or quote
                 // or variable reference
                 if (buf.length() == quoteLimit) {
                     // The */+ follows a 'quoted string'
                     qstart = quoteStart;
                     qlimit = quoteLimit;
                 } else if (buf.length() == varLimit) {
                     // The */+ follows a $variableReference
                     qstart = varStart;
                     qlimit = varLimit;
                 } else {
                     // The */+ follows a single character, possibly
                     // a segment standin
                     qstart = buf.length() - 1;
                     qlimit = qstart + 1;
                 }
                 UnicodeFunctor *m =
                     new StringMatcher(buf, qstart, qlimit, 0, *parser.curData);
                 if (m == NULL) {
                     return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
                 }
                 int32_t min = 0;
                 int32_t max = Quantifier::MAX;
                 switch (c) {
                 case ONE_OR_MORE:
                     min = 1;
                     break;
                 case ZERO_OR_ONE:
                     min = 0;
                     max = 1;
                     break;
                 // case KLEENE_STAR:
                 //    do nothing -- min, max already set
                 }
                 m = new Quantifier(m, min, max);
                 if (m == NULL) {
                     return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
                 }
                 buf.truncate(qstart);
                 buf.append(parser.generateStandInFor(m, status));
             }
             break;
         //------------------------------------------------------
         // Elements allowed ONLY WITHIN segments
         //------------------------------------------------------
         case SEGMENT_CLOSE:
             // assert(isSegment);
             // We're done parsing a segment.
             done = TRUE;
             break;
         //------------------------------------------------------
         // Elements allowed ONLY OUTSIDE segments
         //------------------------------------------------------
         case CONTEXT_ANTE:
             if (ante >= 0) {
                 return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status);
             }
             ante = buf.length();
             break;
         case CONTEXT_POST:
             if (post >= 0) {
                 return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status);
             }
             post = buf.length();
             break;
         case CURSOR_POS:
             if (cursor >= 0) {
                 return syntaxError(U_MULTIPLE_CURSORS, rule, start, status);
             }
             cursor = buf.length();
             break;
         case CURSOR_OFFSET:
             if (cursorOffset < 0) {
                 if (buf.length() > 0) {
                     return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
                 }
                 --cursorOffset;
             } else if (cursorOffset > 0) {
                 if (buf.length() != cursorOffsetPos || cursor >= 0) {
                     return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
                 }
                 ++cursorOffset;
             } else {
                 if (cursor == 0 && buf.length() == 0) {
                     cursorOffset = -1;
                 } else if (cursor < 0) {
                     cursorOffsetPos = buf.length();
                     cursorOffset = 1;
                 } else {
                     return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
                 }
             }
             break;
         //------------------------------------------------------
         // Non-special characters
         //------------------------------------------------------
         default:
             // Disallow unquoted characters other than [0-9A-Za-z]
             // in the printable ASCII range.  These characters are
             // reserved for possible future use.
             if (c >= 0x0021 && c <= 0x007E &&
                 !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) ||
                   (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) ||
                   (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) {
                 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
             }
             buf.append(c);
             break;
         }
     }
     return pos;
 }
 /**
  * Remove context.
  */
 void RuleHalf::removeContext() {
     //text = text.substring(ante < 0 ? 0 : ante,
     //                      post < 0 ? text.length() : post);
     if (post >= 0) {
         text.remove(post);
     }
     if (ante >= 0) {
         text.removeBetween(0, ante);
     }
     ante = post = -1;
     anchorStart = anchorEnd = FALSE;
 }
 /**
  * Return true if this half looks like valid output, that is, does not
  * contain quantifiers or other special input-only elements.
  */
 UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) {
     for (int32_t i=0; i<text.length(); ) {
         UChar32 c = text.char32At(i);
         i += U16_LENGTH(c);
         if (!transParser.parseData->isReplacer(c)) {
             return FALSE;
         }
     }
     return TRUE;
 }
 /**
  * Return true if this half looks like valid input, that is, does not
  * contain functions or other special output-only elements.
  */
 UBool RuleHalf::isValidInput(TransliteratorParser& transParser) {
     for (int32_t i=0; i<text.length(); ) {
         UChar32 c = text.char32At(i);
         i += U16_LENGTH(c);
         if (!transParser.parseData->isMatcher(c)) {
             return FALSE;
         }
     }
     return TRUE;
 }
 //----------------------------------------------------------------------
 // PUBLIC API
 //----------------------------------------------------------------------
 /**
  * Constructor.
  */
 TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) :
 dataVector(statusReturn),
 idBlockVector(statusReturn),
 variablesVector(statusReturn),
 segmentObjects(statusReturn)
 {
     idBlockVector.setDeleter(uprv_deleteUObject);
     curData = NULL;
     compoundFilter = NULL;
     parseData = NULL;
     variableNames.setValueDeleter(uprv_deleteUObject);
 }
 /**
  * Destructor.
  */
 TransliteratorParser::~TransliteratorParser() {
     while (!dataVector.isEmpty())
         delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
     delete compoundFilter;
     delete parseData;
     while (!variablesVector.isEmpty())
         delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
 }
 void
 TransliteratorParser::parse(const UnicodeString& rules,
                             UTransDirection transDirection,
                             UParseError& pe,
                             UErrorCode& ec) {
     if (U_SUCCESS(ec)) {
         parseRules(rules, transDirection, ec);
         pe = parseError;
     }
 }
 /**
  * Return the compound filter parsed by parse().  Caller owns result.
  */
 UnicodeSet* TransliteratorParser::orphanCompoundFilter() {
     UnicodeSet* f = compoundFilter;
     compoundFilter = NULL;
     return f;
 }
 //----------------------------------------------------------------------
 // Private implementation
 //----------------------------------------------------------------------
 /**
  * Parse the given string as a sequence of rules, separated by newline
  * characters ('\n'), and cause this object to implement those rules.  Any
  * previous rules are discarded.  Typically this method is called exactly
  * once, during construction.
  * @exception IllegalArgumentException if there is a syntax error in the
  * rules
  */
 void TransliteratorParser::parseRules(const UnicodeString& rule,
                                       UTransDirection theDirection,
                                       UErrorCode& status)
 {
     // Clear error struct
     uprv_memset(&parseError, 0, sizeof(parseError));
     parseError.line = parseError.offset = -1;
     UBool parsingIDs = TRUE;
     int32_t ruleCount = 0;
     while (!dataVector.isEmpty()) {
         delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
     }
     if (U_FAILURE(status)) {
         return;
     }
     idBlockVector.removeAllElements();
     curData = NULL;
     direction = theDirection;
     ruleCount = 0;
     delete compoundFilter;
     compoundFilter = NULL;
     while (!variablesVector.isEmpty()) {
         delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
     }
     variableNames.removeAll();
     parseData = new ParseData(0, &variablesVector, &variableNames);
     if (parseData == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
     dotStandIn = (UChar) -1;
     UnicodeString *tempstr = NULL; // used for memory allocation error checking
     UnicodeString str; // scratch
     UnicodeString idBlockResult;
     int32_t pos = 0;
     int32_t limit = rule.length();
     // The compound filter offset is an index into idBlockResult.
     // If it is 0, then the compound filter occurred at the start,
     // and it is the offset to the _start_ of the compound filter
     // pattern.  Otherwise it is the offset to the _limit_ of the
     // compound filter pattern within idBlockResult.
     compoundFilter = NULL;
     int32_t compoundFilterOffset = -1;
     while (pos < limit && U_SUCCESS(status)) {
         UChar c = rule.charAt(pos++);
         if (PatternProps::isWhiteSpace(c)) {
             // Ignore leading whitespace.
             continue;
         }
         // Skip lines starting with the comment character
         if (c == RULE_COMMENT_CHAR) {
             pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1;
             if (pos == 0) {
                 break; // No "\n" found; rest of rule is a commnet
             }
             continue; // Either fall out or restart with next line
         }
         // skip empty rules
         if (c == END_OF_RULE)
             continue;
         // keep track of how many rules we've seen
         ++ruleCount;
         // We've found the start of a rule or ID.  c is its first
         // character, and pos points past c.
         --pos;
         // Look for an ID token.  Must have at least ID_TOKEN_LEN + 1
         // chars left.
         if ((pos + ID_TOKEN_LEN + 1) <= limit &&
                 rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) {
             pos += ID_TOKEN_LEN;
             c = rule.charAt(pos);
             while (PatternProps::isWhiteSpace(c) && pos < limit) {
                 ++pos;
                 c = rule.charAt(pos);
             }
             int32_t p = pos;
             if (!parsingIDs) {
                 if (curData != NULL) {
                     if (direction == UTRANS_FORWARD)
                         dataVector.addElement(curData, status);
                     else
                         dataVector.insertElementAt(curData, 0, status);
                     curData = NULL;
                 }
                 parsingIDs = TRUE;
             }
             TransliteratorIDParser::SingleID* id =
                 TransliteratorIDParser::parseSingleID(rule, p, direction, status);
             if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) {
                 // Successful ::ID parse.
                 if (direction == UTRANS_FORWARD) {
                     idBlockResult.append(id->canonID).append(END_OF_RULE);
                 } else {
                     idBlockResult.insert(0, END_OF_RULE);
                     idBlockResult.insert(0, id->canonID);
                 }
             } else {
                 // Couldn't parse an ID.  Try to parse a global filter
                 int32_t withParens = -1;
                 UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL);
                 if (f != NULL) {
                     if (ICU_Utility::parseChar(rule, p, END_OF_RULE)
                         && (direction == UTRANS_FORWARD) == (withParens == 0))
                     {
                         if (compoundFilter != NULL) {
                             // Multiple compound filters
                             syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status);
                             delete f;
                         } else {
                             compoundFilter = f;
                             compoundFilterOffset = ruleCount;
                         }
                     } else {
                         delete f;
                     }
                 } else {
                     // Invalid ::id
                     // Can be parsed as neither an ID nor a global filter
                     syntaxError(U_INVALID_ID, rule, pos, status);
                 }
             }
             delete id;
             pos = p;
         } else {
             if (parsingIDs) {
                 tempstr = new UnicodeString(idBlockResult);
                 // NULL pointer check
                 if (tempstr == NULL) {
                     status = U_MEMORY_ALLOCATION_ERROR;
                     return;
                 }
                 if (direction == UTRANS_FORWARD)
                     idBlockVector.addElement(tempstr, status);
                 else
                     idBlockVector.insertElementAt(tempstr, 0, status);
                 idBlockResult.remove();
                 parsingIDs = FALSE;
                 curData = new TransliterationRuleData(status);
                 // NULL pointer check
                 if (curData == NULL) {
                     status = U_MEMORY_ALLOCATION_ERROR;
                     return;
                 }
                 parseData->data = curData;
                 // By default, rules use part of the private use area
                 // E000..F8FF for variables and other stand-ins.  Currently
                 // the range F000..F8FF is typically sufficient.  The 'use
                 // variable range' pragma allows rule sets to modify this.
                 setVariableRange(0xF000, 0xF8FF, status);
             }
             if (resemblesPragma(rule, pos, limit)) {
                 int32_t ppp = parsePragma(rule, pos, limit, status);
                 if (ppp < 0) {
                     syntaxError(U_MALFORMED_PRAGMA, rule, pos, status);
                 }
                 pos = ppp;
             // Parse a rule
             } else {
                 pos = parseRule(rule, pos, limit, status);
             }
         }
     }
     if (parsingIDs && idBlockResult.length() > 0) {
         tempstr = new UnicodeString(idBlockResult);
         // NULL pointer check
         if (tempstr == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
         if (direction == UTRANS_FORWARD)
             idBlockVector.addElement(tempstr, status);
         else
             idBlockVector.insertElementAt(tempstr, 0, status);
     }
     else if (!parsingIDs && curData != NULL) {
         if (direction == UTRANS_FORWARD)
             dataVector.addElement(curData, status);
         else
             dataVector.insertElementAt(curData, 0, status);
     }
     if (U_SUCCESS(status)) {
         // Convert the set vector to an array
         int32_t i, dataVectorSize = dataVector.size();
         for (i = 0; i < dataVectorSize; i++) {
             TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
             data->variablesLength = variablesVector.size();
             if (data->variablesLength == 0) {
                 data->variables = 0;
             } else {
                 data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*));
                 // NULL pointer check
                 if (data->variables == NULL) {
                     status = U_MEMORY_ALLOCATION_ERROR;
                     return;
                 }
                 data->variablesAreOwned = (i == 0);
             }
             for (int32_t j = 0; j < data->variablesLength; j++) {
                 data->variables[j] =
                     ((UnicodeSet*)variablesVector.elementAt(j));
             }
             data->variableNames.removeAll();
             int32_t pos = -1;
             const UHashElement* he = variableNames.nextElement(pos);
             while (he != NULL) {
                 UnicodeString* tempus = (UnicodeString*)(((UnicodeString*)(he->value.pointer))->clone());
                 if (tempus == NULL) {
                     status = U_MEMORY_ALLOCATION_ERROR;
                     return;
                 }
                 data->variableNames.put(*((UnicodeString*)(he->key.pointer)),
                     tempus, status);
                 he = variableNames.nextElement(pos);
             }
         }
         variablesVector.removeAllElements();   // keeps them from getting deleted when we succeed
         // Index the rules
         if (compoundFilter != NULL) {
             if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) ||
                 (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) {
                 status = U_MISPLACED_COMPOUND_FILTER;
             }
         }
         for (i = 0; i < dataVectorSize; i++) {
             TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
             data->ruleSet.freeze(parseError, status);
         }
         if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) {
             idBlockVector.removeElementAt(0);
         }
     }
 }
 /**
  * Set the variable range to [start, end] (inclusive).
  */
 void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) {
     if (start > end || start < 0 || end > 0xFFFF) {
         status = U_MALFORMED_PRAGMA;
         return;
     }
     curData->variablesBase = (UChar) start;
     if (dataVector.size() == 0) {
         variableNext = (UChar) start;
         variableLimit = (UChar) (end + 1);
     }
 }
 /**
  * Assert that the given character is NOT within the variable range.
  * If it is, return FALSE.  This is neccesary to ensure that the
  * variable range does not overlap characters used in a rule.
  */
 UBool TransliteratorParser::checkVariableRange(UChar32 ch) const {
     return !(ch >= curData->variablesBase && ch < variableLimit);
 }
 /**
  * Set the maximum backup to 'backup', in response to a pragma
  * statement.
  */
 void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) {
     //TODO Finish
 }
 /**
  * Begin normalizing all rules using the given mode, in response
  * to a pragma statement.
  */
 void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) {
     //TODO Finish
 }
 static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use "
 static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;"
 static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;"
 static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;"
 static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;"
 /**
  * Return true if the given rule looks like a pragma.
  * @param pos offset to the first non-whitespace character
  * of the rule.
  * @param limit pointer past the last character of the rule.
  */
 UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) {
     // Must start with /use\s/i
     return ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_USE, 4), NULL) >= 0;
 }
 /**
  * Parse a pragma.  This method assumes resemblesPragma() has
  * already returned true.
  * @param pos offset to the first non-whitespace character
  * of the rule.
  * @param limit pointer past the last character of the rule.
  * @return the position index after the final ';' of the pragma,
  * or -1 on failure.
  */
 int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
     int32_t array[2];
     // resemblesPragma() has already returned true, so we
     // know that pos points to /use\s/i; we can skip 4 characters
     // immediately
     pos += 4;
     // Here are the pragmas we recognize:
     // use variable range 0xE000 0xEFFF;
     // use maximum backup 16;
     // use nfd rules;
     // use nfc rules;
     int p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_VARIABLE_RANGE, -1), array);
     if (p >= 0) {
         setVariableRange(array[0], array[1], status);
         return p;
     }
     p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_MAXIMUM_BACKUP, -1), array);
     if (p >= 0) {
         pragmaMaximumBackup(array[0]);
         return p;
     }
     p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFD_RULES, -1), NULL);
     if (p >= 0) {
         pragmaNormalizeRules(UNORM_NFD);
         return p;
     }
     p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFC_RULES, -1), NULL);
     if (p >= 0) {
         pragmaNormalizeRules(UNORM_NFC);
         return p;
     }
     // Syntax error: unable to parse pragma
     return -1;
 }
 /**
  * MAIN PARSER.  Parse the next rule in the given rule string, starting
  * at pos.  Return the index after the last character parsed.  Do not
  * parse characters at or after limit.
  *
  * Important:  The character at pos must be a non-whitespace character
  * that is not the comment character.
  *
  * This method handles quoting, escaping, and whitespace removal.  It
  * parses the end-of-rule character.  It recognizes context and cursor
  * indicators.  Once it does a lexical breakdown of the rule at pos, it
  * creates a rule object and adds it to our rule list.
  */
 int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
     // Locate the left side, operator, and right side
     int32_t start = pos;
     UChar op = 0;
     int32_t i;
     // Set up segments data
     segmentStandins.truncate(0);
     segmentObjects.removeAllElements();
     // Use pointers to automatics to make swapping possible.
     RuleHalf _left(*this), _right(*this);
     RuleHalf* left = &_left;
     RuleHalf* right = &_right;
     undefinedVariableName.remove();
     pos = left->parse(rule, pos, limit, status);
     if (U_FAILURE(status)) {
         return start;
     }
     if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) {
         return syntaxError(U_MISSING_OPERATOR, rule, start, status);
     }
     ++pos;
     // Found an operator char.  Check for forward-reverse operator.
     if (op == REVERSE_RULE_OP &&
         (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) {
         ++pos;
         op = FWDREV_RULE_OP;
     }
     // Translate alternate op characters.
     switch (op) {
     case ALT_FORWARD_RULE_OP:
         op = FORWARD_RULE_OP;
         break;
     case ALT_REVERSE_RULE_OP:
         op = REVERSE_RULE_OP;
         break;
     case ALT_FWDREV_RULE_OP:
         op = FWDREV_RULE_OP;
         break;
     }
     pos = right->parse(rule, pos, limit, status);
     if (U_FAILURE(status)) {
         return start;
     }
     if (pos < limit) {
         if (rule.charAt(--pos) == END_OF_RULE) {
             ++pos;
         } else {
             // RuleHalf parser must have terminated at an operator
             return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
         }
     }
     if (op == VARIABLE_DEF_OP) {
         // LHS is the name.  RHS is a single character, either a literal
         // or a set (already parsed).  If RHS is longer than one
         // character, it is either a multi-character string, or multiple
         // sets, or a mixture of chars and sets -- syntax error.
         // We expect to see a single undefined variable (the one being
         // defined).
         if (undefinedVariableName.length() == 0) {
             // "Missing '$' or duplicate definition"
             return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status);
         }
         if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) {
             // "Malformed LHS"
             return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
         }
         if (left->anchorStart || left->anchorEnd ||
             right->anchorStart || right->anchorEnd) {
             return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
         }
         // We allow anything on the right, including an empty string.
         UnicodeString* value = new UnicodeString(right->text);
         // NULL pointer check
         if (value == NULL) {
             return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
         }
         variableNames.put(undefinedVariableName, value, status);
         ++variableLimit;
         return pos;
     }
     // If this is not a variable definition rule, we shouldn't have
     // any undefined variable names.
     if (undefinedVariableName.length() != 0) {
         return syntaxError(// "Undefined variable $" + undefinedVariableName,
                     U_UNDEFINED_VARIABLE,
                     rule, start, status);
     }
     // Verify segments
     if (segmentStandins.length() > segmentObjects.size()) {
         syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status);
     }
     for (i=0; i<segmentStandins.length(); ++i) {
         if (segmentStandins.charAt(i) == 0) {
             syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
         }
     }
     for (i=0; i<segmentObjects.size(); ++i) {
         if (segmentObjects.elementAt(i) == NULL) {
             syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
         }
     }
     // If the direction we want doesn't match the rule
     // direction, do nothing.
     if (op != FWDREV_RULE_OP &&
         ((direction == UTRANS_FORWARD) != (op == FORWARD_RULE_OP))) {
         return pos;
     }
     // Transform the rule into a forward rule by swapping the
     // sides if necessary.
     if (direction == UTRANS_REVERSE) {
         left = &_right;
         right = &_left;
     }
     // Remove non-applicable elements in forward-reverse
     // rules.  Bidirectional rules ignore elements that do not
     // apply.
     if (op == FWDREV_RULE_OP) {
         right->removeContext();
         left->cursor = -1;
         left->cursorOffset = 0;
     }
     // Normalize context
     if (left->ante < 0) {
         left->ante = 0;
     }
     if (left->post < 0) {
         left->post = left->text.length();
     }
     // Context is only allowed on the input side.  Cursors are only
     // allowed on the output side.  Segment delimiters can only appear
     // on the left, and references on the right.  Cursor offset
     // cannot appear without an explicit cursor.  Cursor offset
     // cannot place the cursor outside the limits of the context.
     // Anchors are only allowed on the input side.
     if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 ||
         (right->cursorOffset != 0 && right->cursor < 0) ||
         // - The following two checks were used to ensure that the
         // - the cursor offset stayed within the ante- or postcontext.
         // - However, with the addition of quantifiers, we have to
         // - allow arbitrary cursor offsets and do runtime checking.
         //(right->cursorOffset > (left->text.length() - left->post)) ||
         //(-right->cursorOffset > left->ante) ||
         right->anchorStart || right->anchorEnd ||
         !left->isValidInput(*this) || !right->isValidOutput(*this) ||
         left->ante > left->post) {
         return syntaxError(U_MALFORMED_RULE, rule, start, status);
     }
     // Flatten segment objects vector to an array
     UnicodeFunctor** segmentsArray = NULL;
     if (segmentObjects.size() > 0) {
         segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *));
         // Null pointer check
         if (segmentsArray == NULL) {
             return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
         }
         segmentObjects.toArray((void**) segmentsArray);
     }
     TransliterationRule* temptr = new TransliterationRule(
             left->text, left->ante, left->post,
             right->text, right->cursor, right->cursorOffset,
             segmentsArray,
             segmentObjects.size(),
             left->anchorStart, left->anchorEnd,
             curData,
             status);
     //Null pointer check
     if (temptr == NULL) {
         uprv_free(segmentsArray);
         return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
     }
     curData->ruleSet.addRule(temptr, status);
     return pos;
 }
 /**
  * Called by main parser upon syntax error.  Search the rule string
  * for the probable end of the rule.  Of course, if the error is that
  * the end of rule marker is missing, then the rule end will not be found.
  * In any case the rule start will be correctly reported.
  * @param msg error description
  * @param rule pattern string
  * @param start position of first character of current rule
  */
 int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode,
                                           const UnicodeString& rule,
                                           int32_t pos,
                                           UErrorCode& status)
 {
     parseError.offset = pos;
     parseError.line = 0 ; /* we are not using line numbers */
     // for pre-context
     const int32_t LEN = U_PARSE_CONTEXT_LEN - 1;
     int32_t start = uprv_max(pos - LEN, 0);
     int32_t stop  = pos;
     rule.extract(start,stop-start,parseError.preContext);
     //null terminate the buffer
     parseError.preContext[stop-start] = 0;
     //for post-context
     start = pos;
     stop  = uprv_min(pos + LEN, rule.length());
     rule.extract(start,stop-start,parseError.postContext);
     //null terminate the buffer
     parseError.postContext[stop-start]= 0;
     status = (UErrorCode)parseErrorCode;
     return pos;
 }
 /**
  * Parse a UnicodeSet out, store it, and return the stand-in character
  * used to represent it.
  */
 UChar TransliteratorParser::parseSet(const UnicodeString& rule,
                                           ParsePosition& pos,
                                           UErrorCode& status) {
     UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status);
     // Null pointer check
     if (set == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return (UChar)0x0000; // Return empty character with error.
     }
     set->compact();
     return generateStandInFor(set, status);
 }
 /**
  * Generate and return a stand-in for a new UnicodeFunctor.  Store
  * the matcher (adopt it).
  */
 UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) {
     // assert(obj != null);
     // Look up previous stand-in, if any.  This is a short list
     // (typical n is 0, 1, or 2); linear search is optimal.
     for (int32_t i=0; i<variablesVector.size(); ++i) {
         if (variablesVector.elementAt(i) == adopted) { // [sic] pointer comparison
             return (UChar) (curData->variablesBase + i);
         }
     }
     if (variableNext >= variableLimit) {
         delete adopted;
         status = U_VARIABLE_RANGE_EXHAUSTED;
         return 0;
     }
     variablesVector.addElement(adopted, status);
     return variableNext++;
 }
 /**
  * Return the standin for segment seg (1-based).
  */
 UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) {
     // Special character used to indicate an empty spot
     UChar empty = curData->variablesBase - 1;
     while (segmentStandins.length() < seg) {
         segmentStandins.append(empty);
     }
     UChar c = segmentStandins.charAt(seg-1);
     if (c == empty) {
         if (variableNext >= variableLimit) {
             status = U_VARIABLE_RANGE_EXHAUSTED;
             return 0;
         }
         c = variableNext++;
         // Set a placeholder in the master variables vector that will be
         // filled in later by setSegmentObject().  We know that we will get
         // called first because setSegmentObject() will call us.
         variablesVector.addElement((void*) NULL, status);
         segmentStandins.setCharAt(seg-1, c);
     }
     return c;
 }
 /**
  * Set the object for segment seg (1-based).
  */
 void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) {
     // Since we call parseSection() recursively, nested
     // segments will result in segment i+1 getting parsed
     // and stored before segment i; be careful with the
     // vector handling here.
     if (segmentObjects.size() < seg) {
         segmentObjects.setSize(seg, status);
     }
     int32_t index = getSegmentStandin(seg, status) - curData->variablesBase;
     if (segmentObjects.elementAt(seg-1) != NULL ||
         variablesVector.elementAt(index) != NULL) {
         // should never happen
         status = U_INTERNAL_TRANSLITERATOR_ERROR;
         return;
     }
     segmentObjects.setElementAt(adopted, seg-1);
     variablesVector.setElementAt(adopted, index);
 }
 /**
  * Return the stand-in for the dot set.  It is allocated the first
  * time and reused thereafter.
  */
 UChar TransliteratorParser::getDotStandIn(UErrorCode& status) {
     if (dotStandIn == (UChar) -1) {
         UnicodeSet* tempus = new UnicodeSet(UnicodeString(TRUE, DOT_SET, -1), status);
         // Null pointer check.
         if (tempus == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return (UChar)0x0000;
         }
         dotStandIn = generateStandInFor(tempus, status);
     }
     return dotStandIn;
 }
 /**
  * Append the value of the given variable name to the given
  * UnicodeString.
  */
 void TransliteratorParser::appendVariableDef(const UnicodeString& name,
                                                   UnicodeString& buf,
                                                   UErrorCode& status) {
     const UnicodeString* s = (const UnicodeString*) variableNames.get(name);
     if (s == NULL) {
         // We allow one undefined variable so that variable definition
         // statements work.  For the first undefined variable we return
         // the special placeholder variableLimit-1, and save the variable
         // name.
         if (undefinedVariableName.length() == 0) {
             undefinedVariableName = name;
             if (variableNext >= variableLimit) {
                 // throw new RuntimeException("Private use variables exhausted");
                 status = U_ILLEGAL_ARGUMENT_ERROR;
                 return;
             }
             buf.append((UChar) --variableLimit);
         } else {
             //throw new IllegalArgumentException("Undefined variable $"
             //                                   + name);
             status = U_ILLEGAL_ARGUMENT_ERROR;
             return;
         }
     } else {
         buf.append(*s);
     }
 }
 /**
  * Glue method to get around access restrictions in C++.
  */
 /*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) {
     return Transliterator::createBasicInstance(id, canonID);
 }*/
 U_NAMESPACE_END
 U_CAPI int32_t
 utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) {
     U_NAMESPACE_USE
     //const UChar *sourceStart = source;
     const UChar *targetStart = target;
     const UChar *sourceLimit = source+sourceLen;
     UChar *targetLimit = target+sourceLen;
     UChar32 c = 0;
     UBool quoted = FALSE;
     int32_t index;
     uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR);
     /* read the rules into the buffer */
     while (source < sourceLimit)
     {
         index=0;
         U16_NEXT_UNSAFE(source, index, c);
         source+=index;
         if(c == QUOTE) {
             quoted = (UBool)!quoted;
         }
         else if (!quoted) {
             if (c == RULE_COMMENT_CHAR) {
                 /* skip comments and all preceding spaces */
                 while (targetStart < target && *(target - 1) == 0x0020) {
                     target--;
                 }
                 do {
                     c = *(source++);
                 }
                 while (c != CR && c != LF);
             }
             else if (c == ESCAPE) {
                 UChar32   c2 = *source;
                 if (c2 == CR || c2 == LF) {
                     /* A backslash at the end of a line. */
                     /* Since we're stripping lines, ignore the backslash. */
                     source++;
                     continue;
                 }
                 if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */
                     int32_t escapeOffset = 0;
                     UnicodeString escapedStr(source, 5);
                     c2 = escapedStr.unescapeAt(escapeOffset);
                     if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0)
                     {
                         *status = U_PARSE_ERROR;
                         return 0;
                     }
                     if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) {
                         /* It was escaped for a reason. Write what it was suppose to be. */
                         source+=5;
                         c = c2;
                     }
                 }
                 else if (c2 == QUOTE) {
                     /* \' seen. Make sure we don't do anything when we see it again. */
                     quoted = (UBool)!quoted;
                 }
             }
         }
         if (c == CR || c == LF)
         {
             /* ignore spaces carriage returns, and all leading spaces on the next line.
             * and line feed unless in the form \uXXXX
             */
             quoted = FALSE;
             while (source < sourceLimit) {
                 c = *(source);
                 if (c != CR && c != LF && c != 0x0020) {
                     break;
                 }
                 source++;
             }
             continue;
         }
         /* Append UChar * after dissembling if c > 0xffff*/
         index=0;
         U16_APPEND_UNSAFE(target, index, c);
         target+=index;
     }
     if (target < targetLimit) {
         *target = 0;
     }
     return (int32_t)(target-targetStart);
 }
 #endif /* #if !UCONFIG_NO_TRANSLITERATION */

The Tor Browser / annotate

intl/icu/source/i18n/rbt_pars.cpp@fc2d59ddac77 (annotated)

intl/icu/source/i18n/rbt_pars.cpp