michael@0: /* michael@0: ********************************************************************** michael@0: * Copyright (C) 1999-2011, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: ********************************************************************** michael@0: * Date Name Description michael@0: * 11/17/99 aliu Creation. michael@0: ********************************************************************** michael@0: */ michael@0: michael@0: #include "unicode/utypes.h" michael@0: michael@0: #if !UCONFIG_NO_TRANSLITERATION michael@0: michael@0: #include "unicode/uobject.h" michael@0: #include "unicode/parseerr.h" michael@0: #include "unicode/parsepos.h" michael@0: #include "unicode/putil.h" michael@0: #include "unicode/uchar.h" michael@0: #include "unicode/ustring.h" michael@0: #include "unicode/uniset.h" michael@0: #include "unicode/utf16.h" michael@0: #include "cstring.h" michael@0: #include "funcrepl.h" michael@0: #include "hash.h" michael@0: #include "quant.h" michael@0: #include "rbt.h" michael@0: #include "rbt_data.h" michael@0: #include "rbt_pars.h" michael@0: #include "rbt_rule.h" michael@0: #include "strmatch.h" michael@0: #include "strrepl.h" michael@0: #include "unicode/symtable.h" michael@0: #include "tridpars.h" michael@0: #include "uvector.h" michael@0: #include "hash.h" michael@0: #include "patternprops.h" michael@0: #include "util.h" michael@0: #include "cmemory.h" michael@0: #include "uprops.h" michael@0: #include "putilimp.h" michael@0: michael@0: // Operators michael@0: #define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/ michael@0: #define FORWARD_RULE_OP ((UChar)0x003E) /*>*/ michael@0: #define REVERSE_RULE_OP ((UChar)0x003C) /*<*/ michael@0: #define FWDREV_RULE_OP ((UChar)0x007E) /*~*/ // internal rep of <> op michael@0: michael@0: // Other special characters michael@0: #define QUOTE ((UChar)0x0027) /*'*/ michael@0: #define ESCAPE ((UChar)0x005C) /*\*/ michael@0: #define END_OF_RULE ((UChar)0x003B) /*;*/ michael@0: #define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/ michael@0: michael@0: #define SEGMENT_OPEN ((UChar)0x0028) /*(*/ michael@0: #define SEGMENT_CLOSE ((UChar)0x0029) /*)*/ michael@0: #define CONTEXT_ANTE ((UChar)0x007B) /*{*/ michael@0: #define CONTEXT_POST ((UChar)0x007D) /*}*/ michael@0: #define CURSOR_POS ((UChar)0x007C) /*|*/ michael@0: #define CURSOR_OFFSET ((UChar)0x0040) /*@*/ michael@0: #define ANCHOR_START ((UChar)0x005E) /*^*/ michael@0: #define KLEENE_STAR ((UChar)0x002A) /***/ michael@0: #define ONE_OR_MORE ((UChar)0x002B) /*+*/ michael@0: #define ZERO_OR_ONE ((UChar)0x003F) /*?*/ michael@0: michael@0: #define DOT ((UChar)46) /*.*/ michael@0: michael@0: static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]"; michael@0: 91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90, michael@0: 108, 58, 93, 92, 114, 92, 110, 36, 93, 0 michael@0: }; michael@0: michael@0: // A function is denoted &Source-Target/Variant(text) michael@0: #define FUNCTION ((UChar)38) /*&*/ michael@0: michael@0: // Aliases for some of the syntax characters. These are provided so michael@0: // transliteration rules can be expressed in XML without clashing with michael@0: // XML syntax characters '<', '>', and '&'. michael@0: #define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow michael@0: #define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow michael@0: #define ALT_FWDREV_RULE_OP ((UChar)0x2194) // Left Right Arrow michael@0: #define ALT_FUNCTION ((UChar)0x2206) // Increment (~Greek Capital Delta) michael@0: michael@0: // Special characters disallowed at the top level michael@0: static const UChar ILLEGAL_TOP[] = {41,0}; // ")" michael@0: michael@0: // Special characters disallowed within a segment michael@0: static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@" michael@0: michael@0: // Special characters disallowed within a function argument michael@0: static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@" michael@0: michael@0: // By definition, the ANCHOR_END special character is a michael@0: // trailing SymbolTable.SYMBOL_REF character. michael@0: // private static final char ANCHOR_END = '$'; michael@0: michael@0: static const UChar gOPERATORS[] = { // "=><" michael@0: VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, michael@0: ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, michael@0: 0 michael@0: }; michael@0: michael@0: static const UChar HALF_ENDERS[] = { // "=><;" michael@0: VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, michael@0: ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, michael@0: END_OF_RULE, michael@0: 0 michael@0: }; michael@0: michael@0: // These are also used in Transliterator::toRules() michael@0: static const int32_t ID_TOKEN_LEN = 2; michael@0: static const UChar ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':' michael@0: michael@0: /* michael@0: commented out until we do real ::BEGIN/::END functionality michael@0: static const int32_t BEGIN_TOKEN_LEN = 5; michael@0: static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN' michael@0: michael@0: static const int32_t END_TOKEN_LEN = 3; michael@0: static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END' michael@0: */ michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: //---------------------------------------------------------------------- michael@0: // BEGIN ParseData michael@0: //---------------------------------------------------------------------- michael@0: michael@0: /** michael@0: * This class implements the SymbolTable interface. It is used michael@0: * during parsing to give UnicodeSet access to variables that michael@0: * have been defined so far. Note that it uses variablesVector, michael@0: * _not_ data.setVariables. michael@0: */ michael@0: class ParseData : public UMemory, public SymbolTable { michael@0: public: michael@0: const TransliterationRuleData* data; // alias michael@0: michael@0: const UVector* variablesVector; // alias michael@0: michael@0: const Hashtable* variableNames; // alias michael@0: michael@0: ParseData(const TransliterationRuleData* data = 0, michael@0: const UVector* variablesVector = 0, michael@0: const Hashtable* variableNames = 0); michael@0: michael@0: virtual ~ParseData(); michael@0: michael@0: virtual const UnicodeString* lookup(const UnicodeString& s) const; michael@0: michael@0: virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const; michael@0: michael@0: virtual UnicodeString parseReference(const UnicodeString& text, michael@0: ParsePosition& pos, int32_t limit) const; michael@0: /** michael@0: * Return true if the given character is a matcher standin or a plain michael@0: * character (non standin). michael@0: */ michael@0: UBool isMatcher(UChar32 ch); michael@0: michael@0: /** michael@0: * Return true if the given character is a replacer standin or a plain michael@0: * character (non standin). michael@0: */ michael@0: UBool isReplacer(UChar32 ch); michael@0: michael@0: private: michael@0: ParseData(const ParseData &other); // forbid copying of this class michael@0: ParseData &operator=(const ParseData &other); // forbid copying of this class michael@0: }; michael@0: michael@0: ParseData::ParseData(const TransliterationRuleData* d, michael@0: const UVector* sets, michael@0: const Hashtable* vNames) : michael@0: data(d), variablesVector(sets), variableNames(vNames) {} michael@0: michael@0: ParseData::~ParseData() {} michael@0: michael@0: /** michael@0: * Implement SymbolTable API. michael@0: */ michael@0: const UnicodeString* ParseData::lookup(const UnicodeString& name) const { michael@0: return (const UnicodeString*) variableNames->get(name); michael@0: } michael@0: michael@0: /** michael@0: * Implement SymbolTable API. michael@0: */ michael@0: const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const { michael@0: // Note that we cannot use data.lookupSet() because the michael@0: // set array has not been constructed yet. michael@0: const UnicodeFunctor* set = NULL; michael@0: int32_t i = ch - data->variablesBase; michael@0: if (i >= 0 && i < variablesVector->size()) { michael@0: int32_t i = ch - data->variablesBase; michael@0: set = (i < variablesVector->size()) ? michael@0: (UnicodeFunctor*) variablesVector->elementAt(i) : 0; michael@0: } michael@0: return set; michael@0: } michael@0: michael@0: /** michael@0: * Implement SymbolTable API. Parse out a symbol reference michael@0: * name. michael@0: */ michael@0: UnicodeString ParseData::parseReference(const UnicodeString& text, michael@0: ParsePosition& pos, int32_t limit) const { michael@0: int32_t start = pos.getIndex(); michael@0: int32_t i = start; michael@0: UnicodeString result; michael@0: while (i < limit) { michael@0: UChar c = text.charAt(i); michael@0: if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) { michael@0: break; michael@0: } michael@0: ++i; michael@0: } michael@0: if (i == start) { // No valid name chars michael@0: return result; // Indicate failure with empty string michael@0: } michael@0: pos.setIndex(i); michael@0: text.extractBetween(start, i, result); michael@0: return result; michael@0: } michael@0: michael@0: UBool ParseData::isMatcher(UChar32 ch) { michael@0: // Note that we cannot use data.lookup() because the michael@0: // set array has not been constructed yet. michael@0: int32_t i = ch - data->variablesBase; michael@0: if (i >= 0 && i < variablesVector->size()) { michael@0: UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); michael@0: return f != NULL && f->toMatcher() != NULL; michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: /** michael@0: * Return true if the given character is a replacer standin or a plain michael@0: * character (non standin). michael@0: */ michael@0: UBool ParseData::isReplacer(UChar32 ch) { michael@0: // Note that we cannot use data.lookup() because the michael@0: // set array has not been constructed yet. michael@0: int i = ch - data->variablesBase; michael@0: if (i >= 0 && i < variablesVector->size()) { michael@0: UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); michael@0: return f != NULL && f->toReplacer() != NULL; michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: //---------------------------------------------------------------------- michael@0: // BEGIN RuleHalf michael@0: //---------------------------------------------------------------------- michael@0: michael@0: /** michael@0: * A class representing one side of a rule. This class knows how to michael@0: * parse half of a rule. It is tightly coupled to the method michael@0: * RuleBasedTransliterator.Parser.parseRule(). michael@0: */ michael@0: class RuleHalf : public UMemory { michael@0: michael@0: public: michael@0: michael@0: UnicodeString text; michael@0: michael@0: int32_t cursor; // position of cursor in text michael@0: int32_t ante; // position of ante context marker '{' in text michael@0: int32_t post; // position of post context marker '}' in text michael@0: michael@0: // Record the offset to the cursor either to the left or to the michael@0: // right of the key. This is indicated by characters on the output michael@0: // side that allow the cursor to be positioned arbitrarily within michael@0: // the matching text. For example, abc{def} > | @@@ xyz; changes michael@0: // def to xyz and moves the cursor to before abc. Offset characters michael@0: // must be at the start or end, and they cannot move the cursor past michael@0: // the ante- or postcontext text. Placeholders are only valid in michael@0: // output text. The length of the ante and post context is michael@0: // determined at runtime, because of supplementals and quantifiers. michael@0: int32_t cursorOffset; // only nonzero on output side michael@0: michael@0: // Position of first CURSOR_OFFSET on _right_. This will be -1 michael@0: // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc. michael@0: int32_t cursorOffsetPos; michael@0: michael@0: UBool anchorStart; michael@0: UBool anchorEnd; michael@0: michael@0: /** michael@0: * The segment number from 1..n of the next '(' we see michael@0: * during parsing; 1-based. michael@0: */ michael@0: int32_t nextSegmentNumber; michael@0: michael@0: TransliteratorParser& parser; michael@0: michael@0: //-------------------------------------------------- michael@0: // Methods michael@0: michael@0: RuleHalf(TransliteratorParser& parser); michael@0: ~RuleHalf(); michael@0: michael@0: int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status); michael@0: michael@0: int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, michael@0: UnicodeString& buf, michael@0: const UnicodeString& illegal, michael@0: UBool isSegment, michael@0: UErrorCode& status); michael@0: michael@0: /** michael@0: * Remove context. michael@0: */ michael@0: void removeContext(); michael@0: michael@0: /** michael@0: * Return true if this half looks like valid output, that is, does not michael@0: * contain quantifiers or other special input-only elements. michael@0: */ michael@0: UBool isValidOutput(TransliteratorParser& parser); michael@0: michael@0: /** michael@0: * Return true if this half looks like valid input, that is, does not michael@0: * contain functions or other special output-only elements. michael@0: */ michael@0: UBool isValidInput(TransliteratorParser& parser); michael@0: michael@0: int syntaxError(UErrorCode code, michael@0: const UnicodeString& rule, michael@0: int32_t start, michael@0: UErrorCode& status) { michael@0: return parser.syntaxError(code, rule, start, status); michael@0: } michael@0: michael@0: private: michael@0: // Disallowed methods; no impl. michael@0: RuleHalf(const RuleHalf&); michael@0: RuleHalf& operator=(const RuleHalf&); michael@0: }; michael@0: michael@0: RuleHalf::RuleHalf(TransliteratorParser& p) : michael@0: parser(p) michael@0: { michael@0: cursor = -1; michael@0: ante = -1; michael@0: post = -1; michael@0: cursorOffset = 0; michael@0: cursorOffsetPos = 0; michael@0: anchorStart = anchorEnd = FALSE; michael@0: nextSegmentNumber = 1; michael@0: } michael@0: michael@0: RuleHalf::~RuleHalf() { michael@0: } michael@0: michael@0: /** michael@0: * Parse one side of a rule, stopping at either the limit, michael@0: * the END_OF_RULE character, or an operator. michael@0: * @return the index after the terminating character, or michael@0: * if limit was reached, limit michael@0: */ michael@0: int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { michael@0: int32_t start = pos; michael@0: text.truncate(0); michael@0: pos = parseSection(rule, pos, limit, text, UnicodeString(TRUE, ILLEGAL_TOP, -1), FALSE, status); michael@0: michael@0: if (cursorOffset > 0 && cursor != cursorOffsetPos) { michael@0: return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); michael@0: } michael@0: michael@0: return pos; michael@0: } michael@0: michael@0: /** michael@0: * Parse a section of one side of a rule, stopping at either michael@0: * the limit, the END_OF_RULE character, an operator, or a michael@0: * segment close character. This method parses both a michael@0: * top-level rule half and a segment within such a rule half. michael@0: * It calls itself recursively to parse segments and nested michael@0: * segments. michael@0: * @param buf buffer into which to accumulate the rule pattern michael@0: * characters, either literal characters from the rule or michael@0: * standins for UnicodeMatcher objects including segments. michael@0: * @param illegal the set of special characters that is illegal during michael@0: * this parse. michael@0: * @param isSegment if true, then we've already seen a '(' and michael@0: * pos on entry points right after it. Accumulate everything michael@0: * up to the closing ')', put it in a segment matcher object, michael@0: * generate a standin for it, and add the standin to buf. As michael@0: * a side effect, update the segments vector with a reference michael@0: * to the segment matcher. This works recursively for nested michael@0: * segments. If isSegment is false, just accumulate michael@0: * characters into buf. michael@0: * @return the index after the terminating character, or michael@0: * if limit was reached, limit michael@0: */ michael@0: int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, michael@0: UnicodeString& buf, michael@0: const UnicodeString& illegal, michael@0: UBool isSegment, UErrorCode& status) { michael@0: int32_t start = pos; michael@0: ParsePosition pp; michael@0: UnicodeString scratch; michael@0: UBool done = FALSE; michael@0: int32_t quoteStart = -1; // Most recent 'single quoted string' michael@0: int32_t quoteLimit = -1; michael@0: int32_t varStart = -1; // Most recent $variableReference michael@0: int32_t varLimit = -1; michael@0: int32_t bufStart = buf.length(); michael@0: michael@0: while (pos < limit && !done) { michael@0: // Since all syntax characters are in the BMP, fetching michael@0: // 16-bit code units suffices here. michael@0: UChar c = rule.charAt(pos++); michael@0: if (PatternProps::isWhiteSpace(c)) { michael@0: // Ignore whitespace. Note that this is not Unicode michael@0: // spaces, but Java spaces -- a subset, representing michael@0: // whitespace likely to be seen in code. michael@0: continue; michael@0: } michael@0: if (u_strchr(HALF_ENDERS, c) != NULL) { michael@0: if (isSegment) { michael@0: // Unclosed segment michael@0: return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status); michael@0: } michael@0: break; michael@0: } michael@0: if (anchorEnd) { michael@0: // Text after a presumed end anchor is a syntax err michael@0: return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status); michael@0: } michael@0: if (UnicodeSet::resemblesPattern(rule, pos-1)) { michael@0: pp.setIndex(pos-1); // Backup to opening '[' michael@0: buf.append(parser.parseSet(rule, pp, status)); michael@0: if (U_FAILURE(status)) { michael@0: return syntaxError(U_MALFORMED_SET, rule, start, status); michael@0: } michael@0: pos = pp.getIndex(); michael@0: continue; michael@0: } michael@0: // Handle escapes michael@0: if (c == ESCAPE) { michael@0: if (pos == limit) { michael@0: return syntaxError(U_TRAILING_BACKSLASH, rule, start, status); michael@0: } michael@0: UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\' michael@0: if (escaped == (UChar32) -1) { michael@0: return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status); michael@0: } michael@0: if (!parser.checkVariableRange(escaped)) { michael@0: return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status); michael@0: } michael@0: buf.append(escaped); michael@0: continue; michael@0: } michael@0: // Handle quoted matter michael@0: if (c == QUOTE) { michael@0: int32_t iq = rule.indexOf(QUOTE, pos); michael@0: if (iq == pos) { michael@0: buf.append(c); // Parse [''] outside quotes as ['] michael@0: ++pos; michael@0: } else { michael@0: /* This loop picks up a run of quoted text of the michael@0: * form 'aaaa' each time through. If this run michael@0: * hasn't really ended ('aaaa''bbbb') then it keeps michael@0: * looping, each time adding on a new run. When it michael@0: * reaches the final quote it breaks. michael@0: */ michael@0: quoteStart = buf.length(); michael@0: for (;;) { michael@0: if (iq < 0) { michael@0: return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status); michael@0: } michael@0: scratch.truncate(0); michael@0: rule.extractBetween(pos, iq, scratch); michael@0: buf.append(scratch); michael@0: pos = iq+1; michael@0: if (pos < limit && rule.charAt(pos) == QUOTE) { michael@0: // Parse [''] inside quotes as ['] michael@0: iq = rule.indexOf(QUOTE, pos+1); michael@0: // Continue looping michael@0: } else { michael@0: break; michael@0: } michael@0: } michael@0: quoteLimit = buf.length(); michael@0: michael@0: for (iq=quoteStart; iq= 0) { michael@0: syntaxError(U_ILLEGAL_CHARACTER, rule, start, status); michael@0: } michael@0: michael@0: switch (c) { michael@0: michael@0: //------------------------------------------------------ michael@0: // Elements allowed within and out of segments michael@0: //------------------------------------------------------ michael@0: case ANCHOR_START: michael@0: if (buf.length() == 0 && !anchorStart) { michael@0: anchorStart = TRUE; michael@0: } else { michael@0: return syntaxError(U_MISPLACED_ANCHOR_START, michael@0: rule, start, status); michael@0: } michael@0: break; michael@0: case SEGMENT_OPEN: michael@0: { michael@0: // bufSegStart is the offset in buf to the first michael@0: // character of the segment we are parsing. michael@0: int32_t bufSegStart = buf.length(); michael@0: michael@0: // Record segment number now, since nextSegmentNumber michael@0: // will be incremented during the call to parseSection michael@0: // if there are nested segments. michael@0: int32_t segmentNumber = nextSegmentNumber++; // 1-based michael@0: michael@0: // Parse the segment michael@0: pos = parseSection(rule, pos, limit, buf, UnicodeString(TRUE, ILLEGAL_SEG, -1), TRUE, status); michael@0: michael@0: // After parsing a segment, the relevant characters are michael@0: // in buf, starting at offset bufSegStart. Extract them michael@0: // into a string matcher, and replace them with a michael@0: // standin for that matcher. michael@0: StringMatcher* m = michael@0: new StringMatcher(buf, bufSegStart, buf.length(), michael@0: segmentNumber, *parser.curData); michael@0: if (m == NULL) { michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: michael@0: // Record and associate object and segment number michael@0: parser.setSegmentObject(segmentNumber, m, status); michael@0: buf.truncate(bufSegStart); michael@0: buf.append(parser.getSegmentStandin(segmentNumber, status)); michael@0: } michael@0: break; michael@0: case FUNCTION: michael@0: case ALT_FUNCTION: michael@0: { michael@0: int32_t iref = pos; michael@0: TransliteratorIDParser::SingleID* single = michael@0: TransliteratorIDParser::parseFilterID(rule, iref); michael@0: // The next character MUST be a segment open michael@0: if (single == NULL || michael@0: !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) { michael@0: return syntaxError(U_INVALID_FUNCTION, rule, start, status); michael@0: } michael@0: michael@0: Transliterator *t = single->createInstance(); michael@0: delete single; michael@0: if (t == NULL) { michael@0: return syntaxError(U_INVALID_FUNCTION, rule, start, status); michael@0: } michael@0: michael@0: // bufSegStart is the offset in buf to the first michael@0: // character of the segment we are parsing. michael@0: int32_t bufSegStart = buf.length(); michael@0: michael@0: // Parse the segment michael@0: pos = parseSection(rule, iref, limit, buf, UnicodeString(TRUE, ILLEGAL_FUNC, -1), TRUE, status); michael@0: michael@0: // After parsing a segment, the relevant characters are michael@0: // in buf, starting at offset bufSegStart. michael@0: UnicodeString output; michael@0: buf.extractBetween(bufSegStart, buf.length(), output); michael@0: FunctionReplacer *r = michael@0: new FunctionReplacer(t, new StringReplacer(output, parser.curData)); michael@0: if (r == NULL) { michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: michael@0: // Replace the buffer contents with a stand-in michael@0: buf.truncate(bufSegStart); michael@0: buf.append(parser.generateStandInFor(r, status)); michael@0: } michael@0: break; michael@0: case SymbolTable::SYMBOL_REF: michael@0: // Handle variable references and segment references "$1" .. "$9" michael@0: { michael@0: // A variable reference must be followed immediately michael@0: // by a Unicode identifier start and zero or more michael@0: // Unicode identifier part characters, or by a digit michael@0: // 1..9 if it is a segment reference. michael@0: if (pos == limit) { michael@0: // A variable ref character at the end acts as michael@0: // an anchor to the context limit, as in perl. michael@0: anchorEnd = TRUE; michael@0: break; michael@0: } michael@0: // Parse "$1" "$2" .. "$9" .. (no upper limit) michael@0: c = rule.charAt(pos); michael@0: int32_t r = u_digit(c, 10); michael@0: if (r >= 1 && r <= 9) { michael@0: r = ICU_Utility::parseNumber(rule, pos, 10); michael@0: if (r < 0) { michael@0: return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, michael@0: rule, start, status); michael@0: } michael@0: buf.append(parser.getSegmentStandin(r, status)); michael@0: } else { michael@0: pp.setIndex(pos); michael@0: UnicodeString name = parser.parseData-> michael@0: parseReference(rule, pp, limit); michael@0: if (name.length() == 0) { michael@0: // This means the '$' was not followed by a michael@0: // valid name. Try to interpret it as an michael@0: // end anchor then. If this also doesn't work michael@0: // (if we see a following character) then signal michael@0: // an error. michael@0: anchorEnd = TRUE; michael@0: break; michael@0: } michael@0: pos = pp.getIndex(); michael@0: // If this is a variable definition statement, michael@0: // then the LHS variable will be undefined. In michael@0: // that case appendVariableDef() will append the michael@0: // special placeholder char variableLimit-1. michael@0: varStart = buf.length(); michael@0: parser.appendVariableDef(name, buf, status); michael@0: varLimit = buf.length(); michael@0: } michael@0: } michael@0: break; michael@0: case DOT: michael@0: buf.append(parser.getDotStandIn(status)); michael@0: break; michael@0: case KLEENE_STAR: michael@0: case ONE_OR_MORE: michael@0: case ZERO_OR_ONE: michael@0: // Quantifiers. We handle single characters, quoted strings, michael@0: // variable references, and segments. michael@0: // a+ matches aaa michael@0: // 'foo'+ matches foofoofoo michael@0: // $v+ matches xyxyxy if $v == xy michael@0: // (seg)+ matches segsegseg michael@0: { michael@0: if (isSegment && buf.length() == bufStart) { michael@0: // The */+ immediately follows '(' michael@0: return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status); michael@0: } michael@0: michael@0: int32_t qstart, qlimit; michael@0: // The */+ follows an isolated character or quote michael@0: // or variable reference michael@0: if (buf.length() == quoteLimit) { michael@0: // The */+ follows a 'quoted string' michael@0: qstart = quoteStart; michael@0: qlimit = quoteLimit; michael@0: } else if (buf.length() == varLimit) { michael@0: // The */+ follows a $variableReference michael@0: qstart = varStart; michael@0: qlimit = varLimit; michael@0: } else { michael@0: // The */+ follows a single character, possibly michael@0: // a segment standin michael@0: qstart = buf.length() - 1; michael@0: qlimit = qstart + 1; michael@0: } michael@0: michael@0: UnicodeFunctor *m = michael@0: new StringMatcher(buf, qstart, qlimit, 0, *parser.curData); michael@0: if (m == NULL) { michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: int32_t min = 0; michael@0: int32_t max = Quantifier::MAX; michael@0: switch (c) { michael@0: case ONE_OR_MORE: michael@0: min = 1; michael@0: break; michael@0: case ZERO_OR_ONE: michael@0: min = 0; michael@0: max = 1; michael@0: break; michael@0: // case KLEENE_STAR: michael@0: // do nothing -- min, max already set michael@0: } michael@0: m = new Quantifier(m, min, max); michael@0: if (m == NULL) { michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: buf.truncate(qstart); michael@0: buf.append(parser.generateStandInFor(m, status)); michael@0: } michael@0: break; michael@0: michael@0: //------------------------------------------------------ michael@0: // Elements allowed ONLY WITHIN segments michael@0: //------------------------------------------------------ michael@0: case SEGMENT_CLOSE: michael@0: // assert(isSegment); michael@0: // We're done parsing a segment. michael@0: done = TRUE; michael@0: break; michael@0: michael@0: //------------------------------------------------------ michael@0: // Elements allowed ONLY OUTSIDE segments michael@0: //------------------------------------------------------ michael@0: case CONTEXT_ANTE: michael@0: if (ante >= 0) { michael@0: return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status); michael@0: } michael@0: ante = buf.length(); michael@0: break; michael@0: case CONTEXT_POST: michael@0: if (post >= 0) { michael@0: return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status); michael@0: } michael@0: post = buf.length(); michael@0: break; michael@0: case CURSOR_POS: michael@0: if (cursor >= 0) { michael@0: return syntaxError(U_MULTIPLE_CURSORS, rule, start, status); michael@0: } michael@0: cursor = buf.length(); michael@0: break; michael@0: case CURSOR_OFFSET: michael@0: if (cursorOffset < 0) { michael@0: if (buf.length() > 0) { michael@0: return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); michael@0: } michael@0: --cursorOffset; michael@0: } else if (cursorOffset > 0) { michael@0: if (buf.length() != cursorOffsetPos || cursor >= 0) { michael@0: return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); michael@0: } michael@0: ++cursorOffset; michael@0: } else { michael@0: if (cursor == 0 && buf.length() == 0) { michael@0: cursorOffset = -1; michael@0: } else if (cursor < 0) { michael@0: cursorOffsetPos = buf.length(); michael@0: cursorOffset = 1; michael@0: } else { michael@0: return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); michael@0: } michael@0: } michael@0: break; michael@0: michael@0: michael@0: //------------------------------------------------------ michael@0: // Non-special characters michael@0: //------------------------------------------------------ michael@0: default: michael@0: // Disallow unquoted characters other than [0-9A-Za-z] michael@0: // in the printable ASCII range. These characters are michael@0: // reserved for possible future use. michael@0: if (c >= 0x0021 && c <= 0x007E && michael@0: !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) || michael@0: (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) || michael@0: (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) { michael@0: return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); michael@0: } michael@0: buf.append(c); michael@0: break; michael@0: } michael@0: } michael@0: michael@0: return pos; michael@0: } michael@0: michael@0: /** michael@0: * Remove context. michael@0: */ michael@0: void RuleHalf::removeContext() { michael@0: //text = text.substring(ante < 0 ? 0 : ante, michael@0: // post < 0 ? text.length() : post); michael@0: if (post >= 0) { michael@0: text.remove(post); michael@0: } michael@0: if (ante >= 0) { michael@0: text.removeBetween(0, ante); michael@0: } michael@0: ante = post = -1; michael@0: anchorStart = anchorEnd = FALSE; michael@0: } michael@0: michael@0: /** michael@0: * Return true if this half looks like valid output, that is, does not michael@0: * contain quantifiers or other special input-only elements. michael@0: */ michael@0: UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) { michael@0: for (int32_t i=0; iisReplacer(c)) { michael@0: return FALSE; michael@0: } michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: /** michael@0: * Return true if this half looks like valid input, that is, does not michael@0: * contain functions or other special output-only elements. michael@0: */ michael@0: UBool RuleHalf::isValidInput(TransliteratorParser& transParser) { michael@0: for (int32_t i=0; iisMatcher(c)) { michael@0: return FALSE; michael@0: } michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: //---------------------------------------------------------------------- michael@0: // PUBLIC API michael@0: //---------------------------------------------------------------------- michael@0: michael@0: /** michael@0: * Constructor. michael@0: */ michael@0: TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) : michael@0: dataVector(statusReturn), michael@0: idBlockVector(statusReturn), michael@0: variablesVector(statusReturn), michael@0: segmentObjects(statusReturn) michael@0: { michael@0: idBlockVector.setDeleter(uprv_deleteUObject); michael@0: curData = NULL; michael@0: compoundFilter = NULL; michael@0: parseData = NULL; michael@0: variableNames.setValueDeleter(uprv_deleteUObject); michael@0: } michael@0: michael@0: /** michael@0: * Destructor. michael@0: */ michael@0: TransliteratorParser::~TransliteratorParser() { michael@0: while (!dataVector.isEmpty()) michael@0: delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); michael@0: delete compoundFilter; michael@0: delete parseData; michael@0: while (!variablesVector.isEmpty()) michael@0: delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); michael@0: } michael@0: michael@0: void michael@0: TransliteratorParser::parse(const UnicodeString& rules, michael@0: UTransDirection transDirection, michael@0: UParseError& pe, michael@0: UErrorCode& ec) { michael@0: if (U_SUCCESS(ec)) { michael@0: parseRules(rules, transDirection, ec); michael@0: pe = parseError; michael@0: } michael@0: } michael@0: michael@0: /** michael@0: * Return the compound filter parsed by parse(). Caller owns result. michael@0: */ michael@0: UnicodeSet* TransliteratorParser::orphanCompoundFilter() { michael@0: UnicodeSet* f = compoundFilter; michael@0: compoundFilter = NULL; michael@0: return f; michael@0: } michael@0: michael@0: //---------------------------------------------------------------------- michael@0: // Private implementation michael@0: //---------------------------------------------------------------------- michael@0: michael@0: /** michael@0: * Parse the given string as a sequence of rules, separated by newline michael@0: * characters ('\n'), and cause this object to implement those rules. Any michael@0: * previous rules are discarded. Typically this method is called exactly michael@0: * once, during construction. michael@0: * @exception IllegalArgumentException if there is a syntax error in the michael@0: * rules michael@0: */ michael@0: void TransliteratorParser::parseRules(const UnicodeString& rule, michael@0: UTransDirection theDirection, michael@0: UErrorCode& status) michael@0: { michael@0: // Clear error struct michael@0: uprv_memset(&parseError, 0, sizeof(parseError)); michael@0: parseError.line = parseError.offset = -1; michael@0: michael@0: UBool parsingIDs = TRUE; michael@0: int32_t ruleCount = 0; michael@0: michael@0: while (!dataVector.isEmpty()) { michael@0: delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); michael@0: } michael@0: if (U_FAILURE(status)) { michael@0: return; michael@0: } michael@0: michael@0: idBlockVector.removeAllElements(); michael@0: curData = NULL; michael@0: direction = theDirection; michael@0: ruleCount = 0; michael@0: michael@0: delete compoundFilter; michael@0: compoundFilter = NULL; michael@0: michael@0: while (!variablesVector.isEmpty()) { michael@0: delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); michael@0: } michael@0: variableNames.removeAll(); michael@0: parseData = new ParseData(0, &variablesVector, &variableNames); michael@0: if (parseData == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: michael@0: dotStandIn = (UChar) -1; michael@0: michael@0: UnicodeString *tempstr = NULL; // used for memory allocation error checking michael@0: UnicodeString str; // scratch michael@0: UnicodeString idBlockResult; michael@0: int32_t pos = 0; michael@0: int32_t limit = rule.length(); michael@0: michael@0: // The compound filter offset is an index into idBlockResult. michael@0: // If it is 0, then the compound filter occurred at the start, michael@0: // and it is the offset to the _start_ of the compound filter michael@0: // pattern. Otherwise it is the offset to the _limit_ of the michael@0: // compound filter pattern within idBlockResult. michael@0: compoundFilter = NULL; michael@0: int32_t compoundFilterOffset = -1; michael@0: michael@0: while (pos < limit && U_SUCCESS(status)) { michael@0: UChar c = rule.charAt(pos++); michael@0: if (PatternProps::isWhiteSpace(c)) { michael@0: // Ignore leading whitespace. michael@0: continue; michael@0: } michael@0: // Skip lines starting with the comment character michael@0: if (c == RULE_COMMENT_CHAR) { michael@0: pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1; michael@0: if (pos == 0) { michael@0: break; // No "\n" found; rest of rule is a commnet michael@0: } michael@0: continue; // Either fall out or restart with next line michael@0: } michael@0: michael@0: // skip empty rules michael@0: if (c == END_OF_RULE) michael@0: continue; michael@0: michael@0: // keep track of how many rules we've seen michael@0: ++ruleCount; michael@0: michael@0: // We've found the start of a rule or ID. c is its first michael@0: // character, and pos points past c. michael@0: --pos; michael@0: // Look for an ID token. Must have at least ID_TOKEN_LEN + 1 michael@0: // chars left. michael@0: if ((pos + ID_TOKEN_LEN + 1) <= limit && michael@0: rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) { michael@0: pos += ID_TOKEN_LEN; michael@0: c = rule.charAt(pos); michael@0: while (PatternProps::isWhiteSpace(c) && pos < limit) { michael@0: ++pos; michael@0: c = rule.charAt(pos); michael@0: } michael@0: michael@0: int32_t p = pos; michael@0: michael@0: if (!parsingIDs) { michael@0: if (curData != NULL) { michael@0: if (direction == UTRANS_FORWARD) michael@0: dataVector.addElement(curData, status); michael@0: else michael@0: dataVector.insertElementAt(curData, 0, status); michael@0: curData = NULL; michael@0: } michael@0: parsingIDs = TRUE; michael@0: } michael@0: michael@0: TransliteratorIDParser::SingleID* id = michael@0: TransliteratorIDParser::parseSingleID(rule, p, direction, status); michael@0: if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) { michael@0: // Successful ::ID parse. michael@0: michael@0: if (direction == UTRANS_FORWARD) { michael@0: idBlockResult.append(id->canonID).append(END_OF_RULE); michael@0: } else { michael@0: idBlockResult.insert(0, END_OF_RULE); michael@0: idBlockResult.insert(0, id->canonID); michael@0: } michael@0: michael@0: } else { michael@0: // Couldn't parse an ID. Try to parse a global filter michael@0: int32_t withParens = -1; michael@0: UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL); michael@0: if (f != NULL) { michael@0: if (ICU_Utility::parseChar(rule, p, END_OF_RULE) michael@0: && (direction == UTRANS_FORWARD) == (withParens == 0)) michael@0: { michael@0: if (compoundFilter != NULL) { michael@0: // Multiple compound filters michael@0: syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status); michael@0: delete f; michael@0: } else { michael@0: compoundFilter = f; michael@0: compoundFilterOffset = ruleCount; michael@0: } michael@0: } else { michael@0: delete f; michael@0: } michael@0: } else { michael@0: // Invalid ::id michael@0: // Can be parsed as neither an ID nor a global filter michael@0: syntaxError(U_INVALID_ID, rule, pos, status); michael@0: } michael@0: } michael@0: delete id; michael@0: pos = p; michael@0: } else { michael@0: if (parsingIDs) { michael@0: tempstr = new UnicodeString(idBlockResult); michael@0: // NULL pointer check michael@0: if (tempstr == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: if (direction == UTRANS_FORWARD) michael@0: idBlockVector.addElement(tempstr, status); michael@0: else michael@0: idBlockVector.insertElementAt(tempstr, 0, status); michael@0: idBlockResult.remove(); michael@0: parsingIDs = FALSE; michael@0: curData = new TransliterationRuleData(status); michael@0: // NULL pointer check michael@0: if (curData == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: parseData->data = curData; michael@0: michael@0: // By default, rules use part of the private use area michael@0: // E000..F8FF for variables and other stand-ins. Currently michael@0: // the range F000..F8FF is typically sufficient. The 'use michael@0: // variable range' pragma allows rule sets to modify this. michael@0: setVariableRange(0xF000, 0xF8FF, status); michael@0: } michael@0: michael@0: if (resemblesPragma(rule, pos, limit)) { michael@0: int32_t ppp = parsePragma(rule, pos, limit, status); michael@0: if (ppp < 0) { michael@0: syntaxError(U_MALFORMED_PRAGMA, rule, pos, status); michael@0: } michael@0: pos = ppp; michael@0: // Parse a rule michael@0: } else { michael@0: pos = parseRule(rule, pos, limit, status); michael@0: } michael@0: } michael@0: } michael@0: michael@0: if (parsingIDs && idBlockResult.length() > 0) { michael@0: tempstr = new UnicodeString(idBlockResult); michael@0: // NULL pointer check michael@0: if (tempstr == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: if (direction == UTRANS_FORWARD) michael@0: idBlockVector.addElement(tempstr, status); michael@0: else michael@0: idBlockVector.insertElementAt(tempstr, 0, status); michael@0: } michael@0: else if (!parsingIDs && curData != NULL) { michael@0: if (direction == UTRANS_FORWARD) michael@0: dataVector.addElement(curData, status); michael@0: else michael@0: dataVector.insertElementAt(curData, 0, status); michael@0: } michael@0: michael@0: if (U_SUCCESS(status)) { michael@0: // Convert the set vector to an array michael@0: int32_t i, dataVectorSize = dataVector.size(); michael@0: for (i = 0; i < dataVectorSize; i++) { michael@0: TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); michael@0: data->variablesLength = variablesVector.size(); michael@0: if (data->variablesLength == 0) { michael@0: data->variables = 0; michael@0: } else { michael@0: data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*)); michael@0: // NULL pointer check michael@0: if (data->variables == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: data->variablesAreOwned = (i == 0); michael@0: } michael@0: michael@0: for (int32_t j = 0; j < data->variablesLength; j++) { michael@0: data->variables[j] = michael@0: ((UnicodeSet*)variablesVector.elementAt(j)); michael@0: } michael@0: michael@0: data->variableNames.removeAll(); michael@0: int32_t pos = -1; michael@0: const UHashElement* he = variableNames.nextElement(pos); michael@0: while (he != NULL) { michael@0: UnicodeString* tempus = (UnicodeString*)(((UnicodeString*)(he->value.pointer))->clone()); michael@0: if (tempus == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: data->variableNames.put(*((UnicodeString*)(he->key.pointer)), michael@0: tempus, status); michael@0: he = variableNames.nextElement(pos); michael@0: } michael@0: } michael@0: variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed michael@0: michael@0: // Index the rules michael@0: if (compoundFilter != NULL) { michael@0: if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) || michael@0: (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) { michael@0: status = U_MISPLACED_COMPOUND_FILTER; michael@0: } michael@0: } michael@0: michael@0: for (i = 0; i < dataVectorSize; i++) { michael@0: TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); michael@0: data->ruleSet.freeze(parseError, status); michael@0: } michael@0: if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) { michael@0: idBlockVector.removeElementAt(0); michael@0: } michael@0: } michael@0: } michael@0: michael@0: /** michael@0: * Set the variable range to [start, end] (inclusive). michael@0: */ michael@0: void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) { michael@0: if (start > end || start < 0 || end > 0xFFFF) { michael@0: status = U_MALFORMED_PRAGMA; michael@0: return; michael@0: } michael@0: michael@0: curData->variablesBase = (UChar) start; michael@0: if (dataVector.size() == 0) { michael@0: variableNext = (UChar) start; michael@0: variableLimit = (UChar) (end + 1); michael@0: } michael@0: } michael@0: michael@0: /** michael@0: * Assert that the given character is NOT within the variable range. michael@0: * If it is, return FALSE. This is neccesary to ensure that the michael@0: * variable range does not overlap characters used in a rule. michael@0: */ michael@0: UBool TransliteratorParser::checkVariableRange(UChar32 ch) const { michael@0: return !(ch >= curData->variablesBase && ch < variableLimit); michael@0: } michael@0: michael@0: /** michael@0: * Set the maximum backup to 'backup', in response to a pragma michael@0: * statement. michael@0: */ michael@0: void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) { michael@0: //TODO Finish michael@0: } michael@0: michael@0: /** michael@0: * Begin normalizing all rules using the given mode, in response michael@0: * to a pragma statement. michael@0: */ michael@0: void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) { michael@0: //TODO Finish michael@0: } michael@0: michael@0: static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use " michael@0: michael@0: static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;" michael@0: michael@0: static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;" michael@0: michael@0: static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;" michael@0: michael@0: static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;" michael@0: michael@0: /** michael@0: * Return true if the given rule looks like a pragma. michael@0: * @param pos offset to the first non-whitespace character michael@0: * of the rule. michael@0: * @param limit pointer past the last character of the rule. michael@0: */ michael@0: UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) { michael@0: // Must start with /use\s/i michael@0: return ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_USE, 4), NULL) >= 0; michael@0: } michael@0: michael@0: /** michael@0: * Parse a pragma. This method assumes resemblesPragma() has michael@0: * already returned true. michael@0: * @param pos offset to the first non-whitespace character michael@0: * of the rule. michael@0: * @param limit pointer past the last character of the rule. michael@0: * @return the position index after the final ';' of the pragma, michael@0: * or -1 on failure. michael@0: */ michael@0: int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { michael@0: int32_t array[2]; michael@0: michael@0: // resemblesPragma() has already returned true, so we michael@0: // know that pos points to /use\s/i; we can skip 4 characters michael@0: // immediately michael@0: pos += 4; michael@0: michael@0: // Here are the pragmas we recognize: michael@0: // use variable range 0xE000 0xEFFF; michael@0: // use maximum backup 16; michael@0: // use nfd rules; michael@0: // use nfc rules; michael@0: int p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_VARIABLE_RANGE, -1), array); michael@0: if (p >= 0) { michael@0: setVariableRange(array[0], array[1], status); michael@0: return p; michael@0: } michael@0: michael@0: p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_MAXIMUM_BACKUP, -1), array); michael@0: if (p >= 0) { michael@0: pragmaMaximumBackup(array[0]); michael@0: return p; michael@0: } michael@0: michael@0: p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFD_RULES, -1), NULL); michael@0: if (p >= 0) { michael@0: pragmaNormalizeRules(UNORM_NFD); michael@0: return p; michael@0: } michael@0: michael@0: p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFC_RULES, -1), NULL); michael@0: if (p >= 0) { michael@0: pragmaNormalizeRules(UNORM_NFC); michael@0: return p; michael@0: } michael@0: michael@0: // Syntax error: unable to parse pragma michael@0: return -1; michael@0: } michael@0: michael@0: /** michael@0: * MAIN PARSER. Parse the next rule in the given rule string, starting michael@0: * at pos. Return the index after the last character parsed. Do not michael@0: * parse characters at or after limit. michael@0: * michael@0: * Important: The character at pos must be a non-whitespace character michael@0: * that is not the comment character. michael@0: * michael@0: * This method handles quoting, escaping, and whitespace removal. It michael@0: * parses the end-of-rule character. It recognizes context and cursor michael@0: * indicators. Once it does a lexical breakdown of the rule at pos, it michael@0: * creates a rule object and adds it to our rule list. michael@0: */ michael@0: int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { michael@0: // Locate the left side, operator, and right side michael@0: int32_t start = pos; michael@0: UChar op = 0; michael@0: int32_t i; michael@0: michael@0: // Set up segments data michael@0: segmentStandins.truncate(0); michael@0: segmentObjects.removeAllElements(); michael@0: michael@0: // Use pointers to automatics to make swapping possible. michael@0: RuleHalf _left(*this), _right(*this); michael@0: RuleHalf* left = &_left; michael@0: RuleHalf* right = &_right; michael@0: michael@0: undefinedVariableName.remove(); michael@0: pos = left->parse(rule, pos, limit, status); michael@0: if (U_FAILURE(status)) { michael@0: return start; michael@0: } michael@0: michael@0: if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) { michael@0: return syntaxError(U_MISSING_OPERATOR, rule, start, status); michael@0: } michael@0: ++pos; michael@0: michael@0: // Found an operator char. Check for forward-reverse operator. michael@0: if (op == REVERSE_RULE_OP && michael@0: (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) { michael@0: ++pos; michael@0: op = FWDREV_RULE_OP; michael@0: } michael@0: michael@0: // Translate alternate op characters. michael@0: switch (op) { michael@0: case ALT_FORWARD_RULE_OP: michael@0: op = FORWARD_RULE_OP; michael@0: break; michael@0: case ALT_REVERSE_RULE_OP: michael@0: op = REVERSE_RULE_OP; michael@0: break; michael@0: case ALT_FWDREV_RULE_OP: michael@0: op = FWDREV_RULE_OP; michael@0: break; michael@0: } michael@0: michael@0: pos = right->parse(rule, pos, limit, status); michael@0: if (U_FAILURE(status)) { michael@0: return start; michael@0: } michael@0: michael@0: if (pos < limit) { michael@0: if (rule.charAt(--pos) == END_OF_RULE) { michael@0: ++pos; michael@0: } else { michael@0: // RuleHalf parser must have terminated at an operator michael@0: return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); michael@0: } michael@0: } michael@0: michael@0: if (op == VARIABLE_DEF_OP) { michael@0: // LHS is the name. RHS is a single character, either a literal michael@0: // or a set (already parsed). If RHS is longer than one michael@0: // character, it is either a multi-character string, or multiple michael@0: // sets, or a mixture of chars and sets -- syntax error. michael@0: michael@0: // We expect to see a single undefined variable (the one being michael@0: // defined). michael@0: if (undefinedVariableName.length() == 0) { michael@0: // "Missing '$' or duplicate definition" michael@0: return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status); michael@0: } michael@0: if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) { michael@0: // "Malformed LHS" michael@0: return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); michael@0: } michael@0: if (left->anchorStart || left->anchorEnd || michael@0: right->anchorStart || right->anchorEnd) { michael@0: return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); michael@0: } michael@0: // We allow anything on the right, including an empty string. michael@0: UnicodeString* value = new UnicodeString(right->text); michael@0: // NULL pointer check michael@0: if (value == NULL) { michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: variableNames.put(undefinedVariableName, value, status); michael@0: ++variableLimit; michael@0: return pos; michael@0: } michael@0: michael@0: // If this is not a variable definition rule, we shouldn't have michael@0: // any undefined variable names. michael@0: if (undefinedVariableName.length() != 0) { michael@0: return syntaxError(// "Undefined variable $" + undefinedVariableName, michael@0: U_UNDEFINED_VARIABLE, michael@0: rule, start, status); michael@0: } michael@0: michael@0: // Verify segments michael@0: if (segmentStandins.length() > segmentObjects.size()) { michael@0: syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status); michael@0: } michael@0: for (i=0; iremoveContext(); michael@0: left->cursor = -1; michael@0: left->cursorOffset = 0; michael@0: } michael@0: michael@0: // Normalize context michael@0: if (left->ante < 0) { michael@0: left->ante = 0; michael@0: } michael@0: if (left->post < 0) { michael@0: left->post = left->text.length(); michael@0: } michael@0: michael@0: // Context is only allowed on the input side. Cursors are only michael@0: // allowed on the output side. Segment delimiters can only appear michael@0: // on the left, and references on the right. Cursor offset michael@0: // cannot appear without an explicit cursor. Cursor offset michael@0: // cannot place the cursor outside the limits of the context. michael@0: // Anchors are only allowed on the input side. michael@0: if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 || michael@0: (right->cursorOffset != 0 && right->cursor < 0) || michael@0: // - The following two checks were used to ensure that the michael@0: // - the cursor offset stayed within the ante- or postcontext. michael@0: // - However, with the addition of quantifiers, we have to michael@0: // - allow arbitrary cursor offsets and do runtime checking. michael@0: //(right->cursorOffset > (left->text.length() - left->post)) || michael@0: //(-right->cursorOffset > left->ante) || michael@0: right->anchorStart || right->anchorEnd || michael@0: !left->isValidInput(*this) || !right->isValidOutput(*this) || michael@0: left->ante > left->post) { michael@0: michael@0: return syntaxError(U_MALFORMED_RULE, rule, start, status); michael@0: } michael@0: michael@0: // Flatten segment objects vector to an array michael@0: UnicodeFunctor** segmentsArray = NULL; michael@0: if (segmentObjects.size() > 0) { michael@0: segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *)); michael@0: // Null pointer check michael@0: if (segmentsArray == NULL) { michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: segmentObjects.toArray((void**) segmentsArray); michael@0: } michael@0: TransliterationRule* temptr = new TransliterationRule( michael@0: left->text, left->ante, left->post, michael@0: right->text, right->cursor, right->cursorOffset, michael@0: segmentsArray, michael@0: segmentObjects.size(), michael@0: left->anchorStart, left->anchorEnd, michael@0: curData, michael@0: status); michael@0: //Null pointer check michael@0: if (temptr == NULL) { michael@0: uprv_free(segmentsArray); michael@0: return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); michael@0: } michael@0: michael@0: curData->ruleSet.addRule(temptr, status); michael@0: michael@0: return pos; michael@0: } michael@0: michael@0: /** michael@0: * Called by main parser upon syntax error. Search the rule string michael@0: * for the probable end of the rule. Of course, if the error is that michael@0: * the end of rule marker is missing, then the rule end will not be found. michael@0: * In any case the rule start will be correctly reported. michael@0: * @param msg error description michael@0: * @param rule pattern string michael@0: * @param start position of first character of current rule michael@0: */ michael@0: int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode, michael@0: const UnicodeString& rule, michael@0: int32_t pos, michael@0: UErrorCode& status) michael@0: { michael@0: parseError.offset = pos; michael@0: parseError.line = 0 ; /* we are not using line numbers */ michael@0: michael@0: // for pre-context michael@0: const int32_t LEN = U_PARSE_CONTEXT_LEN - 1; michael@0: int32_t start = uprv_max(pos - LEN, 0); michael@0: int32_t stop = pos; michael@0: michael@0: rule.extract(start,stop-start,parseError.preContext); michael@0: //null terminate the buffer michael@0: parseError.preContext[stop-start] = 0; michael@0: michael@0: //for post-context michael@0: start = pos; michael@0: stop = uprv_min(pos + LEN, rule.length()); michael@0: michael@0: rule.extract(start,stop-start,parseError.postContext); michael@0: //null terminate the buffer michael@0: parseError.postContext[stop-start]= 0; michael@0: michael@0: status = (UErrorCode)parseErrorCode; michael@0: return pos; michael@0: michael@0: } michael@0: michael@0: /** michael@0: * Parse a UnicodeSet out, store it, and return the stand-in character michael@0: * used to represent it. michael@0: */ michael@0: UChar TransliteratorParser::parseSet(const UnicodeString& rule, michael@0: ParsePosition& pos, michael@0: UErrorCode& status) { michael@0: UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status); michael@0: // Null pointer check michael@0: if (set == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return (UChar)0x0000; // Return empty character with error. michael@0: } michael@0: set->compact(); michael@0: return generateStandInFor(set, status); michael@0: } michael@0: michael@0: /** michael@0: * Generate and return a stand-in for a new UnicodeFunctor. Store michael@0: * the matcher (adopt it). michael@0: */ michael@0: UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) { michael@0: // assert(obj != null); michael@0: michael@0: // Look up previous stand-in, if any. This is a short list michael@0: // (typical n is 0, 1, or 2); linear search is optimal. michael@0: for (int32_t i=0; ivariablesBase + i); michael@0: } michael@0: } michael@0: michael@0: if (variableNext >= variableLimit) { michael@0: delete adopted; michael@0: status = U_VARIABLE_RANGE_EXHAUSTED; michael@0: return 0; michael@0: } michael@0: variablesVector.addElement(adopted, status); michael@0: return variableNext++; michael@0: } michael@0: michael@0: /** michael@0: * Return the standin for segment seg (1-based). michael@0: */ michael@0: UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) { michael@0: // Special character used to indicate an empty spot michael@0: UChar empty = curData->variablesBase - 1; michael@0: while (segmentStandins.length() < seg) { michael@0: segmentStandins.append(empty); michael@0: } michael@0: UChar c = segmentStandins.charAt(seg-1); michael@0: if (c == empty) { michael@0: if (variableNext >= variableLimit) { michael@0: status = U_VARIABLE_RANGE_EXHAUSTED; michael@0: return 0; michael@0: } michael@0: c = variableNext++; michael@0: // Set a placeholder in the master variables vector that will be michael@0: // filled in later by setSegmentObject(). We know that we will get michael@0: // called first because setSegmentObject() will call us. michael@0: variablesVector.addElement((void*) NULL, status); michael@0: segmentStandins.setCharAt(seg-1, c); michael@0: } michael@0: return c; michael@0: } michael@0: michael@0: /** michael@0: * Set the object for segment seg (1-based). michael@0: */ michael@0: void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) { michael@0: // Since we call parseSection() recursively, nested michael@0: // segments will result in segment i+1 getting parsed michael@0: // and stored before segment i; be careful with the michael@0: // vector handling here. michael@0: if (segmentObjects.size() < seg) { michael@0: segmentObjects.setSize(seg, status); michael@0: } michael@0: int32_t index = getSegmentStandin(seg, status) - curData->variablesBase; michael@0: if (segmentObjects.elementAt(seg-1) != NULL || michael@0: variablesVector.elementAt(index) != NULL) { michael@0: // should never happen michael@0: status = U_INTERNAL_TRANSLITERATOR_ERROR; michael@0: return; michael@0: } michael@0: segmentObjects.setElementAt(adopted, seg-1); michael@0: variablesVector.setElementAt(adopted, index); michael@0: } michael@0: michael@0: /** michael@0: * Return the stand-in for the dot set. It is allocated the first michael@0: * time and reused thereafter. michael@0: */ michael@0: UChar TransliteratorParser::getDotStandIn(UErrorCode& status) { michael@0: if (dotStandIn == (UChar) -1) { michael@0: UnicodeSet* tempus = new UnicodeSet(UnicodeString(TRUE, DOT_SET, -1), status); michael@0: // Null pointer check. michael@0: if (tempus == NULL) { michael@0: status = U_MEMORY_ALLOCATION_ERROR; michael@0: return (UChar)0x0000; michael@0: } michael@0: dotStandIn = generateStandInFor(tempus, status); michael@0: } michael@0: return dotStandIn; michael@0: } michael@0: michael@0: /** michael@0: * Append the value of the given variable name to the given michael@0: * UnicodeString. michael@0: */ michael@0: void TransliteratorParser::appendVariableDef(const UnicodeString& name, michael@0: UnicodeString& buf, michael@0: UErrorCode& status) { michael@0: const UnicodeString* s = (const UnicodeString*) variableNames.get(name); michael@0: if (s == NULL) { michael@0: // We allow one undefined variable so that variable definition michael@0: // statements work. For the first undefined variable we return michael@0: // the special placeholder variableLimit-1, and save the variable michael@0: // name. michael@0: if (undefinedVariableName.length() == 0) { michael@0: undefinedVariableName = name; michael@0: if (variableNext >= variableLimit) { michael@0: // throw new RuntimeException("Private use variables exhausted"); michael@0: status = U_ILLEGAL_ARGUMENT_ERROR; michael@0: return; michael@0: } michael@0: buf.append((UChar) --variableLimit); michael@0: } else { michael@0: //throw new IllegalArgumentException("Undefined variable $" michael@0: // + name); michael@0: status = U_ILLEGAL_ARGUMENT_ERROR; michael@0: return; michael@0: } michael@0: } else { michael@0: buf.append(*s); michael@0: } michael@0: } michael@0: michael@0: /** michael@0: * Glue method to get around access restrictions in C++. michael@0: */ michael@0: /*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) { michael@0: return Transliterator::createBasicInstance(id, canonID); michael@0: }*/ michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: U_CAPI int32_t michael@0: utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) { michael@0: U_NAMESPACE_USE michael@0: michael@0: //const UChar *sourceStart = source; michael@0: const UChar *targetStart = target; michael@0: const UChar *sourceLimit = source+sourceLen; michael@0: UChar *targetLimit = target+sourceLen; michael@0: UChar32 c = 0; michael@0: UBool quoted = FALSE; michael@0: int32_t index; michael@0: michael@0: uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR); michael@0: michael@0: /* read the rules into the buffer */ michael@0: while (source < sourceLimit) michael@0: { michael@0: index=0; michael@0: U16_NEXT_UNSAFE(source, index, c); michael@0: source+=index; michael@0: if(c == QUOTE) { michael@0: quoted = (UBool)!quoted; michael@0: } michael@0: else if (!quoted) { michael@0: if (c == RULE_COMMENT_CHAR) { michael@0: /* skip comments and all preceding spaces */ michael@0: while (targetStart < target && *(target - 1) == 0x0020) { michael@0: target--; michael@0: } michael@0: do { michael@0: c = *(source++); michael@0: } michael@0: while (c != CR && c != LF); michael@0: } michael@0: else if (c == ESCAPE) { michael@0: UChar32 c2 = *source; michael@0: if (c2 == CR || c2 == LF) { michael@0: /* A backslash at the end of a line. */ michael@0: /* Since we're stripping lines, ignore the backslash. */ michael@0: source++; michael@0: continue; michael@0: } michael@0: if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */ michael@0: int32_t escapeOffset = 0; michael@0: UnicodeString escapedStr(source, 5); michael@0: c2 = escapedStr.unescapeAt(escapeOffset); michael@0: michael@0: if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0) michael@0: { michael@0: *status = U_PARSE_ERROR; michael@0: return 0; michael@0: } michael@0: if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) { michael@0: /* It was escaped for a reason. Write what it was suppose to be. */ michael@0: source+=5; michael@0: c = c2; michael@0: } michael@0: } michael@0: else if (c2 == QUOTE) { michael@0: /* \' seen. Make sure we don't do anything when we see it again. */ michael@0: quoted = (UBool)!quoted; michael@0: } michael@0: } michael@0: } michael@0: if (c == CR || c == LF) michael@0: { michael@0: /* ignore spaces carriage returns, and all leading spaces on the next line. michael@0: * and line feed unless in the form \uXXXX michael@0: */ michael@0: quoted = FALSE; michael@0: while (source < sourceLimit) { michael@0: c = *(source); michael@0: if (c != CR && c != LF && c != 0x0020) { michael@0: break; michael@0: } michael@0: source++; michael@0: } michael@0: continue; michael@0: } michael@0: michael@0: /* Append UChar * after dissembling if c > 0xffff*/ michael@0: index=0; michael@0: U16_APPEND_UNSAFE(target, index, c); michael@0: target+=index; michael@0: } michael@0: if (target < targetLimit) { michael@0: *target = 0; michael@0: } michael@0: return (int32_t)(target-targetStart); michael@0: } michael@0: michael@0: #endif /* #if !UCONFIG_NO_TRANSLITERATION */