michael@0: /*
michael@0: **********************************************************************
michael@0: * Copyright (C) 1999-2011, International Business Machines
michael@0: * Corporation and others. All Rights Reserved.
michael@0: **********************************************************************
michael@0: * Date Name Description
michael@0: * 11/17/99 aliu Creation.
michael@0: **********************************************************************
michael@0: */
michael@0:
michael@0: #include "unicode/utypes.h"
michael@0:
michael@0: #if !UCONFIG_NO_TRANSLITERATION
michael@0:
michael@0: #include "unicode/rep.h"
michael@0: #include "unicode/unifilt.h"
michael@0: #include "unicode/uniset.h"
michael@0: #include "unicode/utf16.h"
michael@0: #include "rbt_rule.h"
michael@0: #include "rbt_data.h"
michael@0: #include "cmemory.h"
michael@0: #include "strmatch.h"
michael@0: #include "strrepl.h"
michael@0: #include "util.h"
michael@0: #include "putilimp.h"
michael@0:
michael@0: static const UChar FORWARD_OP[] = {32,62,32,0}; // " > "
michael@0:
michael@0: U_NAMESPACE_BEGIN
michael@0:
michael@0: /**
michael@0: * Construct a new rule with the given input, output text, and other
michael@0: * attributes. A cursor position may be specified for the output text.
michael@0: * @param input input string, including key and optional ante and
michael@0: * post context
michael@0: * @param anteContextPos offset into input to end of ante context, or -1 if
michael@0: * none. Must be <= input.length() if not -1.
michael@0: * @param postContextPos offset into input to start of post context, or -1
michael@0: * if none. Must be <= input.length() if not -1, and must be >=
michael@0: * anteContextPos.
michael@0: * @param output output string
michael@0: * @param cursorPosition offset into output at which cursor is located, or -1 if
michael@0: * none. If less than zero, then the cursor is placed after the
michael@0: * output
; that is, -1 is equivalent to
michael@0: * output.length()
. If greater than
michael@0: * output.length()
then an exception is thrown.
michael@0: * @param segs array of UnicodeFunctors corresponding to input pattern
michael@0: * segments, or null if there are none. The array itself is adopted,
michael@0: * but the pointers within it are not.
michael@0: * @param segsCount number of elements in segs[]
michael@0: * @param anchorStart TRUE if the the rule is anchored on the left to
michael@0: * the context start
michael@0: * @param anchorEnd TRUE if the rule is anchored on the right to the
michael@0: * context limit
michael@0: */
michael@0: TransliterationRule::TransliterationRule(const UnicodeString& input,
michael@0: int32_t anteContextPos, int32_t postContextPos,
michael@0: const UnicodeString& outputStr,
michael@0: int32_t cursorPosition, int32_t cursorOffset,
michael@0: UnicodeFunctor** segs,
michael@0: int32_t segsCount,
michael@0: UBool anchorStart, UBool anchorEnd,
michael@0: const TransliterationRuleData* theData,
michael@0: UErrorCode& status) :
michael@0: UMemory(),
michael@0: segments(0),
michael@0: data(theData) {
michael@0:
michael@0: if (U_FAILURE(status)) {
michael@0: return;
michael@0: }
michael@0: // Do range checks only when warranted to save time
michael@0: if (anteContextPos < 0) {
michael@0: anteContextLength = 0;
michael@0: } else {
michael@0: if (anteContextPos > input.length()) {
michael@0: // throw new IllegalArgumentException("Invalid ante context");
michael@0: status = U_ILLEGAL_ARGUMENT_ERROR;
michael@0: return;
michael@0: }
michael@0: anteContextLength = anteContextPos;
michael@0: }
michael@0: if (postContextPos < 0) {
michael@0: keyLength = input.length() - anteContextLength;
michael@0: } else {
michael@0: if (postContextPos < anteContextLength ||
michael@0: postContextPos > input.length()) {
michael@0: // throw new IllegalArgumentException("Invalid post context");
michael@0: status = U_ILLEGAL_ARGUMENT_ERROR;
michael@0: return;
michael@0: }
michael@0: keyLength = postContextPos - anteContextLength;
michael@0: }
michael@0: if (cursorPosition < 0) {
michael@0: cursorPosition = outputStr.length();
michael@0: } else if (cursorPosition > outputStr.length()) {
michael@0: // throw new IllegalArgumentException("Invalid cursor position");
michael@0: status = U_ILLEGAL_ARGUMENT_ERROR;
michael@0: return;
michael@0: }
michael@0: // We don't validate the segments array. The caller must
michael@0: // guarantee that the segments are well-formed (that is, that
michael@0: // all $n references in the output refer to indices of this
michael@0: // array, and that no array elements are null).
michael@0: this->segments = segs;
michael@0: this->segmentsCount = segsCount;
michael@0:
michael@0: pattern = input;
michael@0: flags = 0;
michael@0: if (anchorStart) {
michael@0: flags |= ANCHOR_START;
michael@0: }
michael@0: if (anchorEnd) {
michael@0: flags |= ANCHOR_END;
michael@0: }
michael@0:
michael@0: anteContext = NULL;
michael@0: if (anteContextLength > 0) {
michael@0: anteContext = new StringMatcher(pattern, 0, anteContextLength,
michael@0: FALSE, *data);
michael@0: /* test for NULL */
michael@0: if (anteContext == 0) {
michael@0: status = U_MEMORY_ALLOCATION_ERROR;
michael@0: return;
michael@0: }
michael@0: }
michael@0:
michael@0: key = NULL;
michael@0: if (keyLength > 0) {
michael@0: key = new StringMatcher(pattern, anteContextLength, anteContextLength + keyLength,
michael@0: FALSE, *data);
michael@0: /* test for NULL */
michael@0: if (key == 0) {
michael@0: status = U_MEMORY_ALLOCATION_ERROR;
michael@0: return;
michael@0: }
michael@0: }
michael@0:
michael@0: int32_t postContextLength = pattern.length() - keyLength - anteContextLength;
michael@0: postContext = NULL;
michael@0: if (postContextLength > 0) {
michael@0: postContext = new StringMatcher(pattern, anteContextLength + keyLength, pattern.length(),
michael@0: FALSE, *data);
michael@0: /* test for NULL */
michael@0: if (postContext == 0) {
michael@0: status = U_MEMORY_ALLOCATION_ERROR;
michael@0: return;
michael@0: }
michael@0: }
michael@0:
michael@0: this->output = new StringReplacer(outputStr, cursorPosition + cursorOffset, data);
michael@0: /* test for NULL */
michael@0: if (this->output == 0) {
michael@0: status = U_MEMORY_ALLOCATION_ERROR;
michael@0: return;
michael@0: }
michael@0: }
michael@0:
michael@0: /**
michael@0: * Copy constructor.
michael@0: */
michael@0: TransliterationRule::TransliterationRule(TransliterationRule& other) :
michael@0: UMemory(other),
michael@0: anteContext(NULL),
michael@0: key(NULL),
michael@0: postContext(NULL),
michael@0: pattern(other.pattern),
michael@0: anteContextLength(other.anteContextLength),
michael@0: keyLength(other.keyLength),
michael@0: flags(other.flags),
michael@0: data(other.data) {
michael@0:
michael@0: segments = NULL;
michael@0: segmentsCount = 0;
michael@0: if (other.segmentsCount > 0) {
michael@0: segments = (UnicodeFunctor **)uprv_malloc(other.segmentsCount * sizeof(UnicodeFunctor *));
michael@0: uprv_memcpy(segments, other.segments, other.segmentsCount*sizeof(segments[0]));
michael@0: }
michael@0:
michael@0: if (other.anteContext != NULL) {
michael@0: anteContext = (StringMatcher*) other.anteContext->clone();
michael@0: }
michael@0: if (other.key != NULL) {
michael@0: key = (StringMatcher*) other.key->clone();
michael@0: }
michael@0: if (other.postContext != NULL) {
michael@0: postContext = (StringMatcher*) other.postContext->clone();
michael@0: }
michael@0: output = other.output->clone();
michael@0: }
michael@0:
michael@0: TransliterationRule::~TransliterationRule() {
michael@0: uprv_free(segments);
michael@0: delete anteContext;
michael@0: delete key;
michael@0: delete postContext;
michael@0: delete output;
michael@0: }
michael@0:
michael@0: /**
michael@0: * Return the preceding context length. This method is needed to
michael@0: * support the Transliterator
method
michael@0: * getMaximumContextLength()
. Internally, this is
michael@0: * implemented as the anteContextLength, optionally plus one if
michael@0: * there is a start anchor. The one character anchor gap is
michael@0: * needed to make repeated incremental transliteration with
michael@0: * anchors work.
michael@0: */
michael@0: int32_t TransliterationRule::getContextLength(void) const {
michael@0: return anteContextLength + ((flags & ANCHOR_START) ? 1 : 0);
michael@0: }
michael@0:
michael@0: /**
michael@0: * Internal method. Returns 8-bit index value for this rule.
michael@0: * This is the low byte of the first character of the key,
michael@0: * unless the first character of the key is a set. If it's a
michael@0: * set, or otherwise can match multiple keys, the index value is -1.
michael@0: */
michael@0: int16_t TransliterationRule::getIndexValue() const {
michael@0: if (anteContextLength == pattern.length()) {
michael@0: // A pattern with just ante context {such as foo)>bar} can
michael@0: // match any key.
michael@0: return -1;
michael@0: }
michael@0: UChar32 c = pattern.char32At(anteContextLength);
michael@0: return (int16_t)(data->lookupMatcher(c) == NULL ? (c & 0xFF) : -1);
michael@0: }
michael@0:
michael@0: /**
michael@0: * Internal method. Returns true if this rule matches the given
michael@0: * index value. The index value is an 8-bit integer, 0..255,
michael@0: * representing the low byte of the first character of the key.
michael@0: * It matches this rule if it matches the first character of the
michael@0: * key, or if the first character of the key is a set, and the set
michael@0: * contains any character with a low byte equal to the index
michael@0: * value. If the rule contains only ante context, as in foo)>bar,
michael@0: * then it will match any key.
michael@0: */
michael@0: UBool TransliterationRule::matchesIndexValue(uint8_t v) const {
michael@0: // Delegate to the key, or if there is none, to the postContext.
michael@0: // If there is neither then we match any key; return true.
michael@0: UnicodeMatcher *m = (key != NULL) ? key : postContext;
michael@0: return (m != NULL) ? m->matchesIndexValue(v) : TRUE;
michael@0: }
michael@0:
michael@0: /**
michael@0: * Return true if this rule masks another rule. If r1 masks r2 then
michael@0: * r1 matches any input string that r2 matches. If r1 masks r2 and r2 masks
michael@0: * r1 then r1 == r2. Examples: "a>x" masks "ab>y". "a>x" masks "a[b]>y".
michael@0: * "[c]a>x" masks "[dc]a>y".
michael@0: */
michael@0: UBool TransliterationRule::masks(const TransliterationRule& r2) const {
michael@0: /* Rule r1 masks rule r2 if the string formed of the
michael@0: * antecontext, key, and postcontext overlaps in the following
michael@0: * way:
michael@0: *
michael@0: * r1: aakkkpppp
michael@0: * r2: aaakkkkkpppp
michael@0: * ^
michael@0: *
michael@0: * The strings must be aligned at the first character of the
michael@0: * key. The length of r1 to the left of the alignment point
michael@0: * must be <= the length of r2 to the left; ditto for the
michael@0: * right. The characters of r1 must equal (or be a superset
michael@0: * of) the corresponding characters of r2. The superset
michael@0: * operation should be performed to check for UnicodeSet
michael@0: * masking.
michael@0: *
michael@0: * Anchors: Two patterns that differ only in anchors only
michael@0: * mask one another if they are exactly equal, and r2 has
michael@0: * all the anchors r1 has (optionally, plus some). Here Y
michael@0: * means the row masks the column, N means it doesn't.
michael@0: *
michael@0: * ab ^ab ab$ ^ab$
michael@0: * ab Y Y Y Y
michael@0: * ^ab N Y N Y
michael@0: * ab$ N N Y Y
michael@0: * ^ab$ N N N Y
michael@0: *
michael@0: * Post context: {a}b masks ab, but not vice versa, since {a}b
michael@0: * matches everything ab matches, and {a}b matches {|a|}b but ab
michael@0: * does not. Pre context is different (a{b} does not align with
michael@0: * ab).
michael@0: */
michael@0:
michael@0: /* LIMITATION of the current mask algorithm: Some rule
michael@0: * maskings are currently not detected. For example,
michael@0: * "{Lu}]a>x" masks "A]a>y". This can be added later. TODO
michael@0: */
michael@0:
michael@0: int32_t len = pattern.length();
michael@0: int32_t left = anteContextLength;
michael@0: int32_t left2 = r2.anteContextLength;
michael@0: int32_t right = len - left;
michael@0: int32_t right2 = r2.pattern.length() - left2;
michael@0: int32_t cachedCompare = r2.pattern.compare(left2 - left, len, pattern);
michael@0:
michael@0: // TODO Clean this up -- some logic might be combinable with the
michael@0: // next statement.
michael@0:
michael@0: // Test for anchor masking
michael@0: if (left == left2 && right == right2 &&
michael@0: keyLength <= r2.keyLength &&
michael@0: 0 == cachedCompare) {
michael@0: // The following boolean logic implements the table above
michael@0: return (flags == r2.flags) ||
michael@0: (!(flags & ANCHOR_START) && !(flags & ANCHOR_END)) ||
michael@0: ((r2.flags & ANCHOR_START) && (r2.flags & ANCHOR_END));
michael@0: }
michael@0:
michael@0: return left <= left2 &&
michael@0: (right < right2 ||
michael@0: (right == right2 && keyLength <= r2.keyLength)) &&
michael@0: (0 == cachedCompare);
michael@0: }
michael@0:
michael@0: static inline int32_t posBefore(const Replaceable& str, int32_t pos) {
michael@0: return (pos > 0) ?
michael@0: pos - U16_LENGTH(str.char32At(pos-1)) :
michael@0: pos - 1;
michael@0: }
michael@0:
michael@0: static inline int32_t posAfter(const Replaceable& str, int32_t pos) {
michael@0: return (pos >= 0 && pos < str.length()) ?
michael@0: pos + U16_LENGTH(str.char32At(pos)) :
michael@0: pos + 1;
michael@0: }
michael@0:
michael@0: /**
michael@0: * Attempt a match and replacement at the given position. Return
michael@0: * the degree of match between this rule and the given text. The
michael@0: * degree of match may be mismatch, a partial match, or a full
michael@0: * match. A mismatch means at least one character of the text
michael@0: * does not match the context or key. A partial match means some
michael@0: * context and key characters match, but the text is not long
michael@0: * enough to match all of them. A full match means all context
michael@0: * and key characters match.
michael@0: *
michael@0: * If a full match is obtained, perform a replacement, update pos,
michael@0: * and return U_MATCH. Otherwise both text and pos are unchanged.
michael@0: *
michael@0: * @param text the text
michael@0: * @param pos the position indices
michael@0: * @param incremental if TRUE, test for partial matches that may
michael@0: * be completed by additional text inserted at pos.limit.
michael@0: * @return one of U_MISMATCH
,
michael@0: * U_PARTIAL_MATCH
, or U_MATCH
. If
michael@0: * incremental is FALSE then U_PARTIAL_MATCH will not be returned.
michael@0: */
michael@0: UMatchDegree TransliterationRule::matchAndReplace(Replaceable& text,
michael@0: UTransPosition& pos,
michael@0: UBool incremental) const {
michael@0: // Matching and replacing are done in one method because the
michael@0: // replacement operation needs information obtained during the
michael@0: // match. Another way to do this is to have the match method
michael@0: // create a match result struct with relevant offsets, and to pass
michael@0: // this into the replace method.
michael@0:
michael@0: // ============================ MATCH ===========================
michael@0:
michael@0: // Reset segment match data
michael@0: if (segments != NULL) {
michael@0: for (int32_t i=0; iresetMatch();
michael@0: }
michael@0: }
michael@0:
michael@0: // int32_t lenDelta, keyLimit;
michael@0: int32_t keyLimit;
michael@0:
michael@0: // ------------------------ Ante Context ------------------------
michael@0:
michael@0: // A mismatch in the ante context, or with the start anchor,
michael@0: // is an outright U_MISMATCH regardless of whether we are
michael@0: // incremental or not.
michael@0: int32_t oText; // offset into 'text'
michael@0: // int32_t newStart = 0;
michael@0: int32_t minOText;
michael@0:
michael@0: // Note (1): We process text in 16-bit code units, rather than
michael@0: // 32-bit code points. This works because stand-ins are
michael@0: // always in the BMP and because we are doing a literal match
michael@0: // operation, which can be done 16-bits at a time.
michael@0:
michael@0: int32_t anteLimit = posBefore(text, pos.contextStart);
michael@0:
michael@0: UMatchDegree match;
michael@0:
michael@0: // Start reverse match at char before pos.start
michael@0: oText = posBefore(text, pos.start);
michael@0:
michael@0: if (anteContext != NULL) {
michael@0: match = anteContext->matches(text, oText, anteLimit, FALSE);
michael@0: if (match != U_MATCH) {
michael@0: return U_MISMATCH;
michael@0: }
michael@0: }
michael@0:
michael@0: minOText = posAfter(text, oText);
michael@0:
michael@0: // ------------------------ Start Anchor ------------------------
michael@0:
michael@0: if (((flags & ANCHOR_START) != 0) && oText != anteLimit) {
michael@0: return U_MISMATCH;
michael@0: }
michael@0:
michael@0: // -------------------- Key and Post Context --------------------
michael@0:
michael@0: oText = pos.start;
michael@0:
michael@0: if (key != NULL) {
michael@0: match = key->matches(text, oText, pos.limit, incremental);
michael@0: if (match != U_MATCH) {
michael@0: return match;
michael@0: }
michael@0: }
michael@0:
michael@0: keyLimit = oText;
michael@0:
michael@0: if (postContext != NULL) {
michael@0: if (incremental && keyLimit == pos.limit) {
michael@0: // The key matches just before pos.limit, and there is
michael@0: // a postContext. Since we are in incremental mode,
michael@0: // we must assume more characters may be inserted at
michael@0: // pos.limit -- this is a partial match.
michael@0: return U_PARTIAL_MATCH;
michael@0: }
michael@0:
michael@0: match = postContext->matches(text, oText, pos.contextLimit, incremental);
michael@0: if (match != U_MATCH) {
michael@0: return match;
michael@0: }
michael@0: }
michael@0:
michael@0: // ------------------------- Stop Anchor ------------------------
michael@0:
michael@0: if (((flags & ANCHOR_END)) != 0) {
michael@0: if (oText != pos.contextLimit) {
michael@0: return U_MISMATCH;
michael@0: }
michael@0: if (incremental) {
michael@0: return U_PARTIAL_MATCH;
michael@0: }
michael@0: }
michael@0:
michael@0: // =========================== REPLACE ==========================
michael@0:
michael@0: // We have a full match. The key is between pos.start and
michael@0: // keyLimit.
michael@0:
michael@0: int32_t newStart;
michael@0: int32_t newLength = output->toReplacer()->replace(text, pos.start, keyLimit, newStart);
michael@0: int32_t lenDelta = newLength - (keyLimit - pos.start);
michael@0:
michael@0: oText += lenDelta;
michael@0: pos.limit += lenDelta;
michael@0: pos.contextLimit += lenDelta;
michael@0: // Restrict new value of start to [minOText, min(oText, pos.limit)].
michael@0: pos.start = uprv_max(minOText, uprv_min(uprv_min(oText, pos.limit), newStart));
michael@0: return U_MATCH;
michael@0: }
michael@0:
michael@0: /**
michael@0: * Create a source string that represents this rule. Append it to the
michael@0: * given string.
michael@0: */
michael@0: UnicodeString& TransliterationRule::toRule(UnicodeString& rule,
michael@0: UBool escapeUnprintable) const {
michael@0:
michael@0: // Accumulate special characters (and non-specials following them)
michael@0: // into quoteBuf. Append quoteBuf, within single quotes, when
michael@0: // a non-quoted element must be inserted.
michael@0: UnicodeString str, quoteBuf;
michael@0:
michael@0: // Do not emit the braces '{' '}' around the pattern if there
michael@0: // is neither anteContext nor postContext.
michael@0: UBool emitBraces =
michael@0: (anteContext != NULL) || (postContext != NULL);
michael@0:
michael@0: // Emit start anchor
michael@0: if ((flags & ANCHOR_START) != 0) {
michael@0: rule.append((UChar)94/*^*/);
michael@0: }
michael@0:
michael@0: // Emit the input pattern
michael@0: ICU_Utility::appendToRule(rule, anteContext, escapeUnprintable, quoteBuf);
michael@0:
michael@0: if (emitBraces) {
michael@0: ICU_Utility::appendToRule(rule, (UChar) 0x007B /*{*/, TRUE, escapeUnprintable, quoteBuf);
michael@0: }
michael@0:
michael@0: ICU_Utility::appendToRule(rule, key, escapeUnprintable, quoteBuf);
michael@0:
michael@0: if (emitBraces) {
michael@0: ICU_Utility::appendToRule(rule, (UChar) 0x007D /*}*/, TRUE, escapeUnprintable, quoteBuf);
michael@0: }
michael@0:
michael@0: ICU_Utility::appendToRule(rule, postContext, escapeUnprintable, quoteBuf);
michael@0:
michael@0: // Emit end anchor
michael@0: if ((flags & ANCHOR_END) != 0) {
michael@0: rule.append((UChar)36/*$*/);
michael@0: }
michael@0:
michael@0: ICU_Utility::appendToRule(rule, UnicodeString(TRUE, FORWARD_OP, 3), TRUE, escapeUnprintable, quoteBuf);
michael@0:
michael@0: // Emit the output pattern
michael@0:
michael@0: ICU_Utility::appendToRule(rule, output->toReplacer()->toReplacerPattern(str, escapeUnprintable),
michael@0: TRUE, escapeUnprintable, quoteBuf);
michael@0:
michael@0: ICU_Utility::appendToRule(rule, (UChar) 0x003B /*;*/, TRUE, escapeUnprintable, quoteBuf);
michael@0:
michael@0: return rule;
michael@0: }
michael@0:
michael@0: void TransliterationRule::setData(const TransliterationRuleData* d) {
michael@0: data = d;
michael@0: if (anteContext != NULL) anteContext->setData(d);
michael@0: if (postContext != NULL) postContext->setData(d);
michael@0: if (key != NULL) key->setData(d);
michael@0: // assert(output != NULL);
michael@0: output->setData(d);
michael@0: // Don't have to do segments since they are in the context or key
michael@0: }
michael@0:
michael@0: /**
michael@0: * Union the set of all characters that may be modified by this rule
michael@0: * into the given set.
michael@0: */
michael@0: void TransliterationRule::addSourceSetTo(UnicodeSet& toUnionTo) const {
michael@0: int32_t limit = anteContextLength + keyLength;
michael@0: for (int32_t i=anteContextLength; ilookupMatcher(ch);
michael@0: if (matcher == NULL) {
michael@0: toUnionTo.add(ch);
michael@0: } else {
michael@0: matcher->addMatchSetTo(toUnionTo);
michael@0: }
michael@0: }
michael@0: }
michael@0:
michael@0: /**
michael@0: * Union the set of all characters that may be emitted by this rule
michael@0: * into the given set.
michael@0: */
michael@0: void TransliterationRule::addTargetSetTo(UnicodeSet& toUnionTo) const {
michael@0: output->toReplacer()->addReplacementSetTo(toUnionTo);
michael@0: }
michael@0:
michael@0: U_NAMESPACE_END
michael@0:
michael@0: #endif /* #if !UCONFIG_NO_TRANSLITERATION */
michael@0:
michael@0: //eof