The Tor Browser: intl/icu/source/i18n/rbt_rule.cpp@fc2d59ddac77 (annotated)

intl/icu/source/i18n/rbt_rule.cpp@fc2d59ddac77 (annotated)

intl/icu/source/i18n/rbt_rule.cpp

Wed, 31 Dec 2014 07:22:50 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 07:22:50 +0100
branch: TOR_BUG_3246
changeset 4: fc2d59ddac77
permissions: -rw-r--r--

Correct previous dual key logic pending first delivery installment.

 /*
  **********************************************************************
  *   Copyright (C) 1999-2011, International Business Machines
  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  *   Date        Name        Description
  *   11/17/99    aliu        Creation.
  **********************************************************************
  */
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_TRANSLITERATION
 #include "unicode/rep.h"
 #include "unicode/unifilt.h"
 #include "unicode/uniset.h"
 #include "unicode/utf16.h"
 #include "rbt_rule.h"
 #include "rbt_data.h"
 #include "cmemory.h"
 #include "strmatch.h"
 #include "strrepl.h"
 #include "util.h"
 #include "putilimp.h"
 static const UChar FORWARD_OP[] = {32,62,32,0}; // " > "
 U_NAMESPACE_BEGIN
 /**
  * Construct a new rule with the given input, output text, and other
  * attributes.  A cursor position may be specified for the output text.
  * @param input input string, including key and optional ante and
  * post context
  * @param anteContextPos offset into input to end of ante context, or -1 if
  * none.  Must be <= input.length() if not -1.
  * @param postContextPos offset into input to start of post context, or -1
  * if none.  Must be <= input.length() if not -1, and must be >=
  * anteContextPos.
  * @param output output string
  * @param cursorPosition offset into output at which cursor is located, or -1 if
  * none.  If less than zero, then the cursor is placed after the
  * <code>output</code>; that is, -1 is equivalent to
  * <code>output.length()</code>.  If greater than
  * <code>output.length()</code> then an exception is thrown.
  * @param segs array of UnicodeFunctors corresponding to input pattern
  * segments, or null if there are none.  The array itself is adopted,
  * but the pointers within it are not.
  * @param segsCount number of elements in segs[]
  * @param anchorStart TRUE if the the rule is anchored on the left to
  * the context start
  * @param anchorEnd TRUE if the rule is anchored on the right to the
  * context limit
  */
 TransliterationRule::TransliterationRule(const UnicodeString& input,
                                          int32_t anteContextPos, int32_t postContextPos,
                                          const UnicodeString& outputStr,
                                          int32_t cursorPosition, int32_t cursorOffset,
                                          UnicodeFunctor** segs,
                                          int32_t segsCount,
                                          UBool anchorStart, UBool anchorEnd,
                                          const TransliterationRuleData* theData,
                                          UErrorCode& status) :
     UMemory(),
     segments(0),
     data(theData) {
     if (U_FAILURE(status)) {
         return;
     }
     // Do range checks only when warranted to save time
     if (anteContextPos < 0) {
         anteContextLength = 0;
     } else {
         if (anteContextPos > input.length()) {
             // throw new IllegalArgumentException("Invalid ante context");
             status = U_ILLEGAL_ARGUMENT_ERROR;
             return;
         }
         anteContextLength = anteContextPos;
     }
     if (postContextPos < 0) {
         keyLength = input.length() - anteContextLength;
     } else {
         if (postContextPos < anteContextLength ||
             postContextPos > input.length()) {
             // throw new IllegalArgumentException("Invalid post context");
             status = U_ILLEGAL_ARGUMENT_ERROR;
             return;
         }
         keyLength = postContextPos - anteContextLength;
     }
     if (cursorPosition < 0) {
         cursorPosition = outputStr.length();
     } else if (cursorPosition > outputStr.length()) {
         // throw new IllegalArgumentException("Invalid cursor position");
         status = U_ILLEGAL_ARGUMENT_ERROR;
         return;
     }
     // We don't validate the segments array.  The caller must
     // guarantee that the segments are well-formed (that is, that
     // all $n references in the output refer to indices of this
     // array, and that no array elements are null).
     this->segments = segs;
     this->segmentsCount = segsCount;
     pattern = input;
     flags = 0;
     if (anchorStart) {
         flags |= ANCHOR_START;
     }
     if (anchorEnd) {
         flags |= ANCHOR_END;
     }
     anteContext = NULL;
     if (anteContextLength > 0) {
         anteContext = new StringMatcher(pattern, 0, anteContextLength,
                                         FALSE, *data);
         /* test for NULL */
         if (anteContext == 0) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
     }
     key = NULL;
     if (keyLength > 0) {
         key = new StringMatcher(pattern, anteContextLength, anteContextLength + keyLength,
                                 FALSE, *data);
         /* test for NULL */
         if (key == 0) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
     }
     int32_t postContextLength = pattern.length() - keyLength - anteContextLength;
     postContext = NULL;
     if (postContextLength > 0) {
         postContext = new StringMatcher(pattern, anteContextLength + keyLength, pattern.length(),
                                         FALSE, *data);
         /* test for NULL */
         if (postContext == 0) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
     }
     this->output = new StringReplacer(outputStr, cursorPosition + cursorOffset, data);
     /* test for NULL */
     if (this->output == 0) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
 }
 /**
  * Copy constructor.
  */
 TransliterationRule::TransliterationRule(TransliterationRule& other) :
     UMemory(other),
     anteContext(NULL),
     key(NULL),
     postContext(NULL),
     pattern(other.pattern),
     anteContextLength(other.anteContextLength),
     keyLength(other.keyLength),
     flags(other.flags),
     data(other.data) {
     segments = NULL;
     segmentsCount = 0;
     if (other.segmentsCount > 0) {
         segments = (UnicodeFunctor **)uprv_malloc(other.segmentsCount * sizeof(UnicodeFunctor *));
         uprv_memcpy(segments, other.segments, other.segmentsCount*sizeof(segments[0]));
     }
     if (other.anteContext != NULL) {
         anteContext = (StringMatcher*) other.anteContext->clone();
     }
     if (other.key != NULL) {
         key = (StringMatcher*) other.key->clone();
     }
     if (other.postContext != NULL) {
         postContext = (StringMatcher*) other.postContext->clone();
     }
     output = other.output->clone();
 }
 TransliterationRule::~TransliterationRule() {
     uprv_free(segments);
     delete anteContext;
     delete key;
     delete postContext;
     delete output;
 }
 /**
  * Return the preceding context length.  This method is needed to
  * support the <code>Transliterator</code> method
  * <code>getMaximumContextLength()</code>.  Internally, this is
  * implemented as the anteContextLength, optionally plus one if
  * there is a start anchor.  The one character anchor gap is
  * needed to make repeated incremental transliteration with
  * anchors work.
  */
 int32_t TransliterationRule::getContextLength(void) const {
     return anteContextLength + ((flags & ANCHOR_START) ? 1 : 0);
 }
 /**
  * Internal method.  Returns 8-bit index value for this rule.
  * This is the low byte of the first character of the key,
  * unless the first character of the key is a set.  If it's a
  * set, or otherwise can match multiple keys, the index value is -1.
  */
 int16_t TransliterationRule::getIndexValue() const {
     if (anteContextLength == pattern.length()) {
         // A pattern with just ante context {such as foo)>bar} can
         // match any key.
         return -1;
     }
     UChar32 c = pattern.char32At(anteContextLength);
     return (int16_t)(data->lookupMatcher(c) == NULL ? (c & 0xFF) : -1);
 }
 /**
  * Internal method.  Returns true if this rule matches the given
  * index value.  The index value is an 8-bit integer, 0..255,
  * representing the low byte of the first character of the key.
  * It matches this rule if it matches the first character of the
  * key, or if the first character of the key is a set, and the set
  * contains any character with a low byte equal to the index
  * value.  If the rule contains only ante context, as in foo)>bar,
  * then it will match any key.
  */
 UBool TransliterationRule::matchesIndexValue(uint8_t v) const {
     // Delegate to the key, or if there is none, to the postContext.
     // If there is neither then we match any key; return true.
     UnicodeMatcher *m = (key != NULL) ? key : postContext;
     return (m != NULL) ? m->matchesIndexValue(v) : TRUE;
 }
 /**
  * Return true if this rule masks another rule.  If r1 masks r2 then
  * r1 matches any input string that r2 matches.  If r1 masks r2 and r2 masks
  * r1 then r1 == r2.  Examples: "a>x" masks "ab>y".  "a>x" masks "a[b]>y".
  * "[c]a>x" masks "[dc]a>y".
  */
 UBool TransliterationRule::masks(const TransliterationRule& r2) const {
     /* Rule r1 masks rule r2 if the string formed of the
      * antecontext, key, and postcontext overlaps in the following
      * way:
      *
      * r1:      aakkkpppp
      * r2:     aaakkkkkpppp
      *            ^
      *
      * The strings must be aligned at the first character of the
      * key.  The length of r1 to the left of the alignment point
      * must be <= the length of r2 to the left; ditto for the
      * right.  The characters of r1 must equal (or be a superset
      * of) the corresponding characters of r2.  The superset
      * operation should be performed to check for UnicodeSet
      * masking.
      *
      * Anchors:  Two patterns that differ only in anchors only
      * mask one another if they are exactly equal, and r2 has
      * all the anchors r1 has (optionally, plus some).  Here Y
      * means the row masks the column, N means it doesn't.
      *
      *         ab   ^ab    ab$  ^ab$
      *   ab    Y     Y     Y     Y
      *  ^ab    N     Y     N     Y
      *   ab$   N     N     Y     Y
      *  ^ab$   N     N     N     Y
      *
      * Post context: {a}b masks ab, but not vice versa, since {a}b
      * matches everything ab matches, and {a}b matches {|a|}b but ab
      * does not.  Pre context is different (a{b} does not align with
      * ab).
      */
     /* LIMITATION of the current mask algorithm: Some rule
      * maskings are currently not detected.  For example,
      * "{Lu}]a>x" masks "A]a>y".  This can be added later. TODO
      */
     int32_t len = pattern.length();
     int32_t left = anteContextLength;
     int32_t left2 = r2.anteContextLength;
     int32_t right = len - left;
     int32_t right2 = r2.pattern.length() - left2;
     int32_t cachedCompare = r2.pattern.compare(left2 - left, len, pattern);
     // TODO Clean this up -- some logic might be combinable with the
     // next statement.
     // Test for anchor masking
     if (left == left2 && right == right2 &&
         keyLength <= r2.keyLength &&
 == cachedCompare) {
         // The following boolean logic implements the table above
         return (flags == r2.flags) ||
             (!(flags & ANCHOR_START) && !(flags & ANCHOR_END)) ||
             ((r2.flags & ANCHOR_START) && (r2.flags & ANCHOR_END));
     }
     return left <= left2 &&
         (right < right2 ||
          (right == right2 && keyLength <= r2.keyLength)) &&
          (0 == cachedCompare);
 }
 static inline int32_t posBefore(const Replaceable& str, int32_t pos) {
     return (pos > 0) ?
         pos - U16_LENGTH(str.char32At(pos-1)) :
         pos - 1;
 }
 static inline int32_t posAfter(const Replaceable& str, int32_t pos) {
     return (pos >= 0 && pos < str.length()) ?
         pos + U16_LENGTH(str.char32At(pos)) :
         pos + 1;
 }
 /**
  * Attempt a match and replacement at the given position.  Return
  * the degree of match between this rule and the given text.  The
  * degree of match may be mismatch, a partial match, or a full
  * match.  A mismatch means at least one character of the text
  * does not match the context or key.  A partial match means some
  * context and key characters match, but the text is not long
  * enough to match all of them.  A full match means all context
  * and key characters match.
  *
  * If a full match is obtained, perform a replacement, update pos,
  * and return U_MATCH.  Otherwise both text and pos are unchanged.
  *
  * @param text the text
  * @param pos the position indices
  * @param incremental if TRUE, test for partial matches that may
  * be completed by additional text inserted at pos.limit.
  * @return one of <code>U_MISMATCH</code>,
  * <code>U_PARTIAL_MATCH</code>, or <code>U_MATCH</code>.  If
  * incremental is FALSE then U_PARTIAL_MATCH will not be returned.
  */
 UMatchDegree TransliterationRule::matchAndReplace(Replaceable& text,
                                                   UTransPosition& pos,
                                                   UBool incremental) const {
     // Matching and replacing are done in one method because the
     // replacement operation needs information obtained during the
     // match.  Another way to do this is to have the match method
     // create a match result struct with relevant offsets, and to pass
     // this into the replace method.
     // ============================ MATCH ===========================
     // Reset segment match data
     if (segments != NULL) {
         for (int32_t i=0; i<segmentsCount; ++i) {
             ((StringMatcher*) segments[i])->resetMatch();
         }
     }
 //    int32_t lenDelta, keyLimit;
     int32_t keyLimit;
     // ------------------------ Ante Context ------------------------
     // A mismatch in the ante context, or with the start anchor,
     // is an outright U_MISMATCH regardless of whether we are
     // incremental or not.
     int32_t oText; // offset into 'text'
 //    int32_t newStart = 0;
     int32_t minOText;
     // Note (1): We process text in 16-bit code units, rather than
     // 32-bit code points.  This works because stand-ins are
     // always in the BMP and because we are doing a literal match
     // operation, which can be done 16-bits at a time.
     int32_t anteLimit = posBefore(text, pos.contextStart);
     UMatchDegree match;
     // Start reverse match at char before pos.start
     oText = posBefore(text, pos.start);
     if (anteContext != NULL) {
         match = anteContext->matches(text, oText, anteLimit, FALSE);
         if (match != U_MATCH) {
             return U_MISMATCH;
         }
     }
     minOText = posAfter(text, oText);
     // ------------------------ Start Anchor ------------------------
     if (((flags & ANCHOR_START) != 0) && oText != anteLimit) {
         return U_MISMATCH;
     }
     // -------------------- Key and Post Context --------------------
     oText = pos.start;
     if (key != NULL) {
         match = key->matches(text, oText, pos.limit, incremental);
         if (match != U_MATCH) {
             return match;
         }
     }
     keyLimit = oText;
     if (postContext != NULL) {
         if (incremental && keyLimit == pos.limit) {
             // The key matches just before pos.limit, and there is
             // a postContext.  Since we are in incremental mode,
             // we must assume more characters may be inserted at
             // pos.limit -- this is a partial match.
             return U_PARTIAL_MATCH;
         }
         match = postContext->matches(text, oText, pos.contextLimit, incremental);
         if (match != U_MATCH) {
             return match;
         }
     }
     // ------------------------- Stop Anchor ------------------------
     if (((flags & ANCHOR_END)) != 0) {
         if (oText != pos.contextLimit) {
             return U_MISMATCH;
         }
         if (incremental) {
             return U_PARTIAL_MATCH;
         }
     }
     // =========================== REPLACE ==========================
     // We have a full match.  The key is between pos.start and
     // keyLimit.
     int32_t newStart;
     int32_t newLength = output->toReplacer()->replace(text, pos.start, keyLimit, newStart);
     int32_t lenDelta = newLength - (keyLimit - pos.start);
     oText += lenDelta;
     pos.limit += lenDelta;
     pos.contextLimit += lenDelta;
     // Restrict new value of start to [minOText, min(oText, pos.limit)].
     pos.start = uprv_max(minOText, uprv_min(uprv_min(oText, pos.limit), newStart));
     return U_MATCH;
 }
 /**
  * Create a source string that represents this rule.  Append it to the
  * given string.
  */
 UnicodeString& TransliterationRule::toRule(UnicodeString& rule,
                                            UBool escapeUnprintable) const {
     // Accumulate special characters (and non-specials following them)
     // into quoteBuf.  Append quoteBuf, within single quotes, when
     // a non-quoted element must be inserted.
     UnicodeString str, quoteBuf;
     // Do not emit the braces '{' '}' around the pattern if there
     // is neither anteContext nor postContext.
     UBool emitBraces =
         (anteContext != NULL) || (postContext != NULL);
     // Emit start anchor
     if ((flags & ANCHOR_START) != 0) {
         rule.append((UChar)94/*^*/);
     }
     // Emit the input pattern
     ICU_Utility::appendToRule(rule, anteContext, escapeUnprintable, quoteBuf);
     if (emitBraces) {
         ICU_Utility::appendToRule(rule, (UChar) 0x007B /*{*/, TRUE, escapeUnprintable, quoteBuf);
     }
     ICU_Utility::appendToRule(rule, key, escapeUnprintable, quoteBuf);
     if (emitBraces) {
         ICU_Utility::appendToRule(rule, (UChar) 0x007D /*}*/, TRUE, escapeUnprintable, quoteBuf);
     }
     ICU_Utility::appendToRule(rule, postContext, escapeUnprintable, quoteBuf);
     // Emit end anchor
     if ((flags & ANCHOR_END) != 0) {
         rule.append((UChar)36/*$*/);
     }
     ICU_Utility::appendToRule(rule, UnicodeString(TRUE, FORWARD_OP, 3), TRUE, escapeUnprintable, quoteBuf);
     // Emit the output pattern
     ICU_Utility::appendToRule(rule, output->toReplacer()->toReplacerPattern(str, escapeUnprintable),
                               TRUE, escapeUnprintable, quoteBuf);
     ICU_Utility::appendToRule(rule, (UChar) 0x003B /*;*/, TRUE, escapeUnprintable, quoteBuf);
     return rule;
 }
 void TransliterationRule::setData(const TransliterationRuleData* d) {
     data = d;
     if (anteContext != NULL) anteContext->setData(d);
     if (postContext != NULL) postContext->setData(d);
     if (key != NULL) key->setData(d);
     // assert(output != NULL);
     output->setData(d);
     // Don't have to do segments since they are in the context or key
 }
 /**
  * Union the set of all characters that may be modified by this rule
  * into the given set.
  */
 void TransliterationRule::addSourceSetTo(UnicodeSet& toUnionTo) const {
     int32_t limit = anteContextLength + keyLength;
     for (int32_t i=anteContextLength; i<limit; ) {
         UChar32 ch = pattern.char32At(i);
         i += U16_LENGTH(ch);
         const UnicodeMatcher* matcher = data->lookupMatcher(ch);
         if (matcher == NULL) {
             toUnionTo.add(ch);
         } else {
             matcher->addMatchSetTo(toUnionTo);
         }
     }
 }
 /**
  * Union the set of all characters that may be emitted by this rule
  * into the given set.
  */
 void TransliterationRule::addTargetSetTo(UnicodeSet& toUnionTo) const {
     output->toReplacer()->addReplacementSetTo(toUnionTo);
 }
 U_NAMESPACE_END
 #endif /* #if !UCONFIG_NO_TRANSLITERATION */
 //eof

The Tor Browser / annotate

intl/icu/source/i18n/rbt_rule.cpp@fc2d59ddac77 (annotated)

intl/icu/source/i18n/rbt_rule.cpp