1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/i18n/rbt_set.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,467 @@
1.4 +/*
1.5 + **********************************************************************
1.6 + * Copyright (C) 1999-2011, International Business Machines
1.7 + * Corporation and others. All Rights Reserved.
1.8 + **********************************************************************
1.9 + * Date Name Description
1.10 + * 11/17/99 aliu Creation.
1.11 + **********************************************************************
1.12 + */
1.13 +
1.14 +#include "unicode/utypes.h"
1.15 +
1.16 +#if !UCONFIG_NO_TRANSLITERATION
1.17 +
1.18 +#include "unicode/unistr.h"
1.19 +#include "unicode/uniset.h"
1.20 +#include "unicode/utf16.h"
1.21 +#include "rbt_set.h"
1.22 +#include "rbt_rule.h"
1.23 +#include "cmemory.h"
1.24 +#include "putilimp.h"
1.25 +
1.26 +U_CDECL_BEGIN
1.27 +static void U_CALLCONV _deleteRule(void *rule) {
1.28 + delete (icu::TransliterationRule *)rule;
1.29 +}
1.30 +U_CDECL_END
1.31 +
1.32 +//----------------------------------------------------------------------
1.33 +// BEGIN Debugging support
1.34 +//----------------------------------------------------------------------
1.35 +
1.36 +// #define DEBUG_RBT
1.37 +
1.38 +#ifdef DEBUG_RBT
1.39 +#include <stdio.h>
1.40 +#include "charstr.h"
1.41 +
1.42 +/**
1.43 + * @param appendTo result is appended to this param.
1.44 + * @param input the string being transliterated
1.45 + * @param pos the index struct
1.46 + */
1.47 +static UnicodeString& _formatInput(UnicodeString &appendTo,
1.48 + const UnicodeString& input,
1.49 + const UTransPosition& pos) {
1.50 + // Output a string of the form aaa{bbb|ccc|ddd}eee, where
1.51 + // the {} indicate the context start and limit, and the ||
1.52 + // indicate the start and limit.
1.53 + if (0 <= pos.contextStart &&
1.54 + pos.contextStart <= pos.start &&
1.55 + pos.start <= pos.limit &&
1.56 + pos.limit <= pos.contextLimit &&
1.57 + pos.contextLimit <= input.length()) {
1.58 +
1.59 + UnicodeString a, b, c, d, e;
1.60 + input.extractBetween(0, pos.contextStart, a);
1.61 + input.extractBetween(pos.contextStart, pos.start, b);
1.62 + input.extractBetween(pos.start, pos.limit, c);
1.63 + input.extractBetween(pos.limit, pos.contextLimit, d);
1.64 + input.extractBetween(pos.contextLimit, input.length(), e);
1.65 + appendTo.append(a).append((UChar)123/*{*/).append(b).
1.66 + append((UChar)124/*|*/).append(c).append((UChar)124/*|*/).append(d).
1.67 + append((UChar)125/*}*/).append(e);
1.68 + } else {
1.69 + appendTo.append("INVALID UTransPosition");
1.70 + //appendTo.append((UnicodeString)"INVALID UTransPosition {cs=" +
1.71 + // pos.contextStart + ", s=" + pos.start + ", l=" +
1.72 + // pos.limit + ", cl=" + pos.contextLimit + "} on " +
1.73 + // input);
1.74 + }
1.75 + return appendTo;
1.76 +}
1.77 +
1.78 +// Append a hex string to the target
1.79 +UnicodeString& _appendHex(uint32_t number,
1.80 + int32_t digits,
1.81 + UnicodeString& target) {
1.82 + static const UChar digitString[] = {
1.83 + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39,
1.84 + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0
1.85 + };
1.86 + while (digits--) {
1.87 + target += digitString[(number >> (digits*4)) & 0xF];
1.88 + }
1.89 + return target;
1.90 +}
1.91 +
1.92 +// Replace nonprintable characters with unicode escapes
1.93 +UnicodeString& _escape(const UnicodeString &source,
1.94 + UnicodeString &target) {
1.95 + for (int32_t i = 0; i < source.length(); ) {
1.96 + UChar32 ch = source.char32At(i);
1.97 + i += U16_LENGTH(ch);
1.98 + if (ch < 0x09 || (ch > 0x0A && ch < 0x20)|| ch > 0x7E) {
1.99 + if (ch <= 0xFFFF) {
1.100 + target += "\\u";
1.101 + _appendHex(ch, 4, target);
1.102 + } else {
1.103 + target += "\\U";
1.104 + _appendHex(ch, 8, target);
1.105 + }
1.106 + } else {
1.107 + target += ch;
1.108 + }
1.109 + }
1.110 + return target;
1.111 +}
1.112 +
1.113 +inline void _debugOut(const char* msg, TransliterationRule* rule,
1.114 + const Replaceable& theText, UTransPosition& pos) {
1.115 + UnicodeString buf(msg, "");
1.116 + if (rule) {
1.117 + UnicodeString r;
1.118 + rule->toRule(r, TRUE);
1.119 + buf.append((UChar)32).append(r);
1.120 + }
1.121 + buf.append(UnicodeString(" => ", ""));
1.122 + UnicodeString* text = (UnicodeString*)&theText;
1.123 + _formatInput(buf, *text, pos);
1.124 + UnicodeString esc;
1.125 + _escape(buf, esc);
1.126 + CharString cbuf(esc);
1.127 + printf("%s\n", (const char*) cbuf);
1.128 +}
1.129 +
1.130 +#else
1.131 +#define _debugOut(msg, rule, theText, pos)
1.132 +#endif
1.133 +
1.134 +//----------------------------------------------------------------------
1.135 +// END Debugging support
1.136 +//----------------------------------------------------------------------
1.137 +
1.138 +// Fill the precontext and postcontext with the patterns of the rules
1.139 +// that are masking one another.
1.140 +static void maskingError(const icu::TransliterationRule& rule1,
1.141 + const icu::TransliterationRule& rule2,
1.142 + UParseError& parseError) {
1.143 + icu::UnicodeString r;
1.144 + int32_t len;
1.145 +
1.146 + parseError.line = parseError.offset = -1;
1.147 +
1.148 + // for pre-context
1.149 + rule1.toRule(r, FALSE);
1.150 + len = uprv_min(r.length(), U_PARSE_CONTEXT_LEN-1);
1.151 + r.extract(0, len, parseError.preContext);
1.152 + parseError.preContext[len] = 0;
1.153 +
1.154 + //for post-context
1.155 + r.truncate(0);
1.156 + rule2.toRule(r, FALSE);
1.157 + len = uprv_min(r.length(), U_PARSE_CONTEXT_LEN-1);
1.158 + r.extract(0, len, parseError.postContext);
1.159 + parseError.postContext[len] = 0;
1.160 +}
1.161 +
1.162 +U_NAMESPACE_BEGIN
1.163 +
1.164 +/**
1.165 + * Construct a new empty rule set.
1.166 + */
1.167 +TransliterationRuleSet::TransliterationRuleSet(UErrorCode& status) : UMemory() {
1.168 + ruleVector = new UVector(&_deleteRule, NULL, status);
1.169 + if (U_FAILURE(status)) {
1.170 + return;
1.171 + }
1.172 + if (ruleVector == NULL) {
1.173 + status = U_MEMORY_ALLOCATION_ERROR;
1.174 + }
1.175 + rules = NULL;
1.176 + maxContextLength = 0;
1.177 +}
1.178 +
1.179 +/**
1.180 + * Copy constructor.
1.181 + */
1.182 +TransliterationRuleSet::TransliterationRuleSet(const TransliterationRuleSet& other) :
1.183 + UMemory(other),
1.184 + ruleVector(0),
1.185 + rules(0),
1.186 + maxContextLength(other.maxContextLength) {
1.187 +
1.188 + int32_t i, len;
1.189 + uprv_memcpy(index, other.index, sizeof(index));
1.190 + UErrorCode status = U_ZERO_ERROR;
1.191 + ruleVector = new UVector(&_deleteRule, NULL, status);
1.192 + if (other.ruleVector != 0 && ruleVector != 0 && U_SUCCESS(status)) {
1.193 + len = other.ruleVector->size();
1.194 + for (i=0; i<len && U_SUCCESS(status); ++i) {
1.195 + TransliterationRule *tempTranslitRule = new TransliterationRule(*(TransliterationRule*)other.ruleVector->elementAt(i));
1.196 + // Null pointer test
1.197 + if (tempTranslitRule == NULL) {
1.198 + status = U_MEMORY_ALLOCATION_ERROR;
1.199 + break;
1.200 + }
1.201 + ruleVector->addElement(tempTranslitRule, status);
1.202 + if (U_FAILURE(status)) {
1.203 + break;
1.204 + }
1.205 + }
1.206 + }
1.207 + if (other.rules != 0 && U_SUCCESS(status)) {
1.208 + UParseError p;
1.209 + freeze(p, status);
1.210 + }
1.211 +}
1.212 +
1.213 +/**
1.214 + * Destructor.
1.215 + */
1.216 +TransliterationRuleSet::~TransliterationRuleSet() {
1.217 + delete ruleVector; // This deletes the contained rules
1.218 + uprv_free(rules);
1.219 +}
1.220 +
1.221 +void TransliterationRuleSet::setData(const TransliterationRuleData* d) {
1.222 + /**
1.223 + * We assume that the ruleset has already been frozen.
1.224 + */
1.225 + int32_t len = index[256]; // see freeze()
1.226 + for (int32_t i=0; i<len; ++i) {
1.227 + rules[i]->setData(d);
1.228 + }
1.229 +}
1.230 +
1.231 +/**
1.232 + * Return the maximum context length.
1.233 + * @return the length of the longest preceding context.
1.234 + */
1.235 +int32_t TransliterationRuleSet::getMaximumContextLength(void) const {
1.236 + return maxContextLength;
1.237 +}
1.238 +
1.239 +/**
1.240 + * Add a rule to this set. Rules are added in order, and order is
1.241 + * significant. The last call to this method must be followed by
1.242 + * a call to <code>freeze()</code> before the rule set is used.
1.243 + *
1.244 + * <p>If freeze() has already been called, calling addRule()
1.245 + * unfreezes the rules, and freeze() must be called again.
1.246 + *
1.247 + * @param adoptedRule the rule to add
1.248 + */
1.249 +void TransliterationRuleSet::addRule(TransliterationRule* adoptedRule,
1.250 + UErrorCode& status) {
1.251 + if (U_FAILURE(status)) {
1.252 + delete adoptedRule;
1.253 + return;
1.254 + }
1.255 + ruleVector->addElement(adoptedRule, status);
1.256 +
1.257 + int32_t len;
1.258 + if ((len = adoptedRule->getContextLength()) > maxContextLength) {
1.259 + maxContextLength = len;
1.260 + }
1.261 +
1.262 + uprv_free(rules);
1.263 + rules = 0;
1.264 +}
1.265 +
1.266 +/**
1.267 + * Check this for masked rules and index it to optimize performance.
1.268 + * The sequence of operations is: (1) add rules to a set using
1.269 + * <code>addRule()</code>; (2) freeze the set using
1.270 + * <code>freeze()</code>; (3) use the rule set. If
1.271 + * <code>addRule()</code> is called after calling this method, it
1.272 + * invalidates this object, and this method must be called again.
1.273 + * That is, <code>freeze()</code> may be called multiple times,
1.274 + * although for optimal performance it shouldn't be.
1.275 + */
1.276 +void TransliterationRuleSet::freeze(UParseError& parseError,UErrorCode& status) {
1.277 + /* Construct the rule array and index table. We reorder the
1.278 + * rules by sorting them into 256 bins. Each bin contains all
1.279 + * rules matching the index value for that bin. A rule
1.280 + * matches an index value if string whose first key character
1.281 + * has a low byte equal to the index value can match the rule.
1.282 + *
1.283 + * Each bin contains zero or more rules, in the same order
1.284 + * they were found originally. However, the total rules in
1.285 + * the bins may exceed the number in the original vector,
1.286 + * since rules that have a variable as their first key
1.287 + * character will generally fall into more than one bin.
1.288 + *
1.289 + * That is, each bin contains all rules that either have that
1.290 + * first index value as their first key character, or have
1.291 + * a set containing the index value as their first character.
1.292 + */
1.293 + int32_t n = ruleVector->size();
1.294 + int32_t j;
1.295 + int16_t x;
1.296 + UVector v(2*n, status); // heuristic; adjust as needed
1.297 +
1.298 + if (U_FAILURE(status)) {
1.299 + return;
1.300 + }
1.301 +
1.302 + /* Precompute the index values. This saves a LOT of time.
1.303 + * Be careful not to call malloc(0).
1.304 + */
1.305 + int16_t* indexValue = (int16_t*) uprv_malloc( sizeof(int16_t) * (n > 0 ? n : 1) );
1.306 + /* test for NULL */
1.307 + if (indexValue == 0) {
1.308 + status = U_MEMORY_ALLOCATION_ERROR;
1.309 + return;
1.310 + }
1.311 + for (j=0; j<n; ++j) {
1.312 + TransliterationRule* r = (TransliterationRule*) ruleVector->elementAt(j);
1.313 + indexValue[j] = r->getIndexValue();
1.314 + }
1.315 + for (x=0; x<256; ++x) {
1.316 + index[x] = v.size();
1.317 + for (j=0; j<n; ++j) {
1.318 + if (indexValue[j] >= 0) {
1.319 + if (indexValue[j] == x) {
1.320 + v.addElement(ruleVector->elementAt(j), status);
1.321 + }
1.322 + } else {
1.323 + // If the indexValue is < 0, then the first key character is
1.324 + // a set, and we must use the more time-consuming
1.325 + // matchesIndexValue check. In practice this happens
1.326 + // rarely, so we seldom tread this code path.
1.327 + TransliterationRule* r = (TransliterationRule*) ruleVector->elementAt(j);
1.328 + if (r->matchesIndexValue((uint8_t)x)) {
1.329 + v.addElement(r, status);
1.330 + }
1.331 + }
1.332 + }
1.333 + }
1.334 + uprv_free(indexValue);
1.335 + index[256] = v.size();
1.336 +
1.337 + /* Freeze things into an array.
1.338 + */
1.339 + uprv_free(rules); // Contains alias pointers
1.340 +
1.341 + /* You can't do malloc(0)! */
1.342 + if (v.size() == 0) {
1.343 + rules = NULL;
1.344 + return;
1.345 + }
1.346 + rules = (TransliterationRule **)uprv_malloc(v.size() * sizeof(TransliterationRule *));
1.347 + /* test for NULL */
1.348 + if (rules == 0) {
1.349 + status = U_MEMORY_ALLOCATION_ERROR;
1.350 + return;
1.351 + }
1.352 + for (j=0; j<v.size(); ++j) {
1.353 + rules[j] = (TransliterationRule*) v.elementAt(j);
1.354 + }
1.355 +
1.356 + // TODO Add error reporting that indicates the rules that
1.357 + // are being masked.
1.358 + //UnicodeString errors;
1.359 +
1.360 + /* Check for masking. This is MUCH faster than our old check,
1.361 + * which was each rule against each following rule, since we
1.362 + * only have to check for masking within each bin now. It's
1.363 + * 256*O(n2^2) instead of O(n1^2), where n1 is the total rule
1.364 + * count, and n2 is the per-bin rule count. But n2<<n1, so
1.365 + * it's a big win.
1.366 + */
1.367 + for (x=0; x<256; ++x) {
1.368 + for (j=index[x]; j<index[x+1]-1; ++j) {
1.369 + TransliterationRule* r1 = rules[j];
1.370 + for (int32_t k=j+1; k<index[x+1]; ++k) {
1.371 + TransliterationRule* r2 = rules[k];
1.372 + if (r1->masks(*r2)) {
1.373 +//| if (errors == null) {
1.374 +//| errors = new StringBuffer();
1.375 +//| } else {
1.376 +//| errors.append("\n");
1.377 +//| }
1.378 +//| errors.append("Rule " + r1 + " masks " + r2);
1.379 + status = U_RULE_MASK_ERROR;
1.380 + maskingError(*r1, *r2, parseError);
1.381 + return;
1.382 + }
1.383 + }
1.384 + }
1.385 + }
1.386 +
1.387 + //if (errors != null) {
1.388 + // throw new IllegalArgumentException(errors.toString());
1.389 + //}
1.390 +}
1.391 +
1.392 +/**
1.393 + * Transliterate the given text with the given UTransPosition
1.394 + * indices. Return TRUE if the transliteration should continue
1.395 + * or FALSE if it should halt (because of a U_PARTIAL_MATCH match).
1.396 + * Note that FALSE is only ever returned if isIncremental is TRUE.
1.397 + * @param text the text to be transliterated
1.398 + * @param pos the position indices, which will be updated
1.399 + * @param incremental if TRUE, assume new text may be inserted
1.400 + * at index.limit, and return FALSE if thre is a partial match.
1.401 + * @return TRUE unless a U_PARTIAL_MATCH has been obtained,
1.402 + * indicating that transliteration should stop until more text
1.403 + * arrives.
1.404 + */
1.405 +UBool TransliterationRuleSet::transliterate(Replaceable& text,
1.406 + UTransPosition& pos,
1.407 + UBool incremental) {
1.408 + int16_t indexByte = (int16_t) (text.char32At(pos.start) & 0xFF);
1.409 + for (int32_t i=index[indexByte]; i<index[indexByte+1]; ++i) {
1.410 + UMatchDegree m = rules[i]->matchAndReplace(text, pos, incremental);
1.411 + switch (m) {
1.412 + case U_MATCH:
1.413 + _debugOut("match", rules[i], text, pos);
1.414 + return TRUE;
1.415 + case U_PARTIAL_MATCH:
1.416 + _debugOut("partial match", rules[i], text, pos);
1.417 + return FALSE;
1.418 + default: /* Ram: added default to make GCC happy */
1.419 + break;
1.420 + }
1.421 + }
1.422 + // No match or partial match from any rule
1.423 + pos.start += U16_LENGTH(text.char32At(pos.start));
1.424 + _debugOut("no match", NULL, text, pos);
1.425 + return TRUE;
1.426 +}
1.427 +
1.428 +/**
1.429 + * Create rule strings that represents this rule set.
1.430 + */
1.431 +UnicodeString& TransliterationRuleSet::toRules(UnicodeString& ruleSource,
1.432 + UBool escapeUnprintable) const {
1.433 + int32_t i;
1.434 + int32_t count = ruleVector->size();
1.435 + ruleSource.truncate(0);
1.436 + for (i=0; i<count; ++i) {
1.437 + if (i != 0) {
1.438 + ruleSource.append((UChar) 0x000A /*\n*/);
1.439 + }
1.440 + TransliterationRule *r =
1.441 + (TransliterationRule*) ruleVector->elementAt(i);
1.442 + r->toRule(ruleSource, escapeUnprintable);
1.443 + }
1.444 + return ruleSource;
1.445 +}
1.446 +
1.447 +/**
1.448 + * Return the set of all characters that may be modified
1.449 + * (getTarget=false) or emitted (getTarget=true) by this set.
1.450 + */
1.451 +UnicodeSet& TransliterationRuleSet::getSourceTargetSet(UnicodeSet& result,
1.452 + UBool getTarget) const
1.453 +{
1.454 + result.clear();
1.455 + int32_t count = ruleVector->size();
1.456 + for (int32_t i=0; i<count; ++i) {
1.457 + TransliterationRule* r =
1.458 + (TransliterationRule*) ruleVector->elementAt(i);
1.459 + if (getTarget) {
1.460 + r->addTargetSetTo(result);
1.461 + } else {
1.462 + r->addSourceSetTo(result);
1.463 + }
1.464 + }
1.465 + return result;
1.466 +}
1.467 +
1.468 +U_NAMESPACE_END
1.469 +
1.470 +#endif /* #if !UCONFIG_NO_TRANSLITERATION */
