1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/i18n/rbt.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,473 @@
1.4 +/*
1.5 +**********************************************************************
1.6 +* Copyright (C) 1999-2007, International Business Machines
1.7 +* Corporation and others. All Rights Reserved.
1.8 +**********************************************************************
1.9 +* Date Name Description
1.10 +* 11/17/99 aliu Creation.
1.11 +**********************************************************************
1.12 +*/
1.13 +#ifndef RBT_H
1.14 +#define RBT_H
1.15 +
1.16 +#include "unicode/utypes.h"
1.17 +
1.18 +#if !UCONFIG_NO_TRANSLITERATION
1.19 +
1.20 +#include "unicode/translit.h"
1.21 +#include "unicode/utypes.h"
1.22 +#include "unicode/parseerr.h"
1.23 +#include "unicode/udata.h"
1.24 +
1.25 +#define U_ICUDATA_TRANSLIT U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "translit"
1.26 +
1.27 +U_NAMESPACE_BEGIN
1.28 +
1.29 +class TransliterationRuleData;
1.30 +
1.31 +/**
1.32 + * <code>RuleBasedTransliterator</code> is a transliterator
1.33 + * that reads a set of rules in order to determine how to perform
1.34 + * translations. Rule sets are stored in resource bundles indexed by
1.35 + * name. Rules within a rule set are separated by semicolons (';').
1.36 + * To include a literal semicolon, prefix it with a backslash ('\').
1.37 + * Whitespace, as defined by <code>Character.isWhitespace()</code>,
1.38 + * is ignored. If the first non-blank character on a line is '#',
1.39 + * the entire line is ignored as a comment. </p>
1.40 + *
1.41 + * <p>Each set of rules consists of two groups, one forward, and one
1.42 + * reverse. This is a convention that is not enforced; rules for one
1.43 + * direction may be omitted, with the result that translations in
1.44 + * that direction will not modify the source text. In addition,
1.45 + * bidirectional forward-reverse rules may be specified for
1.46 + * symmetrical transformations.</p>
1.47 + *
1.48 + * <p><b>Rule syntax</b> </p>
1.49 + *
1.50 + * <p>Rule statements take one of the following forms: </p>
1.51 + *
1.52 + * <dl>
1.53 + * <dt><code>$alefmadda=\u0622;</code></dt>
1.54 + * <dd><strong>Variable definition.</strong> The name on the
1.55 + * left is assigned the text on the right. In this example,
1.56 + * after this statement, instances of the left hand name,
1.57 + * "<code>$alefmadda</code>", will be replaced by
1.58 + * the Unicode character U+0622. Variable names must begin
1.59 + * with a letter and consist only of letters, digits, and
1.60 + * underscores. Case is significant. Duplicate names cause
1.61 + * an exception to be thrown, that is, variables cannot be
1.62 + * redefined. The right hand side may contain well-formed
1.63 + * text of any length, including no text at all ("<code>$empty=;</code>").
1.64 + * The right hand side may contain embedded <code>UnicodeSet</code>
1.65 + * patterns, for example, "<code>$softvowel=[eiyEIY]</code>".</dd>
1.66 + * <dd> </dd>
1.67 + * <dt><code>ai>$alefmadda;</code></dt>
1.68 + * <dd><strong>Forward translation rule.</strong> This rule
1.69 + * states that the string on the left will be changed to the
1.70 + * string on the right when performing forward
1.71 + * transliteration.</dd>
1.72 + * <dt> </dt>
1.73 + * <dt><code>ai<$alefmadda;</code></dt>
1.74 + * <dd><strong>Reverse translation rule.</strong> This rule
1.75 + * states that the string on the right will be changed to
1.76 + * the string on the left when performing reverse
1.77 + * transliteration.</dd>
1.78 + * </dl>
1.79 + *
1.80 + * <dl>
1.81 + * <dt><code>ai<>$alefmadda;</code></dt>
1.82 + * <dd><strong>Bidirectional translation rule.</strong> This
1.83 + * rule states that the string on the right will be changed
1.84 + * to the string on the left when performing forward
1.85 + * transliteration, and vice versa when performing reverse
1.86 + * transliteration.</dd>
1.87 + * </dl>
1.88 + *
1.89 + * <p>Translation rules consist of a <em>match pattern</em> and an <em>output
1.90 + * string</em>. The match pattern consists of literal characters,
1.91 + * optionally preceded by context, and optionally followed by
1.92 + * context. Context characters, like literal pattern characters,
1.93 + * must be matched in the text being transliterated. However, unlike
1.94 + * literal pattern characters, they are not replaced by the output
1.95 + * text. For example, the pattern "<code>abc{def}</code>"
1.96 + * indicates the characters "<code>def</code>" must be
1.97 + * preceded by "<code>abc</code>" for a successful match.
1.98 + * If there is a successful match, "<code>def</code>" will
1.99 + * be replaced, but not "<code>abc</code>". The final '<code>}</code>'
1.100 + * is optional, so "<code>abc{def</code>" is equivalent to
1.101 + * "<code>abc{def}</code>". Another example is "<code>{123}456</code>"
1.102 + * (or "<code>123}456</code>") in which the literal
1.103 + * pattern "<code>123</code>" must be followed by "<code>456</code>".
1.104 + * </p>
1.105 + *
1.106 + * <p>The output string of a forward or reverse rule consists of
1.107 + * characters to replace the literal pattern characters. If the
1.108 + * output string contains the character '<code>|</code>', this is
1.109 + * taken to indicate the location of the <em>cursor</em> after
1.110 + * replacement. The cursor is the point in the text at which the
1.111 + * next replacement, if any, will be applied. The cursor is usually
1.112 + * placed within the replacement text; however, it can actually be
1.113 + * placed into the precending or following context by using the
1.114 + * special character '<code>@</code>'. Examples:</p>
1.115 + *
1.116 + * <blockquote>
1.117 + * <p><code>a {foo} z > | @ bar; # foo -> bar, move cursor
1.118 + * before a<br>
1.119 + * {foo} xyz > bar @@|; # foo -> bar, cursor between
1.120 + * y and z</code></p>
1.121 + * </blockquote>
1.122 + *
1.123 + * <p><b>UnicodeSet</b></p>
1.124 + *
1.125 + * <p><code>UnicodeSet</code> patterns may appear anywhere that
1.126 + * makes sense. They may appear in variable definitions.
1.127 + * Contrariwise, <code>UnicodeSet</code> patterns may themselves
1.128 + * contain variable references, such as "<code>$a=[a-z];$not_a=[^$a]</code>",
1.129 + * or "<code>$range=a-z;$ll=[$range]</code>".</p>
1.130 + *
1.131 + * <p><code>UnicodeSet</code> patterns may also be embedded directly
1.132 + * into rule strings. Thus, the following two rules are equivalent:</p>
1.133 + *
1.134 + * <blockquote>
1.135 + * <p><code>$vowel=[aeiou]; $vowel>'*'; # One way to do this<br>
1.136 + * [aeiou]>'*';
1.137 + * #
1.138 + * Another way</code></p>
1.139 + * </blockquote>
1.140 + *
1.141 + * <p>See {@link UnicodeSet} for more documentation and examples.</p>
1.142 + *
1.143 + * <p><b>Segments</b></p>
1.144 + *
1.145 + * <p>Segments of the input string can be matched and copied to the
1.146 + * output string. This makes certain sets of rules simpler and more
1.147 + * general, and makes reordering possible. For example:</p>
1.148 + *
1.149 + * <blockquote>
1.150 + * <p><code>([a-z]) > $1 $1;
1.151 + * #
1.152 + * double lowercase letters<br>
1.153 + * ([:Lu:]) ([:Ll:]) > $2 $1; # reverse order of Lu-Ll pairs</code></p>
1.154 + * </blockquote>
1.155 + *
1.156 + * <p>The segment of the input string to be copied is delimited by
1.157 + * "<code>(</code>" and "<code>)</code>". Up to
1.158 + * nine segments may be defined. Segments may not overlap. In the
1.159 + * output string, "<code>$1</code>" through "<code>$9</code>"
1.160 + * represent the input string segments, in left-to-right order of
1.161 + * definition.</p>
1.162 + *
1.163 + * <p><b>Anchors</b></p>
1.164 + *
1.165 + * <p>Patterns can be anchored to the beginning or the end of the text. This is done with the
1.166 + * special characters '<code>^</code>' and '<code>$</code>'. For example:</p>
1.167 + *
1.168 + * <blockquote>
1.169 + * <p><code>^ a > 'BEG_A'; # match 'a' at start of text<br>
1.170 + * a > 'A'; # match other instances
1.171 + * of 'a'<br>
1.172 + * z $ > 'END_Z'; # match 'z' at end of text<br>
1.173 + * z > 'Z'; # match other instances
1.174 + * of 'z'</code></p>
1.175 + * </blockquote>
1.176 + *
1.177 + * <p>It is also possible to match the beginning or the end of the text using a <code>UnicodeSet</code>.
1.178 + * This is done by including a virtual anchor character '<code>$</code>' at the end of the
1.179 + * set pattern. Although this is usually the match chafacter for the end anchor, the set will
1.180 + * match either the beginning or the end of the text, depending on its placement. For
1.181 + * example:</p>
1.182 + *
1.183 + * <blockquote>
1.184 + * <p><code>$x = [a-z$]; # match 'a' through 'z' OR anchor<br>
1.185 + * $x 1 > 2; # match '1' after a-z or at the start<br>
1.186 + * 3 $x > 4; # match '3' before a-z or at the end</code></p>
1.187 + * </blockquote>
1.188 + *
1.189 + * <p><b>Example</b> </p>
1.190 + *
1.191 + * <p>The following example rules illustrate many of the features of
1.192 + * the rule language. </p>
1.193 + *
1.194 + * <table border="0" cellpadding="4">
1.195 + * <tr>
1.196 + * <td valign="top">Rule 1.</td>
1.197 + * <td valign="top" nowrap><code>abc{def}>x|y</code></td>
1.198 + * </tr>
1.199 + * <tr>
1.200 + * <td valign="top">Rule 2.</td>
1.201 + * <td valign="top" nowrap><code>xyz>r</code></td>
1.202 + * </tr>
1.203 + * <tr>
1.204 + * <td valign="top">Rule 3.</td>
1.205 + * <td valign="top" nowrap><code>yz>q</code></td>
1.206 + * </tr>
1.207 + * </table>
1.208 + *
1.209 + * <p>Applying these rules to the string "<code>adefabcdefz</code>"
1.210 + * yields the following results: </p>
1.211 + *
1.212 + * <table border="0" cellpadding="4">
1.213 + * <tr>
1.214 + * <td valign="top" nowrap><code>|adefabcdefz</code></td>
1.215 + * <td valign="top">Initial state, no rules match. Advance
1.216 + * cursor.</td>
1.217 + * </tr>
1.218 + * <tr>
1.219 + * <td valign="top" nowrap><code>a|defabcdefz</code></td>
1.220 + * <td valign="top">Still no match. Rule 1 does not match
1.221 + * because the preceding context is not present.</td>
1.222 + * </tr>
1.223 + * <tr>
1.224 + * <td valign="top" nowrap><code>ad|efabcdefz</code></td>
1.225 + * <td valign="top">Still no match. Keep advancing until
1.226 + * there is a match...</td>
1.227 + * </tr>
1.228 + * <tr>
1.229 + * <td valign="top" nowrap><code>ade|fabcdefz</code></td>
1.230 + * <td valign="top">...</td>
1.231 + * </tr>
1.232 + * <tr>
1.233 + * <td valign="top" nowrap><code>adef|abcdefz</code></td>
1.234 + * <td valign="top">...</td>
1.235 + * </tr>
1.236 + * <tr>
1.237 + * <td valign="top" nowrap><code>adefa|bcdefz</code></td>
1.238 + * <td valign="top">...</td>
1.239 + * </tr>
1.240 + * <tr>
1.241 + * <td valign="top" nowrap><code>adefab|cdefz</code></td>
1.242 + * <td valign="top">...</td>
1.243 + * </tr>
1.244 + * <tr>
1.245 + * <td valign="top" nowrap><code>adefabc|defz</code></td>
1.246 + * <td valign="top">Rule 1 matches; replace "<code>def</code>"
1.247 + * with "<code>xy</code>" and back up the cursor
1.248 + * to before the '<code>y</code>'.</td>
1.249 + * </tr>
1.250 + * <tr>
1.251 + * <td valign="top" nowrap><code>adefabcx|yz</code></td>
1.252 + * <td valign="top">Although "<code>xyz</code>" is
1.253 + * present, rule 2 does not match because the cursor is
1.254 + * before the '<code>y</code>', not before the '<code>x</code>'.
1.255 + * Rule 3 does match. Replace "<code>yz</code>"
1.256 + * with "<code>q</code>".</td>
1.257 + * </tr>
1.258 + * <tr>
1.259 + * <td valign="top" nowrap><code>adefabcxq|</code></td>
1.260 + * <td valign="top">The cursor is at the end;
1.261 + * transliteration is complete.</td>
1.262 + * </tr>
1.263 + * </table>
1.264 + *
1.265 + * <p>The order of rules is significant. If multiple rules may match
1.266 + * at some point, the first matching rule is applied. </p>
1.267 + *
1.268 + * <p>Forward and reverse rules may have an empty output string.
1.269 + * Otherwise, an empty left or right hand side of any statement is a
1.270 + * syntax error. </p>
1.271 + *
1.272 + * <p>Single quotes are used to quote any character other than a
1.273 + * digit or letter. To specify a single quote itself, inside or
1.274 + * outside of quotes, use two single quotes in a row. For example,
1.275 + * the rule "<code>'>'>o''clock</code>" changes the
1.276 + * string "<code>></code>" to the string "<code>o'clock</code>".
1.277 + * </p>
1.278 + *
1.279 + * <p><b>Notes</b> </p>
1.280 + *
1.281 + * <p>While a RuleBasedTransliterator is being built, it checks that
1.282 + * the rules are added in proper order. For example, if the rule
1.283 + * "a>x" is followed by the rule "ab>y",
1.284 + * then the second rule will throw an exception. The reason is that
1.285 + * the second rule can never be triggered, since the first rule
1.286 + * always matches anything it matches. In other words, the first
1.287 + * rule <em>masks</em> the second rule. </p>
1.288 + *
1.289 + * @author Alan Liu
1.290 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.291 + */
1.292 +class RuleBasedTransliterator : public Transliterator {
1.293 +private:
1.294 + /**
1.295 + * The data object is immutable, so we can freely share it with
1.296 + * other instances of RBT, as long as we do NOT own this object.
1.297 + * TODO: data is no longer immutable. See bugs #1866, 2155
1.298 + */
1.299 + TransliterationRuleData* fData;
1.300 +
1.301 + /**
1.302 + * If true, we own the data object and must delete it.
1.303 + */
1.304 + UBool isDataOwned;
1.305 +
1.306 +public:
1.307 +
1.308 + /**
1.309 + * Constructs a new transliterator from the given rules.
1.310 + * @param rules rules, separated by ';'
1.311 + * @param direction either FORWARD or REVERSE.
1.312 + * @exception IllegalArgumentException if rules are malformed.
1.313 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.314 + */
1.315 + RuleBasedTransliterator(const UnicodeString& id,
1.316 + const UnicodeString& rules,
1.317 + UTransDirection direction,
1.318 + UnicodeFilter* adoptedFilter,
1.319 + UParseError& parseError,
1.320 + UErrorCode& status);
1.321 +
1.322 + /**
1.323 + * Constructs a new transliterator from the given rules.
1.324 + * @param rules rules, separated by ';'
1.325 + * @param direction either FORWARD or REVERSE.
1.326 + * @exception IllegalArgumentException if rules are malformed.
1.327 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.328 + */
1.329 + /*RuleBasedTransliterator(const UnicodeString& id,
1.330 + const UnicodeString& rules,
1.331 + UTransDirection direction,
1.332 + UnicodeFilter* adoptedFilter,
1.333 + UErrorCode& status);*/
1.334 +
1.335 + /**
1.336 + * Covenience constructor with no filter.
1.337 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.338 + */
1.339 + /*RuleBasedTransliterator(const UnicodeString& id,
1.340 + const UnicodeString& rules,
1.341 + UTransDirection direction,
1.342 + UErrorCode& status);*/
1.343 +
1.344 + /**
1.345 + * Covenience constructor with no filter and FORWARD direction.
1.346 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.347 + */
1.348 + /*RuleBasedTransliterator(const UnicodeString& id,
1.349 + const UnicodeString& rules,
1.350 + UErrorCode& status);*/
1.351 +
1.352 + /**
1.353 + * Covenience constructor with FORWARD direction.
1.354 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.355 + */
1.356 + /*RuleBasedTransliterator(const UnicodeString& id,
1.357 + const UnicodeString& rules,
1.358 + UnicodeFilter* adoptedFilter,
1.359 + UErrorCode& status);*/
1.360 +private:
1.361 +
1.362 + friend class TransliteratorRegistry; // to access TransliterationRuleData convenience ctor
1.363 + /**
1.364 + * Covenience constructor.
1.365 + * @param id the id for the transliterator.
1.366 + * @param theData the rule data for the transliterator.
1.367 + * @param adoptedFilter the filter for the transliterator
1.368 + */
1.369 + RuleBasedTransliterator(const UnicodeString& id,
1.370 + const TransliterationRuleData* theData,
1.371 + UnicodeFilter* adoptedFilter = 0);
1.372 +
1.373 +
1.374 + friend class Transliterator; // to access following ct
1.375 +
1.376 + /**
1.377 + * Internal constructor.
1.378 + * @param id the id for the transliterator.
1.379 + * @param theData the rule data for the transliterator.
1.380 + * @param isDataAdopted determine who will own the 'data' object. True, the caller should not delete 'data'.
1.381 + */
1.382 + RuleBasedTransliterator(const UnicodeString& id,
1.383 + TransliterationRuleData* data,
1.384 + UBool isDataAdopted);
1.385 +
1.386 +public:
1.387 +
1.388 + /**
1.389 + * Copy constructor.
1.390 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.391 + */
1.392 + RuleBasedTransliterator(const RuleBasedTransliterator&);
1.393 +
1.394 + virtual ~RuleBasedTransliterator();
1.395 +
1.396 + /**
1.397 + * Implement Transliterator API.
1.398 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.399 + */
1.400 + virtual Transliterator* clone(void) const;
1.401 +
1.402 +protected:
1.403 + /**
1.404 + * Implements {@link Transliterator#handleTransliterate}.
1.405 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.406 + */
1.407 + virtual void handleTransliterate(Replaceable& text, UTransPosition& offsets,
1.408 + UBool isIncremental) const;
1.409 +
1.410 +public:
1.411 + /**
1.412 + * Return a representation of this transliterator as source rules.
1.413 + * These rules will produce an equivalent transliterator if used
1.414 + * to construct a new transliterator.
1.415 + * @param result the string to receive the rules. Previous
1.416 + * contents will be deleted.
1.417 + * @param escapeUnprintable if TRUE then convert unprintable
1.418 + * character to their hex escape representations, \uxxxx or
1.419 + * \Uxxxxxxxx. Unprintable characters are those other than
1.420 + * U+000A, U+0020..U+007E.
1.421 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.422 + */
1.423 + virtual UnicodeString& toRules(UnicodeString& result,
1.424 + UBool escapeUnprintable) const;
1.425 +
1.426 +protected:
1.427 + /**
1.428 + * Implement Transliterator framework
1.429 + */
1.430 + virtual void handleGetSourceSet(UnicodeSet& result) const;
1.431 +
1.432 +public:
1.433 + /**
1.434 + * Override Transliterator framework
1.435 + */
1.436 + virtual UnicodeSet& getTargetSet(UnicodeSet& result) const;
1.437 +
1.438 + /**
1.439 + * Return the class ID for this class. This is useful only for
1.440 + * comparing to a return value from getDynamicClassID(). For example:
1.441 + * <pre>
1.442 + * . Base* polymorphic_pointer = createPolymorphicObject();
1.443 + * . if (polymorphic_pointer->getDynamicClassID() ==
1.444 + * . Derived::getStaticClassID()) ...
1.445 + * </pre>
1.446 + * @return The class ID for all objects of this class.
1.447 + * @internal Use transliterator factory methods instead since this class will be removed in that release.
1.448 + */
1.449 + U_I18N_API static UClassID U_EXPORT2 getStaticClassID(void);
1.450 +
1.451 + /**
1.452 + * Returns a unique class ID <b>polymorphically</b>. This method
1.453 + * is to implement a simple version of RTTI, since not all C++
1.454 + * compilers support genuine RTTI. Polymorphic operator==() and
1.455 + * clone() methods call this method.
1.456 + *
1.457 + * @return The class ID for this object. All objects of a given
1.458 + * class have the same class ID. Objects of other classes have
1.459 + * different class IDs.
1.460 + */
1.461 + virtual UClassID getDynamicClassID(void) const;
1.462 +
1.463 +private:
1.464 +
1.465 + void _construct(const UnicodeString& rules,
1.466 + UTransDirection direction,
1.467 + UParseError& parseError,
1.468 + UErrorCode& status);
1.469 +};
1.470 +
1.471 +
1.472 +U_NAMESPACE_END
1.473 +
1.474 +#endif /* #if !UCONFIG_NO_TRANSLITERATION */
1.475 +
1.476 +#endif
