The Tor Browser: comparison intl/icu/source/i18n/cpdtrans.cpp

--1:000000000000
+:509447504630
+/*
+**********************************************************************
+*   Copyright (C) 1999-2011, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+**********************************************************************
+*   Date        Name        Description
+*   11/17/99    aliu        Creation.
+**********************************************************************
+*/
+#include "unicode/utypes.h"
+#if !UCONFIG_NO_TRANSLITERATION
+#include "unicode/unifilt.h"
+#include "unicode/uniset.h"
+#include "cpdtrans.h"
+#include "uvector.h"
+#include "tridpars.h"
+#include "cmemory.h"
+// keep in sync with Transliterator
+//static const UChar ID_SEP   = 0x002D; /*-*/
+static const UChar ID_DELIM = 0x003B; /*;*/
+static const UChar NEWLINE  = 10;
+static const UChar COLON_COLON[] = {0x3A, 0x3A, 0}; //"::"
+U_NAMESPACE_BEGIN
+const UChar CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass"
+UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator)
+/**
+* Constructs a new compound transliterator given an array of
+* transliterators.  The array of transliterators may be of any
+* length, including zero or one, however, useful compound
+* transliterators have at least two components.
+* @param transliterators array of <code>Transliterator</code>
+* objects
+* @param transliteratorCount The number of
+* <code>Transliterator</code> objects in transliterators.
+* @param filter the filter.  Any character for which
+* <tt>filter.contains()</tt> returns <tt>false</tt> will not be
+* altered by this transliterator.  If <tt>filter</tt> is
+* <tt>null</tt> then no filtering is applied.
+*/
+CompoundTransliterator::CompoundTransliterator(
+Transliterator* const transliterators[],
+int32_t transliteratorCount,
+UnicodeFilter* adoptedFilter) :
+Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter),
+trans(0), count(0), numAnonymousRBTs(0)  {
+setTransliterators(transliterators, transliteratorCount);
+}
+/**
+* Splits an ID of the form "ID;ID;..." into a compound using each
+* of the IDs.
+* @param id of above form
+* @param forward if false, does the list in reverse order, and
+* takes the inverse of each ID.
+*/
+CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
+UTransDirection direction,
+UnicodeFilter* adoptedFilter,
+UParseError& /*parseError*/,
+UErrorCode& status) :
+Transliterator(id, adoptedFilter),
+trans(0), numAnonymousRBTs(0) {
+// TODO add code for parseError...currently unused, but
+// later may be used by parsing code...
+init(id, direction, TRUE, status);
+}
+CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
+UParseError& /*parseError*/,
+UErrorCode& status) :
+Transliterator(id, 0), // set filter to 0 here!
+trans(0), numAnonymousRBTs(0) {
+// TODO add code for parseError...currently unused, but
+// later may be used by parsing code...
+init(id, UTRANS_FORWARD, TRUE, status);
+}
+/**
+* Private constructor for use of TransliteratorAlias
+*/
+CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID,
+UVector& list,
+UnicodeFilter* adoptedFilter,
+int32_t anonymousRBTs,
+UParseError& /*parseError*/,
+UErrorCode& status) :
+Transliterator(newID, adoptedFilter),
+trans(0), numAnonymousRBTs(anonymousRBTs)
+{
+init(list, UTRANS_FORWARD, FALSE, status);
+}
+/**
+* Private constructor for Transliterator from a vector of
+* transliterators.  The caller is responsible for fixing up the
+* ID.
+*/
+CompoundTransliterator::CompoundTransliterator(UVector& list,
+UParseError& /*parseError*/,
+UErrorCode& status) :
+Transliterator(UnicodeString(), NULL),
+trans(0), numAnonymousRBTs(0)
+{
+// TODO add code for parseError...currently unused, but
+// later may be used by parsing code...
+init(list, UTRANS_FORWARD, FALSE, status);
+// assume caller will fixup ID
+}
+CompoundTransliterator::CompoundTransliterator(UVector& list,
+int32_t anonymousRBTs,
+UParseError& /*parseError*/,
+UErrorCode& status) :
+Transliterator(UnicodeString(), NULL),
+trans(0), numAnonymousRBTs(anonymousRBTs)
+{
+init(list, UTRANS_FORWARD, FALSE, status);
+}
+/**
+* Finish constructing a transliterator: only to be called by
+* constructors.  Before calling init(), set trans and filter to NULL.
+* @param id the id containing ';'-separated entries
+* @param direction either FORWARD or REVERSE
+* @param idSplitPoint the index into id at which the
+* adoptedSplitTransliterator should be inserted, if there is one, or
+* -1 if there is none.
+* @param adoptedSplitTransliterator a transliterator to be inserted
+* before the entry at offset idSplitPoint in the id string.  May be
+* NULL to insert no entry.
+* @param fixReverseID if TRUE, then reconstruct the ID of reverse
+* entries by calling getID() of component entries.  Some constructors
+* do not require this because they apply a facade ID anyway.
+* @param status the error code indicating success or failure
+*/
+void CompoundTransliterator::init(const UnicodeString& id,
+UTransDirection direction,
+UBool fixReverseID,
+UErrorCode& status) {
+// assert(trans == 0);
+if (U_FAILURE(status)) {
+return;
+}
+UVector list(status);
+UnicodeSet* compoundFilter = NULL;
+UnicodeString regenID;
+if (!TransliteratorIDParser::parseCompoundID(id, direction,
+regenID, list, compoundFilter)) {
+status = U_INVALID_ID;
+delete compoundFilter;
+return;
+}
+TransliteratorIDParser::instantiateList(list, status);
+init(list, direction, fixReverseID, status);
+if (compoundFilter != NULL) {
+adoptFilter(compoundFilter);
+}
+}
+/**
+* Finish constructing a transliterator: only to be called by
+* constructors.  Before calling init(), set trans and filter to NULL.
+* @param list a vector of transliterator objects to be adopted.  It
+* should NOT be empty.  The list should be in declared order.  That
+* is, it should be in the FORWARD order; if direction is REVERSE then
+* the list order will be reversed.
+* @param direction either FORWARD or REVERSE
+* @param fixReverseID if TRUE, then reconstruct the ID of reverse
+* entries by calling getID() of component entries.  Some constructors
+* do not require this because they apply a facade ID anyway.
+* @param status the error code indicating success or failure
+*/
+void CompoundTransliterator::init(UVector& list,
+UTransDirection direction,
+UBool fixReverseID,
+UErrorCode& status) {
+// assert(trans == 0);
+// Allocate array
+if (U_SUCCESS(status)) {
+count = list.size();
+trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *));
+/* test for NULL */
+if (trans == 0) {
+status = U_MEMORY_ALLOCATION_ERROR;
+return;
+}
+}
+if (U_FAILURE(status) || trans == 0) {
+// assert(trans == 0);
+return;
+}
+// Move the transliterators from the vector into an array.
+// Reverse the order if necessary.
+int32_t i;
+for (i=0; i<count; ++i) {
+int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i;
+trans[i] = (Transliterator*) list.elementAt(j);
+}
+// If the direction is UTRANS_REVERSE then we may need to fix the
+// ID.
+if (direction == UTRANS_REVERSE && fixReverseID) {
+UnicodeString newID;
+for (i=0; i<count; ++i) {
+if (i > 0) {
+newID.append(ID_DELIM);
+}
+newID.append(trans[i]->getID());
+}
+setID(newID);
+}
+computeMaximumContextLength();
+}
+/**
+* Return the IDs of the given list of transliterators, concatenated
+* with ID_DELIM delimiting them.  Equivalent to the perlish expression
+* join(ID_DELIM, map($_.getID(), transliterators).
+*/
+UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[],
+int32_t transCount) {
+UnicodeString id;
+for (int32_t i=0; i<transCount; ++i) {
+if (i > 0) {
+id.append(ID_DELIM);
+}
+id.append(transliterators[i]->getID());
+}
+return id; // Return temporary
+}
+/**
+* Copy constructor.
+*/
+CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) :
+Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) {
+*this = t;
+}
+/**
+* Destructor
+*/
+CompoundTransliterator::~CompoundTransliterator() {
+freeTransliterators();
+}
+void CompoundTransliterator::freeTransliterators(void) {
+if (trans != 0) {
+for (int32_t i=0; i<count; ++i) {
+delete trans[i];
+}
+uprv_free(trans);
+}
+trans = 0;
+count = 0;
+}
+/**
+* Assignment operator.
+*/
+CompoundTransliterator& CompoundTransliterator::operator=(
+const CompoundTransliterator& t)
+{
+Transliterator::operator=(t);
+int32_t i = 0;
+UBool failed = FALSE;
+if (trans != NULL) {
+for (i=0; i<count; ++i) {
+delete trans[i];
+trans[i] = 0;
+}
+}
+if (t.count > count) {
+if (trans != NULL) {
+uprv_free(trans);
+}
+trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *));
+}
+count = t.count;
+if (trans != NULL) {
+for (i=0; i<count; ++i) {
+trans[i] = t.trans[i]->clone();
+if (trans[i] == NULL) {
+failed = TRUE;
+break;
+}
+}
+}
+// if memory allocation failed delete backwards trans array
+if (failed && i > 0) {
+int32_t n;
+for (n = i-1; n >= 0; n--) {
+uprv_free(trans[n]);
+trans[n] = NULL;
+}
+}
+numAnonymousRBTs = t.numAnonymousRBTs;
+return *this;
+}
+/**
+* Transliterator API.
+*/
+Transliterator* CompoundTransliterator::clone(void) const {
+return new CompoundTransliterator(*this);
+}
+/**
+* Returns the number of transliterators in this chain.
+* @return number of transliterators in this chain.
+*/
+int32_t CompoundTransliterator::getCount(void) const {
+return count;
+}
+/**
+* Returns the transliterator at the given index in this chain.
+* @param index index into chain, from 0 to <code>getCount() - 1</code>
+* @return transliterator at the given index
+*/
+const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const {
+return *trans[index];
+}
+void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[],
+int32_t transCount) {
+Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *));
+if (a == NULL) {
+return;
+}
+int32_t i = 0;
+UBool failed = FALSE;
+for (i=0; i<transCount; ++i) {
+a[i] = transliterators[i]->clone();
+if (a[i] == NULL) {
+failed = TRUE;
+break;
+}
+}
+if (failed && i > 0) {
+int32_t n;
+for (n = i-1; n >= 0; n--) {
+uprv_free(a[n]);
+a[n] = NULL;
+}
+return;
+}
+adoptTransliterators(a, transCount);
+}
+void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[],
+int32_t transCount) {
+// First free trans[] and set count to zero.  Once this is done,
+// orphan the filter.  Set up the new trans[].
+freeTransliterators();
+trans = adoptedTransliterators;
+count = transCount;
+computeMaximumContextLength();
+setID(joinIDs(trans, count));
+}
+/**
+* Append c to buf, unless buf is empty or buf already ends in c.
+*/
+static void _smartAppend(UnicodeString& buf, UChar c) {
+if (buf.length() != 0 &&
+buf.charAt(buf.length() - 1) != c) {
+buf.append(c);
+}
+}
+UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource,
+UBool escapeUnprintable) const {
+// We do NOT call toRules() on our component transliterators, in
+// general.  If we have several rule-based transliterators, this
+// yields a concatenation of the rules -- not what we want.  We do
+// handle compound RBT transliterators specially -- those for which
+// compoundRBTIndex >= 0.  For the transliterator at compoundRBTIndex,
+// we do call toRules() recursively.
+rulesSource.truncate(0);
+if (numAnonymousRBTs >= 1 && getFilter() != NULL) {
+// If we are a compound RBT and if we have a global
+// filter, then emit it at the top.
+UnicodeString pat;
+rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM);
+}
+for (int32_t i=0; i<count; ++i) {
+UnicodeString rule;
+// Anonymous RuleBasedTransliterators (inline rules and
+// ::BEGIN/::END blocks) are given IDs that begin with
+// "%Pass": use toRules() to write all the rules to the output
+// (and insert "::Null;" if we have two in a row)
+if (trans[i]->getID().startsWith(PASS_STRING, 5)) {
+trans[i]->toRules(rule, escapeUnprintable);
+if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5))
+rule = UNICODE_STRING_SIMPLE("::Null;") + rule;
+// we also use toRules() on CompoundTransliterators (which we
+// check for by looking for a semicolon in the ID)-- this gets
+// the list of their child transliterators output in the right
+// format
+} else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) {
+trans[i]->toRules(rule, escapeUnprintable);
+// for everything else, use Transliterator::toRules()
+} else {
+trans[i]->Transliterator::toRules(rule, escapeUnprintable);
+}
+_smartAppend(rulesSource, NEWLINE);
+rulesSource.append(rule);
+_smartAppend(rulesSource, ID_DELIM);
+}
+return rulesSource;
+}
+/**
+* Implement Transliterator framework
+*/
+void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const {
+UnicodeSet set;
+result.clear();
+for (int32_t i=0; i<count; ++i) {
+result.addAll(trans[i]->getSourceSet(set));
+// Take the example of Hiragana-Latin.  This is really
+// Hiragana-Katakana; Katakana-Latin.  The source set of
+// these two is roughly [:Hiragana:] and [:Katakana:].
+// But the source set for the entire transliterator is
+// actually [:Hiragana:] ONLY -- that is, the first
+// non-empty source set.
+// This is a heuristic, and not 100% reliable.
+if (!result.isEmpty()) {
+break;
+}
+}
+}
+/**
+* Override Transliterator framework
+*/
+UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const {
+UnicodeSet set;
+result.clear();
+for (int32_t i=0; i<count; ++i) {
+// This is a heuristic, and not 100% reliable.
+result.addAll(trans[i]->getTargetSet(set));
+}
+return result;
+}
+/**
+* Implements {@link Transliterator#handleTransliterate}.
+*/
+void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index,
+UBool incremental) const {
+/* Call each transliterator with the same contextStart and
+* start, but with the limit as modified
+* by preceding transliterators.  The start index must be
+* reset for each transliterator to give each a chance to
+* transliterate the text.  The initial contextStart index is known
+* to still point to the same place after each transliterator
+* is called because each transliterator will not change the
+* text between contextStart and the initial start index.
+*
+* IMPORTANT: After the first transliterator, each subsequent
+* transliterator only gets to transliterate text committed by
+* preceding transliterators; that is, the start (output
+* value) of transliterator i becomes the limit (input value)
+* of transliterator i+1.  Finally, the overall limit is fixed
+* up before we return.
+*
+* Assumptions we make here:
+* (1) contextStart <= start <= limit <= contextLimit <= text.length()
+* (2) start <= start' <= limit'  ;cursor doesn't move back
+* (3) start <= limit'            ;text before cursor unchanged
+* - start' is the value of start after calling handleKT
+* - limit' is the value of limit after calling handleKT
+*/
+/**
+* Example: 3 transliterators.  This example illustrates the
+* mechanics we need to implement.  C, S, and L are the contextStart,
+* start, and limit.  gl is the globalLimit.  contextLimit is
+* equal to limit throughout.
+*
+* 1. h-u, changes hex to Unicode
+*
+*    4  7  a  d  0      4  7  a
+*    abc/u0061/u    =>  abca/u
+*    C  S       L       C   S L   gl=f->a
+*
+* 2. upup, changes "x" to "XX"
+*
+*    4  7  a       4  7  a
+*    abca/u    =>  abcAA/u
+*    C  SL         C    S
+*                       L    gl=a->b
+* 3. u-h, changes Unicode to hex
+*
+*    4  7  a        4  7  a  d  0  3
+*    abcAA/u    =>  abc/u0041/u0041/u
+*    C  S L         C              S
+*                                  L   gl=b->15
+* 4. return
+*
+*    4  7  a  d  0  3
+*    abc/u0041/u0041/u
+*    C S L
+*/
+if (count < 1) {
+index.start = index.limit;
+return; // Short circuit for empty compound transliterators
+}
+// compoundLimit is the limit value for the entire compound
+// operation.  We overwrite index.limit with the previous
+// index.start.  After each transliteration, we update
+// compoundLimit for insertions or deletions that have happened.
+int32_t compoundLimit = index.limit;
+// compoundStart is the start for the entire compound
+// operation.
+int32_t compoundStart = index.start;
+int32_t delta = 0; // delta in length
+// Give each transliterator a crack at the run of characters.
+// See comments at the top of the method for more detail.
+for (int32_t i=0; i<count; ++i) {
+index.start = compoundStart; // Reset start
+int32_t limit = index.limit;
+if (index.start == index.limit) {
+// Short circuit for empty range
+break;
+}
+trans[i]->filteredTransliterate(text, index, incremental);
+// In a properly written transliterator, start == limit after
+// handleTransliterate() returns when incremental is false.
+// Catch cases where the subclass doesn't do this, and throw
+// an exception.  (Just pinning start to limit is a bad idea,
+// because what's probably happening is that the subclass
+// isn't transliterating all the way to the end, and it should
+// in non-incremental mode.)
+if (!incremental && index.start != index.limit) {
+// We can't throw an exception, so just fudge things
+index.start = index.limit;
+}
+// Cumulative delta for insertions/deletions
+delta += index.limit - limit;
+if (incremental) {
+// In the incremental case, only allow subsequent
+// transliterators to modify what has already been
+// completely processed by prior transliterators.  In the
+// non-incrmental case, allow each transliterator to
+// process the entire text.
+index.limit = index.start;
+}
+}
+compoundLimit += delta;
+// Start is good where it is -- where the last transliterator left
+// it.  Limit needs to be put back where it was, modulo
+// adjustments for deletions/insertions.
+index.limit = compoundLimit;
+}
+/**
+* Sets the length of the longest context required by this transliterator.
+* This is <em>preceding</em> context.
+*/
+void CompoundTransliterator::computeMaximumContextLength(void) {
+int32_t max = 0;
+for (int32_t i=0; i<count; ++i) {
+int32_t len = trans[i]->getMaximumContextLength();
+if (len > max) {
+max = len;
+}
+}
+setMaximumContextLength(max);
+}
+U_NAMESPACE_END
+#endif /* #if !UCONFIG_NO_TRANSLITERATION */
+/* eof */

The Tor Browser / file comparison

comparison: intl/icu/source/i18n/cpdtrans.cpp

intl/icu/source/i18n/cpdtrans.cpp