The Tor Browser: intl/icu/source/i18n/cpdtrans.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/cpdtrans.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/cpdtrans.cpp

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 /*
 **********************************************************************
 *   Copyright (C) 1999-2011, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 **********************************************************************
 *   Date        Name        Description
 *   11/17/99    aliu        Creation.
 **********************************************************************
 */
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_TRANSLITERATION
 #include "unicode/unifilt.h"
 #include "unicode/uniset.h"
 #include "cpdtrans.h"
 #include "uvector.h"
 #include "tridpars.h"
 #include "cmemory.h"
 // keep in sync with Transliterator
 //static const UChar ID_SEP   = 0x002D; /*-*/
 static const UChar ID_DELIM = 0x003B; /*;*/
 static const UChar NEWLINE  = 10;
 static const UChar COLON_COLON[] = {0x3A, 0x3A, 0}; //"::"
 U_NAMESPACE_BEGIN
 const UChar CompoundTransliterator::PASS_STRING[] = { 0x0025, 0x0050, 0x0061, 0x0073, 0x0073, 0 }; // "%Pass"
 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(CompoundTransliterator)
 /**
  * Constructs a new compound transliterator given an array of
  * transliterators.  The array of transliterators may be of any
  * length, including zero or one, however, useful compound
  * transliterators have at least two components.
  * @param transliterators array of <code>Transliterator</code>
  * objects
  * @param transliteratorCount The number of
  * <code>Transliterator</code> objects in transliterators.
  * @param filter the filter.  Any character for which
  * <tt>filter.contains()</tt> returns <tt>false</tt> will not be
  * altered by this transliterator.  If <tt>filter</tt> is
  * <tt>null</tt> then no filtering is applied.
  */
 CompoundTransliterator::CompoundTransliterator(
                            Transliterator* const transliterators[],
                            int32_t transliteratorCount,
                            UnicodeFilter* adoptedFilter) :
     Transliterator(joinIDs(transliterators, transliteratorCount), adoptedFilter),
     trans(0), count(0), numAnonymousRBTs(0)  {
     setTransliterators(transliterators, transliteratorCount);
 }
 /**
  * Splits an ID of the form "ID;ID;..." into a compound using each
  * of the IDs.
  * @param id of above form
  * @param forward if false, does the list in reverse order, and
  * takes the inverse of each ID.
  */
 CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
                               UTransDirection direction,
                               UnicodeFilter* adoptedFilter,
                               UParseError& /*parseError*/,
                               UErrorCode& status) :
     Transliterator(id, adoptedFilter),
     trans(0), numAnonymousRBTs(0) {
     // TODO add code for parseError...currently unused, but
     // later may be used by parsing code...
     init(id, direction, TRUE, status);
 }
 CompoundTransliterator::CompoundTransliterator(const UnicodeString& id,
                               UParseError& /*parseError*/,
                               UErrorCode& status) :
     Transliterator(id, 0), // set filter to 0 here!
     trans(0), numAnonymousRBTs(0) {
     // TODO add code for parseError...currently unused, but
     // later may be used by parsing code...
     init(id, UTRANS_FORWARD, TRUE, status);
 }
 /**
  * Private constructor for use of TransliteratorAlias
  */
 CompoundTransliterator::CompoundTransliterator(const UnicodeString& newID,
                                               UVector& list,
                                               UnicodeFilter* adoptedFilter,
                                               int32_t anonymousRBTs,
                                               UParseError& /*parseError*/,
                                               UErrorCode& status) :
     Transliterator(newID, adoptedFilter),
     trans(0), numAnonymousRBTs(anonymousRBTs)
 {
     init(list, UTRANS_FORWARD, FALSE, status);
 }
 /**
  * Private constructor for Transliterator from a vector of
  * transliterators.  The caller is responsible for fixing up the
  * ID.
  */
 CompoundTransliterator::CompoundTransliterator(UVector& list,
                                                UParseError& /*parseError*/,
                                                UErrorCode& status) :
     Transliterator(UnicodeString(), NULL),
     trans(0), numAnonymousRBTs(0)
 {
     // TODO add code for parseError...currently unused, but
     // later may be used by parsing code...
     init(list, UTRANS_FORWARD, FALSE, status);
     // assume caller will fixup ID
 }
 CompoundTransliterator::CompoundTransliterator(UVector& list,
                                                int32_t anonymousRBTs,
                                                UParseError& /*parseError*/,
                                                UErrorCode& status) :
     Transliterator(UnicodeString(), NULL),
     trans(0), numAnonymousRBTs(anonymousRBTs)
 {
     init(list, UTRANS_FORWARD, FALSE, status);
 }
 /**
  * Finish constructing a transliterator: only to be called by
  * constructors.  Before calling init(), set trans and filter to NULL.
  * @param id the id containing ';'-separated entries
  * @param direction either FORWARD or REVERSE
  * @param idSplitPoint the index into id at which the
  * adoptedSplitTransliterator should be inserted, if there is one, or
  * -1 if there is none.
  * @param adoptedSplitTransliterator a transliterator to be inserted
  * before the entry at offset idSplitPoint in the id string.  May be
  * NULL to insert no entry.
  * @param fixReverseID if TRUE, then reconstruct the ID of reverse
  * entries by calling getID() of component entries.  Some constructors
  * do not require this because they apply a facade ID anyway.
  * @param status the error code indicating success or failure
  */
 void CompoundTransliterator::init(const UnicodeString& id,
                                   UTransDirection direction,
                                   UBool fixReverseID,
                                   UErrorCode& status) {
     // assert(trans == 0);
     if (U_FAILURE(status)) {
         return;
     }
     UVector list(status);
     UnicodeSet* compoundFilter = NULL;
     UnicodeString regenID;
     if (!TransliteratorIDParser::parseCompoundID(id, direction,
                                       regenID, list, compoundFilter)) {
         status = U_INVALID_ID;
         delete compoundFilter;
         return;
     }
     TransliteratorIDParser::instantiateList(list, status);
     init(list, direction, fixReverseID, status);
     if (compoundFilter != NULL) {
         adoptFilter(compoundFilter);
     }
 }
 /**
  * Finish constructing a transliterator: only to be called by
  * constructors.  Before calling init(), set trans and filter to NULL.
  * @param list a vector of transliterator objects to be adopted.  It
  * should NOT be empty.  The list should be in declared order.  That
  * is, it should be in the FORWARD order; if direction is REVERSE then
  * the list order will be reversed.
  * @param direction either FORWARD or REVERSE
  * @param fixReverseID if TRUE, then reconstruct the ID of reverse
  * entries by calling getID() of component entries.  Some constructors
  * do not require this because they apply a facade ID anyway.
  * @param status the error code indicating success or failure
  */
 void CompoundTransliterator::init(UVector& list,
                                   UTransDirection direction,
                                   UBool fixReverseID,
                                   UErrorCode& status) {
     // assert(trans == 0);
     // Allocate array
     if (U_SUCCESS(status)) {
         count = list.size();
         trans = (Transliterator **)uprv_malloc(count * sizeof(Transliterator *));
         /* test for NULL */
         if (trans == 0) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
     }
     if (U_FAILURE(status) || trans == 0) {
          // assert(trans == 0);
         return;
     }
     // Move the transliterators from the vector into an array.
     // Reverse the order if necessary.
     int32_t i;
     for (i=0; i<count; ++i) {
         int32_t j = (direction == UTRANS_FORWARD) ? i : count - 1 - i;
         trans[i] = (Transliterator*) list.elementAt(j);
     }
     // If the direction is UTRANS_REVERSE then we may need to fix the
     // ID.
     if (direction == UTRANS_REVERSE && fixReverseID) {
         UnicodeString newID;
         for (i=0; i<count; ++i) {
             if (i > 0) {
                 newID.append(ID_DELIM);
             }
             newID.append(trans[i]->getID());
         }
         setID(newID);
     }
     computeMaximumContextLength();
 }
 /**
  * Return the IDs of the given list of transliterators, concatenated
  * with ID_DELIM delimiting them.  Equivalent to the perlish expression
  * join(ID_DELIM, map($_.getID(), transliterators).
  */
 UnicodeString CompoundTransliterator::joinIDs(Transliterator* const transliterators[],
                                               int32_t transCount) {
     UnicodeString id;
     for (int32_t i=0; i<transCount; ++i) {
         if (i > 0) {
             id.append(ID_DELIM);
         }
         id.append(transliterators[i]->getID());
     }
     return id; // Return temporary
 }
 /**
  * Copy constructor.
  */
 CompoundTransliterator::CompoundTransliterator(const CompoundTransliterator& t) :
     Transliterator(t), trans(0), count(0), numAnonymousRBTs(-1) {
     *this = t;
 }
 /**
  * Destructor
  */
 CompoundTransliterator::~CompoundTransliterator() {
     freeTransliterators();
 }
 void CompoundTransliterator::freeTransliterators(void) {
     if (trans != 0) {
         for (int32_t i=0; i<count; ++i) {
             delete trans[i];
         }
         uprv_free(trans);
     }
     trans = 0;
     count = 0;
 }
 /**
  * Assignment operator.
  */
 CompoundTransliterator& CompoundTransliterator::operator=(
                                              const CompoundTransliterator& t)
 {
     Transliterator::operator=(t);
     int32_t i = 0;
     UBool failed = FALSE;
     if (trans != NULL) {
         for (i=0; i<count; ++i) {
             delete trans[i];
             trans[i] = 0;
         }
     }
     if (t.count > count) {
         if (trans != NULL) {
             uprv_free(trans);
         }
         trans = (Transliterator **)uprv_malloc(t.count * sizeof(Transliterator *));
     }
     count = t.count;
     if (trans != NULL) {
         for (i=0; i<count; ++i) {
             trans[i] = t.trans[i]->clone();
             if (trans[i] == NULL) {
                 failed = TRUE;
                 break;
             }
         }
     }
     // if memory allocation failed delete backwards trans array
     if (failed && i > 0) {
         int32_t n;
         for (n = i-1; n >= 0; n--) {
             uprv_free(trans[n]);
             trans[n] = NULL;
         }
     }
     numAnonymousRBTs = t.numAnonymousRBTs;
     return *this;
 }
 /**
  * Transliterator API.
  */
 Transliterator* CompoundTransliterator::clone(void) const {
     return new CompoundTransliterator(*this);
 }
 /**
  * Returns the number of transliterators in this chain.
  * @return number of transliterators in this chain.
  */
 int32_t CompoundTransliterator::getCount(void) const {
     return count;
 }
 /**
  * Returns the transliterator at the given index in this chain.
  * @param index index into chain, from 0 to <code>getCount() - 1</code>
  * @return transliterator at the given index
  */
 const Transliterator& CompoundTransliterator::getTransliterator(int32_t index) const {
     return *trans[index];
 }
 void CompoundTransliterator::setTransliterators(Transliterator* const transliterators[],
                                                 int32_t transCount) {
     Transliterator** a = (Transliterator **)uprv_malloc(transCount * sizeof(Transliterator *));
     if (a == NULL) {
         return;
     }
     int32_t i = 0;
     UBool failed = FALSE;
     for (i=0; i<transCount; ++i) {
         a[i] = transliterators[i]->clone();
         if (a[i] == NULL) {
             failed = TRUE;
             break;
         }
     }
     if (failed && i > 0) {
         int32_t n;
         for (n = i-1; n >= 0; n--) {
             uprv_free(a[n]);
             a[n] = NULL;
         }
         return;
     }
     adoptTransliterators(a, transCount);
 }
 void CompoundTransliterator::adoptTransliterators(Transliterator* adoptedTransliterators[],
                                                   int32_t transCount) {
     // First free trans[] and set count to zero.  Once this is done,
     // orphan the filter.  Set up the new trans[].
     freeTransliterators();
     trans = adoptedTransliterators;
     count = transCount;
     computeMaximumContextLength();
     setID(joinIDs(trans, count));
 }
 /**
  * Append c to buf, unless buf is empty or buf already ends in c.
  */
 static void _smartAppend(UnicodeString& buf, UChar c) {
     if (buf.length() != 0 &&
         buf.charAt(buf.length() - 1) != c) {
         buf.append(c);
     }
 }
 UnicodeString& CompoundTransliterator::toRules(UnicodeString& rulesSource,
                                                UBool escapeUnprintable) const {
     // We do NOT call toRules() on our component transliterators, in
     // general.  If we have several rule-based transliterators, this
     // yields a concatenation of the rules -- not what we want.  We do
     // handle compound RBT transliterators specially -- those for which
     // compoundRBTIndex >= 0.  For the transliterator at compoundRBTIndex,
     // we do call toRules() recursively.
     rulesSource.truncate(0);
     if (numAnonymousRBTs >= 1 && getFilter() != NULL) {
         // If we are a compound RBT and if we have a global
         // filter, then emit it at the top.
         UnicodeString pat;
         rulesSource.append(COLON_COLON, 2).append(getFilter()->toPattern(pat, escapeUnprintable)).append(ID_DELIM);
     }
     for (int32_t i=0; i<count; ++i) {
         UnicodeString rule;
         // Anonymous RuleBasedTransliterators (inline rules and
         // ::BEGIN/::END blocks) are given IDs that begin with
         // "%Pass": use toRules() to write all the rules to the output
         // (and insert "::Null;" if we have two in a row)
         if (trans[i]->getID().startsWith(PASS_STRING, 5)) {
             trans[i]->toRules(rule, escapeUnprintable);
             if (numAnonymousRBTs > 1 && i > 0 && trans[i - 1]->getID().startsWith(PASS_STRING, 5))
                 rule = UNICODE_STRING_SIMPLE("::Null;") + rule;
         // we also use toRules() on CompoundTransliterators (which we
         // check for by looking for a semicolon in the ID)-- this gets
         // the list of their child transliterators output in the right
         // format
         } else if (trans[i]->getID().indexOf(ID_DELIM) >= 0) {
             trans[i]->toRules(rule, escapeUnprintable);
         // for everything else, use Transliterator::toRules()
         } else {
             trans[i]->Transliterator::toRules(rule, escapeUnprintable);
         }
         _smartAppend(rulesSource, NEWLINE);
         rulesSource.append(rule);
         _smartAppend(rulesSource, ID_DELIM);
     }
     return rulesSource;
 }
 /**
  * Implement Transliterator framework
  */
 void CompoundTransliterator::handleGetSourceSet(UnicodeSet& result) const {
     UnicodeSet set;
     result.clear();
     for (int32_t i=0; i<count; ++i) {
     result.addAll(trans[i]->getSourceSet(set));
     // Take the example of Hiragana-Latin.  This is really
     // Hiragana-Katakana; Katakana-Latin.  The source set of
     // these two is roughly [:Hiragana:] and [:Katakana:].
     // But the source set for the entire transliterator is
     // actually [:Hiragana:] ONLY -- that is, the first
     // non-empty source set.
     // This is a heuristic, and not 100% reliable.
     if (!result.isEmpty()) {
         break;
     }
     }
 }
 /**
  * Override Transliterator framework
  */
 UnicodeSet& CompoundTransliterator::getTargetSet(UnicodeSet& result) const {
     UnicodeSet set;
     result.clear();
     for (int32_t i=0; i<count; ++i) {
     // This is a heuristic, and not 100% reliable.
     result.addAll(trans[i]->getTargetSet(set));
     }
     return result;
 }
 /**
  * Implements {@link Transliterator#handleTransliterate}.
  */
 void CompoundTransliterator::handleTransliterate(Replaceable& text, UTransPosition& index,
                                                  UBool incremental) const {
     /* Call each transliterator with the same contextStart and
      * start, but with the limit as modified
      * by preceding transliterators.  The start index must be
      * reset for each transliterator to give each a chance to
      * transliterate the text.  The initial contextStart index is known
      * to still point to the same place after each transliterator
      * is called because each transliterator will not change the
      * text between contextStart and the initial start index.
      *
      * IMPORTANT: After the first transliterator, each subsequent
      * transliterator only gets to transliterate text committed by
      * preceding transliterators; that is, the start (output
      * value) of transliterator i becomes the limit (input value)
      * of transliterator i+1.  Finally, the overall limit is fixed
      * up before we return.
      *
      * Assumptions we make here:
      * (1) contextStart <= start <= limit <= contextLimit <= text.length()
      * (2) start <= start' <= limit'  ;cursor doesn't move back
      * (3) start <= limit'            ;text before cursor unchanged
      * - start' is the value of start after calling handleKT
      * - limit' is the value of limit after calling handleKT
      */
     /**
      * Example: 3 transliterators.  This example illustrates the
      * mechanics we need to implement.  C, S, and L are the contextStart,
      * start, and limit.  gl is the globalLimit.  contextLimit is
      * equal to limit throughout.
      *
      * 1. h-u, changes hex to Unicode
      *
      *    4  7  a  d  0      4  7  a
      *    abc/u0061/u    =>  abca/u
      *    C  S       L       C   S L   gl=f->a
      *
      * 2. upup, changes "x" to "XX"
      *
      *    4  7  a       4  7  a
      *    abca/u    =>  abcAA/u
      *    C  SL         C    S
      *                       L    gl=a->b
      * 3. u-h, changes Unicode to hex
      *
      *    4  7  a        4  7  a  d  0  3
      *    abcAA/u    =>  abc/u0041/u0041/u
      *    C  S L         C              S
      *                                  L   gl=b->15
      * 4. return
      *
      *    4  7  a  d  0  3
      *    abc/u0041/u0041/u
      *    C S L
      */
     if (count < 1) {
         index.start = index.limit;
         return; // Short circuit for empty compound transliterators
     }
     // compoundLimit is the limit value for the entire compound
     // operation.  We overwrite index.limit with the previous
     // index.start.  After each transliteration, we update
     // compoundLimit for insertions or deletions that have happened.
     int32_t compoundLimit = index.limit;
     // compoundStart is the start for the entire compound
     // operation.
     int32_t compoundStart = index.start;
     int32_t delta = 0; // delta in length
     // Give each transliterator a crack at the run of characters.
     // See comments at the top of the method for more detail.
     for (int32_t i=0; i<count; ++i) {
         index.start = compoundStart; // Reset start
         int32_t limit = index.limit;
         if (index.start == index.limit) {
             // Short circuit for empty range
             break;
         }
         trans[i]->filteredTransliterate(text, index, incremental);
         // In a properly written transliterator, start == limit after
         // handleTransliterate() returns when incremental is false.
         // Catch cases where the subclass doesn't do this, and throw
         // an exception.  (Just pinning start to limit is a bad idea,
         // because what's probably happening is that the subclass
         // isn't transliterating all the way to the end, and it should
         // in non-incremental mode.)
         if (!incremental && index.start != index.limit) {
             // We can't throw an exception, so just fudge things
             index.start = index.limit;
         }
         // Cumulative delta for insertions/deletions
         delta += index.limit - limit;
         if (incremental) {
             // In the incremental case, only allow subsequent
             // transliterators to modify what has already been
             // completely processed by prior transliterators.  In the
             // non-incrmental case, allow each transliterator to
             // process the entire text.
             index.limit = index.start;
         }
     }
     compoundLimit += delta;
     // Start is good where it is -- where the last transliterator left
     // it.  Limit needs to be put back where it was, modulo
     // adjustments for deletions/insertions.
     index.limit = compoundLimit;
 }
 /**
  * Sets the length of the longest context required by this transliterator.
  * This is <em>preceding</em> context.
  */
 void CompoundTransliterator::computeMaximumContextLength(void) {
     int32_t max = 0;
     for (int32_t i=0; i<count; ++i) {
         int32_t len = trans[i]->getMaximumContextLength();
         if (len > max) {
             max = len;
         }
     }
     setMaximumContextLength(max);
 }
 U_NAMESPACE_END
 #endif /* #if !UCONFIG_NO_TRANSLITERATION */
 /* eof */

The Tor Browser / annotate

intl/icu/source/i18n/cpdtrans.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/cpdtrans.cpp