The Tor Browser: intl/icu/source/i18n/ucoleitr.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/ucoleitr.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/ucoleitr.cpp

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 /*
 ******************************************************************************
 *   Copyright (C) 2001-2011, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 ******************************************************************************
 *
 * File ucoleitr.cpp
 *
 * Modification History:
 *
 * Date        Name        Description
 * 02/15/2001  synwee      Modified all methods to process its own function
 *                         instead of calling the equivalent c++ api (coleitr.h)
 ******************************************************************************/
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_COLLATION
 #include "unicode/ucoleitr.h"
 #include "unicode/ustring.h"
 #include "unicode/sortkey.h"
 #include "unicode/uobject.h"
 #include "ucol_imp.h"
 #include "cmemory.h"
 U_NAMESPACE_USE
 #define BUFFER_LENGTH             100
 #define DEFAULT_BUFFER_SIZE 16
 #define BUFFER_GROW 8
 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
 #define ARRAY_COPY(dst, src, count) uprv_memcpy((void *) (dst), (void *) (src), (count) * sizeof (src)[0])
 #define NEW_ARRAY(type, count) (type *) uprv_malloc((count) * sizeof(type))
 #define GROW_ARRAY(array, newSize) uprv_realloc((void *) (array), (newSize) * sizeof (array)[0])
 #define DELETE_ARRAY(array) uprv_free((void *) (array))
 typedef struct icu::collIterate collIterator;
 struct RCEI
 {
     uint32_t ce;
     int32_t  low;
     int32_t  high;
 };
 U_NAMESPACE_BEGIN
 struct RCEBuffer
 {
     RCEI    defaultBuffer[DEFAULT_BUFFER_SIZE];
     RCEI   *buffer;
     int32_t bufferIndex;
     int32_t bufferSize;
     RCEBuffer();
     ~RCEBuffer();
     UBool empty() const;
     void  put(uint32_t ce, int32_t ixLow, int32_t ixHigh);
     const RCEI *get();
 };
 RCEBuffer::RCEBuffer()
 {
     buffer = defaultBuffer;
     bufferIndex = 0;
     bufferSize = DEFAULT_BUFFER_SIZE;
 }
 RCEBuffer::~RCEBuffer()
 {
     if (buffer != defaultBuffer) {
         DELETE_ARRAY(buffer);
     }
 }
 UBool RCEBuffer::empty() const
 {
     return bufferIndex <= 0;
 }
 void RCEBuffer::put(uint32_t ce, int32_t ixLow, int32_t ixHigh)
 {
     if (bufferIndex >= bufferSize) {
         RCEI *newBuffer = NEW_ARRAY(RCEI, bufferSize + BUFFER_GROW);
         ARRAY_COPY(newBuffer, buffer, bufferSize);
         if (buffer != defaultBuffer) {
             DELETE_ARRAY(buffer);
         }
         buffer = newBuffer;
         bufferSize += BUFFER_GROW;
     }
     buffer[bufferIndex].ce   = ce;
     buffer[bufferIndex].low  = ixLow;
     buffer[bufferIndex].high = ixHigh;
     bufferIndex += 1;
 }
 const RCEI *RCEBuffer::get()
 {
     if (bufferIndex > 0) {
      return &buffer[--bufferIndex];
     }
     return NULL;
 }
 struct PCEI
 {
     uint64_t ce;
     int32_t  low;
     int32_t  high;
 };
 struct PCEBuffer
 {
     PCEI    defaultBuffer[DEFAULT_BUFFER_SIZE];
     PCEI   *buffer;
     int32_t bufferIndex;
     int32_t bufferSize;
     PCEBuffer();
     ~PCEBuffer();
     void  reset();
     UBool empty() const;
     void  put(uint64_t ce, int32_t ixLow, int32_t ixHigh);
     const PCEI *get();
 };
 PCEBuffer::PCEBuffer()
 {
     buffer = defaultBuffer;
     bufferIndex = 0;
     bufferSize = DEFAULT_BUFFER_SIZE;
 }
 PCEBuffer::~PCEBuffer()
 {
     if (buffer != defaultBuffer) {
         DELETE_ARRAY(buffer);
     }
 }
 void PCEBuffer::reset()
 {
     bufferIndex = 0;
 }
 UBool PCEBuffer::empty() const
 {
     return bufferIndex <= 0;
 }
 void PCEBuffer::put(uint64_t ce, int32_t ixLow, int32_t ixHigh)
 {
     if (bufferIndex >= bufferSize) {
         PCEI *newBuffer = NEW_ARRAY(PCEI, bufferSize + BUFFER_GROW);
         ARRAY_COPY(newBuffer, buffer, bufferSize);
         if (buffer != defaultBuffer) {
             DELETE_ARRAY(buffer);
         }
         buffer = newBuffer;
         bufferSize += BUFFER_GROW;
     }
     buffer[bufferIndex].ce   = ce;
     buffer[bufferIndex].low  = ixLow;
     buffer[bufferIndex].high = ixHigh;
     bufferIndex += 1;
 }
 const PCEI *PCEBuffer::get()
 {
     if (bufferIndex > 0) {
      return &buffer[--bufferIndex];
     }
     return NULL;
 }
 /*
  * This inherits from UObject so that
  * it can be allocated by new and the
  * constructor for PCEBuffer is called.
  */
 struct UCollationPCE : public UObject
 {
     PCEBuffer          pceBuffer;
     UCollationStrength strength;
     UBool              toShift;
     UBool              isShifted;
     uint32_t           variableTop;
     UCollationPCE(UCollationElements *elems);
     ~UCollationPCE();
     void init(const UCollator *coll);
     virtual UClassID getDynamicClassID() const;
     static UClassID getStaticClassID();
 };
 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UCollationPCE)
 UCollationPCE::UCollationPCE(UCollationElements *elems)
 {
     init(elems->iteratordata_.coll);
 }
 void UCollationPCE::init(const UCollator *coll)
 {
     UErrorCode status = U_ZERO_ERROR;
     strength    = ucol_getStrength(coll);
     toShift     = ucol_getAttribute(coll, UCOL_ALTERNATE_HANDLING, &status) == UCOL_SHIFTED;
     isShifted   = FALSE;
     variableTop = coll->variableTopValue << 16;
 }
 UCollationPCE::~UCollationPCE()
 {
     // nothing to do
 }
 U_NAMESPACE_END
 inline uint64_t processCE(UCollationElements *elems, uint32_t ce)
 {
     uint64_t primary = 0, secondary = 0, tertiary = 0, quaternary = 0;
     // This is clean, but somewhat slow...
     // We could apply the mask to ce and then
     // just get all three orders...
     switch(elems->pce->strength) {
     default:
         tertiary = ucol_tertiaryOrder(ce);
         /* note fall-through */
     case UCOL_SECONDARY:
         secondary = ucol_secondaryOrder(ce);
         /* note fall-through */
     case UCOL_PRIMARY:
         primary = ucol_primaryOrder(ce);
     }
     // **** This should probably handle continuations too.  ****
     // **** That means that we need 24 bits for the primary ****
     // **** instead of the 16 that we're currently using.   ****
     // **** So we can lay out the 64 bits as: 24.12.12.16.  ****
     // **** Another complication with continuations is that ****
     // **** the *second* CE is marked as a continuation, so ****
     // **** we always have to peek ahead to know how long   ****
     // **** the primary is...                               ****
     if ((elems->pce->toShift && elems->pce->variableTop > ce && primary != 0)
                 || (elems->pce->isShifted && primary == 0)) {
         if (primary == 0) {
             return UCOL_IGNORABLE;
         }
         if (elems->pce->strength >= UCOL_QUATERNARY) {
             quaternary = primary;
         }
         primary = secondary = tertiary = 0;
         elems->pce->isShifted = TRUE;
     } else {
         if (elems->pce->strength >= UCOL_QUATERNARY) {
             quaternary = 0xFFFF;
         }
         elems->pce->isShifted = FALSE;
     }
     return primary << 48 | secondary << 32 | tertiary << 16 | quaternary;
 }
 U_CAPI void U_EXPORT2
 uprv_init_pce(const UCollationElements *elems)
 {
     if (elems->pce != NULL) {
         elems->pce->init(elems->iteratordata_.coll);
     }
 }
 /* public methods ---------------------------------------------------- */
 U_CAPI UCollationElements* U_EXPORT2
 ucol_openElements(const UCollator  *coll,
                   const UChar      *text,
                         int32_t    textLength,
                         UErrorCode *status)
 {
     if (U_FAILURE(*status)) {
         return NULL;
     }
     UCollationElements *result = new UCollationElements;
     if (result == NULL) {
         *status = U_MEMORY_ALLOCATION_ERROR;
         return NULL;
     }
     result->reset_ = TRUE;
     result->isWritable = FALSE;
     result->pce = NULL;
     if (text == NULL) {
         textLength = 0;
     }
     uprv_init_collIterate(coll, text, textLength, &result->iteratordata_, status);
     return result;
 }
 U_CAPI void U_EXPORT2
 ucol_closeElements(UCollationElements *elems)
 {
 	if (elems != NULL) {
 	  collIterate *ci = &elems->iteratordata_;
 	  if (ci->extendCEs) {
 		  uprv_free(ci->extendCEs);
 	  }
 	  if (ci->offsetBuffer) {
 		  uprv_free(ci->offsetBuffer);
 	  }
 	  if (elems->isWritable && elems->iteratordata_.string != NULL)
 	  {
 		uprv_free((UChar *)elems->iteratordata_.string);
 	  }
 	  if (elems->pce != NULL) {
 		  delete elems->pce;
 	  }
 	  delete elems;
 	}
 }
 U_CAPI void U_EXPORT2
 ucol_reset(UCollationElements *elems)
 {
     collIterate *ci = &(elems->iteratordata_);
     elems->reset_   = TRUE;
     ci->pos         = ci->string;
     if ((ci->flags & UCOL_ITER_HASLEN) == 0 || ci->endp == NULL) {
         ci->endp      = ci->string + u_strlen(ci->string);
     }
     ci->CEpos       = ci->toReturn = ci->CEs;
     ci->flags       = (ci->flags & UCOL_FORCE_HAN_IMPLICIT) | UCOL_ITER_HASLEN;
     if (ci->coll->normalizationMode == UCOL_ON) {
         ci->flags |= UCOL_ITER_NORM;
     }
     ci->writableBuffer.remove();
     ci->fcdPosition = NULL;
   //ci->offsetReturn = ci->offsetStore = NULL;
 	ci->offsetRepeatCount = ci->offsetRepeatValue = 0;
 }
 U_CAPI void U_EXPORT2
 ucol_forceHanImplicit(UCollationElements *elems, UErrorCode *status)
 {
     if (U_FAILURE(*status)) {
         return;
     }
     if (elems == NULL) {
         *status = U_ILLEGAL_ARGUMENT_ERROR;
         return;
     }
     elems->iteratordata_.flags |= UCOL_FORCE_HAN_IMPLICIT;
 }
 U_CAPI int32_t U_EXPORT2
 ucol_next(UCollationElements *elems,
           UErrorCode         *status)
 {
     int32_t result;
     if (U_FAILURE(*status)) {
         return UCOL_NULLORDER;
     }
     elems->reset_ = FALSE;
     result = (int32_t)ucol_getNextCE(elems->iteratordata_.coll,
                                      &elems->iteratordata_,
                                      status);
     if (result == UCOL_NO_MORE_CES) {
         result = UCOL_NULLORDER;
     }
     return result;
 }
 U_CAPI int64_t U_EXPORT2
 ucol_nextProcessed(UCollationElements *elems,
                    int32_t            *ixLow,
                    int32_t            *ixHigh,
                    UErrorCode         *status)
 {
     const UCollator *coll = elems->iteratordata_.coll;
     int64_t result = UCOL_IGNORABLE;
     uint32_t low = 0, high = 0;
     if (U_FAILURE(*status)) {
         return UCOL_PROCESSED_NULLORDER;
     }
     if (elems->pce == NULL) {
         elems->pce = new UCollationPCE(elems);
     } else {
         elems->pce->pceBuffer.reset();
     }
     elems->reset_ = FALSE;
     do {
         low = ucol_getOffset(elems);
         uint32_t ce = (uint32_t) ucol_getNextCE(coll, &elems->iteratordata_, status);
         high = ucol_getOffset(elems);
         if (ce == UCOL_NO_MORE_CES) {
              result = UCOL_PROCESSED_NULLORDER;
              break;
         }
         result = processCE(elems, ce);
     } while (result == UCOL_IGNORABLE);
     if (ixLow != NULL) {
         *ixLow = low;
     }
     if (ixHigh != NULL) {
         *ixHigh = high;
     }
     return result;
 }
 U_CAPI int32_t U_EXPORT2
 ucol_previous(UCollationElements *elems,
               UErrorCode         *status)
 {
     if(U_FAILURE(*status)) {
         return UCOL_NULLORDER;
     }
     else
     {
         int32_t result;
         if (elems->reset_ && (elems->iteratordata_.pos == elems->iteratordata_.string)) {
             if (elems->iteratordata_.endp == NULL) {
                 elems->iteratordata_.endp = elems->iteratordata_.string +
                                             u_strlen(elems->iteratordata_.string);
                 elems->iteratordata_.flags |= UCOL_ITER_HASLEN;
             }
             elems->iteratordata_.pos = elems->iteratordata_.endp;
             elems->iteratordata_.fcdPosition = elems->iteratordata_.endp;
         }
         elems->reset_ = FALSE;
         result = (int32_t)ucol_getPrevCE(elems->iteratordata_.coll,
                                          &(elems->iteratordata_),
                                          status);
         if (result == UCOL_NO_MORE_CES) {
             result = UCOL_NULLORDER;
         }
         return result;
     }
 }
 U_CAPI int64_t U_EXPORT2
 ucol_previousProcessed(UCollationElements *elems,
                    int32_t            *ixLow,
                    int32_t            *ixHigh,
                    UErrorCode         *status)
 {
     const UCollator *coll = elems->iteratordata_.coll;
     int64_t result = UCOL_IGNORABLE;
  // int64_t primary = 0, secondary = 0, tertiary = 0, quaternary = 0;
  // UCollationStrength strength = ucol_getStrength(coll);
  //  UBool toShift   = ucol_getAttribute(coll, UCOL_ALTERNATE_HANDLING, status) ==  UCOL_SHIFTED;
  // uint32_t variableTop = coll->variableTopValue;
     int32_t  low = 0, high = 0;
     if (U_FAILURE(*status)) {
         return UCOL_PROCESSED_NULLORDER;
     }
     if (elems->reset_ &&
         (elems->iteratordata_.pos == elems->iteratordata_.string)) {
         if (elems->iteratordata_.endp == NULL) {
             elems->iteratordata_.endp = elems->iteratordata_.string +
                                         u_strlen(elems->iteratordata_.string);
             elems->iteratordata_.flags |= UCOL_ITER_HASLEN;
         }
         elems->iteratordata_.pos = elems->iteratordata_.endp;
         elems->iteratordata_.fcdPosition = elems->iteratordata_.endp;
     }
     if (elems->pce == NULL) {
         elems->pce = new UCollationPCE(elems);
     } else {
       //elems->pce->pceBuffer.reset();
     }
     elems->reset_ = FALSE;
     while (elems->pce->pceBuffer.empty()) {
         // buffer raw CEs up to non-ignorable primary
         RCEBuffer rceb;
         uint32_t ce;
         // **** do we need to reset rceb, or will it always be empty at this point ****
         do {
             high = ucol_getOffset(elems);
             ce   = ucol_getPrevCE(coll, &elems->iteratordata_, status);
             low  = ucol_getOffset(elems);
             if (ce == UCOL_NO_MORE_CES) {
                 if (! rceb.empty()) {
                     break;
                 }
                 goto finish;
             }
             rceb.put(ce, low, high);
         } while ((ce & UCOL_PRIMARYMASK) == 0);
         // process the raw CEs
         while (! rceb.empty()) {
             const RCEI *rcei = rceb.get();
             result = processCE(elems, rcei->ce);
             if (result != UCOL_IGNORABLE) {
                 elems->pce->pceBuffer.put(result, rcei->low, rcei->high);
             }
         }
     }
 finish:
     if (elems->pce->pceBuffer.empty()) {
         // **** Is -1 the right value for ixLow, ixHigh? ****
     	if (ixLow != NULL) {
     		*ixLow = -1;
     	}
     	if (ixHigh != NULL) {
     		*ixHigh = -1
     		;
     	}
         return UCOL_PROCESSED_NULLORDER;
     }
     const PCEI *pcei = elems->pce->pceBuffer.get();
     if (ixLow != NULL) {
         *ixLow = pcei->low;
     }
     if (ixHigh != NULL) {
         *ixHigh = pcei->high;
     }
     return pcei->ce;
 }
 U_CAPI int32_t U_EXPORT2
 ucol_getMaxExpansion(const UCollationElements *elems,
                            int32_t            order)
 {
     uint8_t result;
 #if 0
     UCOL_GETMAXEXPANSION(elems->iteratordata_.coll, (uint32_t)order, result);
 #else
     const UCollator *coll = elems->iteratordata_.coll;
     const uint32_t *start;
     const uint32_t *limit;
     const uint32_t *mid;
           uint32_t strengthMask = 0;
           uint32_t mOrder = (uint32_t) order;
     switch (coll->strength)
     {
     default:
         strengthMask |= UCOL_TERTIARYORDERMASK;
         /* fall through */
     case UCOL_SECONDARY:
         strengthMask |= UCOL_SECONDARYORDERMASK;
         /* fall through */
     case UCOL_PRIMARY:
         strengthMask |= UCOL_PRIMARYORDERMASK;
     }
     mOrder &= strengthMask;
     start = (coll)->endExpansionCE;
     limit = (coll)->lastEndExpansionCE;
     while (start < limit - 1) {
         mid = start + ((limit - start) >> 1);
         if (mOrder <= (*mid & strengthMask)) {
           limit = mid;
         } else {
           start = mid;
         }
     }
     // FIXME: with a masked search, there might be more than one hit,
     // so we need to look forward and backward from the match to find all
     // of the hits...
     if ((*start & strengthMask) == mOrder) {
         result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE));
     } else if ((*limit & strengthMask) == mOrder) {
          result = *(coll->expansionCESize + (limit - coll->endExpansionCE));
    } else if ((mOrder & 0xFFFF) == 0x00C0) {
         result = 2;
    } else {
        result = 1;
    }
 #endif
     return result;
 }
 U_CAPI void U_EXPORT2
 ucol_setText(      UCollationElements *elems,
              const UChar              *text,
                    int32_t            textLength,
                    UErrorCode         *status)
 {
     if (U_FAILURE(*status)) {
         return;
     }
     if (elems->isWritable && elems->iteratordata_.string != NULL)
     {
         uprv_free((UChar *)elems->iteratordata_.string);
     }
     if (text == NULL) {
         textLength = 0;
     }
     elems->isWritable = FALSE;
     /* free offset buffer to avoid memory leak before initializing. */
     ucol_freeOffsetBuffer(&(elems->iteratordata_));
     /* Ensure that previously allocated extendCEs is freed before setting to NULL. */
     if (elems->iteratordata_.extendCEs != NULL) {
         uprv_free(elems->iteratordata_.extendCEs);
     }
     uprv_init_collIterate(elems->iteratordata_.coll, text, textLength,
                           &elems->iteratordata_, status);
     elems->reset_   = TRUE;
 }
 U_CAPI int32_t U_EXPORT2
 ucol_getOffset(const UCollationElements *elems)
 {
   const collIterate *ci = &(elems->iteratordata_);
   if (ci->offsetRepeatCount > 0 && ci->offsetRepeatValue != 0) {
       return ci->offsetRepeatValue;
   }
   if (ci->offsetReturn != NULL) {
       return *ci->offsetReturn;
   }
   // while processing characters in normalization buffer getOffset will
   // return the next non-normalized character.
   // should be inline with the old implementation since the old codes uses
   // nextDecomp in normalizer which also decomposes the string till the
   // first base character is found.
   if (ci->flags & UCOL_ITER_INNORMBUF) {
       if (ci->fcdPosition == NULL) {
         return 0;
       }
       return (int32_t)(ci->fcdPosition - ci->string);
   }
   else {
       return (int32_t)(ci->pos - ci->string);
   }
 }
 U_CAPI void U_EXPORT2
 ucol_setOffset(UCollationElements    *elems,
                int32_t           offset,
                UErrorCode            *status)
 {
     if (U_FAILURE(*status)) {
         return;
     }
     // this methods will clean up any use of the writable buffer and points to
     // the original string
     collIterate *ci = &(elems->iteratordata_);
     ci->pos         = ci->string + offset;
     ci->CEpos       = ci->toReturn = ci->CEs;
     if (ci->flags & UCOL_ITER_INNORMBUF) {
         ci->flags = ci->origFlags;
     }
     if ((ci->flags & UCOL_ITER_HASLEN) == 0) {
         ci->endp  = ci->string + u_strlen(ci->string);
         ci->flags |= UCOL_ITER_HASLEN;
     }
     ci->fcdPosition = NULL;
     elems->reset_ = FALSE;
 	ci->offsetReturn = NULL;
     ci->offsetStore = ci->offsetBuffer;
 	ci->offsetRepeatCount = ci->offsetRepeatValue = 0;
 }
 U_CAPI int32_t U_EXPORT2
 ucol_primaryOrder (int32_t order)
 {
     order &= UCOL_PRIMARYMASK;
     return (order >> UCOL_PRIMARYORDERSHIFT);
 }
 U_CAPI int32_t U_EXPORT2
 ucol_secondaryOrder (int32_t order)
 {
     order &= UCOL_SECONDARYMASK;
     return (order >> UCOL_SECONDARYORDERSHIFT);
 }
 U_CAPI int32_t U_EXPORT2
 ucol_tertiaryOrder (int32_t order)
 {
     return (order & UCOL_TERTIARYMASK);
 }
 void ucol_freeOffsetBuffer(collIterate *s) {
     if (s != NULL && s->offsetBuffer != NULL) {
         uprv_free(s->offsetBuffer);
         s->offsetBuffer = NULL;
         s->offsetBufferSize = 0;
     }
 }
 #endif /* #if !UCONFIG_NO_COLLATION */

The Tor Browser / annotate

intl/icu/source/i18n/ucoleitr.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/ucoleitr.cpp