The Tor Browser: intl/icu/source/common/rbbi.cpp@6474c204b198 (annotated)

intl/icu/source/common/rbbi.cpp@6474c204b198 (annotated)

intl/icu/source/common/rbbi.cpp

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 /*
 ***************************************************************************
 *   Copyright (C) 1999-2013 International Business Machines Corporation
 *   and others. All rights reserved.
 ***************************************************************************
 */
 //
 //  file:  rbbi.c    Contains the implementation of the rule based break iterator
 //                   runtime engine and the API implementation for
 //                   class RuleBasedBreakIterator
 //
 #include "utypeinfo.h"  // for 'typeid' to work
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_BREAK_ITERATION
 #include "unicode/rbbi.h"
 #include "unicode/schriter.h"
 #include "unicode/uchriter.h"
 #include "unicode/udata.h"
 #include "unicode/uclean.h"
 #include "rbbidata.h"
 #include "rbbirb.h"
 #include "cmemory.h"
 #include "cstring.h"
 #include "umutex.h"
 #include "ucln_cmn.h"
 #include "brkeng.h"
 #include "uassert.h"
 #include "uvector.h"
 // if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included.
 #if U_LOCAL_SERVICE_HOOK
 #include "localsvc.h"
 #endif
 #ifdef RBBI_DEBUG
 static UBool fTrace = FALSE;
 #endif
 U_NAMESPACE_BEGIN
 // The state number of the starting state
 #define START_STATE 1
 // The state-transition value indicating "stop"
 #define STOP_STATE  0
 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
 //=======================================================================
 // constructors
 //=======================================================================
 /**
  * Constructs a RuleBasedBreakIterator that uses the already-created
  * tables object that is passed in as a parameter.
  */
 RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
 {
     init();
     fData = new RBBIDataWrapper(data, status); // status checked in constructor
     if (U_FAILURE(status)) {return;}
     if(fData == 0) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
 }
 /**
  * Same as above but does not adopt memory
  */
 RuleBasedBreakIterator::RuleBasedBreakIterator(const RBBIDataHeader* data, enum EDontAdopt, UErrorCode &status)
 {
     init();
     fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status); // status checked in constructor
     if (U_FAILURE(status)) {return;}
     if(fData == 0) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
 }
 //
 //  Construct from precompiled binary rules (tables).  This constructor is public API,
 //  taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules().
 //
 RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
                        uint32_t       ruleLength,
                        UErrorCode     &status) {
     init();
     if (U_FAILURE(status)) {
         return;
     }
     if (compiledRules == NULL || ruleLength < sizeof(RBBIDataHeader)) {
         status = U_ILLEGAL_ARGUMENT_ERROR;
         return;
     }
     const RBBIDataHeader *data = (const RBBIDataHeader *)compiledRules;
     if (data->fLength > ruleLength) {
         status = U_ILLEGAL_ARGUMENT_ERROR;
         return;
     }
     fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status);
     if (U_FAILURE(status)) {return;}
     if(fData == 0) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
 }
 //-------------------------------------------------------------------------------
 //
 //   Constructor   from a UDataMemory handle to precompiled break rules
 //                 stored in an ICU data file.
 //
 //-------------------------------------------------------------------------------
 RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
 {
     init();
     fData = new RBBIDataWrapper(udm, status); // status checked in constructor
     if (U_FAILURE(status)) {return;}
     if(fData == 0) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
 }
 //-------------------------------------------------------------------------------
 //
 //   Constructor       from a set of rules supplied as a string.
 //
 //-------------------------------------------------------------------------------
 RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString  &rules,
                                                 UParseError          &parseError,
                                                 UErrorCode           &status)
 {
     init();
     if (U_FAILURE(status)) {return;}
     RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
         RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status);
     // Note:  This is a bit awkward.  The RBBI ruleBuilder has a factory method that
     //        creates and returns a complete RBBI.  From here, in a constructor, we
     //        can't just return the object created by the builder factory, hence
     //        the assignment of the factory created object to "this".
     if (U_SUCCESS(status)) {
         *this = *bi;
         delete bi;
     }
 }
 //-------------------------------------------------------------------------------
 //
 // Default Constructor.      Create an empty shell that can be set up later.
 //                           Used when creating a RuleBasedBreakIterator from a set
 //                           of rules.
 //-------------------------------------------------------------------------------
 RuleBasedBreakIterator::RuleBasedBreakIterator() {
     init();
 }
 //-------------------------------------------------------------------------------
 //
 //   Copy constructor.  Will produce a break iterator with the same behavior,
 //                      and which iterates over the same text, as the one passed in.
 //
 //-------------------------------------------------------------------------------
 RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
 : BreakIterator(other)
 {
     this->init();
     *this = other;
 }
 /**
  * Destructor
  */
 RuleBasedBreakIterator::~RuleBasedBreakIterator() {
     if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
         // fCharIter was adopted from the outside.
         delete fCharIter;
     }
     fCharIter = NULL;
     delete fSCharIter;
     fCharIter = NULL;
     delete fDCharIter;
     fDCharIter = NULL;
     utext_close(fText);
     if (fData != NULL) {
         fData->removeReference();
         fData = NULL;
     }
     if (fCachedBreakPositions) {
         uprv_free(fCachedBreakPositions);
         fCachedBreakPositions = NULL;
     }
     if (fLanguageBreakEngines) {
         delete fLanguageBreakEngines;
         fLanguageBreakEngines = NULL;
     }
     if (fUnhandledBreakEngine) {
         delete fUnhandledBreakEngine;
         fUnhandledBreakEngine = NULL;
     }
 }
 /**
  * Assignment operator.  Sets this iterator to have the same behavior,
  * and iterate over the same text, as the one passed in.
  */
 RuleBasedBreakIterator&
 RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
     if (this == &that) {
         return *this;
     }
     reset();    // Delete break cache information
     fBreakType = that.fBreakType;
     if (fLanguageBreakEngines != NULL) {
         delete fLanguageBreakEngines;
         fLanguageBreakEngines = NULL;   // Just rebuild for now
     }
     // TODO: clone fLanguageBreakEngines from "that"
     UErrorCode status = U_ZERO_ERROR;
     fText = utext_clone(fText, that.fText, FALSE, TRUE, &status);
     if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
         delete fCharIter;
     }
     fCharIter = NULL;
     if (that.fCharIter != NULL ) {
         // This is a little bit tricky - it will intially appear that
         //  this->fCharIter is adopted, even if that->fCharIter was
         //  not adopted.  That's ok.
         fCharIter = that.fCharIter->clone();
     }
     if (fData != NULL) {
         fData->removeReference();
         fData = NULL;
     }
     if (that.fData != NULL) {
         fData = that.fData->addReference();
     }
     return *this;
 }
 //-----------------------------------------------------------------------------
 //
 //    init()      Shared initialization routine.   Used by all the constructors.
 //                Initializes all fields, leaving the object in a consistent state.
 //
 //-----------------------------------------------------------------------------
 void RuleBasedBreakIterator::init() {
     UErrorCode  status    = U_ZERO_ERROR;
     fText                 = utext_openUChars(NULL, NULL, 0, &status);
     fCharIter             = NULL;
     fSCharIter            = NULL;
     fDCharIter            = NULL;
     fData                 = NULL;
     fLastRuleStatusIndex  = 0;
     fLastStatusIndexValid = TRUE;
     fDictionaryCharCount  = 0;
     fBreakType            = UBRK_WORD;  // Defaulting BreakType to word gives reasonable
                                         //   dictionary behavior for Break Iterators that are
                                         //   built from rules.  Even better would be the ability to
                                         //   declare the type in the rules.
     fCachedBreakPositions    = NULL;
     fLanguageBreakEngines    = NULL;
     fUnhandledBreakEngine    = NULL;
     fNumCachedBreakPositions = 0;
     fPositionInCache         = 0;
 #ifdef RBBI_DEBUG
     static UBool debugInitDone = FALSE;
     if (debugInitDone == FALSE) {
         char *debugEnv = getenv("U_RBBIDEBUG");
         if (debugEnv && uprv_strstr(debugEnv, "trace")) {
             fTrace = TRUE;
         }
         debugInitDone = TRUE;
     }
 #endif
 }
 //-----------------------------------------------------------------------------
 //
 //    clone - Returns a newly-constructed RuleBasedBreakIterator with the same
 //            behavior, and iterating over the same text, as this one.
 //            Virtual function: does the right thing with subclasses.
 //
 //-----------------------------------------------------------------------------
 BreakIterator*
 RuleBasedBreakIterator::clone(void) const {
     return new RuleBasedBreakIterator(*this);
 }
 /**
  * Equality operator.  Returns TRUE if both BreakIterators are of the
  * same class, have the same behavior, and iterate over the same text.
  */
 UBool
 RuleBasedBreakIterator::operator==(const BreakIterator& that) const {
     if (typeid(*this) != typeid(that)) {
         return FALSE;
     }
     const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that;
     if (!utext_equals(fText, that2.fText)) {
         // The two break iterators are operating on different text,
         //   or have a different interation position.
         return FALSE;
     };
     // TODO:  need a check for when in a dictionary region at different offsets.
     if (that2.fData == fData ||
         (fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
             // The two break iterators are using the same rules.
             return TRUE;
         }
     return FALSE;
 }
 /**
  * Compute a hash code for this BreakIterator
  * @return A hash code
  */
 int32_t
 RuleBasedBreakIterator::hashCode(void) const {
     int32_t   hash = 0;
     if (fData != NULL) {
         hash = fData->hashCode();
     }
     return hash;
 }
 void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) {
     if (U_FAILURE(status)) {
         return;
     }
     reset();
     fText = utext_clone(fText, ut, FALSE, TRUE, &status);
     // Set up a dummy CharacterIterator to be returned if anyone
     //   calls getText().  With input from UText, there is no reasonable
     //   way to return a characterIterator over the actual input text.
     //   Return one over an empty string instead - this is the closest
     //   we can come to signaling a failure.
     //   (GetText() is obsolete, this failure is sort of OK)
     if (fDCharIter == NULL) {
         static const UChar c = 0;
         fDCharIter = new UCharCharacterIterator(&c, 0);
         if (fDCharIter == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
     }
     if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
         // existing fCharIter was adopted from the outside.  Delete it now.
         delete fCharIter;
     }
     fCharIter = fDCharIter;
     this->first();
 }
 UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const {
     UText *result = utext_clone(fillIn, fText, FALSE, TRUE, &status);
     return result;
 }
 /**
  * Returns the description used to create this iterator
  */
 const UnicodeString&
 RuleBasedBreakIterator::getRules() const {
     if (fData != NULL) {
         return fData->getRuleSourceString();
     } else {
         static const UnicodeString *s;
         if (s == NULL) {
             // TODO:  something more elegant here.
             //        perhaps API should return the string by value.
             //        Note:  thread unsafe init & leak are semi-ok, better than
             //               what was before.  Sould be cleaned up, though.
             s = new UnicodeString;
         }
         return *s;
     }
 }
 //=======================================================================
 // BreakIterator overrides
 //=======================================================================
 /**
  * Return a CharacterIterator over the text being analyzed.
  */
 CharacterIterator&
 RuleBasedBreakIterator::getText() const {
     return *fCharIter;
 }
 /**
  * Set the iterator to analyze a new piece of text.  This function resets
  * the current iteration position to the beginning of the text.
  * @param newText An iterator over the text to analyze.
  */
 void
 RuleBasedBreakIterator::adoptText(CharacterIterator* newText) {
     // If we are holding a CharacterIterator adopted from a
     //   previous call to this function, delete it now.
     if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
         delete fCharIter;
     }
     fCharIter = newText;
     UErrorCode status = U_ZERO_ERROR;
     reset();
     if (newText==NULL || newText->startIndex() != 0) {
         // startIndex !=0 wants to be an error, but there's no way to report it.
         // Make the iterator text be an empty string.
         fText = utext_openUChars(fText, NULL, 0, &status);
     } else {
         fText = utext_openCharacterIterator(fText, newText, &status);
     }
     this->first();
 }
 /**
  * Set the iterator to analyze a new piece of text.  This function resets
  * the current iteration position to the beginning of the text.
  * @param newText An iterator over the text to analyze.
  */
 void
 RuleBasedBreakIterator::setText(const UnicodeString& newText) {
     UErrorCode status = U_ZERO_ERROR;
     reset();
     fText = utext_openConstUnicodeString(fText, &newText, &status);
     // Set up a character iterator on the string.
     //   Needed in case someone calls getText().
     //  Can not, unfortunately, do this lazily on the (probably never)
     //  call to getText(), because getText is const.
     if (fSCharIter == NULL) {
         fSCharIter = new StringCharacterIterator(newText);
     } else {
         fSCharIter->setText(newText);
     }
     if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
         // old fCharIter was adopted from the outside.  Delete it.
         delete fCharIter;
     }
     fCharIter = fSCharIter;
     this->first();
 }
 /**
  *  Provide a new UText for the input text.  Must reference text with contents identical
  *  to the original.
  *  Intended for use with text data originating in Java (garbage collected) environments
  *  where the data may be moved in memory at arbitrary times.
  */
 RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, UErrorCode &status) {
     if (U_FAILURE(status)) {
         return *this;
     }
     if (input == NULL) {
         status = U_ILLEGAL_ARGUMENT_ERROR;
         return *this;
     }
     int64_t pos = utext_getNativeIndex(fText);
     //  Shallow read-only clone of the new UText into the existing input UText
     fText = utext_clone(fText, input, FALSE, TRUE, &status);
     if (U_FAILURE(status)) {
         return *this;
     }
     utext_setNativeIndex(fText, pos);
     if (utext_getNativeIndex(fText) != pos) {
         // Sanity check.  The new input utext is supposed to have the exact same
         // contents as the old.  If we can't set to the same position, it doesn't.
         // The contents underlying the old utext might be invalid at this point,
         // so it's not safe to check directly.
         status = U_ILLEGAL_ARGUMENT_ERROR;
     }
     return *this;
 }
 /**
  * Sets the current iteration position to the beginning of the text.
  * @return The offset of the beginning of the text.
  */
 int32_t RuleBasedBreakIterator::first(void) {
     reset();
     fLastRuleStatusIndex  = 0;
     fLastStatusIndexValid = TRUE;
     //if (fText == NULL)
     //    return BreakIterator::DONE;
     utext_setNativeIndex(fText, 0);
     return 0;
 }
 /**
  * Sets the current iteration position to the end of the text.
  * @return The text's past-the-end offset.
  */
 int32_t RuleBasedBreakIterator::last(void) {
     reset();
     if (fText == NULL) {
         fLastRuleStatusIndex  = 0;
         fLastStatusIndexValid = TRUE;
         return BreakIterator::DONE;
     }
     fLastStatusIndexValid = FALSE;
     int32_t pos = (int32_t)utext_nativeLength(fText);
     utext_setNativeIndex(fText, pos);
     return pos;
 }
 /**
  * Advances the iterator either forward or backward the specified number of steps.
  * Negative values move backward, and positive values move forward.  This is
  * equivalent to repeatedly calling next() or previous().
  * @param n The number of steps to move.  The sign indicates the direction
  * (negative is backwards, and positive is forwards).
  * @return The character offset of the boundary position n boundaries away from
  * the current one.
  */
 int32_t RuleBasedBreakIterator::next(int32_t n) {
     int32_t result = current();
     while (n > 0) {
         result = next();
         --n;
     }
     while (n < 0) {
         result = previous();
         ++n;
     }
     return result;
 }
 /**
  * Advances the iterator to the next boundary position.
  * @return The position of the first boundary after this one.
  */
 int32_t RuleBasedBreakIterator::next(void) {
     // if we have cached break positions and we're still in the range
     // covered by them, just move one step forward in the cache
     if (fCachedBreakPositions != NULL) {
         if (fPositionInCache < fNumCachedBreakPositions - 1) {
             ++fPositionInCache;
             int32_t pos = fCachedBreakPositions[fPositionInCache];
             utext_setNativeIndex(fText, pos);
             return pos;
         }
         else {
             reset();
         }
     }
     int32_t startPos = current();
     int32_t result = handleNext(fData->fForwardTable);
     if (fDictionaryCharCount > 0) {
         result = checkDictionary(startPos, result, FALSE);
     }
     return result;
 }
 /**
  * Advances the iterator backwards, to the last boundary preceding this one.
  * @return The position of the last boundary position preceding this one.
  */
 int32_t RuleBasedBreakIterator::previous(void) {
     int32_t result;
     int32_t startPos;
     // if we have cached break positions and we're still in the range
     // covered by them, just move one step backward in the cache
     if (fCachedBreakPositions != NULL) {
         if (fPositionInCache > 0) {
             --fPositionInCache;
             // If we're at the beginning of the cache, need to reevaluate the
             // rule status
             if (fPositionInCache <= 0) {
                 fLastStatusIndexValid = FALSE;
             }
             int32_t pos = fCachedBreakPositions[fPositionInCache];
             utext_setNativeIndex(fText, pos);
             return pos;
         }
         else {
             reset();
         }
     }
     // if we're already sitting at the beginning of the text, return DONE
     if (fText == NULL || (startPos = current()) == 0) {
         fLastRuleStatusIndex  = 0;
         fLastStatusIndexValid = TRUE;
         return BreakIterator::DONE;
     }
     if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) {
         result = handlePrevious(fData->fReverseTable);
         if (fDictionaryCharCount > 0) {
             result = checkDictionary(result, startPos, TRUE);
         }
         return result;
     }
     // old rule syntax
     // set things up.  handlePrevious() will back us up to some valid
     // break position before the current position (we back our internal
     // iterator up one step to prevent handlePrevious() from returning
     // the current position), but not necessarily the last one before
     // where we started
     int32_t start = current();
     (void)UTEXT_PREVIOUS32(fText);
     int32_t lastResult    = handlePrevious(fData->fReverseTable);
     if (lastResult == UBRK_DONE) {
         lastResult = 0;
         utext_setNativeIndex(fText, 0);
     }
     result = lastResult;
     int32_t lastTag       = 0;
     UBool   breakTagValid = FALSE;
     // iterate forward from the known break position until we pass our
     // starting point.  The last break position before the starting
     // point is our return value
     for (;;) {
         result         = next();
         if (result == BreakIterator::DONE || result >= start) {
             break;
         }
         lastResult     = result;
         lastTag        = fLastRuleStatusIndex;
         breakTagValid  = TRUE;
     }
     // fLastBreakTag wants to have the value for section of text preceding
     // the result position that we are to return (in lastResult.)  If
     // the backwards rules overshot and the above loop had to do two or more
     // next()s to move up to the desired return position, we will have a valid
     // tag value. But, if handlePrevious() took us to exactly the correct result positon,
     // we wont have a tag value for that position, which is only set by handleNext().
     // set the current iteration position to be the last break position
     // before where we started, and then return that value
     utext_setNativeIndex(fText, lastResult);
     fLastRuleStatusIndex  = lastTag;       // for use by getRuleStatus()
     fLastStatusIndexValid = breakTagValid;
     // No need to check the dictionary; it will have been handled by
     // next()
     return lastResult;
 }
 /**
  * Sets the iterator to refer to the first boundary position following
  * the specified position.
  * @offset The position from which to begin searching for a break position.
  * @return The position of the first break after the current position.
  */
 int32_t RuleBasedBreakIterator::following(int32_t offset) {
     // if we have cached break positions and offset is in the range
     // covered by them, use them
     // TODO: could use binary search
     // TODO: what if offset is outside range, but break is not?
     if (fCachedBreakPositions != NULL) {
         if (offset >= fCachedBreakPositions[0]
                 && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
             fPositionInCache = 0;
             // We are guaranteed not to leave the array due to range test above
             while (offset >= fCachedBreakPositions[fPositionInCache]) {
                 ++fPositionInCache;
             }
             int32_t pos = fCachedBreakPositions[fPositionInCache];
             utext_setNativeIndex(fText, pos);
             return pos;
         }
         else {
             reset();
         }
     }
     // if the offset passed in is already past the end of the text,
     // just return DONE; if it's before the beginning, return the
     // text's starting offset
     fLastRuleStatusIndex  = 0;
     fLastStatusIndexValid = TRUE;
     if (fText == NULL || offset >= utext_nativeLength(fText)) {
         last();
         return next();
     }
     else if (offset < 0) {
         return first();
     }
     // otherwise, set our internal iteration position (temporarily)
     // to the position passed in.  If this is the _beginning_ position,
     // then we can just use next() to get our return value
     int32_t result = 0;
     if (fData->fSafeRevTable != NULL) {
         // new rule syntax
         utext_setNativeIndex(fText, offset);
         // move forward one codepoint to prepare for moving back to a
         // safe point.
         // this handles offset being between a supplementary character
         (void)UTEXT_NEXT32(fText);
         // handlePrevious will move most of the time to < 1 boundary away
         handlePrevious(fData->fSafeRevTable);
         int32_t result = next();
         while (result <= offset) {
             result = next();
         }
         return result;
     }
     if (fData->fSafeFwdTable != NULL) {
         // backup plan if forward safe table is not available
         utext_setNativeIndex(fText, offset);
         (void)UTEXT_PREVIOUS32(fText);
         // handle next will give result >= offset
         handleNext(fData->fSafeFwdTable);
         // previous will give result 0 or 1 boundary away from offset,
         // most of the time
         // we have to
         int32_t oldresult = previous();
         while (oldresult > offset) {
             int32_t result = previous();
             if (result <= offset) {
                 return oldresult;
             }
             oldresult = result;
         }
         int32_t result = next();
         if (result <= offset) {
             return next();
         }
         return result;
     }
     // otherwise, we have to sync up first.  Use handlePrevious() to back
     // up to a known break position before the specified position (if
     // we can determine that the specified position is a break position,
     // we don't back up at all).  This may or may not be the last break
     // position at or before our starting position.  Advance forward
     // from here until we've passed the starting position.  The position
     // we stop on will be the first break position after the specified one.
     // old rule syntax
     utext_setNativeIndex(fText, offset);
     if (offset==0 ||
         (offset==1  && utext_getNativeIndex(fText)==0)) {
         return next();
     }
     result = previous();
     while (result != BreakIterator::DONE && result <= offset) {
         result = next();
     }
     return result;
 }
 /**
  * Sets the iterator to refer to the last boundary position before the
  * specified position.
  * @offset The position to begin searching for a break from.
  * @return The position of the last boundary before the starting position.
  */
 int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
     // if we have cached break positions and offset is in the range
     // covered by them, use them
     if (fCachedBreakPositions != NULL) {
         // TODO: binary search?
         // TODO: What if offset is outside range, but break is not?
         if (offset > fCachedBreakPositions[0]
                 && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
             fPositionInCache = 0;
             while (fPositionInCache < fNumCachedBreakPositions
                    && offset > fCachedBreakPositions[fPositionInCache])
                 ++fPositionInCache;
             --fPositionInCache;
             // If we're at the beginning of the cache, need to reevaluate the
             // rule status
             if (fPositionInCache <= 0) {
                 fLastStatusIndexValid = FALSE;
             }
             utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]);
             return fCachedBreakPositions[fPositionInCache];
         }
         else {
             reset();
         }
     }
     // if the offset passed in is already past the end of the text,
     // just return DONE; if it's before the beginning, return the
     // text's starting offset
     if (fText == NULL || offset > utext_nativeLength(fText)) {
         // return BreakIterator::DONE;
         return last();
     }
     else if (offset < 0) {
         return first();
     }
     // if we start by updating the current iteration position to the
     // position specified by the caller, we can just use previous()
     // to carry out this operation
     if (fData->fSafeFwdTable != NULL) {
         // new rule syntax
         utext_setNativeIndex(fText, offset);
         int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
         if (newOffset != offset) {
             // Will come here if specified offset was not a code point boundary AND
             //   the underlying implmentation is using UText, which snaps any non-code-point-boundary
             //   indices to the containing code point.
             // For breakitereator::preceding only, these non-code-point indices need to be moved
             //   up to refer to the following codepoint.
             (void)UTEXT_NEXT32(fText);
             offset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
         }
         // TODO:  (synwee) would it be better to just check for being in the middle of a surrogate pair,
         //        rather than adjusting the position unconditionally?
         //        (Change would interact with safe rules.)
         // TODO:  change RBBI behavior for off-boundary indices to match that of UText?
         //        affects only preceding(), seems cleaner, but is slightly different.
         (void)UTEXT_PREVIOUS32(fText);
         handleNext(fData->fSafeFwdTable);
         int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
         while (result >= offset) {
             result = previous();
         }
         return result;
     }
     if (fData->fSafeRevTable != NULL) {
         // backup plan if forward safe table is not available
         //  TODO:  check whether this path can be discarded
         //         It's probably OK to say that rules must supply both safe tables
         //            if they use safe tables at all.  We have certainly never described
         //            to anyone how to work with just one safe table.
         utext_setNativeIndex(fText, offset);
         (void)UTEXT_NEXT32(fText);
         // handle previous will give result <= offset
         handlePrevious(fData->fSafeRevTable);
         // next will give result 0 or 1 boundary away from offset,
         // most of the time
         // we have to
         int32_t oldresult = next();
         while (oldresult < offset) {
             int32_t result = next();
             if (result >= offset) {
                 return oldresult;
             }
             oldresult = result;
         }
         int32_t result = previous();
         if (result >= offset) {
             return previous();
         }
         return result;
     }
     // old rule syntax
     utext_setNativeIndex(fText, offset);
     return previous();
 }
 /**
  * Returns true if the specfied position is a boundary position.  As a side
  * effect, leaves the iterator pointing to the first boundary position at
  * or after "offset".
  * @param offset the offset to check.
  * @return True if "offset" is a boundary position.
  */
 UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
     // the beginning index of the iterator is always a boundary position by definition
     if (offset == 0) {
         first();       // For side effects on current position, tag values.
         return TRUE;
     }
     if (offset == (int32_t)utext_nativeLength(fText)) {
         last();       // For side effects on current position, tag values.
         return TRUE;
     }
     // out-of-range indexes are never boundary positions
     if (offset < 0) {
         first();       // For side effects on current position, tag values.
         return FALSE;
     }
     if (offset > utext_nativeLength(fText)) {
         last();        // For side effects on current position, tag values.
         return FALSE;
     }
     // otherwise, we can use following() on the position before the specified
     // one and return true if the position we get back is the one the user
     // specified
     utext_previous32From(fText, offset);
     int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText);
     UBool    result  = following(backOne) == offset;
     return result;
 }
 /**
  * Returns the current iteration position.
  * @return The current iteration position.
  */
 int32_t RuleBasedBreakIterator::current(void) const {
     int32_t  pos = (int32_t)UTEXT_GETNATIVEINDEX(fText);
     return pos;
 }
 //=======================================================================
 // implementation
 //=======================================================================
 //
 // RBBIRunMode  -  the state machine runs an extra iteration at the beginning and end
 //                 of user text.  A variable with this enum type keeps track of where we
 //                 are.  The state machine only fetches user input while in the RUN mode.
 //
 enum RBBIRunMode {
     RBBI_START,     // state machine processing is before first char of input
     RBBI_RUN,       // state machine processing is in the user text
     RBBI_END        // state machine processing is after end of user text.
 };
 //-----------------------------------------------------------------------------------
 //
 //  handleNext(stateTable)
 //     This method is the actual implementation of the rbbi next() method.
 //     This method initializes the state machine to state 1
 //     and advances through the text character by character until we reach the end
 //     of the text or the state machine transitions to state 0.  We update our return
 //     value every time the state machine passes through an accepting state.
 //
 //-----------------------------------------------------------------------------------
 int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
     int32_t             state;
     uint16_t            category        = 0;
     RBBIRunMode         mode;
     RBBIStateTableRow  *row;
     UChar32             c;
     int32_t             lookaheadStatus = 0;
     int32_t             lookaheadTagIdx = 0;
     int32_t             result          = 0;
     int32_t             initialPosition = 0;
     int32_t             lookaheadResult = 0;
     UBool               lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0;
     const char         *tableData       = statetable->fTableData;
     uint32_t            tableRowLen     = statetable->fRowLen;
     #ifdef RBBI_DEBUG
         if (fTrace) {
             RBBIDebugPuts("Handle Next   pos   char  state category");
         }
     #endif
     // No matter what, handleNext alway correctly sets the break tag value.
     fLastStatusIndexValid = TRUE;
     fLastRuleStatusIndex = 0;
     // if we're already at the end of the text, return DONE.
     initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText);
     result          = initialPosition;
     c               = UTEXT_NEXT32(fText);
     if (fData == NULL || c==U_SENTINEL) {
         return BreakIterator::DONE;
     }
     //  Set the initial state for the state machine
     state = START_STATE;
     row = (RBBIStateTableRow *)
             //(statetable->fTableData + (statetable->fRowLen * state));
             (tableData + tableRowLen * state);
     mode     = RBBI_RUN;
     if (statetable->fFlags & RBBI_BOF_REQUIRED) {
         category = 2;
         mode     = RBBI_START;
     }
     // loop until we reach the end of the text or transition to state 0
     //
     for (;;) {
         if (c == U_SENTINEL) {
             // Reached end of input string.
             if (mode == RBBI_END) {
                 // We have already run the loop one last time with the
                 //   character set to the psueudo {eof} value.  Now it is time
                 //   to unconditionally bail out.
                 if (lookaheadResult > result) {
                     // We ran off the end of the string with a pending look-ahead match.
                     // Treat this as if the look-ahead condition had been met, and return
                     //  the match at the / position from the look-ahead rule.
                     result               = lookaheadResult;
                     fLastRuleStatusIndex = lookaheadTagIdx;
                     lookaheadStatus = 0;
                 }
                 break;
             }
             // Run the loop one last time with the fake end-of-input character category.
             mode = RBBI_END;
             category = 1;
         }
         //
         // Get the char category.  An incoming category of 1 or 2 means that
         //      we are preset for doing the beginning or end of input, and
         //      that we shouldn't get a category from an actual text input character.
         //
         if (mode == RBBI_RUN) {
             // look up the current character's character category, which tells us
             // which column in the state table to look at.
             // Note:  the 16 in UTRIE_GET16 refers to the size of the data being returned,
             //        not the size of the character going in, which is a UChar32.
             //
             UTRIE_GET16(&fData->fTrie, c, category);
             // Check the dictionary bit in the character's category.
             //    Counter is only used by dictionary based iterators (subclasses).
             //    Chars that need to be handled by a dictionary have a flag bit set
             //    in their category values.
             //
             if ((category & 0x4000) != 0)  {
                 fDictionaryCharCount++;
                 //  And off the dictionary flag bit.
                 category &= ~0x4000;
             }
         }
        #ifdef RBBI_DEBUG
             if (fTrace) {
                 RBBIDebugPrintf("             %4ld   ", utext_getNativeIndex(fText));
                 if (0x20<=c && c<0x7f) {
                     RBBIDebugPrintf("\"%c\"  ", c);
                 } else {
                     RBBIDebugPrintf("%5x  ", c);
                 }
                 RBBIDebugPrintf("%3d  %3d\n", state, category);
             }
         #endif
         // State Transition - move machine to its next state
         //
         // Note: fNextState is defined as uint16_t[2], but we are casting
         // a generated RBBI table to RBBIStateTableRow and some tables
         // actually have more than 2 categories.
         U_ASSERT(category<fData->fHeader->fCatCount);
         state = row->fNextState[category];  /*Not accessing beyond memory*/
         row = (RBBIStateTableRow *)
             // (statetable->fTableData + (statetable->fRowLen * state));
             (tableData + tableRowLen * state);
         if (row->fAccepting == -1) {
             // Match found, common case.
             if (mode != RBBI_START) {
                 result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
             }
             fLastRuleStatusIndex = row->fTagIdx;   // Remember the break status (tag) values.
         }
         if (row->fLookAhead != 0) {
             if (lookaheadStatus != 0
                 && row->fAccepting == lookaheadStatus) {
                 // Lookahead match is completed.
                 result               = lookaheadResult;
                 fLastRuleStatusIndex = lookaheadTagIdx;
                 lookaheadStatus      = 0;
                 // TODO:  make a standalone hard break in a rule work.
                 if (lookAheadHardBreak) {
                     UTEXT_SETNATIVEINDEX(fText, result);
                     return result;
                 }
                 // Look-ahead completed, but other rules may match further.  Continue on
                 //  TODO:  junk this feature?  I don't think it's used anywhwere.
                 goto continueOn;
             }
             int32_t  r = (int32_t)UTEXT_GETNATIVEINDEX(fText);
             lookaheadResult = r;
             lookaheadStatus = row->fLookAhead;
             lookaheadTagIdx = row->fTagIdx;
             goto continueOn;
         }
         if (row->fAccepting != 0) {
             // Because this is an accepting state, any in-progress look-ahead match
             //   is no longer relavant.  Clear out the pending lookahead status.
             lookaheadStatus = 0;           // clear out any pending look-ahead match.
         }
 continueOn:
         if (state == STOP_STATE) {
             // This is the normal exit from the lookup state machine.
             // We have advanced through the string until it is certain that no
             //   longer match is possible, no matter what characters follow.
             break;
         }
         // Advance to the next character.
         // If this is a beginning-of-input loop iteration, don't advance
         //    the input position.  The next iteration will be processing the
         //    first real input character.
         if (mode == RBBI_RUN) {
             c = UTEXT_NEXT32(fText);
         } else {
             if (mode == RBBI_START) {
                 mode = RBBI_RUN;
             }
         }
     }
     // The state machine is done.  Check whether it found a match...
     // If the iterator failed to advance in the match engine, force it ahead by one.
     //   (This really indicates a defect in the break rules.  They should always match
     //    at least one character.)
     if (result == initialPosition) {
         UTEXT_SETNATIVEINDEX(fText, initialPosition);
         UTEXT_NEXT32(fText);
         result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
     }
     // Leave the iterator at our result position.
     UTEXT_SETNATIVEINDEX(fText, result);
     #ifdef RBBI_DEBUG
         if (fTrace) {
             RBBIDebugPrintf("result = %d\n\n", result);
         }
     #endif
     return result;
 }
 //-----------------------------------------------------------------------------------
 //
 //  handlePrevious()
 //
 //      Iterate backwards, according to the logic of the reverse rules.
 //      This version handles the exact style backwards rules.
 //
 //      The logic of this function is very similar to handleNext(), above.
 //
 //-----------------------------------------------------------------------------------
 int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) {
     int32_t             state;
     uint16_t            category        = 0;
     RBBIRunMode         mode;
     RBBIStateTableRow  *row;
     UChar32             c;
     int32_t             lookaheadStatus = 0;
     int32_t             result          = 0;
     int32_t             initialPosition = 0;
     int32_t             lookaheadResult = 0;
     UBool               lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0;
     #ifdef RBBI_DEBUG
         if (fTrace) {
             RBBIDebugPuts("Handle Previous   pos   char  state category");
         }
     #endif
     // handlePrevious() never gets the rule status.
     // Flag the status as invalid; if the user ever asks for status, we will need
     // to back up, then re-find the break position using handleNext(), which does
     // get the status value.
     fLastStatusIndexValid = FALSE;
     fLastRuleStatusIndex = 0;
     // if we're already at the start of the text, return DONE.
     if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) {
         return BreakIterator::DONE;
     }
     //  Set up the starting char.
     initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText);
     result          = initialPosition;
     c               = UTEXT_PREVIOUS32(fText);
     //  Set the initial state for the state machine
     state = START_STATE;
     row = (RBBIStateTableRow *)
             (statetable->fTableData + (statetable->fRowLen * state));
     category = 3;
     mode     = RBBI_RUN;
     if (statetable->fFlags & RBBI_BOF_REQUIRED) {
         category = 2;
         mode     = RBBI_START;
     }
     // loop until we reach the start of the text or transition to state 0
     //
     for (;;) {
         if (c == U_SENTINEL) {
             // Reached end of input string.
             if (mode == RBBI_END) {
                 // We have already run the loop one last time with the
                 //   character set to the psueudo {eof} value.  Now it is time
                 //   to unconditionally bail out.
                 if (lookaheadResult < result) {
                     // We ran off the end of the string with a pending look-ahead match.
                     // Treat this as if the look-ahead condition had been met, and return
                     //  the match at the / position from the look-ahead rule.
                     result               = lookaheadResult;
                     lookaheadStatus = 0;
                 } else if (result == initialPosition) {
                     // Ran off start, no match found.
                     // move one index one (towards the start, since we are doing a previous())
                     UTEXT_SETNATIVEINDEX(fText, initialPosition);
                     (void)UTEXT_PREVIOUS32(fText);   // TODO:  shouldn't be necessary.  We're already at beginning.  Check.
                 }
                 break;
             }
             // Run the loop one last time with the fake end-of-input character category.
             mode = RBBI_END;
             category = 1;
         }
         //
         // Get the char category.  An incoming category of 1 or 2 means that
         //      we are preset for doing the beginning or end of input, and
         //      that we shouldn't get a category from an actual text input character.
         //
         if (mode == RBBI_RUN) {
             // look up the current character's character category, which tells us
             // which column in the state table to look at.
             // Note:  the 16 in UTRIE_GET16 refers to the size of the data being returned,
             //        not the size of the character going in, which is a UChar32.
             //
             UTRIE_GET16(&fData->fTrie, c, category);
             // Check the dictionary bit in the character's category.
             //    Counter is only used by dictionary based iterators (subclasses).
             //    Chars that need to be handled by a dictionary have a flag bit set
             //    in their category values.
             //
             if ((category & 0x4000) != 0)  {
                 fDictionaryCharCount++;
                 //  And off the dictionary flag bit.
                 category &= ~0x4000;
             }
         }
         #ifdef RBBI_DEBUG
             if (fTrace) {
                 RBBIDebugPrintf("             %4d   ", (int32_t)utext_getNativeIndex(fText));
                 if (0x20<=c && c<0x7f) {
                     RBBIDebugPrintf("\"%c\"  ", c);
                 } else {
                     RBBIDebugPrintf("%5x  ", c);
                 }
                 RBBIDebugPrintf("%3d  %3d\n", state, category);
             }
         #endif
         // State Transition - move machine to its next state
         //
         // Note: fNextState is defined as uint16_t[2], but we are casting
         // a generated RBBI table to RBBIStateTableRow and some tables
         // actually have more than 2 categories.
         U_ASSERT(category<fData->fHeader->fCatCount);
         state = row->fNextState[category];  /*Not accessing beyond memory*/
         row = (RBBIStateTableRow *)
             (statetable->fTableData + (statetable->fRowLen * state));
         if (row->fAccepting == -1) {
             // Match found, common case.
             result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
         }
         if (row->fLookAhead != 0) {
             if (lookaheadStatus != 0
                 && row->fAccepting == lookaheadStatus) {
                 // Lookahead match is completed.
                 result               = lookaheadResult;
                 lookaheadStatus      = 0;
                 // TODO:  make a standalone hard break in a rule work.
                 if (lookAheadHardBreak) {
                     UTEXT_SETNATIVEINDEX(fText, result);
                     return result;
                 }
                 // Look-ahead completed, but other rules may match further.  Continue on
                 //  TODO:  junk this feature?  I don't think it's used anywhwere.
                 goto continueOn;
             }
             int32_t  r = (int32_t)UTEXT_GETNATIVEINDEX(fText);
             lookaheadResult = r;
             lookaheadStatus = row->fLookAhead;
             goto continueOn;
         }
         if (row->fAccepting != 0) {
             // Because this is an accepting state, any in-progress look-ahead match
             //   is no longer relavant.  Clear out the pending lookahead status.
             lookaheadStatus = 0;
         }
 continueOn:
         if (state == STOP_STATE) {
             // This is the normal exit from the lookup state machine.
             // We have advanced through the string until it is certain that no
             //   longer match is possible, no matter what characters follow.
             break;
         }
         // Move (backwards) to the next character to process.
         // If this is a beginning-of-input loop iteration, don't advance
         //    the input position.  The next iteration will be processing the
         //    first real input character.
         if (mode == RBBI_RUN) {
             c = UTEXT_PREVIOUS32(fText);
         } else {
             if (mode == RBBI_START) {
                 mode = RBBI_RUN;
             }
         }
     }
     // The state machine is done.  Check whether it found a match...
     // If the iterator failed to advance in the match engine, force it ahead by one.
     //   (This really indicates a defect in the break rules.  They should always match
     //    at least one character.)
     if (result == initialPosition) {
         UTEXT_SETNATIVEINDEX(fText, initialPosition);
         UTEXT_PREVIOUS32(fText);
         result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
     }
     // Leave the iterator at our result position.
     UTEXT_SETNATIVEINDEX(fText, result);
     #ifdef RBBI_DEBUG
         if (fTrace) {
             RBBIDebugPrintf("result = %d\n\n", result);
         }
     #endif
     return result;
 }
 void
 RuleBasedBreakIterator::reset()
 {
     if (fCachedBreakPositions) {
         uprv_free(fCachedBreakPositions);
     }
     fCachedBreakPositions = NULL;
     fNumCachedBreakPositions = 0;
     fDictionaryCharCount = 0;
     fPositionInCache = 0;
 }
 //-------------------------------------------------------------------------------
 //
 //   getRuleStatus()   Return the break rule tag associated with the current
 //                     iterator position.  If the iterator arrived at its current
 //                     position by iterating forwards, the value will have been
 //                     cached by the handleNext() function.
 //
 //                     If no cached status value is available, the status is
 //                     found by doing a previous() followed by a next(), which
 //                     leaves the iterator where it started, and computes the
 //                     status while doing the next().
 //
 //-------------------------------------------------------------------------------
 void RuleBasedBreakIterator::makeRuleStatusValid() {
     if (fLastStatusIndexValid == FALSE) {
         //  No cached status is available.
         if (fText == NULL || current() == 0) {
             //  At start of text, or there is no text.  Status is always zero.
             fLastRuleStatusIndex = 0;
             fLastStatusIndexValid = TRUE;
         } else {
             //  Not at start of text.  Find status the tedious way.
             int32_t pa = current();
             previous();
             if (fNumCachedBreakPositions > 0) {
                 reset();                // Blow off the dictionary cache
             }
             int32_t pb = next();
             if (pa != pb) {
                 // note: the if (pa != pb) test is here only to eliminate warnings for
                 //       unused local variables on gcc.  Logically, it isn't needed.
                 U_ASSERT(pa == pb);
             }
         }
     }
     U_ASSERT(fLastRuleStatusIndex >= 0  &&  fLastRuleStatusIndex < fData->fStatusMaxIdx);
 }
 int32_t  RuleBasedBreakIterator::getRuleStatus() const {
     RuleBasedBreakIterator *nonConstThis  = (RuleBasedBreakIterator *)this;
     nonConstThis->makeRuleStatusValid();
     // fLastRuleStatusIndex indexes to the start of the appropriate status record
     //                                                 (the number of status values.)
     //   This function returns the last (largest) of the array of status values.
     int32_t  idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex];
     int32_t  tagVal = fData->fRuleStatusTable[idx];
     return tagVal;
 }
 int32_t RuleBasedBreakIterator::getRuleStatusVec(
              int32_t *fillInVec, int32_t capacity, UErrorCode &status)
 {
     if (U_FAILURE(status)) {
         return 0;
     }
     RuleBasedBreakIterator *nonConstThis  = (RuleBasedBreakIterator *)this;
     nonConstThis->makeRuleStatusValid();
     int32_t  numVals = fData->fRuleStatusTable[fLastRuleStatusIndex];
     int32_t  numValsToCopy = numVals;
     if (numVals > capacity) {
         status = U_BUFFER_OVERFLOW_ERROR;
         numValsToCopy = capacity;
     }
     int i;
     for (i=0; i<numValsToCopy; i++) {
         fillInVec[i] = fData->fRuleStatusTable[fLastRuleStatusIndex + i + 1];
     }
     return numVals;
 }
 //-------------------------------------------------------------------------------
 //
 //   getBinaryRules        Access to the compiled form of the rules,
 //                         for use by build system tools that save the data
 //                         for standard iterator types.
 //
 //-------------------------------------------------------------------------------
 const uint8_t  *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) {
     const uint8_t  *retPtr = NULL;
     length = 0;
     if (fData != NULL) {
         retPtr = (const uint8_t *)fData->fHeader;
         length = fData->fHeader->fLength;
     }
     return retPtr;
 }
 BreakIterator *  RuleBasedBreakIterator::createBufferClone(void * /*stackBuffer*/,
                                    int32_t &bufferSize,
                                    UErrorCode &status)
 {
     if (U_FAILURE(status)){
         return NULL;
     }
     if (bufferSize == 0) {
         bufferSize = 1;  // preflighting for deprecated functionality
         return NULL;
     }
     BreakIterator *clonedBI = clone();
     if (clonedBI == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
     } else {
         status = U_SAFECLONE_ALLOCATED_WARNING;
     }
     return (RuleBasedBreakIterator *)clonedBI;
 }
 //-------------------------------------------------------------------------------
 //
 //  isDictionaryChar      Return true if the category lookup for this char
 //                        indicates that it is in the set of dictionary lookup
 //                        chars.
 //
 //                        This function is intended for use by dictionary based
 //                        break iterators.
 //
 //-------------------------------------------------------------------------------
 /*UBool RuleBasedBreakIterator::isDictionaryChar(UChar32   c) {
     if (fData == NULL) {
         return FALSE;
     }
     uint16_t category;
     UTRIE_GET16(&fData->fTrie, c, category);
     return (category & 0x4000) != 0;
 }*/
 //-------------------------------------------------------------------------------
 //
 //  checkDictionary       This function handles all processing of characters in
 //                        the "dictionary" set. It will determine the appropriate
 //                        course of action, and possibly set up a cache in the
 //                        process.
 //
 //-------------------------------------------------------------------------------
 int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos,
                             int32_t endPos,
                             UBool reverse) {
     // Reset the old break cache first.
     reset();
     // note: code segment below assumes that dictionary chars are in the
     // startPos-endPos range
     // value returned should be next character in sequence
     if ((endPos - startPos) <= 1) {
         return (reverse ? startPos : endPos);
     }
     // Bug 5532.  The dictionary code will crash if the input text is UTF-8
     //      because native indexes are different from UTF-16 indexes.
     //      Temporary hack: skip dictionary lookup for UTF-8 encoded text.
     //      It wont give the right breaks, but it's better than a crash.
     //
     //      Check the type of the UText by checking its pFuncs field, which
     //      is UText's function dispatch table.  It will be the same for all
     //      UTF-8 UTexts and different for any other UText type.
     //
     //      We have no other type of UText available with non-UTF-16 native indexing.
     //      This whole check will go away once the dictionary code is fixed.
     static const void *utext_utf8Funcs;
     if (utext_utf8Funcs == NULL) {
         // Cache the UTF-8 UText function pointer value.
         UErrorCode status = U_ZERO_ERROR;
         UText tempUText = UTEXT_INITIALIZER;
         utext_openUTF8(&tempUText, NULL, 0, &status);
         utext_utf8Funcs = tempUText.pFuncs;
         utext_close(&tempUText);
     }
     if (fText->pFuncs == utext_utf8Funcs) {
         return (reverse ? startPos : endPos);
     }
     // Starting from the starting point, scan towards the proposed result,
     // looking for the first dictionary character (which may be the one
     // we're on, if we're starting in the middle of a range).
     utext_setNativeIndex(fText, reverse ? endPos : startPos);
     if (reverse) {
         UTEXT_PREVIOUS32(fText);
     }
     int32_t rangeStart = startPos;
     int32_t rangeEnd = endPos;
     uint16_t    category;
     int32_t     current;
     UErrorCode  status = U_ZERO_ERROR;
     UStack      breaks(status);
     int32_t     foundBreakCount = 0;
     UChar32     c = utext_current32(fText);
     UTRIE_GET16(&fData->fTrie, c, category);
     // Is the character we're starting on a dictionary character? If so, we
     // need to back up to include the entire run; otherwise the results of
     // the break algorithm will differ depending on where we start. Since
     // the result is cached and there is typically a non-dictionary break
     // within a small number of words, there should be little performance impact.
     if (category & 0x4000) {
         if (reverse) {
             do {
                 utext_next32(fText);          // TODO:  recast to work directly with postincrement.
                 c = utext_current32(fText);
                 UTRIE_GET16(&fData->fTrie, c, category);
             } while (c != U_SENTINEL && (category & 0x4000));
             // Back up to the last dictionary character
             rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText);
             if (c == U_SENTINEL) {
                 // c = fText->last32();
                 //   TODO:  why was this if needed?
                 c = UTEXT_PREVIOUS32(fText);
             }
             else {
                 c = UTEXT_PREVIOUS32(fText);
             }
         }
         else {
             do {
                 c = UTEXT_PREVIOUS32(fText);
                 UTRIE_GET16(&fData->fTrie, c, category);
             }
             while (c != U_SENTINEL && (category & 0x4000));
             // Back up to the last dictionary character
             if (c == U_SENTINEL) {
                 // c = fText->first32();
                 c = utext_current32(fText);
             }
             else {
                 utext_next32(fText);
                 c = utext_current32(fText);
             }
             rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);;
         }
         UTRIE_GET16(&fData->fTrie, c, category);
     }
     // Loop through the text, looking for ranges of dictionary characters.
     // For each span, find the appropriate break engine, and ask it to find
     // any breaks within the span.
     // Note: we always do this in the forward direction, so that the break
     // cache is built in the right order.
     if (reverse) {
         utext_setNativeIndex(fText, rangeStart);
         c = utext_current32(fText);
         UTRIE_GET16(&fData->fTrie, c, category);
     }
     while(U_SUCCESS(status)) {
         while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) {
             utext_next32(fText);           // TODO:  tweak for post-increment operation
             c = utext_current32(fText);
             UTRIE_GET16(&fData->fTrie, c, category);
         }
         if (current >= rangeEnd) {
             break;
         }
         // We now have a dictionary character. Get the appropriate language object
         // to deal with it.
         const LanguageBreakEngine *lbe = getLanguageBreakEngine(c);
         // Ask the language object if there are any breaks. It will leave the text
         // pointer on the other side of its range, ready to search for the next one.
         if (lbe != NULL) {
             foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks);
         }
         // Reload the loop variables for the next go-round
         c = utext_current32(fText);
         UTRIE_GET16(&fData->fTrie, c, category);
     }
     // If we found breaks, build a new break cache. The first and last entries must
     // be the original starting and ending position.
     if (foundBreakCount > 0) {
         int32_t totalBreaks = foundBreakCount;
         if (startPos < breaks.elementAti(0)) {
             totalBreaks += 1;
         }
         if (endPos > breaks.peeki()) {
             totalBreaks += 1;
         }
         fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t));
         if (fCachedBreakPositions != NULL) {
             int32_t out = 0;
             fNumCachedBreakPositions = totalBreaks;
             if (startPos < breaks.elementAti(0)) {
                 fCachedBreakPositions[out++] = startPos;
             }
             for (int32_t i = 0; i < foundBreakCount; ++i) {
                 fCachedBreakPositions[out++] = breaks.elementAti(i);
             }
             if (endPos > fCachedBreakPositions[out-1]) {
                 fCachedBreakPositions[out] = endPos;
             }
             // If there are breaks, then by definition, we are replacing the original
             // proposed break by one of the breaks we found. Use following() and
             // preceding() to do the work. They should never recurse in this case.
             if (reverse) {
                 return preceding(endPos);
             }
             else {
                 return following(startPos);
             }
         }
         // If the allocation failed, just fall through to the "no breaks found" case.
     }
     // If we get here, there were no language-based breaks. Set the text pointer
     // to the original proposed break.
     utext_setNativeIndex(fText, reverse ? startPos : endPos);
     return (reverse ? startPos : endPos);
 }
 // defined in ucln_cmn.h
 U_NAMESPACE_END
 static icu::UStack *gLanguageBreakFactories = NULL;
 static icu::UInitOnce gLanguageBreakFactoriesInitOnce = U_INITONCE_INITIALIZER;
 /**
  * Release all static memory held by breakiterator.
  */
 U_CDECL_BEGIN
 static UBool U_CALLCONV breakiterator_cleanup_dict(void) {
     if (gLanguageBreakFactories) {
         delete gLanguageBreakFactories;
         gLanguageBreakFactories = NULL;
     }
     gLanguageBreakFactoriesInitOnce.reset();
     return TRUE;
 }
 U_CDECL_END
 U_CDECL_BEGIN
 static void U_CALLCONV _deleteFactory(void *obj) {
     delete (icu::LanguageBreakFactory *) obj;
 }
 U_CDECL_END
 U_NAMESPACE_BEGIN
 static void U_CALLCONV initLanguageFactories() {
     UErrorCode status = U_ZERO_ERROR;
     U_ASSERT(gLanguageBreakFactories == NULL);
     gLanguageBreakFactories = new UStack(_deleteFactory, NULL, status);
     if (gLanguageBreakFactories != NULL && U_SUCCESS(status)) {
         ICULanguageBreakFactory *builtIn = new ICULanguageBreakFactory(status);
         gLanguageBreakFactories->push(builtIn, status);
 #ifdef U_LOCAL_SERVICE_HOOK
         LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status);
         if (extra != NULL) {
             gLanguageBreakFactories->push(extra, status);
         }
 #endif
     }
     ucln_common_registerCleanup(UCLN_COMMON_BREAKITERATOR_DICT, breakiterator_cleanup_dict);
 }
 static const LanguageBreakEngine*
 getLanguageBreakEngineFromFactory(UChar32 c, int32_t breakType)
 {
     umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories);
     if (gLanguageBreakFactories == NULL) {
         return NULL;
     }
     int32_t i = gLanguageBreakFactories->size();
     const LanguageBreakEngine *lbe = NULL;
     while (--i >= 0) {
         LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i));
         lbe = factory->getEngineFor(c, breakType);
         if (lbe != NULL) {
             break;
         }
     }
     return lbe;
 }
 //-------------------------------------------------------------------------------
 //
 //  getLanguageBreakEngine  Find an appropriate LanguageBreakEngine for the
 //                          the character c.
 //
 //-------------------------------------------------------------------------------
 const LanguageBreakEngine *
 RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
     const LanguageBreakEngine *lbe = NULL;
     UErrorCode status = U_ZERO_ERROR;
     if (fLanguageBreakEngines == NULL) {
         fLanguageBreakEngines = new UStack(status);
         if (fLanguageBreakEngines == NULL || U_FAILURE(status)) {
             delete fLanguageBreakEngines;
             fLanguageBreakEngines = 0;
             return NULL;
         }
     }
     int32_t i = fLanguageBreakEngines->size();
     while (--i >= 0) {
         lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i));
         if (lbe->handles(c, fBreakType)) {
             return lbe;
         }
     }
     // No existing dictionary took the character. See if a factory wants to
     // give us a new LanguageBreakEngine for this character.
     lbe = getLanguageBreakEngineFromFactory(c, fBreakType);
     // If we got one, use it and push it on our stack.
     if (lbe != NULL) {
         fLanguageBreakEngines->push((void *)lbe, status);
         // Even if we can't remember it, we can keep looking it up, so
         // return it even if the push fails.
         return lbe;
     }
     // No engine is forthcoming for this character. Add it to the
     // reject set. Create the reject break engine if needed.
     if (fUnhandledBreakEngine == NULL) {
         fUnhandledBreakEngine = new UnhandledEngine(status);
         if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
         }
         // Put it last so that scripts for which we have an engine get tried
         // first.
         fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status);
         // If we can't insert it, or creation failed, get rid of it
         if (U_FAILURE(status)) {
             delete fUnhandledBreakEngine;
             fUnhandledBreakEngine = 0;
             return NULL;
         }
     }
     // Tell the reject engine about the character; at its discretion, it may
     // add more than just the one character.
     fUnhandledBreakEngine->handleCharacter(c, fBreakType);
     return fUnhandledBreakEngine;
 }
 /*int32_t RuleBasedBreakIterator::getBreakType() const {
     return fBreakType;
 }*/
 void RuleBasedBreakIterator::setBreakType(int32_t type) {
     fBreakType = type;
     reset();
 }
 U_NAMESPACE_END
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */

The Tor Browser / annotate

intl/icu/source/common/rbbi.cpp@6474c204b198 (annotated)

intl/icu/source/common/rbbi.cpp