michael@0: /* michael@0: ******************************************************************************* michael@0: * Copyright (C) 2010-2012, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: ******************************************************************************* michael@0: * file name: ucharstrie.h michael@0: * encoding: US-ASCII michael@0: * tab size: 8 (not used) michael@0: * indentation:4 michael@0: * michael@0: * created on: 2010nov14 michael@0: * created by: Markus W. Scherer michael@0: */ michael@0: michael@0: #ifndef __UCHARSTRIE_H__ michael@0: #define __UCHARSTRIE_H__ michael@0: michael@0: /** michael@0: * \file michael@0: * \brief C++ API: Trie for mapping Unicode strings (or 16-bit-unit sequences) michael@0: * to integer values. michael@0: */ michael@0: michael@0: #include "unicode/utypes.h" michael@0: #include "unicode/unistr.h" michael@0: #include "unicode/uobject.h" michael@0: #include "unicode/ustringtrie.h" michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: class Appendable; michael@0: class UCharsTrieBuilder; michael@0: class UVector32; michael@0: michael@0: /** michael@0: * Light-weight, non-const reader class for a UCharsTrie. michael@0: * Traverses a UChar-serialized data structure with minimal state, michael@0: * for mapping strings (16-bit-unit sequences) to non-negative integer values. michael@0: * michael@0: * This class owns the serialized trie data only if it was constructed by michael@0: * the builder's build() method. michael@0: * The public constructor and the copy constructor only alias the data (only copy the pointer). michael@0: * There is no assignment operator. michael@0: * michael@0: * This class is not intended for public subclassing. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: class U_COMMON_API UCharsTrie : public UMemory { michael@0: public: michael@0: /** michael@0: * Constructs a UCharsTrie reader instance. michael@0: * michael@0: * The trieUChars must contain a copy of a UChar sequence from the UCharsTrieBuilder, michael@0: * starting with the first UChar of that sequence. michael@0: * The UCharsTrie object will not read more UChars than michael@0: * the UCharsTrieBuilder generated in the corresponding build() call. michael@0: * michael@0: * The array is not copied/cloned and must not be modified while michael@0: * the UCharsTrie object is in use. michael@0: * michael@0: * @param trieUChars The UChar array that contains the serialized trie. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UCharsTrie(const UChar *trieUChars) michael@0: : ownedArray_(NULL), uchars_(trieUChars), michael@0: pos_(uchars_), remainingMatchLength_(-1) {} michael@0: michael@0: /** michael@0: * Destructor. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: ~UCharsTrie(); michael@0: michael@0: /** michael@0: * Copy constructor, copies the other trie reader object and its state, michael@0: * but not the UChar array which will be shared. (Shallow copy.) michael@0: * @param other Another UCharsTrie object. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UCharsTrie(const UCharsTrie &other) michael@0: : ownedArray_(NULL), uchars_(other.uchars_), michael@0: pos_(other.pos_), remainingMatchLength_(other.remainingMatchLength_) {} michael@0: michael@0: /** michael@0: * Resets this trie to its initial state. michael@0: * @return *this michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UCharsTrie &reset() { michael@0: pos_=uchars_; michael@0: remainingMatchLength_=-1; michael@0: return *this; michael@0: } michael@0: michael@0: /** michael@0: * UCharsTrie state object, for saving a trie's current state michael@0: * and resetting the trie back to this state later. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: class State : public UMemory { michael@0: public: michael@0: /** michael@0: * Constructs an empty State. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: State() { uchars=NULL; } michael@0: private: michael@0: friend class UCharsTrie; michael@0: michael@0: const UChar *uchars; michael@0: const UChar *pos; michael@0: int32_t remainingMatchLength; michael@0: }; michael@0: michael@0: /** michael@0: * Saves the state of this trie. michael@0: * @param state The State object to hold the trie's state. michael@0: * @return *this michael@0: * @see resetToState michael@0: * @stable ICU 4.8 michael@0: */ michael@0: const UCharsTrie &saveState(State &state) const { michael@0: state.uchars=uchars_; michael@0: state.pos=pos_; michael@0: state.remainingMatchLength=remainingMatchLength_; michael@0: return *this; michael@0: } michael@0: michael@0: /** michael@0: * Resets this trie to the saved state. michael@0: * If the state object contains no state, or the state of a different trie, michael@0: * then this trie remains unchanged. michael@0: * @param state The State object which holds a saved trie state. michael@0: * @return *this michael@0: * @see saveState michael@0: * @see reset michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UCharsTrie &resetToState(const State &state) { michael@0: if(uchars_==state.uchars && uchars_!=NULL) { michael@0: pos_=state.pos; michael@0: remainingMatchLength_=state.remainingMatchLength; michael@0: } michael@0: return *this; michael@0: } michael@0: michael@0: /** michael@0: * Determines whether the string so far matches, whether it has a value, michael@0: * and whether another input UChar can continue a matching string. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult current() const; michael@0: michael@0: /** michael@0: * Traverses the trie from the initial state for this input UChar. michael@0: * Equivalent to reset().next(uchar). michael@0: * @param uchar Input char value. Values below 0 and above 0xffff will never match. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: inline UStringTrieResult first(int32_t uchar) { michael@0: remainingMatchLength_=-1; michael@0: return nextImpl(uchars_, uchar); michael@0: } michael@0: michael@0: /** michael@0: * Traverses the trie from the initial state for the michael@0: * one or two UTF-16 code units for this input code point. michael@0: * Equivalent to reset().nextForCodePoint(cp). michael@0: * @param cp A Unicode code point 0..0x10ffff. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult firstForCodePoint(UChar32 cp); michael@0: michael@0: /** michael@0: * Traverses the trie from the current state for this input UChar. michael@0: * @param uchar Input char value. Values below 0 and above 0xffff will never match. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult next(int32_t uchar); michael@0: michael@0: /** michael@0: * Traverses the trie from the current state for the michael@0: * one or two UTF-16 code units for this input code point. michael@0: * @param cp A Unicode code point 0..0x10ffff. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult nextForCodePoint(UChar32 cp); michael@0: michael@0: /** michael@0: * Traverses the trie from the current state for this string. michael@0: * Equivalent to michael@0: * \code michael@0: * Result result=current(); michael@0: * for(each c in s) michael@0: * if(!USTRINGTRIE_HAS_NEXT(result)) return USTRINGTRIE_NO_MATCH; michael@0: * result=next(c); michael@0: * return result; michael@0: * \endcode michael@0: * @param s A string. Can be NULL if length is 0. michael@0: * @param length The length of the string. Can be -1 if NUL-terminated. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult next(const UChar *s, int32_t length); michael@0: michael@0: /** michael@0: * Returns a matching string's value if called immediately after michael@0: * current()/first()/next() returned USTRINGTRIE_INTERMEDIATE_VALUE or USTRINGTRIE_FINAL_VALUE. michael@0: * getValue() can be called multiple times. michael@0: * michael@0: * Do not call getValue() after USTRINGTRIE_NO_MATCH or USTRINGTRIE_NO_VALUE! michael@0: * @return The value for the string so far. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: inline int32_t getValue() const { michael@0: const UChar *pos=pos_; michael@0: int32_t leadUnit=*pos++; michael@0: // U_ASSERT(leadUnit>=kMinValueLead); michael@0: return leadUnit&kValueIsFinal ? michael@0: readValue(pos, leadUnit&0x7fff) : readNodeValue(pos, leadUnit); michael@0: } michael@0: michael@0: /** michael@0: * Determines whether all strings reachable from the current state michael@0: * map to the same value. michael@0: * @param uniqueValue Receives the unique value, if this function returns TRUE. michael@0: * (output-only) michael@0: * @return TRUE if all strings reachable from the current state michael@0: * map to the same value. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: inline UBool hasUniqueValue(int32_t &uniqueValue) const { michael@0: const UChar *pos=pos_; michael@0: // Skip the rest of a pending linear-match node. michael@0: return pos!=NULL && findUniqueValue(pos+remainingMatchLength_+1, FALSE, uniqueValue); michael@0: } michael@0: michael@0: /** michael@0: * Finds each UChar which continues the string from the current state. michael@0: * That is, each UChar c for which it would be next(c)!=USTRINGTRIE_NO_MATCH now. michael@0: * @param out Each next UChar is appended to this object. michael@0: * @return the number of UChars which continue the string from here michael@0: * @stable ICU 4.8 michael@0: */ michael@0: int32_t getNextUChars(Appendable &out) const; michael@0: michael@0: /** michael@0: * Iterator for all of the (string, value) pairs in a UCharsTrie. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: class U_COMMON_API Iterator : public UMemory { michael@0: public: michael@0: /** michael@0: * Iterates from the root of a UChar-serialized UCharsTrie. michael@0: * @param trieUChars The trie UChars. michael@0: * @param maxStringLength If 0, the iterator returns full strings. michael@0: * Otherwise, the iterator returns strings with this maximum length. michael@0: * @param errorCode Standard ICU error code. Its input value must michael@0: * pass the U_SUCCESS() test, or else the function returns michael@0: * immediately. Check for U_FAILURE() on output or use with michael@0: * function chaining. (See User Guide for details.) michael@0: * @stable ICU 4.8 michael@0: */ michael@0: Iterator(const UChar *trieUChars, int32_t maxStringLength, UErrorCode &errorCode); michael@0: michael@0: /** michael@0: * Iterates from the current state of the specified UCharsTrie. michael@0: * @param trie The trie whose state will be copied for iteration. michael@0: * @param maxStringLength If 0, the iterator returns full strings. michael@0: * Otherwise, the iterator returns strings with this maximum length. michael@0: * @param errorCode Standard ICU error code. Its input value must michael@0: * pass the U_SUCCESS() test, or else the function returns michael@0: * immediately. Check for U_FAILURE() on output or use with michael@0: * function chaining. (See User Guide for details.) michael@0: * @stable ICU 4.8 michael@0: */ michael@0: Iterator(const UCharsTrie &trie, int32_t maxStringLength, UErrorCode &errorCode); michael@0: michael@0: /** michael@0: * Destructor. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: ~Iterator(); michael@0: michael@0: /** michael@0: * Resets this iterator to its initial state. michael@0: * @return *this michael@0: * @stable ICU 4.8 michael@0: */ michael@0: Iterator &reset(); michael@0: michael@0: /** michael@0: * @return TRUE if there are more elements. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UBool hasNext() const; michael@0: michael@0: /** michael@0: * Finds the next (string, value) pair if there is one. michael@0: * michael@0: * If the string is truncated to the maximum length and does not michael@0: * have a real value, then the value is set to -1. michael@0: * In this case, this "not a real value" is indistinguishable from michael@0: * a real value of -1. michael@0: * @param errorCode Standard ICU error code. Its input value must michael@0: * pass the U_SUCCESS() test, or else the function returns michael@0: * immediately. Check for U_FAILURE() on output or use with michael@0: * function chaining. (See User Guide for details.) michael@0: * @return TRUE if there is another element. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UBool next(UErrorCode &errorCode); michael@0: michael@0: /** michael@0: * @return The string for the last successful next(). michael@0: * @stable ICU 4.8 michael@0: */ michael@0: const UnicodeString &getString() const { return str_; } michael@0: /** michael@0: * @return The value for the last successful next(). michael@0: * @stable ICU 4.8 michael@0: */ michael@0: int32_t getValue() const { return value_; } michael@0: michael@0: private: michael@0: UBool truncateAndStop() { michael@0: pos_=NULL; michael@0: value_=-1; // no real value for str michael@0: return TRUE; michael@0: } michael@0: michael@0: const UChar *branchNext(const UChar *pos, int32_t length, UErrorCode &errorCode); michael@0: michael@0: const UChar *uchars_; michael@0: const UChar *pos_; michael@0: const UChar *initialPos_; michael@0: int32_t remainingMatchLength_; michael@0: int32_t initialRemainingMatchLength_; michael@0: UBool skipValue_; // Skip intermediate value which was already delivered. michael@0: michael@0: UnicodeString str_; michael@0: int32_t maxLength_; michael@0: int32_t value_; michael@0: michael@0: // The stack stores pairs of integers for backtracking to another michael@0: // outbound edge of a branch node. michael@0: // The first integer is an offset from uchars_. michael@0: // The second integer has the str_.length() from before the node in bits 15..0, michael@0: // and the remaining branch length in bits 31..16. michael@0: // (We could store the remaining branch length minus 1 in bits 30..16 and not use the sign bit, michael@0: // but the code looks more confusing that way.) michael@0: UVector32 *stack_; michael@0: }; michael@0: michael@0: private: michael@0: friend class UCharsTrieBuilder; michael@0: michael@0: /** michael@0: * Constructs a UCharsTrie reader instance. michael@0: * Unlike the public constructor which just aliases an array, michael@0: * this constructor adopts the builder's array. michael@0: * This constructor is only called by the builder. michael@0: */ michael@0: UCharsTrie(UChar *adoptUChars, const UChar *trieUChars) michael@0: : ownedArray_(adoptUChars), uchars_(trieUChars), michael@0: pos_(uchars_), remainingMatchLength_(-1) {} michael@0: michael@0: // No assignment operator. michael@0: UCharsTrie &operator=(const UCharsTrie &other); michael@0: michael@0: inline void stop() { michael@0: pos_=NULL; michael@0: } michael@0: michael@0: // Reads a compact 32-bit integer. michael@0: // pos is already after the leadUnit, and the lead unit has bit 15 reset. michael@0: static inline int32_t readValue(const UChar *pos, int32_t leadUnit) { michael@0: int32_t value; michael@0: if(leadUnit=kMinTwoUnitValueLead) { michael@0: if(leadUnit>6)-1; michael@0: } else if(leadUnit=kMinTwoUnitNodeValueLead) { michael@0: if(leadUnit=kMinTwoUnitDeltaLead) { michael@0: if(delta==kThreeUnitDeltaLead) { michael@0: delta=(pos[0]<<16)|pos[1]; michael@0: pos+=2; michael@0: } else { michael@0: delta=((delta-kMinTwoUnitDeltaLead)<<16)|*pos++; michael@0: } michael@0: } michael@0: return pos+delta; michael@0: } michael@0: michael@0: static const UChar *skipDelta(const UChar *pos) { michael@0: int32_t delta=*pos++; michael@0: if(delta>=kMinTwoUnitDeltaLead) { michael@0: if(delta==kThreeUnitDeltaLead) { michael@0: pos+=2; michael@0: } else { michael@0: ++pos; michael@0: } michael@0: } michael@0: return pos; michael@0: } michael@0: michael@0: static inline UStringTrieResult valueResult(int32_t node) { michael@0: return (UStringTrieResult)(USTRINGTRIE_INTERMEDIATE_VALUE-(node>>15)); michael@0: } michael@0: michael@0: // Handles a branch node for both next(uchar) and next(string). michael@0: UStringTrieResult branchNext(const UChar *pos, int32_t length, int32_t uchar); michael@0: michael@0: // Requires remainingLength_<0. michael@0: UStringTrieResult nextImpl(const UChar *pos, int32_t uchar); michael@0: michael@0: // Helper functions for hasUniqueValue(). michael@0: // Recursively finds a unique value (or whether there is not a unique one) michael@0: // from a branch. michael@0: static const UChar *findUniqueValueFromBranch(const UChar *pos, int32_t length, michael@0: UBool haveUniqueValue, int32_t &uniqueValue); michael@0: // Recursively finds a unique value (or whether there is not a unique one) michael@0: // starting from a position on a node lead unit. michael@0: static UBool findUniqueValue(const UChar *pos, UBool haveUniqueValue, int32_t &uniqueValue); michael@0: michael@0: // Helper functions for getNextUChars(). michael@0: // getNextUChars() when pos is on a branch node. michael@0: static void getNextBranchUChars(const UChar *pos, int32_t length, Appendable &out); michael@0: michael@0: // UCharsTrie data structure michael@0: // michael@0: // The trie consists of a series of UChar-serialized nodes for incremental michael@0: // Unicode string/UChar sequence matching. (UChar=16-bit unsigned integer) michael@0: // The root node is at the beginning of the trie data. michael@0: // michael@0: // Types of nodes are distinguished by their node lead unit ranges. michael@0: // After each node, except a final-value node, another node follows to michael@0: // encode match values or continue matching further units. michael@0: // michael@0: // Node types: michael@0: // - Final-value node: Stores a 32-bit integer in a compact, variable-length format. michael@0: // The value is for the string/UChar sequence so far. michael@0: // - Match node, optionally with an intermediate value in a different compact format. michael@0: // The value, if present, is for the string/UChar sequence so far. michael@0: // michael@0: // Aside from the value, which uses the node lead unit's high bits: michael@0: // michael@0: // - Linear-match node: Matches a number of units. michael@0: // - Branch node: Branches to other nodes according to the current input unit. michael@0: // The node unit is the length of the branch (number of units to select from) michael@0: // minus 1. It is followed by a sub-node: michael@0: // - If the length is at most kMaxBranchLinearSubNodeLength, then michael@0: // there are length-1 (key, value) pairs and then one more comparison unit. michael@0: // If one of the key units matches, then the value is either a final value for michael@0: // the string so far, or a "jump" delta to the next node. michael@0: // If the last unit matches, then matching continues with the next node. michael@0: // (Values have the same encoding as final-value nodes.) michael@0: // - If the length is greater than kMaxBranchLinearSubNodeLength, then michael@0: // there is one unit and one "jump" delta. michael@0: // If the input unit is less than the sub-node unit, then "jump" by delta to michael@0: // the next sub-node which will have a length of length/2. michael@0: // (The delta has its own compact encoding.) michael@0: // Otherwise, skip the "jump" delta to the next sub-node michael@0: // which will have a length of length-length/2. michael@0: michael@0: // Match-node lead unit values, after masking off intermediate-value bits: michael@0: michael@0: // 0000..002f: Branch node. If node!=0 then the length is node+1, otherwise michael@0: // the length is one more than the next unit. michael@0: michael@0: // For a branch sub-node with at most this many entries, we drop down michael@0: // to a linear search. michael@0: static const int32_t kMaxBranchLinearSubNodeLength=5; michael@0: michael@0: // 0030..003f: Linear-match node, match 1..16 units and continue reading the next node. michael@0: static const int32_t kMinLinearMatch=0x30; michael@0: static const int32_t kMaxLinearMatchLength=0x10; michael@0: michael@0: // Match-node lead unit bits 14..6 for the optional intermediate value. michael@0: // If these bits are 0, then there is no intermediate value. michael@0: // Otherwise, see the *NodeValue* constants below. michael@0: static const int32_t kMinValueLead=kMinLinearMatch+kMaxLinearMatchLength; // 0x0040 michael@0: static const int32_t kNodeTypeMask=kMinValueLead-1; // 0x003f michael@0: michael@0: // A final-value node has bit 15 set. michael@0: static const int32_t kValueIsFinal=0x8000; michael@0: michael@0: // Compact value: After testing and masking off bit 15, use the following thresholds. michael@0: static const int32_t kMaxOneUnitValue=0x3fff; michael@0: michael@0: static const int32_t kMinTwoUnitValueLead=kMaxOneUnitValue+1; // 0x4000 michael@0: static const int32_t kThreeUnitValueLead=0x7fff; michael@0: michael@0: static const int32_t kMaxTwoUnitValue=((kThreeUnitValueLead-kMinTwoUnitValueLead)<<16)-1; // 0x3ffeffff michael@0: michael@0: // Compact intermediate-value integer, lead unit shared with a branch or linear-match node. michael@0: static const int32_t kMaxOneUnitNodeValue=0xff; michael@0: static const int32_t kMinTwoUnitNodeValueLead=kMinValueLead+((kMaxOneUnitNodeValue+1)<<6); // 0x4040 michael@0: static const int32_t kThreeUnitNodeValueLead=0x7fc0; michael@0: michael@0: static const int32_t kMaxTwoUnitNodeValue= michael@0: ((kThreeUnitNodeValueLead-kMinTwoUnitNodeValueLead)<<10)-1; // 0xfdffff michael@0: michael@0: // Compact delta integers. michael@0: static const int32_t kMaxOneUnitDelta=0xfbff; michael@0: static const int32_t kMinTwoUnitDeltaLead=kMaxOneUnitDelta+1; // 0xfc00 michael@0: static const int32_t kThreeUnitDeltaLead=0xffff; michael@0: michael@0: static const int32_t kMaxTwoUnitDelta=((kThreeUnitDeltaLead-kMinTwoUnitDeltaLead)<<16)-1; // 0x03feffff michael@0: michael@0: UChar *ownedArray_; michael@0: michael@0: // Fixed value referencing the UCharsTrie words. michael@0: const UChar *uchars_; michael@0: michael@0: // Iterator variables. michael@0: michael@0: // Pointer to next trie unit to read. NULL if no more matches. michael@0: const UChar *pos_; michael@0: // Remaining length of a linear-match node, minus 1. Negative if not in such a node. michael@0: int32_t remainingMatchLength_; michael@0: }; michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: #endif // __UCHARSTRIE_H__