michael@0: /* michael@0: ******************************************************************************* michael@0: * Copyright (C) 2010-2012, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: ******************************************************************************* michael@0: * file name: bytestrie.h michael@0: * encoding: US-ASCII michael@0: * tab size: 8 (not used) michael@0: * indentation:4 michael@0: * michael@0: * created on: 2010sep25 michael@0: * created by: Markus W. Scherer michael@0: */ michael@0: michael@0: #ifndef __BYTESTRIE_H__ michael@0: #define __BYTESTRIE_H__ michael@0: michael@0: /** michael@0: * \file michael@0: * \brief C++ API: Trie for mapping byte sequences to integer values. michael@0: */ michael@0: michael@0: #include "unicode/utypes.h" michael@0: #include "unicode/stringpiece.h" michael@0: #include "unicode/uobject.h" michael@0: #include "unicode/ustringtrie.h" michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: class ByteSink; michael@0: class BytesTrieBuilder; michael@0: class CharString; michael@0: class UVector32; michael@0: michael@0: /** michael@0: * Light-weight, non-const reader class for a BytesTrie. michael@0: * Traverses a byte-serialized data structure with minimal state, michael@0: * for mapping byte sequences to non-negative integer values. michael@0: * michael@0: * This class owns the serialized trie data only if it was constructed by michael@0: * the builder's build() method. michael@0: * The public constructor and the copy constructor only alias the data (only copy the pointer). michael@0: * There is no assignment operator. michael@0: * michael@0: * This class is not intended for public subclassing. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: class U_COMMON_API BytesTrie : public UMemory { michael@0: public: michael@0: /** michael@0: * Constructs a BytesTrie reader instance. michael@0: * michael@0: * The trieBytes must contain a copy of a byte sequence from the BytesTrieBuilder, michael@0: * starting with the first byte of that sequence. michael@0: * The BytesTrie object will not read more bytes than michael@0: * the BytesTrieBuilder generated in the corresponding build() call. michael@0: * michael@0: * The array is not copied/cloned and must not be modified while michael@0: * the BytesTrie object is in use. michael@0: * michael@0: * @param trieBytes The byte array that contains the serialized trie. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: BytesTrie(const void *trieBytes) michael@0: : ownedArray_(NULL), bytes_(static_cast(trieBytes)), michael@0: pos_(bytes_), remainingMatchLength_(-1) {} michael@0: michael@0: /** michael@0: * Destructor. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: ~BytesTrie(); michael@0: michael@0: /** michael@0: * Copy constructor, copies the other trie reader object and its state, michael@0: * but not the byte array which will be shared. (Shallow copy.) michael@0: * @param other Another BytesTrie object. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: BytesTrie(const BytesTrie &other) michael@0: : ownedArray_(NULL), bytes_(other.bytes_), michael@0: pos_(other.pos_), remainingMatchLength_(other.remainingMatchLength_) {} michael@0: michael@0: /** michael@0: * Resets this trie to its initial state. michael@0: * @return *this michael@0: * @stable ICU 4.8 michael@0: */ michael@0: BytesTrie &reset() { michael@0: pos_=bytes_; michael@0: remainingMatchLength_=-1; michael@0: return *this; michael@0: } michael@0: michael@0: /** michael@0: * BytesTrie state object, for saving a trie's current state michael@0: * and resetting the trie back to this state later. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: class State : public UMemory { michael@0: public: michael@0: /** michael@0: * Constructs an empty State. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: State() { bytes=NULL; } michael@0: private: michael@0: friend class BytesTrie; michael@0: michael@0: const uint8_t *bytes; michael@0: const uint8_t *pos; michael@0: int32_t remainingMatchLength; michael@0: }; michael@0: michael@0: /** michael@0: * Saves the state of this trie. michael@0: * @param state The State object to hold the trie's state. michael@0: * @return *this michael@0: * @see resetToState michael@0: * @stable ICU 4.8 michael@0: */ michael@0: const BytesTrie &saveState(State &state) const { michael@0: state.bytes=bytes_; michael@0: state.pos=pos_; michael@0: state.remainingMatchLength=remainingMatchLength_; michael@0: return *this; michael@0: } michael@0: michael@0: /** michael@0: * Resets this trie to the saved state. michael@0: * If the state object contains no state, or the state of a different trie, michael@0: * then this trie remains unchanged. michael@0: * @param state The State object which holds a saved trie state. michael@0: * @return *this michael@0: * @see saveState michael@0: * @see reset michael@0: * @stable ICU 4.8 michael@0: */ michael@0: BytesTrie &resetToState(const State &state) { michael@0: if(bytes_==state.bytes && bytes_!=NULL) { michael@0: pos_=state.pos; michael@0: remainingMatchLength_=state.remainingMatchLength; michael@0: } michael@0: return *this; michael@0: } michael@0: michael@0: /** michael@0: * Determines whether the byte sequence so far matches, whether it has a value, michael@0: * and whether another input byte can continue a matching byte sequence. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult current() const; michael@0: michael@0: /** michael@0: * Traverses the trie from the initial state for this input byte. michael@0: * Equivalent to reset().next(inByte). michael@0: * @param inByte Input byte value. Values -0x100..-1 are treated like 0..0xff. michael@0: * Values below -0x100 and above 0xff will never match. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: inline UStringTrieResult first(int32_t inByte) { michael@0: remainingMatchLength_=-1; michael@0: if(inByte<0) { michael@0: inByte+=0x100; michael@0: } michael@0: return nextImpl(bytes_, inByte); michael@0: } michael@0: michael@0: /** michael@0: * Traverses the trie from the current state for this input byte. michael@0: * @param inByte Input byte value. Values -0x100..-1 are treated like 0..0xff. michael@0: * Values below -0x100 and above 0xff will never match. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult next(int32_t inByte); michael@0: michael@0: /** michael@0: * Traverses the trie from the current state for this byte sequence. michael@0: * Equivalent to michael@0: * \code michael@0: * Result result=current(); michael@0: * for(each c in s) michael@0: * if(!USTRINGTRIE_HAS_NEXT(result)) return USTRINGTRIE_NO_MATCH; michael@0: * result=next(c); michael@0: * return result; michael@0: * \endcode michael@0: * @param s A string or byte sequence. Can be NULL if length is 0. michael@0: * @param length The length of the byte sequence. Can be -1 if NUL-terminated. michael@0: * @return The match/value Result. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UStringTrieResult next(const char *s, int32_t length); michael@0: michael@0: /** michael@0: * Returns a matching byte sequence's value if called immediately after michael@0: * current()/first()/next() returned USTRINGTRIE_INTERMEDIATE_VALUE or USTRINGTRIE_FINAL_VALUE. michael@0: * getValue() can be called multiple times. michael@0: * michael@0: * Do not call getValue() after USTRINGTRIE_NO_MATCH or USTRINGTRIE_NO_VALUE! michael@0: * @return The value for the byte sequence so far. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: inline int32_t getValue() const { michael@0: const uint8_t *pos=pos_; michael@0: int32_t leadByte=*pos++; michael@0: // U_ASSERT(leadByte>=kMinValueLead); michael@0: return readValue(pos, leadByte>>1); michael@0: } michael@0: michael@0: /** michael@0: * Determines whether all byte sequences reachable from the current state michael@0: * map to the same value. michael@0: * @param uniqueValue Receives the unique value, if this function returns TRUE. michael@0: * (output-only) michael@0: * @return TRUE if all byte sequences reachable from the current state michael@0: * map to the same value. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: inline UBool hasUniqueValue(int32_t &uniqueValue) const { michael@0: const uint8_t *pos=pos_; michael@0: // Skip the rest of a pending linear-match node. michael@0: return pos!=NULL && findUniqueValue(pos+remainingMatchLength_+1, FALSE, uniqueValue); michael@0: } michael@0: michael@0: /** michael@0: * Finds each byte which continues the byte sequence from the current state. michael@0: * That is, each byte b for which it would be next(b)!=USTRINGTRIE_NO_MATCH now. michael@0: * @param out Each next byte is appended to this object. michael@0: * (Only uses the out.Append(s, length) method.) michael@0: * @return the number of bytes which continue the byte sequence from here michael@0: * @stable ICU 4.8 michael@0: */ michael@0: int32_t getNextBytes(ByteSink &out) const; michael@0: michael@0: /** michael@0: * Iterator for all of the (byte sequence, value) pairs in a BytesTrie. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: class U_COMMON_API Iterator : public UMemory { michael@0: public: michael@0: /** michael@0: * Iterates from the root of a byte-serialized BytesTrie. michael@0: * @param trieBytes The trie bytes. michael@0: * @param maxStringLength If 0, the iterator returns full strings/byte sequences. michael@0: * Otherwise, the iterator returns strings with this maximum length. michael@0: * @param errorCode Standard ICU error code. Its input value must michael@0: * pass the U_SUCCESS() test, or else the function returns michael@0: * immediately. Check for U_FAILURE() on output or use with michael@0: * function chaining. (See User Guide for details.) michael@0: * @stable ICU 4.8 michael@0: */ michael@0: Iterator(const void *trieBytes, int32_t maxStringLength, UErrorCode &errorCode); michael@0: michael@0: /** michael@0: * Iterates from the current state of the specified BytesTrie. michael@0: * @param trie The trie whose state will be copied for iteration. michael@0: * @param maxStringLength If 0, the iterator returns full strings/byte sequences. michael@0: * Otherwise, the iterator returns strings with this maximum length. michael@0: * @param errorCode Standard ICU error code. Its input value must michael@0: * pass the U_SUCCESS() test, or else the function returns michael@0: * immediately. Check for U_FAILURE() on output or use with michael@0: * function chaining. (See User Guide for details.) michael@0: * @stable ICU 4.8 michael@0: */ michael@0: Iterator(const BytesTrie &trie, int32_t maxStringLength, UErrorCode &errorCode); michael@0: michael@0: /** michael@0: * Destructor. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: ~Iterator(); michael@0: michael@0: /** michael@0: * Resets this iterator to its initial state. michael@0: * @return *this michael@0: * @stable ICU 4.8 michael@0: */ michael@0: Iterator &reset(); michael@0: michael@0: /** michael@0: * @return TRUE if there are more elements. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UBool hasNext() const; michael@0: michael@0: /** michael@0: * Finds the next (byte sequence, value) pair if there is one. michael@0: * michael@0: * If the byte sequence is truncated to the maximum length and does not michael@0: * have a real value, then the value is set to -1. michael@0: * In this case, this "not a real value" is indistinguishable from michael@0: * a real value of -1. michael@0: * @param errorCode Standard ICU error code. Its input value must michael@0: * pass the U_SUCCESS() test, or else the function returns michael@0: * immediately. Check for U_FAILURE() on output or use with michael@0: * function chaining. (See User Guide for details.) michael@0: * @return TRUE if there is another element. michael@0: * @stable ICU 4.8 michael@0: */ michael@0: UBool next(UErrorCode &errorCode); michael@0: michael@0: /** michael@0: * @return The NUL-terminated byte sequence for the last successful next(). michael@0: * @stable ICU 4.8 michael@0: */ michael@0: const StringPiece &getString() const { return sp_; } michael@0: /** michael@0: * @return The value for the last successful next(). michael@0: * @stable ICU 4.8 michael@0: */ michael@0: int32_t getValue() const { return value_; } michael@0: michael@0: private: michael@0: UBool truncateAndStop(); michael@0: michael@0: const uint8_t *branchNext(const uint8_t *pos, int32_t length, UErrorCode &errorCode); michael@0: michael@0: const uint8_t *bytes_; michael@0: const uint8_t *pos_; michael@0: const uint8_t *initialPos_; michael@0: int32_t remainingMatchLength_; michael@0: int32_t initialRemainingMatchLength_; michael@0: michael@0: CharString *str_; michael@0: StringPiece sp_; michael@0: int32_t maxLength_; michael@0: int32_t value_; michael@0: michael@0: // The stack stores pairs of integers for backtracking to another michael@0: // outbound edge of a branch node. michael@0: // The first integer is an offset from bytes_. michael@0: // The second integer has the str_->length() from before the node in bits 15..0, michael@0: // and the remaining branch length in bits 24..16. (Bits 31..25 are unused.) michael@0: // (We could store the remaining branch length minus 1 in bits 23..16 and not use bits 31..24, michael@0: // but the code looks more confusing that way.) michael@0: UVector32 *stack_; michael@0: }; michael@0: michael@0: private: michael@0: friend class BytesTrieBuilder; michael@0: michael@0: /** michael@0: * Constructs a BytesTrie reader instance. michael@0: * Unlike the public constructor which just aliases an array, michael@0: * this constructor adopts the builder's array. michael@0: * This constructor is only called by the builder. michael@0: */ michael@0: BytesTrie(void *adoptBytes, const void *trieBytes) michael@0: : ownedArray_(static_cast(adoptBytes)), michael@0: bytes_(static_cast(trieBytes)), michael@0: pos_(bytes_), remainingMatchLength_(-1) {} michael@0: michael@0: // No assignment operator. michael@0: BytesTrie &operator=(const BytesTrie &other); michael@0: michael@0: inline void stop() { michael@0: pos_=NULL; michael@0: } michael@0: michael@0: // Reads a compact 32-bit integer. michael@0: // pos is already after the leadByte, and the lead byte is already shifted right by 1. michael@0: static int32_t readValue(const uint8_t *pos, int32_t leadByte); michael@0: static inline const uint8_t *skipValue(const uint8_t *pos, int32_t leadByte) { michael@0: // U_ASSERT(leadByte>=kMinValueLead); michael@0: if(leadByte>=(kMinTwoByteValueLead<<1)) { michael@0: if(leadByte<(kMinThreeByteValueLead<<1)) { michael@0: ++pos; michael@0: } else if(leadByte<(kFourByteValueLead<<1)) { michael@0: pos+=2; michael@0: } else { michael@0: pos+=3+((leadByte>>1)&1); michael@0: } michael@0: } michael@0: return pos; michael@0: } michael@0: static inline const uint8_t *skipValue(const uint8_t *pos) { michael@0: int32_t leadByte=*pos++; michael@0: return skipValue(pos, leadByte); michael@0: } michael@0: michael@0: // Reads a jump delta and jumps. michael@0: static const uint8_t *jumpByDelta(const uint8_t *pos); michael@0: michael@0: static inline const uint8_t *skipDelta(const uint8_t *pos) { michael@0: int32_t delta=*pos++; michael@0: if(delta>=kMinTwoByteDeltaLead) { michael@0: if(delta>8)+1; // 0x6c michael@0: static const int32_t kFourByteValueLead=0x7e; michael@0: michael@0: // A little more than Unicode code points. (0x11ffff) michael@0: static const int32_t kMaxThreeByteValue=((kFourByteValueLead-kMinThreeByteValueLead)<<16)-1; michael@0: michael@0: static const int32_t kFiveByteValueLead=0x7f; michael@0: michael@0: // Compact delta integers. michael@0: static const int32_t kMaxOneByteDelta=0xbf; michael@0: static const int32_t kMinTwoByteDeltaLead=kMaxOneByteDelta+1; // 0xc0 michael@0: static const int32_t kMinThreeByteDeltaLead=0xf0; michael@0: static const int32_t kFourByteDeltaLead=0xfe; michael@0: static const int32_t kFiveByteDeltaLead=0xff; michael@0: michael@0: static const int32_t kMaxTwoByteDelta=((kMinThreeByteDeltaLead-kMinTwoByteDeltaLead)<<8)-1; // 0x2fff michael@0: static const int32_t kMaxThreeByteDelta=((kFourByteDeltaLead-kMinThreeByteDeltaLead)<<16)-1; // 0xdffff michael@0: michael@0: uint8_t *ownedArray_; michael@0: michael@0: // Fixed value referencing the BytesTrie bytes. michael@0: const uint8_t *bytes_; michael@0: michael@0: // Iterator variables. michael@0: michael@0: // Pointer to next trie byte to read. NULL if no more matches. michael@0: const uint8_t *pos_; michael@0: // Remaining length of a linear-match node, minus 1. Negative if not in such a node. michael@0: int32_t remainingMatchLength_; michael@0: }; michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: #endif // __BYTESTRIE_H__