The Tor Browser / file comparison
comparison: intl/icu/source/common/dictionarydata.h
intl/icu/source/common/dictionarydata.h
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:1369fdf80328 |
|---|---|
| 1 /* | |
| 2 ******************************************************************************* | |
| 3 * Copyright (C) 2013, International Business Machines | |
| 4 * Corporation and others. All Rights Reserved. | |
| 5 ******************************************************************************* | |
| 6 * dictionarydata.h | |
| 7 * | |
| 8 * created on: 2012may31 | |
| 9 * created by: Markus W. Scherer & Maxime Serrano | |
| 10 */ | |
| 11 | |
| 12 #ifndef __DICTIONARYDATA_H__ | |
| 13 #define __DICTIONARYDATA_H__ | |
| 14 | |
| 15 #include "unicode/utypes.h" | |
| 16 | |
| 17 #if !UCONFIG_NO_BREAK_ITERATION | |
| 18 | |
| 19 #include "unicode/utext.h" | |
| 20 #include "unicode/udata.h" | |
| 21 #include "udataswp.h" | |
| 22 #include "unicode/uobject.h" | |
| 23 #include "unicode/ustringtrie.h" | |
| 24 | |
| 25 U_NAMESPACE_BEGIN | |
| 26 | |
| 27 class UCharsTrie; | |
| 28 class BytesTrie; | |
| 29 | |
| 30 class U_COMMON_API DictionaryData : public UMemory { | |
| 31 public: | |
| 32 static const int32_t TRIE_TYPE_BYTES; // = 0; | |
| 33 static const int32_t TRIE_TYPE_UCHARS; // = 1; | |
| 34 static const int32_t TRIE_TYPE_MASK; // = 7; | |
| 35 static const int32_t TRIE_HAS_VALUES; // = 8; | |
| 36 | |
| 37 static const int32_t TRANSFORM_NONE; // = 0; | |
| 38 static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000; | |
| 39 static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000; | |
| 40 static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff; | |
| 41 | |
| 42 enum { | |
| 43 // Byte offsets from the start of the data, after the generic header. | |
| 44 IX_STRING_TRIE_OFFSET, | |
| 45 IX_RESERVED1_OFFSET, | |
| 46 IX_RESERVED2_OFFSET, | |
| 47 IX_TOTAL_SIZE, | |
| 48 | |
| 49 // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc. | |
| 50 IX_TRIE_TYPE, | |
| 51 // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc. | |
| 52 IX_TRANSFORM, | |
| 53 | |
| 54 IX_RESERVED6, | |
| 55 IX_RESERVED7, | |
| 56 IX_COUNT | |
| 57 }; | |
| 58 }; | |
| 59 | |
| 60 /** | |
| 61 * Wrapper class around generic dictionaries, implementing matches(). | |
| 62 * getType() should return a TRIE_TYPE_??? constant from DictionaryData. | |
| 63 * | |
| 64 * All implementations of this interface must be thread-safe if they are to be used inside of the | |
| 65 * dictionary-based break iteration code. | |
| 66 */ | |
| 67 class U_COMMON_API DictionaryMatcher : public UMemory { | |
| 68 public: | |
| 69 virtual ~DictionaryMatcher(); | |
| 70 // this should emulate CompactTrieDictionary::matches() | |
| 71 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count, | |
| 72 int32_t limit, int32_t *values = NULL) const = 0; | |
| 73 /** @return DictionaryData::TRIE_TYPE_XYZ */ | |
| 74 virtual int32_t getType() const = 0; | |
| 75 }; | |
| 76 | |
| 77 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary | |
| 78 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher { | |
| 79 public: | |
| 80 // constructs a new UCharsDictionaryMatcher. | |
| 81 // The UDataMemory * will be closed on this object's destruction. | |
| 82 UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { } | |
| 83 virtual ~UCharsDictionaryMatcher(); | |
| 84 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count, | |
| 85 int32_t limit, int32_t *values = NULL) const; | |
| 86 virtual int32_t getType() const; | |
| 87 private: | |
| 88 const UChar *characters; | |
| 89 UDataMemory *file; | |
| 90 }; | |
| 91 | |
| 92 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary | |
| 93 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher { | |
| 94 public: | |
| 95 // constructs a new BytesTrieDictionaryMatcher | |
| 96 // the transform constant should be the constant read from the file, not a masked version! | |
| 97 // the UDataMemory * fed in here will be closed on this object's destruction | |
| 98 BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f) | |
| 99 : characters(c), transformConstant(t), file(f) { } | |
| 100 virtual ~BytesDictionaryMatcher(); | |
| 101 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count, | |
| 102 int32_t limit, int32_t *values = NULL) const; | |
| 103 virtual int32_t getType() const; | |
| 104 private: | |
| 105 UChar32 transform(UChar32 c) const; | |
| 106 | |
| 107 const char *characters; | |
| 108 int32_t transformConstant; | |
| 109 UDataMemory *file; | |
| 110 }; | |
| 111 | |
| 112 U_NAMESPACE_END | |
| 113 | |
| 114 U_CAPI int32_t U_EXPORT2 | |
| 115 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode); | |
| 116 | |
| 117 /** | |
| 118 * Format of dictionary .dict data files. | |
| 119 * Format version 1.0. | |
| 120 * | |
| 121 * A dictionary .dict data file contains a byte-serialized BytesTrie or | |
| 122 * a UChars-serialized UCharsTrie. | |
| 123 * Such files are used in dictionary-based break iteration (DBBI). | |
| 124 * | |
| 125 * For a BytesTrie, a transformation type is specified for | |
| 126 * transforming Unicode strings into byte sequences. | |
| 127 * | |
| 128 * A .dict file begins with a standard ICU data file header | |
| 129 * (DataHeader, see ucmndata.h and unicode/udata.h). | |
| 130 * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0). | |
| 131 * | |
| 132 * After the header, the file contains the following parts. | |
| 133 * Constants are defined in the DictionaryData class. | |
| 134 * | |
| 135 * For the data structure of BytesTrie & UCharsTrie see | |
| 136 * http://site.icu-project.org/design/struct/tries | |
| 137 * and the bytestrie.h and ucharstrie.h header files. | |
| 138 * | |
| 139 * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4; | |
| 140 * | |
| 141 * The first four indexes are byte offsets in ascending order. | |
| 142 * Each byte offset marks the start of the next part in the data file, | |
| 143 * and the end of the previous one. | |
| 144 * When two consecutive byte offsets are the same, then the corresponding part is empty. | |
| 145 * Byte offsets are offsets from after the header, | |
| 146 * that is, from the beginning of the indexes[]. | |
| 147 * Each part starts at an offset with proper alignment for its data. | |
| 148 * If necessary, the previous part may include padding bytes to achieve this alignment. | |
| 149 * | |
| 150 * trieType=indexes[IX_TRIE_TYPE] defines the trie type. | |
| 151 * transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation. | |
| 152 * If the transformation type is TRANSFORM_TYPE_OFFSET, | |
| 153 * then the lower 21 bits contain the offset code point. | |
| 154 * Each code point c is mapped to byte b = (c - offset). | |
| 155 * Code points outside the range offset..(offset+0xff) cannot be mapped | |
| 156 * and do not occur in the dictionary. | |
| 157 * | |
| 158 * stringTrie; -- a serialized BytesTrie or UCharsTrie | |
| 159 * | |
| 160 * The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType), | |
| 161 * or it maps all strings to 0 (TRIE_HAS_VALUES bit not set). | |
| 162 */ | |
| 163 | |
| 164 #endif /* !UCONFIG_NO_BREAK_ITERATION */ | |
| 165 #endif /* __DICTIONARYDATA_H__ */ |
