The Tor Browser / annotate

intl/icu/source/common/dictionarydata.h@6474c204b198 (annotated)

intl/icu/source/common/dictionarydata.h

Wed, 31 Dec 2014 06:09:35 +0100

author
Michael Schloh von Bennewitz <michael@schloh.com>
date
Wed, 31 Dec 2014 06:09:35 +0100
changeset 0
6474c204b198
permissions
-rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

michael@0 1 /*
michael@0 2 *******************************************************************************
michael@0 3 * Copyright (C) 2013, International Business Machines
michael@0 4 * Corporation and others. All Rights Reserved.
michael@0 5 *******************************************************************************
michael@0 6 * dictionarydata.h
michael@0 7 *
michael@0 8 * created on: 2012may31
michael@0 9 * created by: Markus W. Scherer & Maxime Serrano
michael@0 10 */
michael@0 11
michael@0 12 #ifndef __DICTIONARYDATA_H__
michael@0 13 #define __DICTIONARYDATA_H__
michael@0 14
michael@0 15 #include "unicode/utypes.h"
michael@0 16
michael@0 17 #if !UCONFIG_NO_BREAK_ITERATION
michael@0 18
michael@0 19 #include "unicode/utext.h"
michael@0 20 #include "unicode/udata.h"
michael@0 21 #include "udataswp.h"
michael@0 22 #include "unicode/uobject.h"
michael@0 23 #include "unicode/ustringtrie.h"
michael@0 24
michael@0 25 U_NAMESPACE_BEGIN
michael@0 26
michael@0 27 class UCharsTrie;
michael@0 28 class BytesTrie;
michael@0 29
michael@0 30 class U_COMMON_API DictionaryData : public UMemory {
michael@0 31 public:
michael@0 32 static const int32_t TRIE_TYPE_BYTES; // = 0;
michael@0 33 static const int32_t TRIE_TYPE_UCHARS; // = 1;
michael@0 34 static const int32_t TRIE_TYPE_MASK; // = 7;
michael@0 35 static const int32_t TRIE_HAS_VALUES; // = 8;
michael@0 36
michael@0 37 static const int32_t TRANSFORM_NONE; // = 0;
michael@0 38 static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000;
michael@0 39 static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000;
michael@0 40 static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff;
michael@0 41
michael@0 42 enum {
michael@0 43 // Byte offsets from the start of the data, after the generic header.
michael@0 44 IX_STRING_TRIE_OFFSET,
michael@0 45 IX_RESERVED1_OFFSET,
michael@0 46 IX_RESERVED2_OFFSET,
michael@0 47 IX_TOTAL_SIZE,
michael@0 48
michael@0 49 // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc.
michael@0 50 IX_TRIE_TYPE,
michael@0 51 // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc.
michael@0 52 IX_TRANSFORM,
michael@0 53
michael@0 54 IX_RESERVED6,
michael@0 55 IX_RESERVED7,
michael@0 56 IX_COUNT
michael@0 57 };
michael@0 58 };
michael@0 59
michael@0 60 /**
michael@0 61 * Wrapper class around generic dictionaries, implementing matches().
michael@0 62 * getType() should return a TRIE_TYPE_??? constant from DictionaryData.
michael@0 63 *
michael@0 64 * All implementations of this interface must be thread-safe if they are to be used inside of the
michael@0 65 * dictionary-based break iteration code.
michael@0 66 */
michael@0 67 class U_COMMON_API DictionaryMatcher : public UMemory {
michael@0 68 public:
michael@0 69 virtual ~DictionaryMatcher();
michael@0 70 // this should emulate CompactTrieDictionary::matches()
michael@0 71 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
michael@0 72 int32_t limit, int32_t *values = NULL) const = 0;
michael@0 73 /** @return DictionaryData::TRIE_TYPE_XYZ */
michael@0 74 virtual int32_t getType() const = 0;
michael@0 75 };
michael@0 76
michael@0 77 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary
michael@0 78 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher {
michael@0 79 public:
michael@0 80 // constructs a new UCharsDictionaryMatcher.
michael@0 81 // The UDataMemory * will be closed on this object's destruction.
michael@0 82 UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { }
michael@0 83 virtual ~UCharsDictionaryMatcher();
michael@0 84 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
michael@0 85 int32_t limit, int32_t *values = NULL) const;
michael@0 86 virtual int32_t getType() const;
michael@0 87 private:
michael@0 88 const UChar *characters;
michael@0 89 UDataMemory *file;
michael@0 90 };
michael@0 91
michael@0 92 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary
michael@0 93 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher {
michael@0 94 public:
michael@0 95 // constructs a new BytesTrieDictionaryMatcher
michael@0 96 // the transform constant should be the constant read from the file, not a masked version!
michael@0 97 // the UDataMemory * fed in here will be closed on this object's destruction
michael@0 98 BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f)
michael@0 99 : characters(c), transformConstant(t), file(f) { }
michael@0 100 virtual ~BytesDictionaryMatcher();
michael@0 101 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
michael@0 102 int32_t limit, int32_t *values = NULL) const;
michael@0 103 virtual int32_t getType() const;
michael@0 104 private:
michael@0 105 UChar32 transform(UChar32 c) const;
michael@0 106
michael@0 107 const char *characters;
michael@0 108 int32_t transformConstant;
michael@0 109 UDataMemory *file;
michael@0 110 };
michael@0 111
michael@0 112 U_NAMESPACE_END
michael@0 113
michael@0 114 U_CAPI int32_t U_EXPORT2
michael@0 115 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode);
michael@0 116
michael@0 117 /**
michael@0 118 * Format of dictionary .dict data files.
michael@0 119 * Format version 1.0.
michael@0 120 *
michael@0 121 * A dictionary .dict data file contains a byte-serialized BytesTrie or
michael@0 122 * a UChars-serialized UCharsTrie.
michael@0 123 * Such files are used in dictionary-based break iteration (DBBI).
michael@0 124 *
michael@0 125 * For a BytesTrie, a transformation type is specified for
michael@0 126 * transforming Unicode strings into byte sequences.
michael@0 127 *
michael@0 128 * A .dict file begins with a standard ICU data file header
michael@0 129 * (DataHeader, see ucmndata.h and unicode/udata.h).
michael@0 130 * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0).
michael@0 131 *
michael@0 132 * After the header, the file contains the following parts.
michael@0 133 * Constants are defined in the DictionaryData class.
michael@0 134 *
michael@0 135 * For the data structure of BytesTrie & UCharsTrie see
michael@0 136 * http://site.icu-project.org/design/struct/tries
michael@0 137 * and the bytestrie.h and ucharstrie.h header files.
michael@0 138 *
michael@0 139 * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4;
michael@0 140 *
michael@0 141 * The first four indexes are byte offsets in ascending order.
michael@0 142 * Each byte offset marks the start of the next part in the data file,
michael@0 143 * and the end of the previous one.
michael@0 144 * When two consecutive byte offsets are the same, then the corresponding part is empty.
michael@0 145 * Byte offsets are offsets from after the header,
michael@0 146 * that is, from the beginning of the indexes[].
michael@0 147 * Each part starts at an offset with proper alignment for its data.
michael@0 148 * If necessary, the previous part may include padding bytes to achieve this alignment.
michael@0 149 *
michael@0 150 * trieType=indexes[IX_TRIE_TYPE] defines the trie type.
michael@0 151 * transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation.
michael@0 152 * If the transformation type is TRANSFORM_TYPE_OFFSET,
michael@0 153 * then the lower 21 bits contain the offset code point.
michael@0 154 * Each code point c is mapped to byte b = (c - offset).
michael@0 155 * Code points outside the range offset..(offset+0xff) cannot be mapped
michael@0 156 * and do not occur in the dictionary.
michael@0 157 *
michael@0 158 * stringTrie; -- a serialized BytesTrie or UCharsTrie
michael@0 159 *
michael@0 160 * The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType),
michael@0 161 * or it maps all strings to 0 (TRIE_HAS_VALUES bit not set).
michael@0 162 */
michael@0 163
michael@0 164 #endif /* !UCONFIG_NO_BREAK_ITERATION */
michael@0 165 #endif /* __DICTIONARYDATA_H__ */

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial