The Tor Browser: intl/icu/source/common/dictionarydata.h@fc2d59ddac77

     1 /*

     2 *******************************************************************************

     3 * Copyright (C) 2013, International Business Machines

     4 * Corporation and others.  All Rights Reserved.

     5 *******************************************************************************

     6 * dictionarydata.h

7 *

     8 * created on: 2012may31

     9 * created by: Markus W. Scherer & Maxime Serrano

    10 */

    12 #ifndef __DICTIONARYDATA_H__

    13 #define __DICTIONARYDATA_H__

    15 #include "unicode/utypes.h"

    17 #if !UCONFIG_NO_BREAK_ITERATION

    19 #include "unicode/utext.h"

    20 #include "unicode/udata.h"

    21 #include "udataswp.h"

    22 #include "unicode/uobject.h"

    23 #include "unicode/ustringtrie.h"

    25 U_NAMESPACE_BEGIN

    27 class UCharsTrie;

    28 class BytesTrie;

    30 class U_COMMON_API DictionaryData : public UMemory {

    31 public:

    32     static const int32_t TRIE_TYPE_BYTES; // = 0;

    33     static const int32_t TRIE_TYPE_UCHARS; // = 1;

    34     static const int32_t TRIE_TYPE_MASK; // = 7;

    35     static const int32_t TRIE_HAS_VALUES; // = 8;

    37     static const int32_t TRANSFORM_NONE; // = 0;

    38     static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000;

    39     static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000;

    40     static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff;

    42     enum {

    43         // Byte offsets from the start of the data, after the generic header.

    44         IX_STRING_TRIE_OFFSET,

    45         IX_RESERVED1_OFFSET,

    46         IX_RESERVED2_OFFSET,

    47         IX_TOTAL_SIZE,

    49         // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc.

    50         IX_TRIE_TYPE,

    51         // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc.

    52         IX_TRANSFORM,

    54         IX_RESERVED6,

    55         IX_RESERVED7,

    56         IX_COUNT

    57     };

    58 };

    60 /**

    61  * Wrapper class around generic dictionaries, implementing matches().

    62  * getType() should return a TRIE_TYPE_??? constant from DictionaryData.

    63  *

    64  * All implementations of this interface must be thread-safe if they are to be used inside of the

    65  * dictionary-based break iteration code.

    66  */

    67 class U_COMMON_API DictionaryMatcher : public UMemory {

    68 public:

    69     virtual ~DictionaryMatcher();

    70     // this should emulate CompactTrieDictionary::matches()

    71     virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,

    72                             int32_t limit, int32_t *values = NULL) const = 0;

    73     /** @return DictionaryData::TRIE_TYPE_XYZ */

    74     virtual int32_t getType() const = 0;

    75 };

    77 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary

    78 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher {

    79 public:

    80     // constructs a new UCharsDictionaryMatcher.

    81     // The UDataMemory * will be closed on this object's destruction.

    82     UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { }

    83     virtual ~UCharsDictionaryMatcher();

    84     virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,

    85                             int32_t limit, int32_t *values = NULL) const;

    86     virtual int32_t getType() const;

    87 private:

    88     const UChar *characters;

    89     UDataMemory *file;

    90 };

    92 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary

    93 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher {

    94 public:

    95     // constructs a new BytesTrieDictionaryMatcher

    96     // the transform constant should be the constant read from the file, not a masked version!

    97     // the UDataMemory * fed in here will be closed on this object's destruction

    98     BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f)

    99             : characters(c), transformConstant(t), file(f) { }

   100     virtual ~BytesDictionaryMatcher();

   101     virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,

   102                             int32_t limit, int32_t *values = NULL) const;

   103     virtual int32_t getType() const;

   104 private:

   105     UChar32 transform(UChar32 c) const;

   107     const char *characters;

   108     int32_t transformConstant;

   109     UDataMemory *file;

   110 };

   112 U_NAMESPACE_END

   114 U_CAPI int32_t U_EXPORT2

   115 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode);

   117 /**

   118  * Format of dictionary .dict data files.

   119  * Format version 1.0.

   120  *

   121  * A dictionary .dict data file contains a byte-serialized BytesTrie or

   122  * a UChars-serialized UCharsTrie.

   123  * Such files are used in dictionary-based break iteration (DBBI).

   124  *

   125  * For a BytesTrie, a transformation type is specified for

   126  * transforming Unicode strings into byte sequences.

   127  *

   128  * A .dict file begins with a standard ICU data file header

   129  * (DataHeader, see ucmndata.h and unicode/udata.h).

   130  * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0).

   131  *

   132  * After the header, the file contains the following parts.

   133  * Constants are defined in the DictionaryData class.

   134  *

   135  * For the data structure of BytesTrie & UCharsTrie see

   136  * http://site.icu-project.org/design/struct/tries

   137  * and the bytestrie.h and ucharstrie.h header files.

   138  *

   139  * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4;

   140  *

   141  *      The first four indexes are byte offsets in ascending order.

   142  *      Each byte offset marks the start of the next part in the data file,

   143  *      and the end of the previous one.

   144  *      When two consecutive byte offsets are the same, then the corresponding part is empty.

   145  *      Byte offsets are offsets from after the header,

   146  *      that is, from the beginning of the indexes[].

   147  *      Each part starts at an offset with proper alignment for its data.

   148  *      If necessary, the previous part may include padding bytes to achieve this alignment.

   149  *

   150  *      trieType=indexes[IX_TRIE_TYPE] defines the trie type.

   151  *      transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation.

   152  *          If the transformation type is TRANSFORM_TYPE_OFFSET,

   153  *          then the lower 21 bits contain the offset code point.

   154  *          Each code point c is mapped to byte b = (c - offset).

   155  *          Code points outside the range offset..(offset+0xff) cannot be mapped

   156  *          and do not occur in the dictionary.

   157  *

   158  * stringTrie; -- a serialized BytesTrie or UCharsTrie

   159  *

   160  *      The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType),

   161  *      or it maps all strings to 0 (TRIE_HAS_VALUES bit not set).

   162  */

   164 #endif  /* !UCONFIG_NO_BREAK_ITERATION */

   165 #endif  /* __DICTIONARYDATA_H__ */

The Tor Browser / file revision