1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/common/dictionarydata.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,165 @@
1.4 +/*
1.5 +*******************************************************************************
1.6 +* Copyright (C) 2013, International Business Machines
1.7 +* Corporation and others. All Rights Reserved.
1.8 +*******************************************************************************
1.9 +* dictionarydata.h
1.10 +*
1.11 +* created on: 2012may31
1.12 +* created by: Markus W. Scherer & Maxime Serrano
1.13 +*/
1.14 +
1.15 +#ifndef __DICTIONARYDATA_H__
1.16 +#define __DICTIONARYDATA_H__
1.17 +
1.18 +#include "unicode/utypes.h"
1.19 +
1.20 +#if !UCONFIG_NO_BREAK_ITERATION
1.21 +
1.22 +#include "unicode/utext.h"
1.23 +#include "unicode/udata.h"
1.24 +#include "udataswp.h"
1.25 +#include "unicode/uobject.h"
1.26 +#include "unicode/ustringtrie.h"
1.27 +
1.28 +U_NAMESPACE_BEGIN
1.29 +
1.30 +class UCharsTrie;
1.31 +class BytesTrie;
1.32 +
1.33 +class U_COMMON_API DictionaryData : public UMemory {
1.34 +public:
1.35 + static const int32_t TRIE_TYPE_BYTES; // = 0;
1.36 + static const int32_t TRIE_TYPE_UCHARS; // = 1;
1.37 + static const int32_t TRIE_TYPE_MASK; // = 7;
1.38 + static const int32_t TRIE_HAS_VALUES; // = 8;
1.39 +
1.40 + static const int32_t TRANSFORM_NONE; // = 0;
1.41 + static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000;
1.42 + static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000;
1.43 + static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff;
1.44 +
1.45 + enum {
1.46 + // Byte offsets from the start of the data, after the generic header.
1.47 + IX_STRING_TRIE_OFFSET,
1.48 + IX_RESERVED1_OFFSET,
1.49 + IX_RESERVED2_OFFSET,
1.50 + IX_TOTAL_SIZE,
1.51 +
1.52 + // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc.
1.53 + IX_TRIE_TYPE,
1.54 + // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc.
1.55 + IX_TRANSFORM,
1.56 +
1.57 + IX_RESERVED6,
1.58 + IX_RESERVED7,
1.59 + IX_COUNT
1.60 + };
1.61 +};
1.62 +
1.63 +/**
1.64 + * Wrapper class around generic dictionaries, implementing matches().
1.65 + * getType() should return a TRIE_TYPE_??? constant from DictionaryData.
1.66 + *
1.67 + * All implementations of this interface must be thread-safe if they are to be used inside of the
1.68 + * dictionary-based break iteration code.
1.69 + */
1.70 +class U_COMMON_API DictionaryMatcher : public UMemory {
1.71 +public:
1.72 + virtual ~DictionaryMatcher();
1.73 + // this should emulate CompactTrieDictionary::matches()
1.74 + virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
1.75 + int32_t limit, int32_t *values = NULL) const = 0;
1.76 + /** @return DictionaryData::TRIE_TYPE_XYZ */
1.77 + virtual int32_t getType() const = 0;
1.78 +};
1.79 +
1.80 +// Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary
1.81 +class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher {
1.82 +public:
1.83 + // constructs a new UCharsDictionaryMatcher.
1.84 + // The UDataMemory * will be closed on this object's destruction.
1.85 + UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { }
1.86 + virtual ~UCharsDictionaryMatcher();
1.87 + virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
1.88 + int32_t limit, int32_t *values = NULL) const;
1.89 + virtual int32_t getType() const;
1.90 +private:
1.91 + const UChar *characters;
1.92 + UDataMemory *file;
1.93 +};
1.94 +
1.95 +// Implementation of the DictionaryMatcher interface for a BytesTrie dictionary
1.96 +class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher {
1.97 +public:
1.98 + // constructs a new BytesTrieDictionaryMatcher
1.99 + // the transform constant should be the constant read from the file, not a masked version!
1.100 + // the UDataMemory * fed in here will be closed on this object's destruction
1.101 + BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f)
1.102 + : characters(c), transformConstant(t), file(f) { }
1.103 + virtual ~BytesDictionaryMatcher();
1.104 + virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
1.105 + int32_t limit, int32_t *values = NULL) const;
1.106 + virtual int32_t getType() const;
1.107 +private:
1.108 + UChar32 transform(UChar32 c) const;
1.109 +
1.110 + const char *characters;
1.111 + int32_t transformConstant;
1.112 + UDataMemory *file;
1.113 +};
1.114 +
1.115 +U_NAMESPACE_END
1.116 +
1.117 +U_CAPI int32_t U_EXPORT2
1.118 +udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode);
1.119 +
1.120 +/**
1.121 + * Format of dictionary .dict data files.
1.122 + * Format version 1.0.
1.123 + *
1.124 + * A dictionary .dict data file contains a byte-serialized BytesTrie or
1.125 + * a UChars-serialized UCharsTrie.
1.126 + * Such files are used in dictionary-based break iteration (DBBI).
1.127 + *
1.128 + * For a BytesTrie, a transformation type is specified for
1.129 + * transforming Unicode strings into byte sequences.
1.130 + *
1.131 + * A .dict file begins with a standard ICU data file header
1.132 + * (DataHeader, see ucmndata.h and unicode/udata.h).
1.133 + * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0).
1.134 + *
1.135 + * After the header, the file contains the following parts.
1.136 + * Constants are defined in the DictionaryData class.
1.137 + *
1.138 + * For the data structure of BytesTrie & UCharsTrie see
1.139 + * http://site.icu-project.org/design/struct/tries
1.140 + * and the bytestrie.h and ucharstrie.h header files.
1.141 + *
1.142 + * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4;
1.143 + *
1.144 + * The first four indexes are byte offsets in ascending order.
1.145 + * Each byte offset marks the start of the next part in the data file,
1.146 + * and the end of the previous one.
1.147 + * When two consecutive byte offsets are the same, then the corresponding part is empty.
1.148 + * Byte offsets are offsets from after the header,
1.149 + * that is, from the beginning of the indexes[].
1.150 + * Each part starts at an offset with proper alignment for its data.
1.151 + * If necessary, the previous part may include padding bytes to achieve this alignment.
1.152 + *
1.153 + * trieType=indexes[IX_TRIE_TYPE] defines the trie type.
1.154 + * transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation.
1.155 + * If the transformation type is TRANSFORM_TYPE_OFFSET,
1.156 + * then the lower 21 bits contain the offset code point.
1.157 + * Each code point c is mapped to byte b = (c - offset).
1.158 + * Code points outside the range offset..(offset+0xff) cannot be mapped
1.159 + * and do not occur in the dictionary.
1.160 + *
1.161 + * stringTrie; -- a serialized BytesTrie or UCharsTrie
1.162 + *
1.163 + * The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType),
1.164 + * or it maps all strings to 0 (TRIE_HAS_VALUES bit not set).
1.165 + */
1.166 +
1.167 +#endif /* !UCONFIG_NO_BREAK_ITERATION */
1.168 +#endif /* __DICTIONARYDATA_H__ */
