michael@0: /* michael@0: ******************************************************************************* michael@0: * michael@0: * Copyright (C) 1998-2013, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: * michael@0: ******************************************************************************* michael@0: * michael@0: * Private implementation header for C collation michael@0: * file name: ucol_imp.h michael@0: * encoding: US-ASCII michael@0: * tab size: 8 (not used) michael@0: * indentation:4 michael@0: * michael@0: * created on: 2000dec11 michael@0: * created by: Vladimir Weinstein michael@0: * michael@0: * Modification history michael@0: * Date Name Comments michael@0: * 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr michael@0: * 02/27/2001 synwee Added getMaxExpansion data structure in UCollator michael@0: * 03/02/2001 synwee Added UCOL_IMPLICIT_CE michael@0: * 03/12/2001 synwee Added pointer start to collIterate. michael@0: */ michael@0: michael@0: #ifndef UCOL_IMP_H michael@0: #define UCOL_IMP_H michael@0: michael@0: #include "unicode/utypes.h" michael@0: #ifdef __cplusplus michael@0: # include "unicode/utf16.h" michael@0: #endif michael@0: michael@0: #define UCA_DATA_TYPE "icu" michael@0: #define UCA_DATA_NAME "ucadata" michael@0: #define INVC_DATA_TYPE "icu" michael@0: #define INVC_DATA_NAME "invuca" michael@0: michael@0: /** michael@0: * Convenience string denoting the Collation data tree michael@0: * @internal ICU 3.0 michael@0: */ michael@0: #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll" michael@0: michael@0: #if !UCONFIG_NO_COLLATION michael@0: michael@0: #ifdef __cplusplus michael@0: #include "unicode/normalizer2.h" michael@0: #include "unicode/unistr.h" michael@0: #endif michael@0: #include "unicode/ucol.h" michael@0: #include "ucol_data.h" michael@0: #include "utrie.h" michael@0: #include "cmemory.h" michael@0: michael@0: /* This is the internal header file which contains important declarations for michael@0: * the collation framework. michael@0: * Ready to use collators are stored as binary images. Both UCA and tailorings michael@0: * share the same binary format. Individual files (currently only UCA) have a michael@0: * udata header in front of the image and should be opened using udata_open. michael@0: * Tailoring images are currently stored inside resource bundles and are intialized michael@0: * through ucol_open API. michael@0: * michael@0: * The following describes the formats for collation binaries michael@0: * (UCA & tailorings) and for the inverse UCA table. michael@0: * Substructures are described in the collation design document at michael@0: * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm michael@0: * michael@0: * ------------------------------------------------------------- michael@0: * michael@0: * Here is the format of binary collation image. michael@0: * michael@0: * Physical order of structures: michael@0: * - header (UCATableHeader) michael@0: * - options (UColOptionSet) michael@0: * - expansions (CE[]) michael@0: * - contractions (UChar[contractionSize] + CE[contractionSize]) michael@0: * - serialized UTrie with mappings of code points to CEs michael@0: * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount]) michael@0: * - two bit sets for backward processing in strcoll (identical prefixes) michael@0: * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE]) michael@0: * - UCA constants (UCAConstants) michael@0: * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth]) michael@0: * michael@0: * UCATableHeader fields: michael@0: * michael@0: * int32_t size; - image size in bytes michael@0: * michael@0: * Offsets to interesting data. All offsets are in bytes. michael@0: * to get the address add to the header address and cast properly. michael@0: * Some offsets are zero if the corresponding structures are empty. michael@0: * michael@0: * Tailoring binaries that only set options and contain no mappings etc. michael@0: * will have all offsets 0 except for the options and expansion offsets, michael@0: * which give the position and length of the options array. michael@0: * michael@0: * uint32_t options; - offset to default collator options (UColOptionSet *), michael@0: * a set of 32-bit values. See declaration of UColOptionSet for more details michael@0: * michael@0: * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges michael@0: * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store michael@0: * important constant values that are defined in the UCA and used for building and runtime. michael@0: * michael@0: * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth], michael@0: * containing contractions from the UCA. These are needed in the build process to copy UCA contractions michael@0: * in case the base contraction symbol is tailored. michael@0: * michael@0: * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3) michael@0: * michael@0: * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such. michael@0: * Used as a primary lookup table for collation elements. michael@0: * michael@0: * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0. michael@0: * michael@0: * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents michael@0: * are aligned with the contents of contractionCEs table. 0 if no contractions. michael@0: * michael@0: * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the michael@0: * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table. michael@0: * 0 if no contractions. michael@0: * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs). michael@0: * michael@0: * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion michael@0: * and last CEs in expansions. michael@0: * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *). michael@0: * Never 0. michael@0: * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *) michael@0: * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION michael@0: * for the usage model michael@0: * michael@0: * These two offsets point to byte tables that are used in the backup heuristics. michael@0: * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function. michael@0: * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP. michael@0: * michael@0: * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos michael@0: * (formatVersion 2.3) michael@0: * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition. michael@0: * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3) michael@0: * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3) michael@0: * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3) michael@0: * michael@0: * Various version fields michael@0: * UVersionInfo version; - version 4 uint8_t michael@0: * UVersionInfo UCAVersion; - version 4 uint8_t michael@0: * UVersionInfo UCDVersion; - version 4 uint8_t michael@0: * UVersionInfo formatVersion; - version of the format of the collation binary michael@0: * same formatVersion as in ucadata.icu's UDataInfo header michael@0: * (formatVersion 2.3) michael@0: * michael@0: * uint32_t offset to the reordering code to lead CE byte remapping table michael@0: * uint32_t offset to the lead CE byte to reordering code mapping table michael@0: * michael@0: * uint8_t reserved[76]; - currently unused michael@0: * michael@0: * ------------------------------------------------------------- michael@0: * michael@0: * Inverse UCA is used for constructing collators from rules. It is always an individual file michael@0: * and always has a UDataInfo header. michael@0: * here is the structure: michael@0: * michael@0: * uint32_t byteSize; - size of inverse UCA image in bytes michael@0: * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows) michael@0: * uint32_t contsSize; - size of continuation table (number of UChars in table) michael@0: * michael@0: * uint32_t table; - offset to inverse table (uint32_t *) michael@0: * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation michael@0: * the third value is either a code unit (if there is only one code unit for element) or an index to continuation michael@0: * (number of code units combined with an index). michael@0: * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final michael@0: * code unit sequence for a CE is terminated by FFFE. michael@0: * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE. michael@0: * michael@0: * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t michael@0: * uint8_t padding[8]; - padding 8 uint8_t michael@0: * Header is followed by the table and continuation table. michael@0: */ michael@0: michael@0: /* definition of UCOL_HEADER_MAGIC moved to common/ucol_data.h */ michael@0: michael@0: /* UDataInfo for UCA mapping table */ michael@0: /* dataFormat="UCol" */ michael@0: #define UCA_DATA_FORMAT_0 ((uint8_t)0x55) michael@0: #define UCA_DATA_FORMAT_1 ((uint8_t)0x43) michael@0: #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f) michael@0: #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c) michael@0: michael@0: #define UCA_FORMAT_VERSION_0 ((uint8_t)3) michael@0: #define UCA_FORMAT_VERSION_1 0 michael@0: #define UCA_FORMAT_VERSION_2 ((uint8_t)0) michael@0: #define UCA_FORMAT_VERSION_3 ((uint8_t)0) michael@0: michael@0: /* UDataInfo for inverse UCA table */ michael@0: /* dataFormat="InvC" */ michael@0: #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49) michael@0: #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E) michael@0: #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76) michael@0: #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43) michael@0: michael@0: #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2) michael@0: #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1) michael@0: #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0) michael@0: #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0) michael@0: michael@0: /* This is the size of the stack allocated buffer for sortkey generation and similar operations */ michael@0: /* if it is too small, heap allocation will occur.*/ michael@0: /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */ michael@0: #define UCOL_MAX_BUFFER 128 michael@0: michael@0: #define UCOL_NORMALIZATION_GROWTH 2 michael@0: #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH michael@0: michael@0: /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */ michael@0: /* Sometimes we already have a writable buffer (like in case of normalized strings). */ michael@0: /* michael@0: you can change this value to any value >= 4 if you need memory - michael@0: it will affect the performance, though, since we're going to malloc. michael@0: Note 3 is the minimum value for Thai collation and 4 is the michael@0: minimum number for special Jamo michael@0: */ michael@0: #define UCOL_WRITABLE_BUFFER_SIZE 256 michael@0: michael@0: /* This is the size of the buffer for expansion CE's */ michael@0: /* In reality we should not have to deal with expm sequences longer then 16 */ michael@0: /* you can change this value if you need memory */ michael@0: /* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */ michael@0: /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */ michael@0: /* This is the longest expansion sequence we can handle without bombing out */ michael@0: #define UCOL_EXPAND_CE_BUFFER_SIZE 64 michael@0: michael@0: /* This is the size to increase the buffer for expansion CE's */ michael@0: #define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64 michael@0: michael@0: michael@0: /* Unsafe UChar hash table table size. */ michael@0: /* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */ michael@0: /* hashing the rest of the chars. Size in bytes */ michael@0: #define UCOL_UNSAFECP_TABLE_SIZE 1056 michael@0: /* mask value down to "some power of two"-1 */ michael@0: /* number of bits, not num of bytes. */ michael@0: #define UCOL_UNSAFECP_TABLE_MASK 0x1fff michael@0: michael@0: michael@0: /* flags bits for collIterate.flags */ michael@0: /* */ michael@0: /* NORM - set for incremental normalize of source string */ michael@0: #define UCOL_ITER_NORM 1 michael@0: michael@0: #define UCOL_ITER_HASLEN 2 michael@0: michael@0: /* UCOL_ITER_INNORMBUF - set if the "pos" is in */ michael@0: /* the writable side buffer, handling */ michael@0: /* incrementally normalized characters. */ michael@0: #define UCOL_ITER_INNORMBUF 4 michael@0: michael@0: /* UCOL_ITER_ALLOCATED - set if this iterator has */ michael@0: /* malloced storage to expand a buffer. */ michael@0: #define UCOL_ITER_ALLOCATED 8 michael@0: /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */ michael@0: #define UCOL_HIRAGANA_Q 16 michael@0: /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */ michael@0: /* otherwise set to false */ michael@0: #define UCOL_WAS_HIRAGANA 32 michael@0: /* UCOL_USE_ITERATOR - set this if collIterate uses a */ michael@0: /* character iterator instead of simply accessing string */ michael@0: /* by index */ michael@0: #define UCOL_USE_ITERATOR 64 michael@0: michael@0: #define UCOL_FORCE_HAN_IMPLICIT 128 michael@0: michael@0: #define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300 michael@0: michael@0: #ifdef __cplusplus michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: typedef struct collIterate : public UMemory { michael@0: const UChar *string; /* Original string */ michael@0: /* UChar *start; Pointer to the start of the source string. Either points to string michael@0: or to writableBuffer */ michael@0: const UChar *endp; /* string end ptr. Is undefined for null terminated strings */ michael@0: const UChar *pos; /* This is position in the string. Can be to original or writable buf */ michael@0: michael@0: uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */ michael@0: uint32_t *CEpos; /* This is the position to which we have stored processed CEs */ michael@0: michael@0: int32_t *offsetReturn; /* This is the offset to return, if non-NULL */ michael@0: int32_t *offsetStore; /* This is the pointer for storing offsets */ michael@0: int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */ michael@0: int32_t offsetRepeatValue; /* offset value to repeat */ michael@0: michael@0: UnicodeString writableBuffer; michael@0: const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ michael@0: const UCollator *coll; michael@0: const Normalizer2 *nfd; michael@0: uint8_t flags; michael@0: uint8_t origFlags; michael@0: uint32_t *extendCEs; /* This is use if CEs is not big enough */ michael@0: int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */ michael@0: uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */ michael@0: michael@0: int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */ michael@0: int32_t offsetBufferSize; /* The size of the offset buffer */ michael@0: michael@0: UCharIterator *iterator; michael@0: /*int32_t iteratorIndex;*/ michael@0: michael@0: // The offsetBuffer should probably be a UVector32, but helper functions michael@0: // are an improvement over duplicated code. michael@0: void appendOffset(int32_t offset, UErrorCode &errorCode); michael@0: } collIterate; michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: #else michael@0: michael@0: typedef struct collIterate collIterate; michael@0: michael@0: #endif michael@0: michael@0: #define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0)) michael@0: #define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet))) michael@0: michael@0: /* michael@0: struct used internally in getSpecial*CE. michael@0: data similar to collIterate. michael@0: */ michael@0: struct collIterateState { michael@0: const UChar *pos; /* This is position in the string. Can be to original or writable buf */ michael@0: const UChar *returnPos; michael@0: const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ michael@0: const UChar *bufferaddress; /* address of the normalization buffer */ michael@0: int32_t buffersize; michael@0: uint8_t flags; michael@0: uint8_t origFlags; michael@0: uint32_t iteratorIndex; michael@0: int32_t iteratorMove; michael@0: }; michael@0: michael@0: U_CAPI void U_EXPORT2 michael@0: uprv_init_collIterate(const UCollator *collator, michael@0: const UChar *sourceString, int32_t sourceLen, michael@0: U_NAMESPACE_QUALIFIER collIterate *s, UErrorCode *status); michael@0: michael@0: /* Internal functions for C test code. */ michael@0: U_CAPI U_NAMESPACE_QUALIFIER collIterate * U_EXPORT2 michael@0: uprv_new_collIterate(UErrorCode *status); michael@0: michael@0: U_CAPI void U_EXPORT2 michael@0: uprv_delete_collIterate(U_NAMESPACE_QUALIFIER collIterate *s); michael@0: michael@0: /* @return s->pos == s->endp */ michael@0: U_CAPI UBool U_EXPORT2 michael@0: uprv_collIterateAtEnd(U_NAMESPACE_QUALIFIER collIterate *s); michael@0: michael@0: #ifdef __cplusplus michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: struct UCollationPCE; michael@0: typedef struct UCollationPCE UCollationPCE; michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: struct UCollationElements : public icu::UMemory michael@0: { michael@0: /** michael@0: * Struct wrapper for source data michael@0: */ michael@0: icu::collIterate iteratordata_; michael@0: /** michael@0: * Indicates if this data has been reset. michael@0: */ michael@0: UBool reset_; michael@0: /** michael@0: * Indicates if the data should be deleted. michael@0: */ michael@0: UBool isWritable; michael@0: michael@0: /** michael@0: * Data for getNextProcessed, getPreviousProcessed. michael@0: */ michael@0: icu::UCollationPCE *pce; michael@0: }; michael@0: michael@0: #else michael@0: /*opaque type*/ michael@0: struct UCollationElements; michael@0: #endif michael@0: michael@0: U_CAPI void U_EXPORT2 michael@0: uprv_init_pce(const struct UCollationElements *elems); michael@0: michael@0: #define UCOL_LEVELTERMINATOR 1 michael@0: michael@0: /* mask off anything but primary order */ michael@0: #define UCOL_PRIMARYORDERMASK 0xffff0000 michael@0: /* mask off anything but secondary order */ michael@0: #define UCOL_SECONDARYORDERMASK 0x0000ff00 michael@0: /* mask off anything but tertiary order */ michael@0: #define UCOL_TERTIARYORDERMASK 0x000000ff michael@0: /* primary order shift */ michael@0: #define UCOL_PRIMARYORDERSHIFT 16 michael@0: /* secondary order shift */ michael@0: #define UCOL_SECONDARYORDERSHIFT 8 michael@0: michael@0: #define UCOL_BYTE_SIZE_MASK 0xFF michael@0: michael@0: #define UCOL_CASE_BYTE_START 0x80 michael@0: #define UCOL_CASE_SHIFT_START 7 michael@0: michael@0: #define UCOL_IGNORABLE 0 michael@0: michael@0: /* get weights from a CE */ michael@0: #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT) michael@0: #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT) michael@0: #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK) michael@0: michael@0: /** michael@0: * Determine if a character is a Thai vowel (which sorts after michael@0: * its base consonant). michael@0: */ michael@0: #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \ michael@0: (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0))) michael@0: michael@0: /** michael@0: * Determine if a character is a Thai base consonant michael@0: */ michael@0: #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01) michael@0: michael@0: #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \ michael@0: (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \ michael@0: (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8))) michael@0: michael@0: /* Han character ranges */ michael@0: #define UCOL_FIRST_HAN 0x4E00 michael@0: #define UCOL_LAST_HAN 0x9FFF michael@0: #define UCOL_FIRST_HAN_A 0x3400 michael@0: #define UCOL_LAST_HAN_A 0x4DBF michael@0: #define UCOL_FIRST_HAN_COMPAT 0xFAE0 michael@0: #define UCOL_LAST_HAN_COMPAT 0xFA2F michael@0: michael@0: /* Han extension B is in plane 2 */ michael@0: #define UCOL_FIRST_HAN_B 0x20000 michael@0: #define UCOL_LAST_HAN_B 0x2A6DF michael@0: michael@0: /* Hangul range */ michael@0: #define UCOL_FIRST_HANGUL 0xAC00 michael@0: #define UCOL_LAST_HANGUL 0xD7AF michael@0: michael@0: /* Jamo ranges */ michael@0: #define UCOL_FIRST_L_JAMO 0x1100 michael@0: #define UCOL_FIRST_V_JAMO 0x1161 michael@0: #define UCOL_FIRST_T_JAMO 0x11A8 michael@0: #define UCOL_LAST_T_JAMO 0x11F9 michael@0: michael@0: michael@0: #if 0 michael@0: /* initializes collIterate structure */ michael@0: /* made as macro to speed up things */ michael@0: #define init_collIterate(collator, sourceString, sourceLen, s) { \ michael@0: (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \ michael@0: (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \ michael@0: (s)->CEpos = (s)->toReturn = (s)->CEs; \ michael@0: (s)->isThai = TRUE; \ michael@0: (s)->writableBuffer = (s)->stackWritableBuffer; \ michael@0: (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \ michael@0: (s)->coll = (collator); \ michael@0: (s)->fcdPosition = 0; \ michael@0: (s)->flags = 0; \ michael@0: if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \ michael@0: } michael@0: #endif michael@0: michael@0: michael@0: michael@0: /* michael@0: * Macro to get the maximum size of an expansion ending with the argument ce. michael@0: * Used in the Boyer Moore algorithm. michael@0: * Note for tailoring, the UCA maxexpansion table has been merged. michael@0: * Hence we only have to search the tailored collator only. michael@0: * @param coll const UCollator pointer michael@0: * @param order last collation element of the expansion sequence michael@0: * @param result size of the longest expansion with argument collation element michael@0: * as the last element michael@0: */ michael@0: #define UCOL_GETMAXEXPANSION(coll, order, result) { \ michael@0: const uint32_t *start; \ michael@0: const uint32_t *limit; \ michael@0: const uint32_t *mid; \ michael@0: start = (coll)->endExpansionCE; \ michael@0: limit = (coll)->lastEndExpansionCE; \ michael@0: while (start < limit - 1) { \ michael@0: mid = start + ((limit - start) >> 1); \ michael@0: if ((order) <= *mid) { \ michael@0: limit = mid; \ michael@0: } \ michael@0: else { \ michael@0: start = mid; \ michael@0: } \ michael@0: } \ michael@0: if (*start == order) { \ michael@0: result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \ michael@0: } \ michael@0: else if (*limit == order) { \ michael@0: result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \ michael@0: } \ michael@0: else if ((order & 0xFFFF) == 0x00C0) { \ michael@0: result = 2; \ michael@0: } \ michael@0: else { \ michael@0: result = 1; \ michael@0: } \ michael@0: } michael@0: michael@0: U_CFUNC michael@0: uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, michael@0: U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status); michael@0: michael@0: U_CFUNC michael@0: uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE, michael@0: U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status); michael@0: U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, michael@0: U_NAMESPACE_QUALIFIER collIterate *collationSource, UErrorCode *status); michael@0: U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll, michael@0: U_NAMESPACE_QUALIFIER collIterate *collationSource, michael@0: UErrorCode *status); michael@0: /* get some memory */ michael@0: void *ucol_getABuffer(const UCollator *coll, uint32_t size); michael@0: michael@0: #ifdef __cplusplus michael@0: michael@0: U_NAMESPACE_BEGIN michael@0: michael@0: class CollationKey; michael@0: class SortKeyByteSink; michael@0: michael@0: U_NAMESPACE_END michael@0: michael@0: /* function used by C++ getCollationKey to prevent restarting the calculation */ michael@0: U_CFUNC int32_t michael@0: ucol_getCollationKey(const UCollator *coll, michael@0: const UChar *source, int32_t sourceLength, michael@0: icu::CollationKey &key, michael@0: UErrorCode &errorCode); michael@0: michael@0: typedef void U_CALLCONV michael@0: SortKeyGenerator(const UCollator *coll, michael@0: const UChar *source, michael@0: int32_t sourceLength, michael@0: icu::SortKeyByteSink &result, michael@0: UErrorCode *status); michael@0: michael@0: /* worker function for generating sortkeys */ michael@0: U_CFUNC michael@0: void U_CALLCONV michael@0: ucol_calcSortKey(const UCollator *coll, michael@0: const UChar *source, michael@0: int32_t sourceLength, michael@0: icu::SortKeyByteSink &result, michael@0: UErrorCode *status); michael@0: michael@0: U_CFUNC michael@0: void U_CALLCONV michael@0: ucol_calcSortKeySimpleTertiary(const UCollator *coll, michael@0: const UChar *source, michael@0: int32_t sourceLength, michael@0: icu::SortKeyByteSink &result, michael@0: UErrorCode *status); michael@0: michael@0: #else michael@0: michael@0: typedef void U_CALLCONV michael@0: SortKeyGenerator(const UCollator *coll, michael@0: const UChar *source, michael@0: int32_t sourceLength, michael@0: void *result, michael@0: UErrorCode *status); michael@0: michael@0: #endif michael@0: michael@0: /** michael@0: * Used to set requested and valid locales on a collator returned by the collator michael@0: * service. michael@0: */ michael@0: U_CFUNC void U_EXPORT2 michael@0: ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt); michael@0: michael@0: #define UCOL_SPECIAL_FLAG 0xF0000000 michael@0: #define UCOL_TAG_SHIFT 24 michael@0: #define UCOL_TAG_MASK 0x0F000000 michael@0: #define INIT_EXP_TABLE_SIZE 1024 michael@0: #define UCOL_NOT_FOUND 0xF0000000 michael@0: #define UCOL_EXPANSION 0xF1000000 michael@0: #define UCOL_CONTRACTION 0xF2000000 michael@0: #define UCOL_THAI 0xF3000000 michael@0: #define UCOL_UNMARKED 0x03 michael@0: #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f michael@0: michael@0: /* Bit mask for primary collation strength. */ michael@0: #define UCOL_PRIMARYMASK 0xFFFF0000 michael@0: michael@0: /* Bit mask for secondary collation strength. */ michael@0: #define UCOL_SECONDARYMASK 0x0000FF00 michael@0: michael@0: /* Bit mask for tertiary collation strength. */ michael@0: #define UCOL_TERTIARYMASK 0x000000FF michael@0: michael@0: /** michael@0: * Internal. michael@0: * This indicates the last element in a UCollationElements has been consumed. michael@0: * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs. michael@0: */ michael@0: #define UCOL_NO_MORE_CES 0x00010101 michael@0: #define UCOL_NO_MORE_CES_PRIMARY 0x00010000 michael@0: #define UCOL_NO_MORE_CES_SECONDARY 0x00000100 michael@0: #define UCOL_NO_MORE_CES_TERTIARY 0x00000001 michael@0: michael@0: #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF) michael@0: michael@0: #define UCOL_UPPER_CASE 0x80 michael@0: #define UCOL_MIXED_CASE 0x40 michael@0: #define UCOL_LOWER_CASE 0x00 michael@0: michael@0: #define UCOL_CONTINUATION_MARKER 0xC0 michael@0: #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F michael@0: michael@0: #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER) michael@0: #define isFlagged(CE) (((CE) & 0x80) == 0x80) michael@0: #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0) michael@0: michael@0: #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT) michael@0: #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG)) michael@0: #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG)) michael@0: #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<>4) michael@0: #define getExpansionCount(CE) ((CE)&0xF) michael@0: #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0) michael@0: michael@0: /* StringSearch internal use */ michael@0: #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF) michael@0: #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL) michael@0: #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs) michael@0: #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs) michael@0: #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset) michael@0: #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn) michael@0: #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces) michael@0: michael@0: /* This is an enum that lists magic special byte values from the fractional UCA. michael@0: * See also http://site.icu-project.org/design/collation/bytes */ michael@0: /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */ michael@0: michael@0: enum { michael@0: UCOL_BYTE_ZERO = 0x00, michael@0: UCOL_BYTE_LEVEL_SEPARATOR = 0x01, michael@0: UCOL_BYTE_SORTKEY_GLUE = 0x02, michael@0: UCOL_BYTE_SHIFT_PREFIX = 0x03, michael@0: UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX, michael@0: UCOL_BYTE_FIRST_TAILORED = 0x04, michael@0: UCOL_BYTE_COMMON = 0x05, michael@0: UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON, michael@0: /* TODO: Make the following values dynamic since they change with almost every UCA version. */ michael@0: UCOL_CODAN_PLACEHOLDER = 0x12, michael@0: UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x5B, michael@0: UCOL_BYTE_UNSHIFTED_MAX = 0xFF michael@0: }; michael@0: michael@0: #if 0 michael@0: #define UCOL_RESET_TOP_VALUE 0x9F000303 michael@0: #define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705 michael@0: #define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05 michael@0: #define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0 michael@0: #define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000 michael@0: #define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500 michael@0: #define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000 michael@0: #define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000 michael@0: #define UCOL_FIRST_VARIABLE 0x05070505 michael@0: #define UCOL_LAST_VARIABLE 0x179B0505 michael@0: #define UCOL_FIRST_NON_VARIABLE 0x1A200505 michael@0: #define UCOL_LAST_NON_VARIABLE 0x7B41058F michael@0: michael@0: #define UCOL_NEXT_TOP_VALUE 0xE8960303 michael@0: #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905 michael@0: #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303 michael@0: #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705 michael@0: #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500 michael@0: #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000 michael@0: #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000 michael@0: #define UCOL_NEXT_FIRST_VARIABLE 0x05090505 michael@0: #define UCOL_NEXT_LAST_VARIABLE 0x1A200505 michael@0: michael@0: #define PRIMARY_IMPLICIT_MIN 0xE8000000 michael@0: #define PRIMARY_IMPLICIT_MAX 0xF0000000 michael@0: #endif michael@0: michael@0: /* These constants can be changed - sortkey size is affected by them */ michael@0: #define UCOL_PROPORTION2 0.5 michael@0: #define UCOL_PROPORTION3 0.667 michael@0: michael@0: /* These values come from the UCA */ michael@0: #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON michael@0: #define UCOL_COMMON_TOP2 0x86u michael@0: #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1) michael@0: michael@0: #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80 michael@0: #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40 michael@0: #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85 michael@0: #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45 michael@0: #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5 michael@0: michael@0: /* These values come from the UCA */ michael@0: #define UCOL_COMMON_BOT3 0x05 michael@0: michael@0: #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86; michael@0: #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3; michael@0: michael@0: #define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2) michael@0: #define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2) michael@0: michael@0: michael@0: #define UCOL_COMMON2 UCOL_COMMON_BOT2 michael@0: #define UCOL_COMMON3_UPPERFIRST 0xC5 michael@0: #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3 michael@0: michael@0: #define UCOL_COMMON4 0xFF michael@0: michael@0: /* constants for case level/case first handling */ michael@0: /* used to instantiate UCollators fields in ucol_updateInternalState */ michael@0: #define UCOL_CASE_SWITCH 0xC0 michael@0: #define UCOL_NO_CASE_SWITCH 0x00 michael@0: michael@0: #define UCOL_REMOVE_CASE 0x3F michael@0: #define UCOL_KEEP_CASE 0xFF michael@0: michael@0: #define UCOL_CASE_BIT_MASK 0xC0 michael@0: michael@0: #define UCOL_TERT_CASE_MASK 0xFF michael@0: michael@0: #define UCOL_ENDOFLATINONERANGE 0xFF michael@0: #define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50) michael@0: #define UCOL_BAIL_OUT_CE 0xFF000000 michael@0: michael@0: michael@0: typedef enum { michael@0: NOT_FOUND_TAG = 0, michael@0: EXPANSION_TAG = 1, /* This code point results in an expansion */ michael@0: CONTRACTION_TAG = 2, /* Start of a contraction */ michael@0: THAI_TAG = 3, /* Thai character - do the reordering */ michael@0: CHARSET_TAG = 4, /* Charset processing, not yet implemented */ michael@0: SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */ michael@0: HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/ michael@0: LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/ michael@0: TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/ michael@0: CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/ michael@0: IMPLICIT_TAG = 10, michael@0: SPEC_PROC_TAG = 11, michael@0: /* ICU 2.1 */ michael@0: LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */ michael@0: /* It fits in a single 32 bit CE and is used instead of expansion to save */ michael@0: /* space without affecting the performance (hopefully) */ michael@0: michael@0: DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */ michael@0: michael@0: CE_TAGS_COUNT michael@0: } UColCETags; michael@0: michael@0: /* michael@0: ***************************************************************************************** michael@0: * set to zero michael@0: * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF michael@0: ****************************************************************************************** michael@0: */ michael@0: michael@0: typedef struct { michael@0: uint32_t variableTopValue; michael@0: /*UColAttributeValue*/ int32_t frenchCollation; michael@0: /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/ michael@0: /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */ michael@0: /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */ michael@0: /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */ michael@0: /*UColAttributeValue*/ int32_t strength; /* attribute for strength */ michael@0: /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */ michael@0: /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */ michael@0: uint32_t reserved[15]; /* for future use */ michael@0: } UColOptionSet; michael@0: michael@0: typedef struct { michael@0: uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ michael@0: uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ michael@0: uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/ michael@0: uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/ michael@0: uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/ michael@0: uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/ michael@0: uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/ michael@0: uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/ michael@0: uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/ michael@0: uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/ michael@0: uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/ michael@0: uint32_t UCA_FIRST_IMPLICIT[2]; michael@0: uint32_t UCA_LAST_IMPLICIT[2]; michael@0: uint32_t UCA_FIRST_TRAILING[2]; michael@0: uint32_t UCA_LAST_TRAILING[2]; michael@0: michael@0: #if 0 michael@0: uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/ michael@0: uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/ michael@0: uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/ michael@0: uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/ michael@0: uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/ michael@0: uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/ michael@0: uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/ michael@0: uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/ michael@0: uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/ michael@0: #endif michael@0: michael@0: uint32_t UCA_PRIMARY_TOP_MIN; michael@0: uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/ michael@0: uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/ michael@0: uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/ michael@0: uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/ michael@0: uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/ michael@0: uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/ michael@0: } UCAConstants; michael@0: michael@0: /* definition of UCATableHeader moved to common/ucol_data.h */ michael@0: michael@0: #define U_UNKNOWN_STATE 0 michael@0: #define U_COLLATOR_STATE 0x01 michael@0: #define U_STATE_LIMIT 0x02 michael@0: michael@0: /* This is the first structure in a state */ michael@0: /* it should be machine independent */ michael@0: typedef struct { michael@0: /* this structure is supposed to be readable on all the platforms.*/ michael@0: /* first 2 fields hold the size of the structure in a platform independent way */ michael@0: uint8_t sizeLo; michael@0: uint8_t sizeHi; michael@0: /* identifying the writing platform */ michael@0: uint8_t isBigEndian; michael@0: /* see U_CHARSET_FAMILY values in utypes.h */ michael@0: uint8_t charsetFamily; michael@0: /* version of ICU this state structure comes from */ michael@0: uint8_t icuVersion[4]; michael@0: /* What is the data following this state */ michael@0: uint8_t type; michael@0: /* more stuff to come, keep it on 16 byte boundary */ michael@0: uint8_t reserved[7]; michael@0: } UStateStruct; michael@0: michael@0: /* This structure follows UStatusStruct */ michael@0: /* and contains data specific for the collators */ michael@0: /* Endianess needs to be decided before accessing this structure */ michael@0: /* However, it's size IS endianess independent */ michael@0: typedef struct { michael@0: /* size of this structure */ michael@0: uint8_t sizeLo; michael@0: uint8_t sizeHi; michael@0: /* This state is followed by the frozen tailoring */ michael@0: uint8_t containsTailoring; michael@0: /* This state is followed by the frozen UCA */ michael@0: uint8_t containsUCA; michael@0: /* Version info - the same one */ michael@0: uint8_t versionInfo[4]; michael@0: michael@0: /* for charset CEs */ michael@0: uint8_t charsetName[32]; michael@0: /* this is the resolved locale name*/ michael@0: uint8_t locale[32]; michael@0: michael@0: /* Attributes. Open ended */ michael@0: /* all the following will be moved to uint32_t because of portability */ michael@0: /* variable top value */ michael@0: uint32_t variableTopValue; michael@0: /* attribute for handling variable elements*/ michael@0: uint32_t /*UColAttributeValue*/ alternateHandling; michael@0: /* how to handle secondary weights */ michael@0: uint32_t /*UColAttributeValue*/ frenchCollation; michael@0: /* who goes first, lower case or uppercase */ michael@0: uint32_t /*UColAttributeValue*/ caseFirst; michael@0: /* do we have an extra case level */ michael@0: uint32_t /*UColAttributeValue*/ caseLevel; michael@0: /* attribute for normalization */ michael@0: uint32_t /*UColAttributeValue*/ normalizationMode; michael@0: /* attribute for strength */ michael@0: uint32_t /*UColAttributeValue*/ strength; michael@0: /* to be immediately 16 byte aligned */ michael@0: uint8_t reserved[12]; michael@0: } UColStateStruct; michael@0: michael@0: #define UCOL_INV_SIZEMASK 0xFFF00000 michael@0: #define UCOL_INV_OFFSETMASK 0x000FFFFF michael@0: #define UCOL_INV_SHIFTVALUE 20 michael@0: michael@0: U_CDECL_BEGIN michael@0: michael@0: /* definition of InverseUCATableHeader moved to common/ucol_data.h */ michael@0: michael@0: typedef void U_CALLCONV michael@0: ResourceCleaner(UCollator *coll); michael@0: michael@0: michael@0: struct UCollator { michael@0: UColOptionSet *options; michael@0: SortKeyGenerator *sortKeyGen; michael@0: uint32_t *latinOneCEs; michael@0: char* actualLocale; michael@0: char* validLocale; michael@0: char* requestedLocale; michael@0: const UChar *rules; michael@0: const UChar *ucaRules; michael@0: const UCollator *UCA; michael@0: const UCATableHeader *image; michael@0: UTrie mapping; michael@0: const uint32_t *latinOneMapping; michael@0: const uint32_t *expansion; michael@0: const UChar *contractionIndex; michael@0: const uint32_t *contractionCEs; michael@0: michael@0: const uint32_t *endExpansionCE; /* array of last ces in an expansion ce. michael@0: corresponds to expansionCESize */ michael@0: const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */ michael@0: const uint8_t *expansionCESize; /* array of the maximum size of a michael@0: expansion ce with the last ce michael@0: corresponding to endExpansionCE, michael@0: terminated with a null */ michael@0: const uint8_t *unsafeCP; /* unsafe code points hashtable */ michael@0: const uint8_t *contrEndCP; /* Contraction ending chars hash table */ michael@0: UChar minUnsafeCP; /* Smallest unsafe Code Point. */ michael@0: UChar minContrEndCP; /* Smallest code point at end of a contraction */ michael@0: michael@0: int32_t rulesLength; michael@0: int32_t latinOneTableLen; michael@0: michael@0: uint32_t variableTopValue; michael@0: UColAttributeValue frenchCollation; michael@0: UColAttributeValue alternateHandling; /* attribute for handling variable elements*/ michael@0: UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */ michael@0: UColAttributeValue caseLevel; /* do we have an extra case level */ michael@0: UColAttributeValue normalizationMode; /* attribute for normalization */ michael@0: UColAttributeValue strength; /* attribute for strength */ michael@0: UColAttributeValue hiraganaQ; /* attribute for Hiragana */ michael@0: UColAttributeValue numericCollation; michael@0: UBool variableTopValueisDefault; michael@0: UBool frenchCollationisDefault; michael@0: UBool alternateHandlingisDefault; /* attribute for handling variable elements*/ michael@0: UBool caseFirstisDefault; /* who goes first, lower case or uppercase */ michael@0: UBool caseLevelisDefault; /* do we have an extra case level */ michael@0: UBool normalizationModeisDefault; /* attribute for normalization */ michael@0: UBool strengthisDefault; /* attribute for strength */ michael@0: UBool hiraganaQisDefault; /* attribute for Hiragana */ michael@0: UBool numericCollationisDefault; michael@0: UBool hasRealData; /* some collators have only options, like French, no rules */ michael@0: /* to speed up things, we use the UCA image, but we don't want it */ michael@0: /* to run around */ michael@0: michael@0: UBool freeOnClose; michael@0: UBool freeOptionsOnClose; michael@0: UBool freeRulesOnClose; michael@0: UBool freeImageOnClose; michael@0: UBool freeDefaultReorderCodesOnClose; michael@0: UBool freeReorderCodesOnClose; michael@0: UBool freeLeadBytePermutationTableOnClose; michael@0: michael@0: UBool latinOneUse; michael@0: UBool latinOneRegenTable; michael@0: UBool latinOneFailed; michael@0: michael@0: int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */ michael@0: uint8_t caseSwitch; michael@0: uint8_t tertiaryCommon; michael@0: uint8_t tertiaryMask; michael@0: uint8_t tertiaryTop; /* Upper range when compressing */ michael@0: uint8_t tertiaryBottom; /* Upper range when compressing */ michael@0: uint8_t tertiaryTopCount; michael@0: uint8_t tertiaryBottomCount; michael@0: michael@0: UVersionInfo dataVersion; /* Data info of UCA table */ michael@0: int32_t* defaultReorderCodes; michael@0: int32_t defaultReorderCodesLength; michael@0: int32_t* reorderCodes; michael@0: int32_t reorderCodesLength; michael@0: uint8_t* leadBytePermutationTable; michael@0: void *delegate; /* if non-null: C++ object to delegate all API calls to. */ michael@0: }; michael@0: michael@0: U_CDECL_END michael@0: michael@0: /* various internal functions */ michael@0: michael@0: /* do not close UCA returned by ucol_initUCA! */ michael@0: U_CFUNC michael@0: UCollator* ucol_initUCA(UErrorCode *status); michael@0: michael@0: U_CFUNC michael@0: UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status); michael@0: michael@0: U_CFUNC michael@0: void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); michael@0: michael@0: U_CFUNC michael@0: UCollator* ucol_open_internal(const char* loc, UErrorCode* status); michael@0: michael@0: #if 0 michael@0: U_CFUNC michael@0: void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); michael@0: #endif michael@0: michael@0: U_CFUNC michael@0: void ucol_updateInternalState(UCollator *coll, UErrorCode *status); michael@0: michael@0: U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status); michael@0: U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status); michael@0: michael@0: U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status); michael@0: michael@0: U_CAPI void U_EXPORT2 michael@0: uprv_uca_initImplicitConstants(UErrorCode *status); michael@0: michael@0: U_CAPI uint32_t U_EXPORT2 michael@0: uprv_uca_getImplicitFromRaw(UChar32 cp); michael@0: michael@0: /*U_CFUNC uint32_t U_EXPORT2 michael@0: uprv_uca_getImplicitPrimary(UChar32 cp);*/ michael@0: michael@0: U_CAPI UChar32 U_EXPORT2 michael@0: uprv_uca_getRawFromImplicit(uint32_t implicit); michael@0: michael@0: U_CAPI UChar32 U_EXPORT2 michael@0: uprv_uca_getRawFromCodePoint(UChar32 i); michael@0: michael@0: U_CAPI UChar32 U_EXPORT2 michael@0: uprv_uca_getCodePointFromRaw(UChar32 i); michael@0: michael@0: typedef const UChar* GetCollationRulesFunction(void* context, const char* locale, const char* type, int32_t* pLength, UErrorCode* status); michael@0: michael@0: U_CAPI UCollator* U_EXPORT2 michael@0: ucol_openRulesForImport( const UChar *rules, michael@0: int32_t rulesLength, michael@0: UColAttributeValue normalizationMode, michael@0: UCollationStrength strength, michael@0: UParseError *parseError, michael@0: GetCollationRulesFunction importFunc, michael@0: void* context, michael@0: UErrorCode *status); michael@0: michael@0: michael@0: U_CFUNC void U_EXPORT2 michael@0: ucol_buildPermutationTable(UCollator *coll, UErrorCode *status); michael@0: michael@0: U_CFUNC int U_EXPORT2 michael@0: ucol_getLeadBytesForReorderCode(const UCollator *uca, int reorderCode, uint16_t* returnLeadBytes, int returnCapacity); michael@0: michael@0: U_CFUNC int U_EXPORT2 michael@0: ucol_getReorderCodesForLeadByte(const UCollator *uca, int leadByte, int16_t* returnReorderCodes, int returnCapacity); michael@0: michael@0: #ifdef __cplusplus michael@0: /* michael@0: * Test whether a character is potentially "unsafe" for use as a collation michael@0: * starting point. Unsafe chars are those with combining class != 0 plus michael@0: * those that are the 2nd thru nth character in a contraction sequence. michael@0: * michael@0: * Function is in header file because it's used in both collation and string search, michael@0: * and needs to be inline for performance. michael@0: */ michael@0: static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) { michael@0: int32_t hash; michael@0: uint8_t htbyte; michael@0: michael@0: if (c < coll->minUnsafeCP) { michael@0: return FALSE; michael@0: } michael@0: michael@0: hash = c; michael@0: if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) { michael@0: if(U16_IS_SURROGATE(c)) { michael@0: /* Lead or trail surrogate */ michael@0: /* These are always considered unsafe. */ michael@0: return TRUE; michael@0: } michael@0: hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256; michael@0: } michael@0: htbyte = coll->unsafeCP[hash>>3]; michael@0: return ((htbyte >> (hash & 7)) & 1); michael@0: } michael@0: #endif /* __cplusplus */ michael@0: michael@0: /* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */ michael@0: void ucol_freeOffsetBuffer(U_NAMESPACE_QUALIFIER collIterate *s); michael@0: michael@0: #endif /* #if !UCONFIG_NO_COLLATION */ michael@0: michael@0: #endif