michael@0: /* michael@0: ****************************************************************************** michael@0: * michael@0: * Copyright (C) 2000-2013, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: * michael@0: ****************************************************************************** michael@0: * file name: ucnvmbcs.h michael@0: * encoding: US-ASCII michael@0: * tab size: 8 (not used) michael@0: * indentation:4 michael@0: * michael@0: * created on: 2000jul07 michael@0: * created by: Markus W. Scherer michael@0: */ michael@0: michael@0: #ifndef __UCNVMBCS_H__ michael@0: #define __UCNVMBCS_H__ michael@0: michael@0: #include "unicode/utypes.h" michael@0: michael@0: #if !UCONFIG_NO_CONVERSION michael@0: michael@0: #include "unicode/ucnv.h" michael@0: #include "ucnv_cnv.h" michael@0: #include "ucnv_ext.h" michael@0: michael@0: /** michael@0: * ICU conversion (.cnv) data file structure, following the usual UDataInfo michael@0: * header. michael@0: * michael@0: * Format version: 6.2 michael@0: * michael@0: * struct UConverterStaticData -- struct containing the converter name, IBM CCSID, michael@0: * min/max bytes per character, etc. michael@0: * see ucnv_bld.h michael@0: * michael@0: * -------------------- michael@0: * michael@0: * The static data is followed by conversionType-specific data structures. michael@0: * At the moment, there are only variations of MBCS converters. They all have michael@0: * the same toUnicode structures, while the fromUnicode structures for SBCS michael@0: * differ from those for other MBCS-style converters. michael@0: * michael@0: * _MBCSHeader.version 5 is optional and not backward-compatible michael@0: * (as usual for changes in the major version field). michael@0: * michael@0: * Versions 5.m work like versions 4.m except: michael@0: * - The _MBCSHeader has variable length (and is always longer than in version 4). michael@0: * See the struct _MBCSHeader further description below. michael@0: * - There is a set of flags which indicate further incompatible changes. michael@0: * (Reader code must reject the file if it does not recognize them all.) michael@0: * - In particular, one of these flags indicates that most of the fromUnicode michael@0: * data is missing and must be reconstituted from the toUnicode data michael@0: * and from the utf8Friendly mbcsIndex at load time. michael@0: * (This only works with a utf8Friendly table.) michael@0: * In this case, makeconv may increase maxFastUChar automatically to U+FFFF. michael@0: * michael@0: * The first of these versions is 5.3, which is like 4.3 except for the differences above. michael@0: * michael@0: * When possible, makeconv continues to generate version 4.m files. michael@0: * michael@0: * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4) michael@0: * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details. michael@0: * michael@0: * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures michael@0: * slightly and optionally adds a table for conversion to MBCS (non-SBCS) michael@0: * charsets. michael@0: * michael@0: * The modifications are to make the data utf8Friendly. Not every 4.3 file michael@0: * file contains utf8Friendly data. michael@0: * It is utf8Friendly if _MBCSHeader.version[2]!=0. michael@0: * In this case, the data structures are utf8Friendly up to the code point michael@0: * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff) michael@0: * michael@0: * A utf8Friendly file has fromUnicode stage 3 entries for code points up to michael@0: * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from michael@0: * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries. michael@0: * michael@0: * In addition, a utf8Friendly MBCS file contains an additional michael@0: * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; michael@0: * which replaces the stage 1 and 2 tables for indexing with bits from the michael@0: * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table, michael@0: * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks michael@0: * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to michael@0: * the extension data structure. This also allows for faster roundtrip michael@0: * conversion from UTF-16. michael@0: * michael@0: * SBCS files do not contain an additional sbcsIndex[] array because the michael@0: * proportional size increase would be noticeable, but the runtime michael@0: * code builds one for the code point range for which the runtime conversion michael@0: * code is optimized. michael@0: * michael@0: * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv michael@0: * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in michael@0: * the file, a larger maxFastUChar only affects stage 3 block allocation size michael@0: * and is free in empty blocks. (Larger blocks with sparse contents cause larger michael@0: * files.) U+1FFF includes almost all of the small scripts. michael@0: * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with michael@0: * 0xe0. This includes most scripts with legacy SBCS charsets. michael@0: * The initial runtime implementation using 4.3 files only builds an sbcsIndex michael@0: * for code points up to U+0FFF. michael@0: * michael@0: * For MBCS, maxFastUChar should be at least U+D7FF (=initial value). michael@0: * This boundary is convenient because practically all of the commonly used michael@0: * characters are below it, and because it is the boundary to surrogate michael@0: * code points, above which special handling is necessary anyway. michael@0: * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.) michael@0: * michael@0: * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be michael@0: * useful especially for conversion from UTF-8 when the input can be assumed michael@0: * to be valid, because the surrogate range would then not have to be michael@0: * checked. michael@0: * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value michael@0: * overflow because with the all-unassigned block 0 and nearly full mappings michael@0: * from the BMP it is theoretically possible that an index into stage 3 michael@0: * exceeds 16 bits.) michael@0: * michael@0: * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. michael@0: * If it is present, then an ICU version reading header versions 4.0 or 4.1 michael@0: * will be able to use the base table and ignore the extension. michael@0: * michael@0: * The unicodeMask in the static data is part of the base table data structure. michael@0: * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the michael@0: * fromUnicode stage 1 array. michael@0: * The static data unicodeMask refers only to the base table's properties if michael@0: * a base table is included. michael@0: * In an extension-only file, the static data unicodeMask is 0. michael@0: * The extension data indexes have a separate field with the unicodeMask flags. michael@0: * michael@0: * MBCS-style data structure following the static data. michael@0: * Offsets are counted in bytes from the beginning of the MBCS header structure. michael@0: * Details about usage in comments in ucnvmbcs.c. michael@0: * michael@0: * struct _MBCSHeader (see the definition in this header file below) michael@0: * contains 32-bit fields as follows: michael@0: * 8 values: michael@0: * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0) michael@0: * 1 uint32_t countStates michael@0: * 2 uint32_t countToUFallbacks michael@0: * 3 uint32_t offsetToUCodeUnits michael@0: * 4 uint32_t offsetFromUTable michael@0: * 5 uint32_t offsetFromUBytes michael@0: * 6 uint32_t flags, bits: michael@0: * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher michael@0: * 0 for older versions and if michael@0: * there is not extension structure michael@0: * 7.. 0 outputType michael@0: * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher michael@0: * counts bytes in fromUBytes[] michael@0: * michael@0: * New and required in version 5: michael@0: * 8 uint32_t options, bits: michael@0: * 31..16 reserved for flags that can be added without breaking michael@0: * backward compatibility michael@0: * 15.. 6 reserved for flags whose addition will break michael@0: * backward compatibility michael@0: * 6 MBCS_OPT_FROM_U -- if set, michael@0: * then most of the fromUnicode data is omitted; michael@0: * fullStage2Length is present and the missing michael@0: * bottom part of stage 2 must be reconstituted from michael@0: * the toUnicode data; michael@0: * stage 3 is missing completely as well; michael@0: * not used for SBCS tables michael@0: * 5.. 0 length of the _MBCSHeader (number of uint32_t) michael@0: * michael@0: * New and optional in version 5: michael@0: * 9 uint32_t fullStage2Length: used if MBCS_OPT_FROM_U is set michael@0: * specifies the full length of stage 2 michael@0: * including the omitted part michael@0: * michael@0: * if(outputType==MBCS_OUTPUT_EXT_ONLY) { michael@0: * -- base table name for extension-only table michael@0: * char baseTableName[variable]; -- with NUL plus padding for 4-alignment michael@0: * michael@0: * -- all _MBCSHeader fields except for version and flags are 0 michael@0: * } else { michael@0: * -- normal base table with optional extension michael@0: * michael@0: * int32_t stateTable[countStates][256]; michael@0: * michael@0: * struct _MBCSToUFallback { (fallbacks are sorted by offset) michael@0: * uint32_t offset; michael@0: * UChar32 codePoint; michael@0: * } toUFallbacks[countToUFallbacks]; michael@0: * michael@0: * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2]; michael@0: * (padded to an even number of units) michael@0: * michael@0: * -- stage 1 tables michael@0: * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) { michael@0: * -- stage 1 table for all of Unicode michael@0: * uint16_t fromUTable[0x440]; (32-bit-aligned) michael@0: * } else { michael@0: * -- BMP-only tables have a smaller stage 1 table michael@0: * uint16_t fromUTable[0x40]; (32-bit-aligned) michael@0: * } michael@0: * michael@0: * -- stage 2 tables michael@0: * length determined by top of stage 1 and bottom of stage 3 tables michael@0: * if(outputType==MBCS_OUTPUT_1) { michael@0: * -- SBCS: pure indexes michael@0: * uint16_t stage 2 indexes[?]; michael@0: * } else { michael@0: * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes michael@0: * uint32_t stage 2 flags and indexes[?]; michael@0: * if(options&MBCS_OPT_NO_FROM_U) { michael@0: * stage 2 really has length fullStage2Length michael@0: * and the omitted lower part must be reconstituted from michael@0: * the toUnicode data michael@0: * } michael@0: * } michael@0: * michael@0: * -- stage 3 tables with byte results michael@0: * if(outputType==MBCS_OUTPUT_1) { michael@0: * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c michael@0: * uint16_t fromUBytes[fromUBytesLength/2]; michael@0: * } else if(!(options&MBCS_OPT_NO_FROM_U)) { michael@0: * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c michael@0: * uint8_t fromUBytes[fromUBytesLength]; or michael@0: * uint16_t fromUBytes[fromUBytesLength/2]; or michael@0: * uint32_t fromUBytes[fromUBytesLength/4]; michael@0: * } else { michael@0: * fromUBytes[] must be reconstituted from the toUnicode data michael@0: * } michael@0: * michael@0: * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher michael@0: * if(outputType!=MBCS_OUTPUT_1 && michael@0: * _MBCSHeader.version[1]>=3 && michael@0: * (maxFastUChar=_MBCSHeader.version[2])!=0 michael@0: * ) { michael@0: * maxFastUChar=(maxFastUChar<<8)|0xff; michael@0: * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; michael@0: * } michael@0: * } michael@0: * michael@0: * -- extension table, details see ucnv_ext.h michael@0: * int32_t indexes[>=32]; ... michael@0: */ michael@0: michael@0: /* MBCS converter data and state -------------------------------------------- */ michael@0: michael@0: enum { michael@0: MBCS_MAX_STATE_COUNT=128 michael@0: }; michael@0: michael@0: /** michael@0: * MBCS action codes for conversions to Unicode. michael@0: * These values are in bits 23..20 of the state table entries. michael@0: */ michael@0: enum { michael@0: MBCS_STATE_VALID_DIRECT_16, michael@0: MBCS_STATE_VALID_DIRECT_20, michael@0: michael@0: MBCS_STATE_FALLBACK_DIRECT_16, michael@0: MBCS_STATE_FALLBACK_DIRECT_20, michael@0: michael@0: MBCS_STATE_VALID_16, michael@0: MBCS_STATE_VALID_16_PAIR, michael@0: michael@0: MBCS_STATE_UNASSIGNED, michael@0: MBCS_STATE_ILLEGAL, michael@0: michael@0: MBCS_STATE_CHANGE_ONLY michael@0: }; michael@0: michael@0: /* Macros for state table entries */ michael@0: #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset)) michael@0: #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset)) michael@0: #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset)) michael@0: michael@0: #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value)) michael@0: #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000) michael@0: #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L)) michael@0: #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value)) michael@0: #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value)) michael@0: michael@0: #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L)) michael@0: michael@0: #define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) michael@0: michael@0: #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0) michael@0: #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0) michael@0: michael@0: #define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24) michael@0: #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff) michael@0: michael@0: #define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) michael@0: #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000) michael@0: #define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf) michael@0: #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff) michael@0: #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry) michael@0: michael@0: #define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0) michael@0: michael@0: /* single-byte fromUnicode: get the 16-bit result word */ michael@0: #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ] michael@0: michael@0: /* single-byte fromUnicode using the sbcsIndex */ michael@0: #define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] michael@0: michael@0: /* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */ michael@0: #define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ] michael@0: michael@0: /* multi-byte fromUnicode: get the 32-bit stage 2 entry */ michael@0: #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] michael@0: #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0) michael@0: michael@0: #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] michael@0: #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] michael@0: michael@0: #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3) michael@0: michael@0: /* double-byte fromUnicode using the mbcsIndex */ michael@0: #define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] michael@0: michael@0: /* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */ michael@0: #define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ] michael@0: michael@0: michael@0: /** michael@0: * MBCS output types for conversions from Unicode. michael@0: * These per-converter types determine the storage method in stage 3 of the lookup table, michael@0: * mostly how many bytes are stored per entry. michael@0: */ michael@0: enum { michael@0: MBCS_OUTPUT_1, /* 0 */ michael@0: MBCS_OUTPUT_2, /* 1 */ michael@0: MBCS_OUTPUT_3, /* 2 */ michael@0: MBCS_OUTPUT_4, /* 3 */ michael@0: michael@0: MBCS_OUTPUT_3_EUC=8, /* 8 */ michael@0: MBCS_OUTPUT_4_EUC, /* 9 */ michael@0: michael@0: MBCS_OUTPUT_2_SISO=12, /* c */ michael@0: MBCS_OUTPUT_2_HZ, /* d */ michael@0: michael@0: MBCS_OUTPUT_EXT_ONLY, /* e */ michael@0: michael@0: MBCS_OUTPUT_COUNT, michael@0: michael@0: MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */ michael@0: }; michael@0: michael@0: /** michael@0: * Fallbacks to Unicode are stored outside the normal state table and code point structures michael@0: * in a vector of items of this type. They are sorted by offset. michael@0: */ michael@0: typedef struct { michael@0: uint32_t offset; michael@0: UChar32 codePoint; michael@0: } _MBCSToUFallback; michael@0: michael@0: /** Constants for fast and UTF-8-friendly conversion. */ michael@0: enum { michael@0: SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */ michael@0: SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */ michael@0: MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */ michael@0: MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */ michael@0: }; michael@0: michael@0: /** michael@0: * This is the MBCS part of the UConverterTable union (a runtime data structure). michael@0: * It keeps all the per-converter data and points into the loaded mapping tables. michael@0: * michael@0: * utf8Friendly data structures added with _MBCSHeader.version 4.3 michael@0: */ michael@0: typedef struct UConverterMBCSTable { michael@0: /* toUnicode */ michael@0: uint8_t countStates, dbcsOnlyState, stateTableOwned; michael@0: uint32_t countToUFallbacks; michael@0: michael@0: const int32_t (*stateTable)/*[countStates]*/[256]; michael@0: int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */ michael@0: const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/; michael@0: const _MBCSToUFallback *toUFallbacks; michael@0: michael@0: /* fromUnicode */ michael@0: const uint16_t *fromUnicodeTable; michael@0: const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */ michael@0: uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */ michael@0: const uint8_t *fromUnicodeBytes; michael@0: uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ michael@0: uint32_t fromUBytesLength; michael@0: uint8_t outputType, unicodeMask; michael@0: UBool utf8Friendly; /* for utf8Friendly data */ michael@0: UChar maxFastUChar; /* for utf8Friendly data */ michael@0: michael@0: /* roundtrips */ michael@0: uint32_t asciiRoundtrips; michael@0: michael@0: /* reconstituted data that was omitted from the .cnv file */ michael@0: uint8_t *reconstitutedData; michael@0: michael@0: /* converter name for swaplfnl */ michael@0: char *swapLFNLName; michael@0: michael@0: /* extension data */ michael@0: struct UConverterSharedData *baseSharedData; michael@0: const int32_t *extIndexes; michael@0: } UConverterMBCSTable; michael@0: michael@0: #define UCNV_MBCS_TABLE_INITIALIZER { \ michael@0: /* toUnicode */ \ michael@0: 0, 0, 0, \ michael@0: 0, \ michael@0: \ michael@0: NULL, \ michael@0: NULL, \ michael@0: NULL, \ michael@0: NULL, \ michael@0: \ michael@0: /* fromUnicode */ \ michael@0: NULL, \ michael@0: NULL, \ michael@0: { 0 }, \ michael@0: NULL, \ michael@0: NULL, \ michael@0: 0, \ michael@0: 0, 0, \ michael@0: FALSE, \ michael@0: 0, \ michael@0: \ michael@0: /* roundtrips */ \ michael@0: 0, \ michael@0: \ michael@0: /* reconstituted data that was omitted from the .cnv file */ \ michael@0: NULL, \ michael@0: \ michael@0: /* converter name for swaplfnl */ \ michael@0: NULL, \ michael@0: \ michael@0: /* extension data */ \ michael@0: NULL, \ michael@0: NULL \ michael@0: } michael@0: michael@0: enum { michael@0: MBCS_OPT_LENGTH_MASK=0x3f, michael@0: MBCS_OPT_NO_FROM_U=0x40, michael@0: /* michael@0: * If any of the following options bits are set, michael@0: * then the file must be rejected. michael@0: */ michael@0: MBCS_OPT_INCOMPATIBLE_MASK=0xffc0, michael@0: /* michael@0: * Remove bits from this mask as more options are recognized michael@0: * by all implementations that use this constant. michael@0: */ michael@0: MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80 michael@0: }; michael@0: michael@0: enum { michael@0: MBCS_HEADER_V4_LENGTH=8, michael@0: MBCS_HEADER_V5_MIN_LENGTH=9 michael@0: }; michael@0: michael@0: /** michael@0: * MBCS data header. See data format description above. michael@0: */ michael@0: typedef struct { michael@0: UVersionInfo version; michael@0: uint32_t countStates, michael@0: countToUFallbacks, michael@0: offsetToUCodeUnits, michael@0: offsetFromUTable, michael@0: offsetFromUBytes, michael@0: flags, michael@0: fromUBytesLength; michael@0: michael@0: /* new and required in version 5 */ michael@0: uint32_t options; michael@0: michael@0: /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */ michael@0: uint32_t fullStage2Length; /* number of 32-bit units */ michael@0: } _MBCSHeader; michael@0: michael@0: #define UCNV_MBCS_HEADER_INITIALIZER { { 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 } michael@0: michael@0: /* michael@0: * This is a simple version of _MBCSGetNextUChar() that is used michael@0: * by other converter implementations. michael@0: * It only returns an "assigned" result if it consumes the entire input. michael@0: * It does not use state from the converter, nor error codes. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It handles conversion extensions but not GB 18030. michael@0: * michael@0: * Return value: michael@0: * U+fffe unassigned michael@0: * U+ffff illegal michael@0: * otherwise the Unicode code point michael@0: */ michael@0: U_CFUNC UChar32 michael@0: ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, michael@0: const char *source, int32_t length, michael@0: UBool useFallback); michael@0: michael@0: /** michael@0: * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It does not handle conversion extensions (_extToU()). michael@0: */ michael@0: U_CFUNC UChar32 michael@0: ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, michael@0: uint8_t b, UBool useFallback); michael@0: michael@0: /** michael@0: * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte. michael@0: * It works for single-byte, single-state codepages that only map michael@0: * to and from BMP code points, and it always michael@0: * returns fallback values. michael@0: */ michael@0: #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \ michael@0: (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)]) michael@0: michael@0: /** michael@0: * This is an internal function that allows other converter implementations michael@0: * to check whether a byte is a lead byte. michael@0: */ michael@0: U_CFUNC UBool michael@0: ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte); michael@0: michael@0: /** This is a macro version of _MBCSIsLeadByte(). */ michael@0: #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \ michael@0: (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)]) michael@0: michael@0: /* michael@0: * This is another simple conversion function for internal use by other michael@0: * conversion implementations. michael@0: * It does not use the converter state nor call callbacks. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It handles conversion extensions but not GB 18030. michael@0: * michael@0: * It converts one single Unicode code point into codepage bytes, encoded michael@0: * as one 32-bit value. The function returns the number of bytes in *pValue: michael@0: * 1..4 the number of bytes in *pValue michael@0: * 0 unassigned (*pValue undefined) michael@0: * -1 illegal (currently not used, *pValue undefined) michael@0: * michael@0: * *pValue will contain the resulting bytes with the last byte in bits 7..0, michael@0: * the second to last byte in bits 15..8, etc. michael@0: * Currently, the function assumes but does not check that 0<=c<=0x10ffff. michael@0: */ michael@0: U_CFUNC int32_t michael@0: ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, michael@0: UChar32 c, uint32_t *pValue, michael@0: UBool useFallback); michael@0: michael@0: /** michael@0: * This version of _MBCSFromUChar32() is optimized for single-byte codepages. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * michael@0: * It returns the codepage byte for the code point, or -1 if it is unassigned. michael@0: */ michael@0: U_CFUNC int32_t michael@0: ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, michael@0: UChar32 c, michael@0: UBool useFallback); michael@0: michael@0: /** michael@0: * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but michael@0: * we cheat a little about the type, returning the old types if appropriate. michael@0: */ michael@0: U_CFUNC UConverterType michael@0: ucnv_MBCSGetType(const UConverter* converter); michael@0: michael@0: U_CFUNC void michael@0: ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode); michael@0: U_CFUNC void michael@0: ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: /* michael@0: * Internal function returning a UnicodeSet for toUnicode() conversion. michael@0: * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. michael@0: * In the future, if we add support for fallback sets, this function michael@0: * needs to be updated. michael@0: * Handles extensions. michael@0: * Does not empty the set first. michael@0: */ michael@0: U_CFUNC void michael@0: ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: /* michael@0: * Same as ucnv_MBCSGetUnicodeSetForUnicode() but michael@0: * the set can be filtered by encoding scheme. michael@0: * Used by stateful converters which share regular conversion tables michael@0: * but only use a subset of their mappings. michael@0: */ michael@0: U_CFUNC void michael@0: ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UConverterSetFilter filter, michael@0: UErrorCode *pErrorCode); michael@0: michael@0: #endif michael@0: michael@0: #endif