michael@0: /* michael@0: ****************************************************************************** michael@0: * michael@0: * Copyright (C) 2000-2013, International Business Machines michael@0: * Corporation and others. All Rights Reserved. michael@0: * michael@0: ****************************************************************************** michael@0: * file name: ucnvmbcs.c michael@0: * encoding: US-ASCII michael@0: * tab size: 8 (not used) michael@0: * indentation:4 michael@0: * michael@0: * created on: 2000jul03 michael@0: * created by: Markus W. Scherer michael@0: * michael@0: * The current code in this file replaces the previous implementation michael@0: * of conversion code from multi-byte codepages to Unicode and back. michael@0: * This implementation supports the following: michael@0: * - legacy variable-length codepages with up to 4 bytes per character michael@0: * - all Unicode code points (up to 0x10ffff) michael@0: * - efficient distinction of unassigned vs. illegal byte sequences michael@0: * - it is possible in fromUnicode() to directly deal with simple michael@0: * stateful encodings (used for EBCDIC_STATEFUL) michael@0: * - it is possible to convert Unicode code points michael@0: * to a single zero byte (but not as a fallback except for SBCS) michael@0: * michael@0: * Remaining limitations in fromUnicode: michael@0: * - byte sequences must not have leading zero bytes michael@0: * - except for SBCS codepages: no fallback mapping from Unicode to a zero byte michael@0: * - limitation to up to 4 bytes per character michael@0: * michael@0: * ICU 2.8 (late 2003) adds a secondary data structure which lifts some of these michael@0: * limitations and adds m:n character mappings and other features. michael@0: * See ucnv_ext.h for details. michael@0: * michael@0: * Change history: michael@0: * michael@0: * 5/6/2001 Ram Moved MBCS_SINGLE_RESULT_FROM_U,MBCS_STAGE_2_FROM_U, michael@0: * MBCS_VALUE_2_FROM_STAGE_2, MBCS_VALUE_4_FROM_STAGE_2 michael@0: * macros to ucnvmbcs.h file michael@0: */ michael@0: michael@0: #include "unicode/utypes.h" michael@0: michael@0: #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION michael@0: michael@0: #include "unicode/ucnv.h" michael@0: #include "unicode/ucnv_cb.h" michael@0: #include "unicode/udata.h" michael@0: #include "unicode/uset.h" michael@0: #include "unicode/utf8.h" michael@0: #include "unicode/utf16.h" michael@0: #include "ucnv_bld.h" michael@0: #include "ucnvmbcs.h" michael@0: #include "ucnv_ext.h" michael@0: #include "ucnv_cnv.h" michael@0: #include "cmemory.h" michael@0: #include "cstring.h" michael@0: #include "cmutex.h" michael@0: michael@0: /* control optimizations according to the platform */ michael@0: #define MBCS_UNROLL_SINGLE_TO_BMP 1 michael@0: #define MBCS_UNROLL_SINGLE_FROM_BMP 0 michael@0: michael@0: /* michael@0: * _MBCSHeader versions 5.3 & 4.3 michael@0: * (Note that the _MBCSHeader version is in addition to the converter formatVersion.) michael@0: * michael@0: * This version is optional. Version 5 is used for incompatible data format changes. michael@0: * makeconv will continue to generate version 4 files if possible. michael@0: * michael@0: * Changes from version 4: michael@0: * michael@0: * The main difference is an additional _MBCSHeader field with michael@0: * - the length (number of uint32_t) of the _MBCSHeader michael@0: * - flags for further incompatible data format changes michael@0: * - flags for further, backward compatible data format changes michael@0: * michael@0: * The MBCS_OPT_FROM_U flag indicates that most of the fromUnicode data is omitted from michael@0: * the file and needs to be reconstituted at load time. michael@0: * This requires a utf8Friendly format with an additional mbcsIndex table for fast michael@0: * (and UTF-8-friendly) fromUnicode conversion for Unicode code points up to maxFastUChar. michael@0: * (For details about these structures see below, and see ucnvmbcs.h.) michael@0: * michael@0: * utf8Friendly also implies that the fromUnicode mappings are stored in ascending order michael@0: * of the Unicode code points. (This requires that the .ucm file has the |0 etc. michael@0: * precision markers for all mappings.) michael@0: * michael@0: * All fallbacks have been moved to the extension table, leaving only roundtrips in the michael@0: * omitted data that can be reconstituted from the toUnicode data. michael@0: * michael@0: * Of the stage 2 table, the part corresponding to maxFastUChar and below is omitted. michael@0: * With only roundtrip mappings in the base fromUnicode data, this part is fully michael@0: * redundant with the mbcsIndex and will be reconstituted from that (also using the michael@0: * stage 1 table which contains the information about how stage 2 was compacted). michael@0: * michael@0: * The rest of the stage 2 table, the part for code points above maxFastUChar, michael@0: * is stored in the file and will be appended to the reconstituted part. michael@0: * michael@0: * The entire fromUBytes array is omitted from the file and will be reconstitued. michael@0: * This is done by enumerating all toUnicode roundtrip mappings, performing michael@0: * each mapping (using the stage 1 and reconstituted stage 2 tables) and michael@0: * writing instead of reading the byte values. michael@0: * michael@0: * _MBCSHeader version 4.3 michael@0: * michael@0: * Change from version 4.2: michael@0: * - Optional utf8Friendly data structures, with 64-entry stage 3 block michael@0: * allocation for parts of the BMP, and an additional mbcsIndex in non-SBCS michael@0: * files which can be used instead of stages 1 & 2. michael@0: * Faster lookups for roundtrips from most commonly used characters, michael@0: * and lookups from UTF-8 byte sequences with a natural bit distribution. michael@0: * See ucnvmbcs.h for more details. michael@0: * michael@0: * Change from version 4.1: michael@0: * - Added an optional extension table structure at the end of the .cnv file. michael@0: * It is present if the upper bits of the header flags field contains a non-zero michael@0: * byte offset to it. michael@0: * Files that contain only a conversion table and no base table michael@0: * use the special outputType MBCS_OUTPUT_EXT_ONLY. michael@0: * These contain the base table name between the MBCS header and the extension michael@0: * data. michael@0: * michael@0: * Change from version 4.0: michael@0: * - Replace header.reserved with header.fromUBytesLength so that all michael@0: * fields in the data have length. michael@0: * michael@0: * Changes from version 3 (for performance improvements): michael@0: * - new bit distribution for state table entries michael@0: * - reordered action codes michael@0: * - new data structure for single-byte fromUnicode michael@0: * + stage 2 only contains indexes michael@0: * + stage 3 stores 16 bits per character with classification bits 15..8 michael@0: * - no multiplier for stage 1 entries michael@0: * - stage 2 for non-single-byte codepages contains the index and the flags in michael@0: * one 32-bit value michael@0: * - 2-byte and 4-byte fromUnicode results are stored directly as 16/32-bit integers michael@0: * michael@0: * For more details about old versions of the MBCS data structure, see michael@0: * the corresponding versions of this file. michael@0: * michael@0: * Converting stateless codepage data ---------------------------------------*** michael@0: * (or codepage data with simple states) to Unicode. michael@0: * michael@0: * Data structure and algorithm for converting from complex legacy codepages michael@0: * to Unicode. (Designed before 2000-may-22.) michael@0: * michael@0: * The basic idea is that the structure of legacy codepages can be described michael@0: * with state tables. michael@0: * When reading a byte stream, each input byte causes a state transition. michael@0: * Some transitions result in the output of a code point, some result in michael@0: * "unassigned" or "illegal" output. michael@0: * This is used here for character conversion. michael@0: * michael@0: * The data structure begins with a state table consisting of a row michael@0: * per state, with 256 entries (columns) per row for each possible input michael@0: * byte value. michael@0: * Each entry is 32 bits wide, with two formats distinguished by michael@0: * the sign bit (bit 31): michael@0: * michael@0: * One format for transitional entries (bit 31 not set) for non-final bytes, and michael@0: * one format for final entries (bit 31 set). michael@0: * Both formats contain the number of the next state in the same bit michael@0: * positions. michael@0: * State 0 is the initial state. michael@0: * michael@0: * Most of the time, the offset values of subsequent states are added michael@0: * up to a scalar value. This value will eventually be the index of michael@0: * the Unicode code point in a table that follows the state table. michael@0: * The effect is that the code points for final state table rows michael@0: * are contiguous. The code points of final state rows follow each other michael@0: * in the order of the references to those final states by previous michael@0: * states, etc. michael@0: * michael@0: * For some terminal states, the offset is itself the output Unicode michael@0: * code point (16 bits for a BMP code point or 20 bits for a supplementary michael@0: * code point (stored as code point minus 0x10000 so that 20 bits are enough). michael@0: * For others, the code point in the Unicode table is stored with either michael@0: * one or two code units: one for BMP code points, two for a pair of michael@0: * surrogates. michael@0: * All code points for a final state entry take up the same number of code michael@0: * units, regardless of whether they all actually _use_ the same number michael@0: * of code units. This is necessary for simple array access. michael@0: * michael@0: * An additional feature comes in with what in ICU is called "fallback" michael@0: * mappings: michael@0: * michael@0: * In addition to round-trippable, precise, 1:1 mappings, there are often michael@0: * mappings defined between similar, though not the same, characters. michael@0: * Typically, such mappings occur only in fromUnicode mapping tables because michael@0: * Unicode has a superset repertoire of most other codepages. However, it michael@0: * is possible to provide such mappings in the toUnicode tables, too. michael@0: * In this case, the fallback mappings are partly integrated into the michael@0: * general state tables because the structure of the encoding includes their michael@0: * byte sequences. michael@0: * For final entries in an initial state, fallback mappings are stored in michael@0: * the entry itself like with roundtrip mappings. michael@0: * For other final entries, they are stored in the code units table if michael@0: * the entry is for a pair of code units. michael@0: * For single-unit results in the code units table, there is no space to michael@0: * alternatively hold a fallback mapping; in this case, the code unit michael@0: * is stored as U+fffe (unassigned), and the fallback mapping needs to michael@0: * be looked up by the scalar offset value in a separate table. michael@0: * michael@0: * "Unassigned" state entries really mean "structurally unassigned", michael@0: * i.e., such a byte sequence will never have a mapping result. michael@0: * michael@0: * The interpretation of the bits in each entry is as follows: michael@0: * michael@0: * Bit 31 not set, not a terminal entry ("transitional"): michael@0: * 30..24 next state michael@0: * 23..0 offset delta, to be added up michael@0: * michael@0: * Bit 31 set, terminal ("final") entry: michael@0: * 30..24 next state (regardless of action code) michael@0: * 23..20 action code: michael@0: * action codes 0 and 1 result in precise-mapping Unicode code points michael@0: * 0 valid byte sequence michael@0: * 19..16 not used, 0 michael@0: * 15..0 16-bit Unicode BMP code point michael@0: * never U+fffe or U+ffff michael@0: * 1 valid byte sequence michael@0: * 19..0 20-bit Unicode supplementary code point michael@0: * never U+fffe or U+ffff michael@0: * michael@0: * action codes 2 and 3 result in fallback (unidirectional-mapping) Unicode code points michael@0: * 2 valid byte sequence (fallback) michael@0: * 19..16 not used, 0 michael@0: * 15..0 16-bit Unicode BMP code point as fallback result michael@0: * 3 valid byte sequence (fallback) michael@0: * 19..0 20-bit Unicode supplementary code point as fallback result michael@0: * michael@0: * action codes 4 and 5 may result in roundtrip/fallback/unassigned/illegal results michael@0: * depending on the code units they result in michael@0: * 4 valid byte sequence michael@0: * 19..9 not used, 0 michael@0: * 8..0 final offset delta michael@0: * pointing to one 16-bit code unit which may be michael@0: * fffe unassigned -- look for a fallback for this offset michael@0: * ffff illegal michael@0: * 5 valid byte sequence michael@0: * 19..9 not used, 0 michael@0: * 8..0 final offset delta michael@0: * pointing to two 16-bit code units michael@0: * (typically UTF-16 surrogates) michael@0: * the result depends on the first code unit as follows: michael@0: * 0000..d7ff roundtrip BMP code point (1st alone) michael@0: * d800..dbff roundtrip surrogate pair (1st, 2nd) michael@0: * dc00..dfff fallback surrogate pair (1st-400, 2nd) michael@0: * e000 roundtrip BMP code point (2nd alone) michael@0: * e001 fallback BMP code point (2nd alone) michael@0: * fffe unassigned michael@0: * ffff illegal michael@0: * (the final offset deltas are at most 255 * 2, michael@0: * times 2 because of storing code unit pairs) michael@0: * michael@0: * 6 unassigned byte sequence michael@0: * 19..16 not used, 0 michael@0: * 15..0 16-bit Unicode BMP code point U+fffe (new with version 2) michael@0: * this does not contain a final offset delta because the main michael@0: * purpose of this action code is to save scalar offset values; michael@0: * therefore, fallback values cannot be assigned to byte michael@0: * sequences that result in this action code michael@0: * 7 illegal byte sequence michael@0: * 19..16 not used, 0 michael@0: * 15..0 16-bit Unicode BMP code point U+ffff (new with version 2) michael@0: * 8 state change only michael@0: * 19..0 not used, 0 michael@0: * useful for state changes in simple stateful encodings, michael@0: * at Shift-In/Shift-Out codes michael@0: * michael@0: * michael@0: * 9..15 reserved for future use michael@0: * current implementations will only perform a state change michael@0: * and ignore bits 19..0 michael@0: * michael@0: * An encoding with contiguous ranges of unassigned byte sequences, like michael@0: * Shift-JIS and especially EUC-TW, can be stored efficiently by having michael@0: * at least two states for the trail bytes: michael@0: * One trail byte state that results in code points, and one that only michael@0: * has "unassigned" and "illegal" terminal states. michael@0: * michael@0: * Note: partly by accident, this data structure supports simple stateful michael@0: * encodings without any additional logic. michael@0: * Currently, only simple Shift-In/Shift-Out schemes are handled with michael@0: * appropriate state tables (especially EBCDIC_STATEFUL!). michael@0: * michael@0: * MBCS version 2 added: michael@0: * unassigned and illegal action codes have U+fffe and U+ffff michael@0: * instead of unused bits; this is useful for _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP() michael@0: * michael@0: * Converting from Unicode to codepage bytes --------------------------------*** michael@0: * michael@0: * The conversion data structure for fromUnicode is designed for the known michael@0: * structure of Unicode. It maps from 21-bit code points (0..0x10ffff) to michael@0: * a sequence of 1..4 bytes, in addition to a flag that indicates if there is michael@0: * a roundtrip mapping. michael@0: * michael@0: * The lookup is done with a 3-stage trie, using 11/6/4 bits for stage 1/2/3 michael@0: * like in the character properties table. michael@0: * The beginning of the trie is at offsetFromUTable, the beginning of stage 3 michael@0: * with the resulting bytes is at offsetFromUBytes. michael@0: * michael@0: * Beginning with version 4, single-byte codepages have a significantly different michael@0: * trie compared to other codepages. michael@0: * In all cases, the entry in stage 1 is directly the index of the block of michael@0: * 64 entries in stage 2. michael@0: * michael@0: * Single-byte lookup: michael@0: * michael@0: * Stage 2 only contains 16-bit indexes directly to the 16-blocks in stage 3. michael@0: * Stage 3 contains one 16-bit word per result: michael@0: * Bits 15..8 indicate the kind of result: michael@0: * f roundtrip result michael@0: * c fallback result from private-use code point michael@0: * 8 fallback result from other code points michael@0: * 0 unassigned michael@0: * Bits 7..0 contain the codepage byte. A zero byte is always possible. michael@0: * michael@0: * In version 4.3, the runtime code can build an sbcsIndex for a utf8Friendly michael@0: * file. For 2-byte UTF-8 byte sequences and some 3-byte sequences the lookup michael@0: * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. michael@0: * ASCII code points can be looked up with a linear array access into stage 3. michael@0: * See maxFastUChar and other details in ucnvmbcs.h. michael@0: * michael@0: * Multi-byte lookup: michael@0: * michael@0: * Stage 2 contains a 32-bit word for each 16-block in stage 3: michael@0: * Bits 31..16 contain flags for which stage 3 entries contain roundtrip results michael@0: * test: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) michael@0: * If this test is false, then a non-zero result will be interpreted as michael@0: * a fallback mapping. michael@0: * Bits 15..0 contain the index to stage 3, which must be multiplied by 16*(bytes per char) michael@0: * michael@0: * Stage 3 contains 2, 3, or 4 bytes per result. michael@0: * 2 or 4 bytes are stored as uint16_t/uint32_t in platform endianness, michael@0: * while 3 bytes are stored as bytes in big-endian order. michael@0: * Leading zero bytes are ignored, and the number of bytes is counted. michael@0: * A zero byte mapping result is possible as a roundtrip result. michael@0: * For some output types, the actual result is processed from this; michael@0: * see ucnv_MBCSFromUnicodeWithOffsets(). michael@0: * michael@0: * Note that stage 1 always contains 0x440=1088 entries (0x440==0x110000>>10), michael@0: * or (version 3 and up) for BMP-only codepages, it contains 64 entries. michael@0: * michael@0: * In version 4.3, a utf8Friendly file contains an mbcsIndex table. michael@0: * For 2-byte UTF-8 byte sequences and most 3-byte sequences the lookup michael@0: * becomes a 2-stage (single-index) trie lookup with 6 bits for stage 3. michael@0: * ASCII code points can be looked up with a linear array access into stage 3. michael@0: * See maxFastUChar, mbcsIndex and other details in ucnvmbcs.h. michael@0: * michael@0: * In version 3, stage 2 blocks may overlap by multiples of the multiplier michael@0: * for compaction. michael@0: * In version 4, stage 2 blocks (and for single-byte codepages, stage 3 blocks) michael@0: * may overlap by any number of entries. michael@0: * michael@0: * MBCS version 2 added: michael@0: * the converter checks for known output types, which allows michael@0: * adding new ones without crashing an unaware converter michael@0: */ michael@0: michael@0: static const UConverterImpl _SBCSUTF8Impl; michael@0: static const UConverterImpl _DBCSUTF8Impl; michael@0: michael@0: /* GB 18030 data ------------------------------------------------------------ */ michael@0: michael@0: /* helper macros for linear values for GB 18030 four-byte sequences */ michael@0: #define LINEAR_18030(a, b, c, d) ((((a)*10+(b))*126L+(c))*10L+(d)) michael@0: michael@0: #define LINEAR_18030_BASE LINEAR_18030(0x81, 0x30, 0x81, 0x30) michael@0: michael@0: #define LINEAR(x) LINEAR_18030(x>>24, (x>>16)&0xff, (x>>8)&0xff, x&0xff) michael@0: michael@0: /* michael@0: * Some ranges of GB 18030 where both the Unicode code points and the michael@0: * GB four-byte sequences are contiguous and are handled algorithmically by michael@0: * the special callback functions below. michael@0: * The values are start & end of Unicode & GB codes. michael@0: * michael@0: * Note that single surrogates are not mapped by GB 18030 michael@0: * as of the re-released mapping tables from 2000-nov-30. michael@0: */ michael@0: static const uint32_t michael@0: gb18030Ranges[14][4]={ michael@0: {0x10000, 0x10FFFF, LINEAR(0x90308130), LINEAR(0xE3329A35)}, michael@0: {0x9FA6, 0xD7FF, LINEAR(0x82358F33), LINEAR(0x8336C738)}, michael@0: {0x0452, 0x1E3E, LINEAR(0x8130D330), LINEAR(0x8135F436)}, michael@0: {0x1E40, 0x200F, LINEAR(0x8135F438), LINEAR(0x8136A531)}, michael@0: {0xE865, 0xF92B, LINEAR(0x8336D030), LINEAR(0x84308534)}, michael@0: {0x2643, 0x2E80, LINEAR(0x8137A839), LINEAR(0x8138FD38)}, michael@0: {0xFA2A, 0xFE2F, LINEAR(0x84309C38), LINEAR(0x84318537)}, michael@0: {0x3CE1, 0x4055, LINEAR(0x8231D438), LINEAR(0x8232AF32)}, michael@0: {0x361B, 0x3917, LINEAR(0x8230A633), LINEAR(0x8230F237)}, michael@0: {0x49B8, 0x4C76, LINEAR(0x8234A131), LINEAR(0x8234E733)}, michael@0: {0x4160, 0x4336, LINEAR(0x8232C937), LINEAR(0x8232F837)}, michael@0: {0x478E, 0x4946, LINEAR(0x8233E838), LINEAR(0x82349638)}, michael@0: {0x44D7, 0x464B, LINEAR(0x8233A339), LINEAR(0x8233C931)}, michael@0: {0xFFE6, 0xFFFF, LINEAR(0x8431A234), LINEAR(0x8431A439)} michael@0: }; michael@0: michael@0: /* bit flag for UConverter.options indicating GB 18030 special handling */ michael@0: #define _MBCS_OPTION_GB18030 0x8000 michael@0: michael@0: /* bit flag for UConverter.options indicating KEIS,JEF,JIF special handling */ michael@0: #define _MBCS_OPTION_KEIS 0x01000 michael@0: #define _MBCS_OPTION_JEF 0x02000 michael@0: #define _MBCS_OPTION_JIPS 0x04000 michael@0: michael@0: #define KEIS_SO_CHAR_1 0x0A michael@0: #define KEIS_SO_CHAR_2 0x42 michael@0: #define KEIS_SI_CHAR_1 0x0A michael@0: #define KEIS_SI_CHAR_2 0x41 michael@0: michael@0: #define JEF_SO_CHAR 0x28 michael@0: #define JEF_SI_CHAR 0x29 michael@0: michael@0: #define JIPS_SO_CHAR_1 0x1A michael@0: #define JIPS_SO_CHAR_2 0x70 michael@0: #define JIPS_SI_CHAR_1 0x1A michael@0: #define JIPS_SI_CHAR_2 0x71 michael@0: michael@0: enum SISO_Option { michael@0: SI, michael@0: SO michael@0: }; michael@0: typedef enum SISO_Option SISO_Option; michael@0: michael@0: static int32_t getSISOBytes(SISO_Option option, uint32_t cnvOption, uint8_t *value) { michael@0: int32_t SISOLength = 0; michael@0: michael@0: switch (option) { michael@0: case SI: michael@0: if ((cnvOption&_MBCS_OPTION_KEIS)!=0) { michael@0: value[0] = KEIS_SI_CHAR_1; michael@0: value[1] = KEIS_SI_CHAR_2; michael@0: SISOLength = 2; michael@0: } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) { michael@0: value[0] = JEF_SI_CHAR; michael@0: SISOLength = 1; michael@0: } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) { michael@0: value[0] = JIPS_SI_CHAR_1; michael@0: value[1] = JIPS_SI_CHAR_2; michael@0: SISOLength = 2; michael@0: } else { michael@0: value[0] = UCNV_SI; michael@0: SISOLength = 1; michael@0: } michael@0: break; michael@0: case SO: michael@0: if ((cnvOption&_MBCS_OPTION_KEIS)!=0) { michael@0: value[0] = KEIS_SO_CHAR_1; michael@0: value[1] = KEIS_SO_CHAR_2; michael@0: SISOLength = 2; michael@0: } else if ((cnvOption&_MBCS_OPTION_JEF)!=0) { michael@0: value[0] = JEF_SO_CHAR; michael@0: SISOLength = 1; michael@0: } else if ((cnvOption&_MBCS_OPTION_JIPS)!=0) { michael@0: value[0] = JIPS_SO_CHAR_1; michael@0: value[1] = JIPS_SO_CHAR_2; michael@0: SISOLength = 2; michael@0: } else { michael@0: value[0] = UCNV_SO; michael@0: SISOLength = 1; michael@0: } michael@0: break; michael@0: default: michael@0: /* Should never happen. */ michael@0: break; michael@0: } michael@0: michael@0: return SISOLength; michael@0: } michael@0: michael@0: /* Miscellaneous ------------------------------------------------------------ */ michael@0: michael@0: /** michael@0: * Callback from ucnv_MBCSEnumToUnicode(), takes 32 mappings from michael@0: * consecutive sequences of bytes, starting from the one encoded in value, michael@0: * to Unicode code points. (Multiple mappings to reduce per-function call overhead.) michael@0: * Does not currently support m:n mappings or reverse fallbacks. michael@0: * This function will not be called for sequences of bytes with leading zeros. michael@0: * michael@0: * @param context an opaque pointer, as passed into ucnv_MBCSEnumToUnicode() michael@0: * @param value contains 1..4 bytes of the first byte sequence, right-aligned michael@0: * @param codePoints resulting Unicode code points, or negative if a byte sequence does michael@0: * not map to anything michael@0: * @return TRUE to continue enumeration, FALSE to stop michael@0: */ michael@0: typedef UBool U_CALLCONV michael@0: UConverterEnumToUCallback(const void *context, uint32_t value, UChar32 codePoints[32]); michael@0: michael@0: /* similar to ucnv_MBCSGetNextUChar() but recursive */ michael@0: static UBool michael@0: enumToU(UConverterMBCSTable *mbcsTable, int8_t stateProps[], michael@0: int32_t state, uint32_t offset, michael@0: uint32_t value, michael@0: UConverterEnumToUCallback *callback, const void *context, michael@0: UErrorCode *pErrorCode) { michael@0: UChar32 codePoints[32]; michael@0: const int32_t *row; michael@0: const uint16_t *unicodeCodeUnits; michael@0: UChar32 anyCodePoints; michael@0: int32_t b, limit; michael@0: michael@0: row=mbcsTable->stateTable[state]; michael@0: unicodeCodeUnits=mbcsTable->unicodeCodeUnits; michael@0: michael@0: value<<=8; michael@0: anyCodePoints=-1; /* becomes non-negative if there is a mapping */ michael@0: michael@0: b=(stateProps[state]&0x38)<<2; michael@0: if(b==0 && stateProps[state]>=0x40) { michael@0: /* skip byte sequences with leading zeros because they are not stored in the fromUnicode table */ michael@0: codePoints[0]=U_SENTINEL; michael@0: b=1; michael@0: } michael@0: limit=((stateProps[state]&7)+1)<<5; michael@0: while(b=0) { michael@0: /* recurse to a state with non-ignorable actions */ michael@0: if(!enumToU( michael@0: mbcsTable, stateProps, nextState, michael@0: offset+MBCS_ENTRY_TRANSITION_OFFSET(entry), michael@0: value|(uint32_t)b, michael@0: callback, context, michael@0: pErrorCode)) { michael@0: return FALSE; michael@0: } michael@0: } michael@0: codePoints[b&0x1f]=U_SENTINEL; michael@0: } else { michael@0: UChar32 c; michael@0: int32_t action; michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=MBCS_ENTRY_FINAL_ACTION(entry); michael@0: if(action==MBCS_STATE_VALID_DIRECT_16) { michael@0: /* output BMP code point */ michael@0: c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: } else if(action==MBCS_STATE_VALID_16) { michael@0: int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[finalOffset]; michael@0: if(c<0xfffe) { michael@0: /* output BMP code point */ michael@0: } else { michael@0: c=U_SENTINEL; michael@0: } michael@0: } else if(action==MBCS_STATE_VALID_16_PAIR) { michael@0: int32_t finalOffset=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[finalOffset++]; michael@0: if(c<0xd800) { michael@0: /* output BMP code point below 0xd800 */ michael@0: } else if(c<=0xdbff) { michael@0: /* output roundtrip or fallback supplementary code point */ michael@0: c=((c&0x3ff)<<10)+unicodeCodeUnits[finalOffset]+(0x10000-0xdc00); michael@0: } else if(c==0xe000) { michael@0: /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ michael@0: c=unicodeCodeUnits[finalOffset]; michael@0: } else { michael@0: c=U_SENTINEL; michael@0: } michael@0: } else if(action==MBCS_STATE_VALID_DIRECT_20) { michael@0: /* output supplementary code point */ michael@0: c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); michael@0: } else { michael@0: c=U_SENTINEL; michael@0: } michael@0: michael@0: codePoints[b&0x1f]=c; michael@0: anyCodePoints&=c; michael@0: } michael@0: if(((++b)&0x1f)==0) { michael@0: if(anyCodePoints>=0) { michael@0: if(!callback(context, value|(uint32_t)(b-0x20), codePoints)) { michael@0: return FALSE; michael@0: } michael@0: anyCodePoints=-1; michael@0: } michael@0: } michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: /* michael@0: * Only called if stateProps[state]==-1. michael@0: * A recursive call may do stateProps[state]|=0x40 if this state is the target of an michael@0: * MBCS_STATE_CHANGE_ONLY. michael@0: */ michael@0: static int8_t michael@0: getStateProp(const int32_t (*stateTable)[256], int8_t stateProps[], int state) { michael@0: const int32_t *row; michael@0: int32_t min, max, entry, nextState; michael@0: michael@0: row=stateTable[state]; michael@0: stateProps[state]=0; michael@0: michael@0: /* find first non-ignorable state */ michael@0: for(min=0;; ++min) { michael@0: entry=row[min]; michael@0: nextState=MBCS_ENTRY_STATE(entry); michael@0: if(stateProps[nextState]==-1) { michael@0: getStateProp(stateTable, stateProps, nextState); michael@0: } michael@0: if(MBCS_ENTRY_IS_TRANSITION(entry)) { michael@0: if(stateProps[nextState]>=0) { michael@0: break; michael@0: } michael@0: } else if(MBCS_ENTRY_FINAL_ACTION(entry)>5)<<3); michael@0: michael@0: /* find last non-ignorable state */ michael@0: for(max=0xff; min=0) { michael@0: break; michael@0: } michael@0: } else if(MBCS_ENTRY_FINAL_ACTION(entry)>5); michael@0: michael@0: /* recurse further and collect direct-state information */ michael@0: while(min<=max) { michael@0: entry=row[min]; michael@0: nextState=MBCS_ENTRY_STATE(entry); michael@0: if(stateProps[nextState]==-1) { michael@0: getStateProp(stateTable, stateProps, nextState); michael@0: } michael@0: if(MBCS_ENTRY_IS_FINAL(entry)) { michael@0: stateProps[nextState]|=0x40; michael@0: if(MBCS_ENTRY_FINAL_ACTION(entry)<=MBCS_STATE_FALLBACK_DIRECT_20) { michael@0: stateProps[state]|=0x40; michael@0: } michael@0: } michael@0: ++min; michael@0: } michael@0: return stateProps[state]; michael@0: } michael@0: michael@0: /* michael@0: * Internal function enumerating the toUnicode data of an MBCS converter. michael@0: * Currently only used for reconstituting data for a MBCS_OPT_NO_FROM_U michael@0: * table, but could also be used for a future ucnv_getUnicodeSet() option michael@0: * that includes reverse fallbacks (after updating this function's implementation). michael@0: * Currently only handles roundtrip mappings. michael@0: * Does not currently handle extensions. michael@0: */ michael@0: static void michael@0: ucnv_MBCSEnumToUnicode(UConverterMBCSTable *mbcsTable, michael@0: UConverterEnumToUCallback *callback, const void *context, michael@0: UErrorCode *pErrorCode) { michael@0: /* michael@0: * Properties for each state, to speed up the enumeration. michael@0: * Ignorable actions are unassigned/illegal/state-change-only: michael@0: * They do not lead to mappings. michael@0: * michael@0: * Bits 7..6: michael@0: * 1 direct/initial state (stateful converters have multiple) michael@0: * 0 non-initial state with transitions or with non-ignorable result actions michael@0: * -1 final state with only ignorable actions michael@0: * michael@0: * Bits 5..3: michael@0: * The lowest byte value with non-ignorable actions is michael@0: * value<<5 (rounded down). michael@0: * michael@0: * Bits 2..0: michael@0: * The highest byte value with non-ignorable actions is michael@0: * (value<<5)&0x1f (rounded up). michael@0: */ michael@0: int8_t stateProps[MBCS_MAX_STATE_COUNT]; michael@0: int32_t state; michael@0: michael@0: uprv_memset(stateProps, -1, sizeof(stateProps)); michael@0: michael@0: /* recurse from state 0 and set all stateProps */ michael@0: getStateProp(mbcsTable->stateTable, stateProps, 0); michael@0: michael@0: for(state=0; statecountStates; ++state) { michael@0: /*if(stateProps[state]==-1) { michael@0: printf("unused/unreachable %d\n", state); michael@0: }*/ michael@0: if(stateProps[state]>=0x40) { michael@0: /* start from each direct state */ michael@0: enumToU( michael@0: mbcsTable, stateProps, state, 0, 0, michael@0: callback, context, michael@0: pErrorCode); michael@0: } michael@0: } michael@0: } michael@0: michael@0: U_CFUNC void michael@0: ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UConverterSetFilter filter, michael@0: UErrorCode *pErrorCode) { michael@0: const UConverterMBCSTable *mbcsTable; michael@0: const uint16_t *table; michael@0: michael@0: uint32_t st3; michael@0: uint16_t st1, maxStage1, st2; michael@0: michael@0: UChar32 c; michael@0: michael@0: /* enumerate the from-Unicode trie table */ michael@0: mbcsTable=&sharedData->mbcs; michael@0: table=mbcsTable->fromUnicodeTable; michael@0: if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { michael@0: maxStage1=0x440; michael@0: } else { michael@0: maxStage1=0x40; michael@0: } michael@0: michael@0: c=0; /* keep track of the current code point while enumerating */ michael@0: michael@0: if(mbcsTable->outputType==MBCS_OUTPUT_1) { michael@0: const uint16_t *stage2, *stage3, *results; michael@0: uint16_t minValue; michael@0: michael@0: results=(const uint16_t *)mbcsTable->fromUnicodeBytes; michael@0: michael@0: /* michael@0: * Set a threshold variable for selecting which mappings to use. michael@0: * See ucnv_MBCSSingleFromBMPWithOffsets() and michael@0: * MBCS_SINGLE_RESULT_FROM_U() for details. michael@0: */ michael@0: if(which==UCNV_ROUNDTRIP_SET) { michael@0: /* use only roundtrips */ michael@0: minValue=0xf00; michael@0: } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ { michael@0: /* use all roundtrip and fallback results */ michael@0: minValue=0x800; michael@0: } michael@0: michael@0: for(st1=0; st1maxStage1) { michael@0: stage2=table+st2; michael@0: for(st2=0; st2<64; ++st2) { michael@0: if((st3=stage2[st2])!=0) { michael@0: /* read the stage 3 block */ michael@0: stage3=results+st3; michael@0: michael@0: do { michael@0: if(*stage3++>=minValue) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: } while((++c&0xf)!=0); michael@0: } else { michael@0: c+=16; /* empty stage 3 block */ michael@0: } michael@0: } michael@0: } else { michael@0: c+=1024; /* empty stage 2 block */ michael@0: } michael@0: } michael@0: } else { michael@0: const uint32_t *stage2; michael@0: const uint8_t *stage3, *bytes; michael@0: uint32_t st3Multiplier; michael@0: uint32_t value; michael@0: UBool useFallback; michael@0: michael@0: bytes=mbcsTable->fromUnicodeBytes; michael@0: michael@0: useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET); michael@0: michael@0: switch(mbcsTable->outputType) { michael@0: case MBCS_OUTPUT_3: michael@0: case MBCS_OUTPUT_4_EUC: michael@0: st3Multiplier=3; michael@0: break; michael@0: case MBCS_OUTPUT_4: michael@0: st3Multiplier=4; michael@0: break; michael@0: default: michael@0: st3Multiplier=2; michael@0: break; michael@0: } michael@0: michael@0: for(st1=0; st1(maxStage1>>1)) { michael@0: stage2=(const uint32_t *)table+st2; michael@0: for(st2=0; st2<64; ++st2) { michael@0: if((st3=stage2[st2])!=0) { michael@0: /* read the stage 3 block */ michael@0: stage3=bytes+st3Multiplier*16*(uint32_t)(uint16_t)st3; michael@0: michael@0: /* get the roundtrip flags for the stage 3 block */ michael@0: st3>>=16; michael@0: michael@0: /* michael@0: * Add code points for which the roundtrip flag is set, michael@0: * or which map to non-zero bytes if we use fallbacks. michael@0: * See ucnv_MBCSFromUnicodeWithOffsets() for details. michael@0: */ michael@0: switch(filter) { michael@0: case UCNV_SET_FILTER_NONE: michael@0: do { michael@0: if(st3&1) { michael@0: sa->add(sa->set, c); michael@0: stage3+=st3Multiplier; michael@0: } else if(useFallback) { michael@0: uint8_t b=0; michael@0: switch(st3Multiplier) { michael@0: case 4: michael@0: b|=*stage3++; michael@0: case 3: /*fall through*/ michael@0: b|=*stage3++; michael@0: case 2: /*fall through*/ michael@0: b|=stage3[0]|stage3[1]; michael@0: stage3+=2; michael@0: default: michael@0: break; michael@0: } michael@0: if(b!=0) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: } michael@0: st3>>=1; michael@0: } while((++c&0xf)!=0); michael@0: break; michael@0: case UCNV_SET_FILTER_DBCS_ONLY: michael@0: /* Ignore single-byte results (<0x100). */ michael@0: do { michael@0: if(((st3&1)!=0 || useFallback) && *((const uint16_t *)stage3)>=0x100) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: st3>>=1; michael@0: stage3+=2; /* +=st3Multiplier */ michael@0: } while((++c&0xf)!=0); michael@0: break; michael@0: case UCNV_SET_FILTER_2022_CN: michael@0: /* Only add code points that map to CNS 11643 planes 1 & 2 for non-EXT ISO-2022-CN. */ michael@0: do { michael@0: if(((st3&1)!=0 || useFallback) && ((value=*stage3)==0x81 || value==0x82)) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: st3>>=1; michael@0: stage3+=3; /* +=st3Multiplier */ michael@0: } while((++c&0xf)!=0); michael@0: break; michael@0: case UCNV_SET_FILTER_SJIS: michael@0: /* Only add code points that map to Shift-JIS codes corresponding to JIS X 0208. */ michael@0: do { michael@0: if(((st3&1)!=0 || useFallback) && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: st3>>=1; michael@0: stage3+=2; /* +=st3Multiplier */ michael@0: } while((++c&0xf)!=0); michael@0: break; michael@0: case UCNV_SET_FILTER_GR94DBCS: michael@0: /* Only add code points that map to ISO 2022 GR 94 DBCS codes (each byte A1..FE). */ michael@0: do { michael@0: if( ((st3&1)!=0 || useFallback) && michael@0: (uint16_t)((value=*((const uint16_t *)stage3)) - 0xa1a1)<=(0xfefe - 0xa1a1) && michael@0: (uint8_t)(value-0xa1)<=(0xfe - 0xa1) michael@0: ) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: st3>>=1; michael@0: stage3+=2; /* +=st3Multiplier */ michael@0: } while((++c&0xf)!=0); michael@0: break; michael@0: case UCNV_SET_FILTER_HZ: michael@0: /* Only add code points that are suitable for HZ DBCS (lead byte A1..FD). */ michael@0: do { michael@0: if( ((st3&1)!=0 || useFallback) && michael@0: (uint16_t)((value=*((const uint16_t *)stage3))-0xa1a1)<=(0xfdfe - 0xa1a1) && michael@0: (uint8_t)(value-0xa1)<=(0xfe - 0xa1) michael@0: ) { michael@0: sa->add(sa->set, c); michael@0: } michael@0: st3>>=1; michael@0: stage3+=2; /* +=st3Multiplier */ michael@0: } while((++c&0xf)!=0); michael@0: break; michael@0: default: michael@0: *pErrorCode=U_INTERNAL_PROGRAM_ERROR; michael@0: return; michael@0: } michael@0: } else { michael@0: c+=16; /* empty stage 3 block */ michael@0: } michael@0: } michael@0: } else { michael@0: c+=1024; /* empty stage 2 block */ michael@0: } michael@0: } michael@0: } michael@0: michael@0: ucnv_extGetUnicodeSet(sharedData, sa, which, filter, pErrorCode); michael@0: } michael@0: michael@0: U_CFUNC void michael@0: ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UErrorCode *pErrorCode) { michael@0: ucnv_MBCSGetFilteredUnicodeSetForUnicode( michael@0: sharedData, sa, which, michael@0: sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? michael@0: UCNV_SET_FILTER_DBCS_ONLY : michael@0: UCNV_SET_FILTER_NONE, michael@0: pErrorCode); michael@0: } michael@0: michael@0: static void michael@0: ucnv_MBCSGetUnicodeSet(const UConverter *cnv, michael@0: const USetAdder *sa, michael@0: UConverterUnicodeSet which, michael@0: UErrorCode *pErrorCode) { michael@0: if(cnv->options&_MBCS_OPTION_GB18030) { michael@0: sa->addRange(sa->set, 0, 0xd7ff); michael@0: sa->addRange(sa->set, 0xe000, 0x10ffff); michael@0: } else { michael@0: ucnv_MBCSGetUnicodeSetForUnicode(cnv->sharedData, sa, which, pErrorCode); michael@0: } michael@0: } michael@0: michael@0: /* conversion extensions for input not in the main table -------------------- */ michael@0: michael@0: /* michael@0: * Hardcoded extension handling for GB 18030. michael@0: * Definition of LINEAR macros and gb18030Ranges see near the beginning of the file. michael@0: * michael@0: * In the future, conversion extensions may handle m:n mappings and delta tables, michael@0: * see http://source.icu-project.org/repos/icu/icuhtml/trunk/design/conversion/conversion_extensions.html michael@0: * michael@0: * If an input character cannot be mapped, then these functions set an error michael@0: * code. The framework will then call the callback function. michael@0: */ michael@0: michael@0: /* michael@0: * @return if(U_FAILURE) return the code point for cnv->fromUChar32 michael@0: * else return 0 after output has been written to the target michael@0: */ michael@0: static UChar32 michael@0: _extFromU(UConverter *cnv, const UConverterSharedData *sharedData, michael@0: UChar32 cp, michael@0: const UChar **source, const UChar *sourceLimit, michael@0: uint8_t **target, const uint8_t *targetLimit, michael@0: int32_t **offsets, int32_t sourceIndex, michael@0: UBool flush, michael@0: UErrorCode *pErrorCode) { michael@0: const int32_t *cx; michael@0: michael@0: cnv->useSubChar1=FALSE; michael@0: michael@0: if( (cx=sharedData->mbcs.extIndexes)!=NULL && michael@0: ucnv_extInitialMatchFromU( michael@0: cnv, cx, michael@0: cp, source, sourceLimit, michael@0: (char **)target, (char *)targetLimit, michael@0: offsets, sourceIndex, michael@0: flush, michael@0: pErrorCode) michael@0: ) { michael@0: return 0; /* an extension mapping handled the input */ michael@0: } michael@0: michael@0: /* GB 18030 */ michael@0: if((cnv->options&_MBCS_OPTION_GB18030)!=0) { michael@0: const uint32_t *range; michael@0: int32_t i; michael@0: michael@0: range=gb18030Ranges[0]; michael@0: for(i=0; itoUBytes[0..length[ michael@0: * @return if(U_FAILURE) return the length (toULength, byteIndex) for the input michael@0: * else return 0 after output has been written to the target michael@0: */ michael@0: static int8_t michael@0: _extToU(UConverter *cnv, const UConverterSharedData *sharedData, michael@0: int8_t length, michael@0: const uint8_t **source, const uint8_t *sourceLimit, michael@0: UChar **target, const UChar *targetLimit, michael@0: int32_t **offsets, int32_t sourceIndex, michael@0: UBool flush, michael@0: UErrorCode *pErrorCode) { michael@0: const int32_t *cx; michael@0: michael@0: if( (cx=sharedData->mbcs.extIndexes)!=NULL && michael@0: ucnv_extInitialMatchToU( michael@0: cnv, cx, michael@0: length, (const char **)source, (const char *)sourceLimit, michael@0: target, targetLimit, michael@0: offsets, sourceIndex, michael@0: flush, michael@0: pErrorCode) michael@0: ) { michael@0: return 0; /* an extension mapping handled the input */ michael@0: } michael@0: michael@0: /* GB 18030 */ michael@0: if(length==4 && (cnv->options&_MBCS_OPTION_GB18030)!=0) { michael@0: const uint32_t *range; michael@0: uint32_t linear; michael@0: int32_t i; michael@0: michael@0: linear=LINEAR_18030(cnv->toUBytes[0], cnv->toUBytes[1], cnv->toUBytes[2], cnv->toUBytes[3]); michael@0: range=gb18030Ranges[0]; michael@0: for(i=0; iNL ------------------------------------------------------ */ michael@0: michael@0: /* michael@0: * This code modifies a standard EBCDIC<->Unicode mapping table for michael@0: * OS/390 (z/OS) Unix System Services (Open Edition). michael@0: * The difference is in the mapping of Line Feed and New Line control codes: michael@0: * Standard EBCDIC maps michael@0: * michael@0: * \x25 |0 michael@0: * \x15 |0 michael@0: * michael@0: * but OS/390 USS EBCDIC swaps the control codes for LF and NL, michael@0: * mapping michael@0: * michael@0: * \x15 |0 michael@0: * \x25 |0 michael@0: * michael@0: * This code modifies a loaded standard EBCDIC<->Unicode mapping table michael@0: * by copying it into allocated memory and swapping the LF and NL values. michael@0: * It allows to support the same EBCDIC charset in both versions without michael@0: * duplicating the entire installed table. michael@0: */ michael@0: michael@0: /* standard EBCDIC codes */ michael@0: #define EBCDIC_LF 0x25 michael@0: #define EBCDIC_NL 0x15 michael@0: michael@0: /* standard EBCDIC codes with roundtrip flag as stored in Unicode-to-single-byte tables */ michael@0: #define EBCDIC_RT_LF 0xf25 michael@0: #define EBCDIC_RT_NL 0xf15 michael@0: michael@0: /* Unicode code points */ michael@0: #define U_LF 0x0a michael@0: #define U_NL 0x85 michael@0: michael@0: static UBool michael@0: _EBCDICSwapLFNL(UConverterSharedData *sharedData, UErrorCode *pErrorCode) { michael@0: UConverterMBCSTable *mbcsTable; michael@0: michael@0: const uint16_t *table, *results; michael@0: const uint8_t *bytes; michael@0: michael@0: int32_t (*newStateTable)[256]; michael@0: uint16_t *newResults; michael@0: uint8_t *p; michael@0: char *name; michael@0: michael@0: uint32_t stage2Entry; michael@0: uint32_t size, sizeofFromUBytes; michael@0: michael@0: mbcsTable=&sharedData->mbcs; michael@0: michael@0: table=mbcsTable->fromUnicodeTable; michael@0: bytes=mbcsTable->fromUnicodeBytes; michael@0: results=(const uint16_t *)bytes; michael@0: michael@0: /* michael@0: * Check that this is an EBCDIC table with SBCS portion - michael@0: * SBCS or EBCDIC_STATEFUL with standard EBCDIC LF and NL mappings. michael@0: * michael@0: * If not, ignore the option. Options are always ignored if they do not apply. michael@0: */ michael@0: if(!( michael@0: (mbcsTable->outputType==MBCS_OUTPUT_1 || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) && michael@0: mbcsTable->stateTable[0][EBCDIC_LF]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF) && michael@0: mbcsTable->stateTable[0][EBCDIC_NL]==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL) michael@0: )) { michael@0: return FALSE; michael@0: } michael@0: michael@0: if(mbcsTable->outputType==MBCS_OUTPUT_1) { michael@0: if(!( michael@0: EBCDIC_RT_LF==MBCS_SINGLE_RESULT_FROM_U(table, results, U_LF) && michael@0: EBCDIC_RT_NL==MBCS_SINGLE_RESULT_FROM_U(table, results, U_NL) michael@0: )) { michael@0: return FALSE; michael@0: } michael@0: } else /* MBCS_OUTPUT_2_SISO */ { michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); michael@0: if(!( michael@0: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_LF)!=0 && michael@0: EBCDIC_LF==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_LF) michael@0: )) { michael@0: return FALSE; michael@0: } michael@0: michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); michael@0: if(!( michael@0: MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, U_NL)!=0 && michael@0: EBCDIC_NL==MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, U_NL) michael@0: )) { michael@0: return FALSE; michael@0: } michael@0: } michael@0: michael@0: if(mbcsTable->fromUBytesLength>0) { michael@0: /* michael@0: * We _know_ the number of bytes in the fromUnicodeBytes array michael@0: * starting with header.version 4.1. michael@0: */ michael@0: sizeofFromUBytes=mbcsTable->fromUBytesLength; michael@0: } else { michael@0: /* michael@0: * Otherwise: michael@0: * There used to be code to enumerate the fromUnicode michael@0: * trie and find the highest entry, but it was removed in ICU 3.2 michael@0: * because it was not tested and caused a low code coverage number. michael@0: * See Jitterbug 3674. michael@0: * This affects only some .cnv file formats with a header.version michael@0: * below 4.1, and only when swaplfnl is requested. michael@0: * michael@0: * ucnvmbcs.c revision 1.99 is the last one with the michael@0: * ucnv_MBCSSizeofFromUBytes() function. michael@0: */ michael@0: *pErrorCode=U_INVALID_FORMAT_ERROR; michael@0: return FALSE; michael@0: } michael@0: michael@0: /* michael@0: * The table has an appropriate format. michael@0: * Allocate and build michael@0: * - a modified to-Unicode state table michael@0: * - a modified from-Unicode output array michael@0: * - a converter name string with the swap option appended michael@0: */ michael@0: size= michael@0: mbcsTable->countStates*1024+ michael@0: sizeofFromUBytes+ michael@0: UCNV_MAX_CONVERTER_NAME_LENGTH+20; michael@0: p=(uint8_t *)uprv_malloc(size); michael@0: if(p==NULL) { michael@0: *pErrorCode=U_MEMORY_ALLOCATION_ERROR; michael@0: return FALSE; michael@0: } michael@0: michael@0: /* copy and modify the to-Unicode state table */ michael@0: newStateTable=(int32_t (*)[256])p; michael@0: uprv_memcpy(newStateTable, mbcsTable->stateTable, mbcsTable->countStates*1024); michael@0: michael@0: newStateTable[0][EBCDIC_LF]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_NL); michael@0: newStateTable[0][EBCDIC_NL]=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, U_LF); michael@0: michael@0: /* copy and modify the from-Unicode result table */ michael@0: newResults=(uint16_t *)newStateTable[mbcsTable->countStates]; michael@0: uprv_memcpy(newResults, bytes, sizeofFromUBytes); michael@0: michael@0: /* conveniently, the table access macros work on the left side of expressions */ michael@0: if(mbcsTable->outputType==MBCS_OUTPUT_1) { michael@0: MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_LF)=EBCDIC_RT_NL; michael@0: MBCS_SINGLE_RESULT_FROM_U(table, newResults, U_NL)=EBCDIC_RT_LF; michael@0: } else /* MBCS_OUTPUT_2_SISO */ { michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, U_LF); michael@0: MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_LF)=EBCDIC_NL; michael@0: michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, U_NL); michael@0: MBCS_VALUE_2_FROM_STAGE_2(newResults, stage2Entry, U_NL)=EBCDIC_LF; michael@0: } michael@0: michael@0: /* set the canonical converter name */ michael@0: name=(char *)newResults+sizeofFromUBytes; michael@0: uprv_strcpy(name, sharedData->staticData->name); michael@0: uprv_strcat(name, UCNV_SWAP_LFNL_OPTION_STRING); michael@0: michael@0: /* set the pointers */ michael@0: umtx_lock(NULL); michael@0: if(mbcsTable->swapLFNLStateTable==NULL) { michael@0: mbcsTable->swapLFNLStateTable=newStateTable; michael@0: mbcsTable->swapLFNLFromUnicodeBytes=(uint8_t *)newResults; michael@0: mbcsTable->swapLFNLName=name; michael@0: michael@0: newStateTable=NULL; michael@0: } michael@0: umtx_unlock(NULL); michael@0: michael@0: /* release the allocated memory if another thread beat us to it */ michael@0: if(newStateTable!=NULL) { michael@0: uprv_free(newStateTable); michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: /* reconstitute omitted fromUnicode data ------------------------------------ */ michael@0: michael@0: /* for details, compare with genmbcs.c MBCSAddFromUnicode() and transformEUC() */ michael@0: static UBool U_CALLCONV michael@0: writeStage3Roundtrip(const void *context, uint32_t value, UChar32 codePoints[32]) { michael@0: UConverterMBCSTable *mbcsTable=(UConverterMBCSTable *)context; michael@0: const uint16_t *table; michael@0: uint32_t *stage2; michael@0: uint8_t *bytes, *p; michael@0: UChar32 c; michael@0: int32_t i, st3; michael@0: michael@0: table=mbcsTable->fromUnicodeTable; michael@0: bytes=(uint8_t *)mbcsTable->fromUnicodeBytes; michael@0: michael@0: /* for EUC outputTypes, modify the value like genmbcs.c's transformEUC() */ michael@0: switch(mbcsTable->outputType) { michael@0: case MBCS_OUTPUT_3_EUC: michael@0: if(value<=0xffff) { michael@0: /* short sequences are stored directly */ michael@0: /* code set 0 or 1 */ michael@0: } else if(value<=0x8effff) { michael@0: /* code set 2 */ michael@0: value&=0x7fff; michael@0: } else /* first byte is 0x8f */ { michael@0: /* code set 3 */ michael@0: value&=0xff7f; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4_EUC: michael@0: if(value<=0xffffff) { michael@0: /* short sequences are stored directly */ michael@0: /* code set 0 or 1 */ michael@0: } else if(value<=0x8effffff) { michael@0: /* code set 2 */ michael@0: value&=0x7fffff; michael@0: } else /* first byte is 0x8f */ { michael@0: /* code set 3 */ michael@0: value&=0xff7fff; michael@0: } michael@0: break; michael@0: default: michael@0: break; michael@0: } michael@0: michael@0: for(i=0; i<=0x1f; ++value, ++i) { michael@0: c=codePoints[i]; michael@0: if(c<0) { michael@0: continue; michael@0: } michael@0: michael@0: /* locate the stage 2 & 3 data */ michael@0: stage2=((uint32_t *)table)+table[c>>10]+((c>>4)&0x3f); michael@0: p=bytes; michael@0: st3=(int32_t)(uint16_t)*stage2*16+(c&0xf); michael@0: michael@0: /* write the codepage bytes into stage 3 */ michael@0: switch(mbcsTable->outputType) { michael@0: case MBCS_OUTPUT_3: michael@0: case MBCS_OUTPUT_4_EUC: michael@0: p+=st3*3; michael@0: p[0]=(uint8_t)(value>>16); michael@0: p[1]=(uint8_t)(value>>8); michael@0: p[2]=(uint8_t)value; michael@0: break; michael@0: case MBCS_OUTPUT_4: michael@0: ((uint32_t *)p)[st3]=value; michael@0: break; michael@0: default: michael@0: /* 2 bytes per character */ michael@0: ((uint16_t *)p)[st3]=(uint16_t)value; michael@0: break; michael@0: } michael@0: michael@0: /* set the roundtrip flag */ michael@0: *stage2|=(1UL<<(16+(c&0xf))); michael@0: } michael@0: return TRUE; michael@0: } michael@0: michael@0: static void michael@0: reconstituteData(UConverterMBCSTable *mbcsTable, michael@0: uint32_t stage1Length, uint32_t stage2Length, michael@0: uint32_t fullStage2Length, /* lengths are numbers of units, not bytes */ michael@0: UErrorCode *pErrorCode) { michael@0: uint16_t *stage1; michael@0: uint32_t *stage2; michael@0: uint32_t dataLength=stage1Length*2+fullStage2Length*4+mbcsTable->fromUBytesLength; michael@0: mbcsTable->reconstitutedData=(uint8_t *)uprv_malloc(dataLength); michael@0: if(mbcsTable->reconstitutedData==NULL) { michael@0: *pErrorCode=U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: uprv_memset(mbcsTable->reconstitutedData, 0, dataLength); michael@0: michael@0: /* copy existing data and reroute the pointers */ michael@0: stage1=(uint16_t *)mbcsTable->reconstitutedData; michael@0: uprv_memcpy(stage1, mbcsTable->fromUnicodeTable, stage1Length*2); michael@0: michael@0: stage2=(uint32_t *)(stage1+stage1Length); michael@0: uprv_memcpy(stage2+(fullStage2Length-stage2Length), michael@0: mbcsTable->fromUnicodeTable+stage1Length, michael@0: stage2Length*4); michael@0: michael@0: mbcsTable->fromUnicodeTable=stage1; michael@0: mbcsTable->fromUnicodeBytes=(uint8_t *)(stage2+fullStage2Length); michael@0: michael@0: /* indexes into stage 2 count from the bottom of the fromUnicodeTable */ michael@0: stage2=(uint32_t *)stage1; michael@0: michael@0: /* reconstitute the initial part of stage 2 from the mbcsIndex */ michael@0: { michael@0: int32_t stageUTF8Length=((int32_t)mbcsTable->maxFastUChar+1)>>6; michael@0: int32_t stageUTF8Index=0; michael@0: int32_t st1, st2, st3, i; michael@0: michael@0: for(st1=0; stageUTF8IndexmbcsIndex[stageUTF8Index++]; michael@0: if(st3!=0) { michael@0: /* an stage 2 entry's index is per stage 3 16-block, not per stage 3 entry */ michael@0: st3>>=4; michael@0: /* michael@0: * 4 stage 2 entries point to 4 consecutive stage 3 16-blocks which are michael@0: * allocated together as a single 64-block for access from the mbcsIndex michael@0: */ michael@0: stage2[st2++]=st3++; michael@0: stage2[st2++]=st3++; michael@0: stage2[st2++]=st3++; michael@0: stage2[st2++]=st3; michael@0: } else { michael@0: /* no stage 3 block, skip */ michael@0: st2+=4; michael@0: } michael@0: } michael@0: } else { michael@0: /* no stage 2 block, skip */ michael@0: stageUTF8Index+=16; michael@0: } michael@0: } michael@0: } michael@0: michael@0: /* reconstitute fromUnicodeBytes with roundtrips from toUnicode data */ michael@0: ucnv_MBCSEnumToUnicode(mbcsTable, writeStage3Roundtrip, mbcsTable, pErrorCode); michael@0: } michael@0: michael@0: /* MBCS setup functions ----------------------------------------------------- */ michael@0: michael@0: static void michael@0: ucnv_MBCSLoad(UConverterSharedData *sharedData, michael@0: UConverterLoadArgs *pArgs, michael@0: const uint8_t *raw, michael@0: UErrorCode *pErrorCode) { michael@0: UDataInfo info; michael@0: UConverterMBCSTable *mbcsTable=&sharedData->mbcs; michael@0: _MBCSHeader *header=(_MBCSHeader *)raw; michael@0: uint32_t offset; michael@0: uint32_t headerLength; michael@0: UBool noFromU=FALSE; michael@0: michael@0: if(header->version[0]==4) { michael@0: headerLength=MBCS_HEADER_V4_LENGTH; michael@0: } else if(header->version[0]==5 && header->version[1]>=3 && michael@0: (header->options&MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK)==0) { michael@0: headerLength=header->options&MBCS_OPT_LENGTH_MASK; michael@0: noFromU=(UBool)((header->options&MBCS_OPT_NO_FROM_U)!=0); michael@0: } else { michael@0: *pErrorCode=U_INVALID_TABLE_FORMAT; michael@0: return; michael@0: } michael@0: michael@0: mbcsTable->outputType=(uint8_t)header->flags; michael@0: if(noFromU && mbcsTable->outputType==MBCS_OUTPUT_1) { michael@0: *pErrorCode=U_INVALID_TABLE_FORMAT; michael@0: return; michael@0: } michael@0: michael@0: /* extension data, header version 4.2 and higher */ michael@0: offset=header->flags>>8; michael@0: if(offset!=0) { michael@0: mbcsTable->extIndexes=(const int32_t *)(raw+offset); michael@0: } michael@0: michael@0: if(mbcsTable->outputType==MBCS_OUTPUT_EXT_ONLY) { michael@0: UConverterLoadArgs args={ 0 }; michael@0: UConverterSharedData *baseSharedData; michael@0: const int32_t *extIndexes; michael@0: const char *baseName; michael@0: michael@0: /* extension-only file, load the base table and set values appropriately */ michael@0: if((extIndexes=mbcsTable->extIndexes)==NULL) { michael@0: /* extension-only file without extension */ michael@0: *pErrorCode=U_INVALID_TABLE_FORMAT; michael@0: return; michael@0: } michael@0: michael@0: if(pArgs->nestedLoads!=1) { michael@0: /* an extension table must not be loaded as a base table */ michael@0: *pErrorCode=U_INVALID_TABLE_FILE; michael@0: return; michael@0: } michael@0: michael@0: /* load the base table */ michael@0: baseName=(const char *)header+headerLength*4; michael@0: if(0==uprv_strcmp(baseName, sharedData->staticData->name)) { michael@0: /* forbid loading this same extension-only file */ michael@0: *pErrorCode=U_INVALID_TABLE_FORMAT; michael@0: return; michael@0: } michael@0: michael@0: /* TODO parse package name out of the prefix of the base name in the extension .cnv file? */ michael@0: args.size=sizeof(UConverterLoadArgs); michael@0: args.nestedLoads=2; michael@0: args.onlyTestIsLoadable=pArgs->onlyTestIsLoadable; michael@0: args.reserved=pArgs->reserved; michael@0: args.options=pArgs->options; michael@0: args.pkg=pArgs->pkg; michael@0: args.name=baseName; michael@0: baseSharedData=ucnv_load(&args, pErrorCode); michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: return; michael@0: } michael@0: if( baseSharedData->staticData->conversionType!=UCNV_MBCS || michael@0: baseSharedData->mbcs.baseSharedData!=NULL michael@0: ) { michael@0: ucnv_unload(baseSharedData); michael@0: *pErrorCode=U_INVALID_TABLE_FORMAT; michael@0: return; michael@0: } michael@0: if(pArgs->onlyTestIsLoadable) { michael@0: /* michael@0: * Exit as soon as we know that we can load the converter michael@0: * and the format is valid and supported. michael@0: * The worst that can happen in the following code is a memory michael@0: * allocation error. michael@0: */ michael@0: ucnv_unload(baseSharedData); michael@0: return; michael@0: } michael@0: michael@0: /* copy the base table data */ michael@0: uprv_memcpy(mbcsTable, &baseSharedData->mbcs, sizeof(UConverterMBCSTable)); michael@0: michael@0: /* overwrite values with relevant ones for the extension converter */ michael@0: mbcsTable->baseSharedData=baseSharedData; michael@0: mbcsTable->extIndexes=extIndexes; michael@0: michael@0: /* michael@0: * It would be possible to share the swapLFNL data with a base converter, michael@0: * but the generated name would have to be different, and the memory michael@0: * would have to be free'd only once. michael@0: * It is easier to just create the data for the extension converter michael@0: * separately when it is requested. michael@0: */ michael@0: mbcsTable->swapLFNLStateTable=NULL; michael@0: mbcsTable->swapLFNLFromUnicodeBytes=NULL; michael@0: mbcsTable->swapLFNLName=NULL; michael@0: michael@0: /* michael@0: * The reconstitutedData must be deleted only when the base converter michael@0: * is unloaded. michael@0: */ michael@0: mbcsTable->reconstitutedData=NULL; michael@0: michael@0: /* michael@0: * Set a special, runtime-only outputType if the extension converter michael@0: * is a DBCS version of a base converter that also maps single bytes. michael@0: */ michael@0: if( sharedData->staticData->conversionType==UCNV_DBCS || michael@0: (sharedData->staticData->conversionType==UCNV_MBCS && michael@0: sharedData->staticData->minBytesPerChar>=2) michael@0: ) { michael@0: if(baseSharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO) { michael@0: /* the base converter is SI/SO-stateful */ michael@0: int32_t entry; michael@0: michael@0: /* get the dbcs state from the state table entry for SO=0x0e */ michael@0: entry=mbcsTable->stateTable[0][0xe]; michael@0: if( MBCS_ENTRY_IS_FINAL(entry) && michael@0: MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_CHANGE_ONLY && michael@0: MBCS_ENTRY_FINAL_STATE(entry)!=0 michael@0: ) { michael@0: mbcsTable->dbcsOnlyState=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); michael@0: michael@0: mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; michael@0: } michael@0: } else if( michael@0: baseSharedData->staticData->conversionType==UCNV_MBCS && michael@0: baseSharedData->staticData->minBytesPerChar==1 && michael@0: baseSharedData->staticData->maxBytesPerChar==2 && michael@0: mbcsTable->countStates<=127 michael@0: ) { michael@0: /* non-stateful base converter, need to modify the state table */ michael@0: int32_t (*newStateTable)[256]; michael@0: int32_t *state; michael@0: int32_t i, count; michael@0: michael@0: /* allocate a new state table and copy the base state table contents */ michael@0: count=mbcsTable->countStates; michael@0: newStateTable=(int32_t (*)[256])uprv_malloc((count+1)*1024); michael@0: if(newStateTable==NULL) { michael@0: ucnv_unload(baseSharedData); michael@0: *pErrorCode=U_MEMORY_ALLOCATION_ERROR; michael@0: return; michael@0: } michael@0: michael@0: uprv_memcpy(newStateTable, mbcsTable->stateTable, count*1024); michael@0: michael@0: /* change all final single-byte entries to go to a new all-illegal state */ michael@0: state=newStateTable[0]; michael@0: for(i=0; i<256; ++i) { michael@0: if(MBCS_ENTRY_IS_FINAL(state[i])) { michael@0: state[i]=MBCS_ENTRY_TRANSITION(count, 0); michael@0: } michael@0: } michael@0: michael@0: /* build the new all-illegal state */ michael@0: state=newStateTable[count]; michael@0: for(i=0; i<256; ++i) { michael@0: state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0); michael@0: } michael@0: mbcsTable->stateTable=(const int32_t (*)[256])newStateTable; michael@0: mbcsTable->countStates=(uint8_t)(count+1); michael@0: mbcsTable->stateTableOwned=TRUE; michael@0: michael@0: mbcsTable->outputType=MBCS_OUTPUT_DBCS_ONLY; michael@0: } michael@0: } michael@0: michael@0: /* michael@0: * unlike below for files with base tables, do not get the unicodeMask michael@0: * from the sharedData; instead, use the base table's unicodeMask, michael@0: * which we copied in the memcpy above; michael@0: * this is necessary because the static data unicodeMask, especially michael@0: * the UCNV_HAS_SUPPLEMENTARY flag, is part of the base table data michael@0: */ michael@0: } else { michael@0: /* conversion file with a base table; an additional extension table is optional */ michael@0: /* make sure that the output type is known */ michael@0: switch(mbcsTable->outputType) { michael@0: case MBCS_OUTPUT_1: michael@0: case MBCS_OUTPUT_2: michael@0: case MBCS_OUTPUT_3: michael@0: case MBCS_OUTPUT_4: michael@0: case MBCS_OUTPUT_3_EUC: michael@0: case MBCS_OUTPUT_4_EUC: michael@0: case MBCS_OUTPUT_2_SISO: michael@0: /* OK */ michael@0: break; michael@0: default: michael@0: *pErrorCode=U_INVALID_TABLE_FORMAT; michael@0: return; michael@0: } michael@0: if(pArgs->onlyTestIsLoadable) { michael@0: /* michael@0: * Exit as soon as we know that we can load the converter michael@0: * and the format is valid and supported. michael@0: * The worst that can happen in the following code is a memory michael@0: * allocation error. michael@0: */ michael@0: return; michael@0: } michael@0: michael@0: mbcsTable->countStates=(uint8_t)header->countStates; michael@0: mbcsTable->countToUFallbacks=header->countToUFallbacks; michael@0: mbcsTable->stateTable=(const int32_t (*)[256])(raw+headerLength*4); michael@0: mbcsTable->toUFallbacks=(const _MBCSToUFallback *)(mbcsTable->stateTable+header->countStates); michael@0: mbcsTable->unicodeCodeUnits=(const uint16_t *)(raw+header->offsetToUCodeUnits); michael@0: michael@0: mbcsTable->fromUnicodeTable=(const uint16_t *)(raw+header->offsetFromUTable); michael@0: mbcsTable->fromUnicodeBytes=(const uint8_t *)(raw+header->offsetFromUBytes); michael@0: mbcsTable->fromUBytesLength=header->fromUBytesLength; michael@0: michael@0: /* michael@0: * converter versions 6.1 and up contain a unicodeMask that is michael@0: * used here to select the most efficient function implementations michael@0: */ michael@0: info.size=sizeof(UDataInfo); michael@0: udata_getInfo((UDataMemory *)sharedData->dataMemory, &info); michael@0: if(info.formatVersion[0]>6 || (info.formatVersion[0]==6 && info.formatVersion[1]>=1)) { michael@0: /* mask off possible future extensions to be safe */ michael@0: mbcsTable->unicodeMask=(uint8_t)(sharedData->staticData->unicodeMask&3); michael@0: } else { michael@0: /* for older versions, assume worst case: contains anything possible (prevent over-optimizations) */ michael@0: mbcsTable->unicodeMask=UCNV_HAS_SUPPLEMENTARY|UCNV_HAS_SURROGATES; michael@0: } michael@0: michael@0: /* michael@0: * _MBCSHeader.version 4.3 adds utf8Friendly data structures. michael@0: * Check for the header version, SBCS vs. MBCS, and for whether the michael@0: * data structures are optimized for code points as high as what the michael@0: * runtime code is designed for. michael@0: * The implementation does not handle mapping tables with entries for michael@0: * unpaired surrogates. michael@0: */ michael@0: if( header->version[1]>=3 && michael@0: (mbcsTable->unicodeMask&UCNV_HAS_SURROGATES)==0 && michael@0: (mbcsTable->countStates==1 ? michael@0: (header->version[2]>=(SBCS_FAST_MAX>>8)) : michael@0: (header->version[2]>=(MBCS_FAST_MAX>>8)) michael@0: ) michael@0: ) { michael@0: mbcsTable->utf8Friendly=TRUE; michael@0: michael@0: if(mbcsTable->countStates==1) { michael@0: /* michael@0: * SBCS: Stage 3 is allocated in 64-entry blocks for U+0000..SBCS_FAST_MAX or higher. michael@0: * Build a table with indexes to each block, to be used instead of michael@0: * the regular stage 1/2 table. michael@0: */ michael@0: int32_t i; michael@0: for(i=0; i<(SBCS_FAST_LIMIT>>6); ++i) { michael@0: mbcsTable->sbcsIndex[i]=mbcsTable->fromUnicodeTable[mbcsTable->fromUnicodeTable[i>>4]+((i<<2)&0x3c)]; michael@0: } michael@0: /* set SBCS_FAST_MAX to reflect the reach of sbcsIndex[] even if header->version[2]>(SBCS_FAST_MAX>>8) */ michael@0: mbcsTable->maxFastUChar=SBCS_FAST_MAX; michael@0: } else { michael@0: /* michael@0: * MBCS: Stage 3 is allocated in 64-entry blocks for U+0000..MBCS_FAST_MAX or higher. michael@0: * The .cnv file is prebuilt with an additional stage table with indexes michael@0: * to each block. michael@0: */ michael@0: mbcsTable->mbcsIndex=(const uint16_t *) michael@0: (mbcsTable->fromUnicodeBytes+ michael@0: (noFromU ? 0 : mbcsTable->fromUBytesLength)); michael@0: mbcsTable->maxFastUChar=(((UChar)header->version[2])<<8)|0xff; michael@0: } michael@0: } michael@0: michael@0: /* calculate a bit set of 4 ASCII characters per bit that round-trip to ASCII bytes */ michael@0: { michael@0: uint32_t asciiRoundtrips=0xffffffff; michael@0: int32_t i; michael@0: michael@0: for(i=0; i<0x80; ++i) { michael@0: if(mbcsTable->stateTable[0][i]!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, i)) { michael@0: asciiRoundtrips&=~((uint32_t)1<<(i>>2)); michael@0: } michael@0: } michael@0: mbcsTable->asciiRoundtrips=asciiRoundtrips; michael@0: } michael@0: michael@0: if(noFromU) { michael@0: uint32_t stage1Length= michael@0: mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY ? michael@0: 0x440 : 0x40; michael@0: uint32_t stage2Length= michael@0: (header->offsetFromUBytes-header->offsetFromUTable)/4- michael@0: stage1Length/2; michael@0: reconstituteData(mbcsTable, stage1Length, stage2Length, header->fullStage2Length, pErrorCode); michael@0: } michael@0: } michael@0: michael@0: /* Set the impl pointer here so that it is set for both extension-only and base tables. */ michael@0: if(mbcsTable->utf8Friendly) { michael@0: if(mbcsTable->countStates==1) { michael@0: sharedData->impl=&_SBCSUTF8Impl; michael@0: } else { michael@0: if(mbcsTable->outputType==MBCS_OUTPUT_2) { michael@0: sharedData->impl=&_DBCSUTF8Impl; michael@0: } michael@0: } michael@0: } michael@0: michael@0: if(mbcsTable->outputType==MBCS_OUTPUT_DBCS_ONLY || mbcsTable->outputType==MBCS_OUTPUT_2_SISO) { michael@0: /* michael@0: * MBCS_OUTPUT_DBCS_ONLY: No SBCS mappings, therefore ASCII does not roundtrip. michael@0: * MBCS_OUTPUT_2_SISO: Bypass the ASCII fastpath to handle prevLength correctly. michael@0: */ michael@0: mbcsTable->asciiRoundtrips=0; michael@0: } michael@0: } michael@0: michael@0: static void michael@0: ucnv_MBCSUnload(UConverterSharedData *sharedData) { michael@0: UConverterMBCSTable *mbcsTable=&sharedData->mbcs; michael@0: michael@0: if(mbcsTable->swapLFNLStateTable!=NULL) { michael@0: uprv_free(mbcsTable->swapLFNLStateTable); michael@0: } michael@0: if(mbcsTable->stateTableOwned) { michael@0: uprv_free((void *)mbcsTable->stateTable); michael@0: } michael@0: if(mbcsTable->baseSharedData!=NULL) { michael@0: ucnv_unload(mbcsTable->baseSharedData); michael@0: } michael@0: if(mbcsTable->reconstitutedData!=NULL) { michael@0: uprv_free(mbcsTable->reconstitutedData); michael@0: } michael@0: } michael@0: michael@0: static void michael@0: ucnv_MBCSOpen(UConverter *cnv, michael@0: UConverterLoadArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverterMBCSTable *mbcsTable; michael@0: const int32_t *extIndexes; michael@0: uint8_t outputType; michael@0: int8_t maxBytesPerUChar; michael@0: michael@0: if(pArgs->onlyTestIsLoadable) { michael@0: return; michael@0: } michael@0: michael@0: mbcsTable=&cnv->sharedData->mbcs; michael@0: outputType=mbcsTable->outputType; michael@0: michael@0: if(outputType==MBCS_OUTPUT_DBCS_ONLY) { michael@0: /* the swaplfnl option does not apply, remove it */ michael@0: cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL; michael@0: } michael@0: michael@0: if((pArgs->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: /* do this because double-checked locking is broken */ michael@0: UBool isCached; michael@0: michael@0: umtx_lock(NULL); michael@0: isCached=mbcsTable->swapLFNLStateTable!=NULL; michael@0: umtx_unlock(NULL); michael@0: michael@0: if(!isCached) { michael@0: if(!_EBCDICSwapLFNL(cnv->sharedData, pErrorCode)) { michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: return; /* something went wrong */ michael@0: } michael@0: michael@0: /* the option does not apply, remove it */ michael@0: cnv->options=pArgs->options&=~UCNV_OPTION_SWAP_LFNL; michael@0: } michael@0: } michael@0: } michael@0: michael@0: if(uprv_strstr(pArgs->name, "18030")!=NULL) { michael@0: if(uprv_strstr(pArgs->name, "gb18030")!=NULL || uprv_strstr(pArgs->name, "GB18030")!=NULL) { michael@0: /* set a flag for GB 18030 mode, which changes the callback behavior */ michael@0: cnv->options|=_MBCS_OPTION_GB18030; michael@0: } michael@0: } else if((uprv_strstr(pArgs->name, "KEIS")!=NULL) || (uprv_strstr(pArgs->name, "keis")!=NULL)) { michael@0: /* set a flag for KEIS converter, which changes the SI/SO character sequence */ michael@0: cnv->options|=_MBCS_OPTION_KEIS; michael@0: } else if((uprv_strstr(pArgs->name, "JEF")!=NULL) || (uprv_strstr(pArgs->name, "jef")!=NULL)) { michael@0: /* set a flag for JEF converter, which changes the SI/SO character sequence */ michael@0: cnv->options|=_MBCS_OPTION_JEF; michael@0: } else if((uprv_strstr(pArgs->name, "JIPS")!=NULL) || (uprv_strstr(pArgs->name, "jips")!=NULL)) { michael@0: /* set a flag for JIPS converter, which changes the SI/SO character sequence */ michael@0: cnv->options|=_MBCS_OPTION_JIPS; michael@0: } michael@0: michael@0: /* fix maxBytesPerUChar depending on outputType and options etc. */ michael@0: if(outputType==MBCS_OUTPUT_2_SISO) { michael@0: cnv->maxBytesPerUChar=3; /* SO+DBCS */ michael@0: } michael@0: michael@0: extIndexes=mbcsTable->extIndexes; michael@0: if(extIndexes!=NULL) { michael@0: maxBytesPerUChar=(int8_t)UCNV_GET_MAX_BYTES_PER_UCHAR(extIndexes); michael@0: if(outputType==MBCS_OUTPUT_2_SISO) { michael@0: ++maxBytesPerUChar; /* SO + multiple DBCS */ michael@0: } michael@0: michael@0: if(maxBytesPerUChar>cnv->maxBytesPerUChar) { michael@0: cnv->maxBytesPerUChar=maxBytesPerUChar; michael@0: } michael@0: } michael@0: michael@0: #if 0 michael@0: /* michael@0: * documentation of UConverter fields used for status michael@0: * all of these fields are (re)set to 0 by ucnv_bld.c and ucnv_reset() michael@0: */ michael@0: michael@0: /* toUnicode */ michael@0: cnv->toUnicodeStatus=0; /* offset */ michael@0: cnv->mode=0; /* state */ michael@0: cnv->toULength=0; /* byteIndex */ michael@0: michael@0: /* fromUnicode */ michael@0: cnv->fromUChar32=0; michael@0: cnv->fromUnicodeStatus=1; /* prevLength */ michael@0: #endif michael@0: } michael@0: michael@0: static const char * michael@0: ucnv_MBCSGetName(const UConverter *cnv) { michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0 && cnv->sharedData->mbcs.swapLFNLName!=NULL) { michael@0: return cnv->sharedData->mbcs.swapLFNLName; michael@0: } else { michael@0: return cnv->sharedData->staticData->name; michael@0: } michael@0: } michael@0: michael@0: /* MBCS-to-Unicode conversion functions ------------------------------------- */ michael@0: michael@0: static UChar32 michael@0: ucnv_MBCSGetFallback(UConverterMBCSTable *mbcsTable, uint32_t offset) { michael@0: const _MBCSToUFallback *toUFallbacks; michael@0: uint32_t i, start, limit; michael@0: michael@0: limit=mbcsTable->countToUFallbacks; michael@0: if(limit>0) { michael@0: /* do a binary search for the fallback mapping */ michael@0: toUFallbacks=mbcsTable->toUFallbacks; michael@0: start=0; michael@0: while(startconverter; michael@0: source=(const uint8_t *)pArgs->source; michael@0: sourceLimit=(const uint8_t *)pArgs->sourceLimit; michael@0: target=pArgs->target; michael@0: targetLimit=pArgs->targetLimit; michael@0: offsets=pArgs->offsets; michael@0: michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; michael@0: } else { michael@0: stateTable=cnv->sharedData->mbcs.stateTable; michael@0: } michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: sourceIndex=0; michael@0: michael@0: /* conversion loop */ michael@0: while(source=targetLimit) { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: michael@0: entry=stateTable[0][*source++]; michael@0: /* MBCS_ENTRY_IS_FINAL(entry) */ michael@0: michael@0: /* test the most common case first */ michael@0: if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { michael@0: /* output BMP code point */ michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: michael@0: /* normal end of action codes: prepare for a new character */ michael@0: ++sourceIndex; michael@0: continue; michael@0: } michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_VALID_DIRECT_20 || michael@0: (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) michael@0: ) { michael@0: entry=MBCS_ENTRY_FINAL_VALUE(entry); michael@0: /* output surrogate pair */ michael@0: *target++=(UChar)(0xd800|(UChar)(entry>>10)); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); michael@0: if(targetUCharErrorBuffer[0]=c; michael@0: cnv->UCharErrorBufferLength=1; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: michael@0: ++sourceIndex; michael@0: continue; michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv)) { michael@0: /* output BMP code point */ michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: michael@0: ++sourceIndex; michael@0: continue; michael@0: } michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: /* just fall through */ michael@0: } else if(action==MBCS_STATE_ILLEGAL) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } else { michael@0: /* reserved, must never occur */ michael@0: ++sourceIndex; michael@0: continue; michael@0: } michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* callback(illegal) */ michael@0: break; michael@0: } else /* unassigned sequences indicated with byteIndex>0 */ { michael@0: /* try an extension mapping */ michael@0: pArgs->source=(const char *)source; michael@0: cnv->toUBytes[0]=*(source-1); michael@0: cnv->toULength=_extToU(cnv, cnv->sharedData, michael@0: 1, &source, sourceLimit, michael@0: &target, targetLimit, michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: sourceIndex+=1+(int32_t)(source-(const uint8_t *)pArgs->source); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } michael@0: } michael@0: } michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=(const char *)source; michael@0: pArgs->target=target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: /* michael@0: * This version of ucnv_MBCSSingleToUnicodeWithOffsets() is optimized for single-byte, single-state codepages michael@0: * that only map to and from the BMP. michael@0: * In addition to single-byte optimizations, the offset calculations michael@0: * become much easier. michael@0: */ michael@0: static void michael@0: ucnv_MBCSSingleToBMPWithOffsets(UConverterToUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const uint8_t *source, *sourceLimit, *lastSource; michael@0: UChar *target; michael@0: int32_t targetCapacity, length; michael@0: int32_t *offsets; michael@0: michael@0: const int32_t (*stateTable)[256]; michael@0: michael@0: int32_t sourceIndex; michael@0: michael@0: int32_t entry; michael@0: uint8_t action; michael@0: michael@0: /* set up the local pointers */ michael@0: cnv=pArgs->converter; michael@0: source=(const uint8_t *)pArgs->source; michael@0: sourceLimit=(const uint8_t *)pArgs->sourceLimit; michael@0: target=pArgs->target; michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); michael@0: offsets=pArgs->offsets; michael@0: michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; michael@0: } else { michael@0: stateTable=cnv->sharedData->mbcs.stateTable; michael@0: } michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: sourceIndex=0; michael@0: lastSource=source; michael@0: michael@0: /* michael@0: * since the conversion here is 1:1 UChar:uint8_t, we need only one counter michael@0: * for the minimum of the sourceLength and targetCapacity michael@0: */ michael@0: length=(int32_t)(sourceLimit-source); michael@0: if(length=16) { michael@0: int32_t count, loops, oredEntries; michael@0: michael@0: loops=count=targetCapacity>>4; michael@0: do { michael@0: oredEntries=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: oredEntries|=entry=stateTable[0][*source++]; michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: michael@0: /* were all 16 entries really valid? */ michael@0: if(!MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(oredEntries)) { michael@0: /* no, return to the first of these 16 */ michael@0: source-=16; michael@0: target-=16; michael@0: break; michael@0: } michael@0: } while(--count>0); michael@0: count=loops-count; michael@0: targetCapacity-=16*count; michael@0: michael@0: if(offsets!=NULL) { michael@0: lastSource+=16*count; michael@0: while(count>0) { michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: --count; michael@0: } michael@0: } michael@0: } michael@0: #endif michael@0: michael@0: /* conversion loop */ michael@0: while(targetCapacity > 0 && source < sourceLimit) { michael@0: entry=stateTable[0][*source++]; michael@0: /* MBCS_ENTRY_IS_FINAL(entry) */ michael@0: michael@0: /* test the most common case first */ michael@0: if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { michael@0: /* output BMP code point */ michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: --targetCapacity; michael@0: continue; michael@0: } michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv)) { michael@0: /* output BMP code point */ michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: --targetCapacity; michael@0: continue; michael@0: } michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: /* just fall through */ michael@0: } else if(action==MBCS_STATE_ILLEGAL) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } else { michael@0: /* reserved, must never occur */ michael@0: continue; michael@0: } michael@0: michael@0: /* set offsets since the start or the last extension */ michael@0: if(offsets!=NULL) { michael@0: int32_t count=(int32_t)(source-lastSource); michael@0: michael@0: /* predecrement: do not set the offset for the callback-causing character */ michael@0: while(--count>0) { michael@0: *offsets++=sourceIndex++; michael@0: } michael@0: /* offset and sourceIndex are now set for the current character */ michael@0: } michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* callback(illegal) */ michael@0: break; michael@0: } else /* unassigned sequences indicated with byteIndex>0 */ { michael@0: /* try an extension mapping */ michael@0: lastSource=source; michael@0: cnv->toUBytes[0]=*(source-1); michael@0: cnv->toULength=_extToU(cnv, cnv->sharedData, michael@0: 1, &source, sourceLimit, michael@0: &target, pArgs->targetLimit, michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: sourceIndex+=1+(int32_t)(source-lastSource); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-target); michael@0: length=(int32_t)(sourceLimit-source); michael@0: if(length=pArgs->targetLimit) { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: } michael@0: michael@0: /* set offsets since the start or the last callback */ michael@0: if(offsets!=NULL) { michael@0: size_t count=source-lastSource; michael@0: while(count>0) { michael@0: *offsets++=sourceIndex++; michael@0: --count; michael@0: } michael@0: } michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=(const char *)source; michael@0: pArgs->target=target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: static UBool michael@0: hasValidTrailBytes(const int32_t (*stateTable)[256], uint8_t state) { michael@0: const int32_t *row=stateTable[state]; michael@0: int32_t b, entry; michael@0: /* First test for final entries in this state for some commonly valid byte values. */ michael@0: entry=row[0xa1]; michael@0: if( !MBCS_ENTRY_IS_TRANSITION(entry) && michael@0: MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL michael@0: ) { michael@0: return TRUE; michael@0: } michael@0: entry=row[0x41]; michael@0: if( !MBCS_ENTRY_IS_TRANSITION(entry) && michael@0: MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL michael@0: ) { michael@0: return TRUE; michael@0: } michael@0: /* Then test for final entries in this state. */ michael@0: for(b=0; b<=0xff; ++b) { michael@0: entry=row[b]; michael@0: if( !MBCS_ENTRY_IS_TRANSITION(entry) && michael@0: MBCS_ENTRY_FINAL_ACTION(entry)!=MBCS_STATE_ILLEGAL michael@0: ) { michael@0: return TRUE; michael@0: } michael@0: } michael@0: /* Then recurse for transition entries. */ michael@0: for(b=0; b<=0xff; ++b) { michael@0: entry=row[b]; michael@0: if( MBCS_ENTRY_IS_TRANSITION(entry) && michael@0: hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry)) michael@0: ) { michael@0: return TRUE; michael@0: } michael@0: } michael@0: return FALSE; michael@0: } michael@0: michael@0: /* michael@0: * Is byte b a single/lead byte in this state? michael@0: * Recurse for transition states, because here we don't want to say that michael@0: * b is a lead byte if all byte sequences that start with b are illegal. michael@0: */ michael@0: static UBool michael@0: isSingleOrLead(const int32_t (*stateTable)[256], uint8_t state, UBool isDBCSOnly, uint8_t b) { michael@0: const int32_t *row=stateTable[state]; michael@0: int32_t entry=row[b]; michael@0: if(MBCS_ENTRY_IS_TRANSITION(entry)) { /* lead byte */ michael@0: return hasValidTrailBytes(stateTable, (uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry)); michael@0: } else { michael@0: uint8_t action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_CHANGE_ONLY && isDBCSOnly) { michael@0: return FALSE; /* SI/SO are illegal for DBCS-only conversion */ michael@0: } else { michael@0: return action!=MBCS_STATE_ILLEGAL; michael@0: } michael@0: } michael@0: } michael@0: michael@0: U_CFUNC void michael@0: ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const uint8_t *source, *sourceLimit; michael@0: UChar *target; michael@0: const UChar *targetLimit; michael@0: int32_t *offsets; michael@0: michael@0: const int32_t (*stateTable)[256]; michael@0: const uint16_t *unicodeCodeUnits; michael@0: michael@0: uint32_t offset; michael@0: uint8_t state; michael@0: int8_t byteIndex; michael@0: uint8_t *bytes; michael@0: michael@0: int32_t sourceIndex, nextSourceIndex; michael@0: michael@0: int32_t entry; michael@0: UChar c; michael@0: uint8_t action; michael@0: michael@0: /* use optimized function if possible */ michael@0: cnv=pArgs->converter; michael@0: michael@0: if(cnv->preToULength>0) { michael@0: /* michael@0: * pass sourceIndex=-1 because we continue from an earlier buffer michael@0: * in the future, this may change with continuous offsets michael@0: */ michael@0: ucnv_extContinueMatchToU(cnv, pArgs, -1, pErrorCode); michael@0: michael@0: if(U_FAILURE(*pErrorCode) || cnv->preToULength<0) { michael@0: return; michael@0: } michael@0: } michael@0: michael@0: if(cnv->sharedData->mbcs.countStates==1) { michael@0: if(!(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { michael@0: ucnv_MBCSSingleToBMPWithOffsets(pArgs, pErrorCode); michael@0: } else { michael@0: ucnv_MBCSSingleToUnicodeWithOffsets(pArgs, pErrorCode); michael@0: } michael@0: return; michael@0: } michael@0: michael@0: /* set up the local pointers */ michael@0: source=(const uint8_t *)pArgs->source; michael@0: sourceLimit=(const uint8_t *)pArgs->sourceLimit; michael@0: target=pArgs->target; michael@0: targetLimit=pArgs->targetLimit; michael@0: offsets=pArgs->offsets; michael@0: michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; michael@0: } else { michael@0: stateTable=cnv->sharedData->mbcs.stateTable; michael@0: } michael@0: unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; michael@0: michael@0: /* get the converter state from UConverter */ michael@0: offset=cnv->toUnicodeStatus; michael@0: byteIndex=cnv->toULength; michael@0: bytes=cnv->toUBytes; michael@0: michael@0: /* michael@0: * if we are in the SBCS state for a DBCS-only converter, michael@0: * then load the DBCS state from the MBCS data michael@0: * (dbcsOnlyState==0 if it is not a DBCS-only converter) michael@0: */ michael@0: if((state=(uint8_t)(cnv->mode))==0) { michael@0: state=cnv->sharedData->mbcs.dbcsOnlyState; michael@0: } michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: sourceIndex=byteIndex==0 ? 0 : -1; michael@0: nextSourceIndex=0; michael@0: michael@0: /* conversion loop */ michael@0: while(source=targetLimit) { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: michael@0: if(byteIndex==0) { michael@0: /* optimized loop for 1/2-byte input and BMP output */ michael@0: if(offsets==NULL) { michael@0: do { michael@0: entry=stateTable[state][*source]; michael@0: if(MBCS_ENTRY_IS_TRANSITION(entry)) { michael@0: state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); michael@0: offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); michael@0: michael@0: ++source; michael@0: if( source=sourceLimit) { michael@0: break; michael@0: } michael@0: if(target>=targetLimit) { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: michael@0: ++nextSourceIndex; michael@0: bytes[byteIndex++]=*source++; michael@0: } else /* byteIndex>0 */ { michael@0: ++nextSourceIndex; michael@0: entry=stateTable[state][bytes[byteIndex++]=*source++]; michael@0: } michael@0: michael@0: if(MBCS_ENTRY_IS_TRANSITION(entry)) { michael@0: state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); michael@0: offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); michael@0: continue; michael@0: } michael@0: michael@0: /* save the previous state for proper extension mapping with SI/SO-stateful converters */ michael@0: cnv->mode=state; michael@0: michael@0: /* set the next state early so that we can reuse the entry variable */ michael@0: state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_VALID_16) { michael@0: offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[offset]; michael@0: if(c<0xfffe) { michael@0: /* output BMP code point */ michael@0: *target++=c; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: } else if(c==0xfffe) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv) && (entry=(int32_t)ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { michael@0: /* output fallback BMP code point */ michael@0: *target++=(UChar)entry; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: } michael@0: } else { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } michael@0: } else if(action==MBCS_STATE_VALID_DIRECT_16) { michael@0: /* output BMP code point */ michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: } else if(action==MBCS_STATE_VALID_16_PAIR) { michael@0: offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[offset++]; michael@0: if(c<0xd800) { michael@0: /* output BMP code point below 0xd800 */ michael@0: *target++=c; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { michael@0: /* output roundtrip or fallback surrogate pair */ michael@0: *target++=(UChar)(c&0xdbff); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: if(targetUCharErrorBuffer[0]=unicodeCodeUnits[offset]; michael@0: cnv->UCharErrorBufferLength=1; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: michael@0: offset=0; michael@0: break; michael@0: } michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { michael@0: /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ michael@0: *target++=unicodeCodeUnits[offset]; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: } else if(c==0xffff) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } michael@0: } else if(action==MBCS_STATE_VALID_DIRECT_20 || michael@0: (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) michael@0: ) { michael@0: entry=MBCS_ENTRY_FINAL_VALUE(entry); michael@0: /* output surrogate pair */ michael@0: *target++=(UChar)(0xd800|(UChar)(entry>>10)); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: c=(UChar)(0xdc00|(UChar)(entry&0x3ff)); michael@0: if(targetUCharErrorBuffer[0]=c; michael@0: cnv->UCharErrorBufferLength=1; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: michael@0: offset=0; michael@0: break; michael@0: } michael@0: } else if(action==MBCS_STATE_CHANGE_ONLY) { michael@0: /* michael@0: * This serves as a state change without any output. michael@0: * It is useful for reading simple stateful encodings, michael@0: * for example using just Shift-In/Shift-Out codes. michael@0: * The 21 unused bits may later be used for more sophisticated michael@0: * state transitions. michael@0: */ michael@0: if(cnv->sharedData->mbcs.dbcsOnlyState==0) { michael@0: byteIndex=0; michael@0: } else { michael@0: /* SI/SO are illegal for DBCS-only conversion */ michael@0: state=(uint8_t)(cnv->mode); /* restore the previous state */ michael@0: michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv)) { michael@0: /* output BMP code point */ michael@0: *target++=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: byteIndex=0; michael@0: } michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: /* just fall through */ michael@0: } else if(action==MBCS_STATE_ILLEGAL) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } else { michael@0: /* reserved, must never occur */ michael@0: byteIndex=0; michael@0: } michael@0: michael@0: /* end of action codes: prepare for a new character */ michael@0: offset=0; michael@0: michael@0: if(byteIndex==0) { michael@0: sourceIndex=nextSourceIndex; michael@0: } else if(U_FAILURE(*pErrorCode)) { michael@0: /* callback(illegal) */ michael@0: if(byteIndex>1) { michael@0: /* michael@0: * Ticket 5691: consistent illegal sequences: michael@0: * - We include at least the first byte in the illegal sequence. michael@0: * - If any of the non-initial bytes could be the start of a character, michael@0: * we stop the illegal sequence before the first one of those. michael@0: */ michael@0: UBool isDBCSOnly=(UBool)(cnv->sharedData->mbcs.dbcsOnlyState!=0); michael@0: int8_t i; michael@0: for(i=1; michael@0: isource); michael@0: byteIndex=i; /* length of reported illegal byte sequence */ michael@0: if(backOutDistance<=bytesFromThisBuffer) { michael@0: source-=backOutDistance; michael@0: } else { michael@0: /* Back out bytes from the previous buffer: Need to replay them. */ michael@0: cnv->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance); michael@0: /* preToULength is negative! */ michael@0: uprv_memcpy(cnv->preToU, bytes+i, -cnv->preToULength); michael@0: source=(const uint8_t *)pArgs->source; michael@0: } michael@0: } michael@0: } michael@0: break; michael@0: } else /* unassigned sequences indicated with byteIndex>0 */ { michael@0: /* try an extension mapping */ michael@0: pArgs->source=(const char *)source; michael@0: byteIndex=_extToU(cnv, cnv->sharedData, michael@0: byteIndex, &source, sourceLimit, michael@0: &target, targetLimit, michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: sourceIndex=nextSourceIndex+=(int32_t)(source-(const uint8_t *)pArgs->source); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } michael@0: } michael@0: } michael@0: michael@0: /* set the converter state back into UConverter */ michael@0: cnv->toUnicodeStatus=offset; michael@0: cnv->mode=state; michael@0: cnv->toULength=byteIndex; michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=(const char *)source; michael@0: pArgs->target=target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: /* michael@0: * This version of ucnv_MBCSGetNextUChar() is optimized for single-byte, single-state codepages. michael@0: * We still need a conversion loop in case we find reserved action codes, which are to be ignored. michael@0: */ michael@0: static UChar32 michael@0: ucnv_MBCSSingleGetNextUChar(UConverterToUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const int32_t (*stateTable)[256]; michael@0: const uint8_t *source, *sourceLimit; michael@0: michael@0: int32_t entry; michael@0: uint8_t action; michael@0: michael@0: /* set up the local pointers */ michael@0: cnv=pArgs->converter; michael@0: source=(const uint8_t *)pArgs->source; michael@0: sourceLimit=(const uint8_t *)pArgs->sourceLimit; michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; michael@0: } else { michael@0: stateTable=cnv->sharedData->mbcs.stateTable; michael@0: } michael@0: michael@0: /* conversion loop */ michael@0: while(sourcesource=(const char *)source; michael@0: michael@0: if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { michael@0: /* output BMP code point */ michael@0: return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: } michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if( action==MBCS_STATE_VALID_DIRECT_20 || michael@0: (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) michael@0: ) { michael@0: /* output supplementary code point */ michael@0: return (UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv)) { michael@0: /* output BMP code point */ michael@0: return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: } michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: /* just fall through */ michael@0: } else if(action==MBCS_STATE_ILLEGAL) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } else { michael@0: /* reserved, must never occur */ michael@0: continue; michael@0: } michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* callback(illegal) */ michael@0: break; michael@0: } else /* unassigned sequence */ { michael@0: /* defer to the generic implementation */ michael@0: pArgs->source=(const char *)source-1; michael@0: return UCNV_GET_NEXT_UCHAR_USE_TO_U; michael@0: } michael@0: } michael@0: michael@0: /* no output because of empty input or only state changes */ michael@0: *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; michael@0: return 0xffff; michael@0: } michael@0: michael@0: /* michael@0: * Version of _MBCSToUnicodeWithOffsets() optimized for single-character michael@0: * conversion without offset handling. michael@0: * michael@0: * When a character does not have a mapping to Unicode, then we return to the michael@0: * generic ucnv_getNextUChar() code for extension/GB 18030 and error/callback michael@0: * handling. michael@0: * We also defer to the generic code in other complicated cases and have them michael@0: * ultimately handled by _MBCSToUnicodeWithOffsets() itself. michael@0: * michael@0: * All normal mappings and errors are handled here. michael@0: */ michael@0: static UChar32 michael@0: ucnv_MBCSGetNextUChar(UConverterToUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const uint8_t *source, *sourceLimit, *lastSource; michael@0: michael@0: const int32_t (*stateTable)[256]; michael@0: const uint16_t *unicodeCodeUnits; michael@0: michael@0: uint32_t offset; michael@0: uint8_t state; michael@0: michael@0: int32_t entry; michael@0: UChar32 c; michael@0: uint8_t action; michael@0: michael@0: /* use optimized function if possible */ michael@0: cnv=pArgs->converter; michael@0: michael@0: if(cnv->preToULength>0) { michael@0: /* use the generic code in ucnv_getNextUChar() to continue with a partial match */ michael@0: return UCNV_GET_NEXT_UCHAR_USE_TO_U; michael@0: } michael@0: michael@0: if(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SURROGATES) { michael@0: /* michael@0: * Using the generic ucnv_getNextUChar() code lets us deal correctly michael@0: * with the rare case of a codepage that maps single surrogates michael@0: * without adding the complexity to this already complicated function here. michael@0: */ michael@0: return UCNV_GET_NEXT_UCHAR_USE_TO_U; michael@0: } else if(cnv->sharedData->mbcs.countStates==1) { michael@0: return ucnv_MBCSSingleGetNextUChar(pArgs, pErrorCode); michael@0: } michael@0: michael@0: /* set up the local pointers */ michael@0: source=lastSource=(const uint8_t *)pArgs->source; michael@0: sourceLimit=(const uint8_t *)pArgs->sourceLimit; michael@0: michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: stateTable=(const int32_t (*)[256])cnv->sharedData->mbcs.swapLFNLStateTable; michael@0: } else { michael@0: stateTable=cnv->sharedData->mbcs.stateTable; michael@0: } michael@0: unicodeCodeUnits=cnv->sharedData->mbcs.unicodeCodeUnits; michael@0: michael@0: /* get the converter state from UConverter */ michael@0: offset=cnv->toUnicodeStatus; michael@0: michael@0: /* michael@0: * if we are in the SBCS state for a DBCS-only converter, michael@0: * then load the DBCS state from the MBCS data michael@0: * (dbcsOnlyState==0 if it is not a DBCS-only converter) michael@0: */ michael@0: if((state=(uint8_t)(cnv->mode))==0) { michael@0: state=cnv->sharedData->mbcs.dbcsOnlyState; michael@0: } michael@0: michael@0: /* conversion loop */ michael@0: c=U_SENTINEL; michael@0: while(sourcemode=state; michael@0: michael@0: /* set the next state early so that we can reuse the entry variable */ michael@0: state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); /* typically 0 */ michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_VALID_DIRECT_16) { michael@0: /* output BMP code point */ michael@0: c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: break; michael@0: } else if(action==MBCS_STATE_VALID_16) { michael@0: offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[offset]; michael@0: if(c<0xfffe) { michael@0: /* output BMP code point */ michael@0: break; michael@0: } else if(c==0xfffe) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv) && (c=ucnv_MBCSGetFallback(&cnv->sharedData->mbcs, offset))!=0xfffe) { michael@0: break; michael@0: } michael@0: } else { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } michael@0: } else if(action==MBCS_STATE_VALID_16_PAIR) { michael@0: offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[offset++]; michael@0: if(c<0xd800) { michael@0: /* output BMP code point below 0xd800 */ michael@0: break; michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { michael@0: /* output roundtrip or fallback supplementary code point */ michael@0: c=((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00); michael@0: break; michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { michael@0: /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ michael@0: c=unicodeCodeUnits[offset]; michael@0: break; michael@0: } else if(c==0xffff) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } michael@0: } else if(action==MBCS_STATE_VALID_DIRECT_20 || michael@0: (action==MBCS_STATE_FALLBACK_DIRECT_20 && UCNV_TO_U_USE_FALLBACK(cnv)) michael@0: ) { michael@0: /* output supplementary code point */ michael@0: c=(UChar32)(MBCS_ENTRY_FINAL_VALUE(entry)+0x10000); michael@0: break; michael@0: } else if(action==MBCS_STATE_CHANGE_ONLY) { michael@0: /* michael@0: * This serves as a state change without any output. michael@0: * It is useful for reading simple stateful encodings, michael@0: * for example using just Shift-In/Shift-Out codes. michael@0: * The 21 unused bits may later be used for more sophisticated michael@0: * state transitions. michael@0: */ michael@0: if(cnv->sharedData->mbcs.dbcsOnlyState!=0) { michael@0: /* SI/SO are illegal for DBCS-only conversion */ michael@0: state=(uint8_t)(cnv->mode); /* restore the previous state */ michael@0: michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(UCNV_TO_U_USE_FALLBACK(cnv)) { michael@0: /* output BMP code point */ michael@0: c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: break; michael@0: } michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: /* just fall through */ michael@0: } else if(action==MBCS_STATE_ILLEGAL) { michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: } else { michael@0: /* reserved (must never occur), or only state change */ michael@0: offset=0; michael@0: lastSource=source; michael@0: continue; michael@0: } michael@0: michael@0: /* end of action codes: prepare for a new character */ michael@0: offset=0; michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* callback(illegal) */ michael@0: break; michael@0: } else /* unassigned sequence */ { michael@0: /* defer to the generic implementation */ michael@0: cnv->toUnicodeStatus=0; michael@0: cnv->mode=state; michael@0: pArgs->source=(const char *)lastSource; michael@0: return UCNV_GET_NEXT_UCHAR_USE_TO_U; michael@0: } michael@0: } michael@0: } michael@0: michael@0: if(c<0) { michael@0: if(U_SUCCESS(*pErrorCode) && source==sourceLimit && lastSourcetoUBytes; michael@0: cnv->toULength=(int8_t)(source-lastSource); michael@0: do { michael@0: *bytes++=*lastSource++; michael@0: } while(lastSourcesharedData->mbcs.dbcsOnlyState!=0); michael@0: uint8_t *bytes=cnv->toUBytes; michael@0: *bytes++=*lastSource++; /* first byte */ michael@0: if(lastSource==source) { michael@0: cnv->toULength=1; michael@0: } else /* lastSourcetoULength=i; michael@0: source=lastSource; michael@0: } michael@0: } else { michael@0: /* no output because of empty input or only state changes */ michael@0: *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; michael@0: } michael@0: c=0xffff; michael@0: } michael@0: michael@0: /* set the converter state back into UConverter, ready for a new character */ michael@0: cnv->toUnicodeStatus=0; michael@0: cnv->mode=state; michael@0: michael@0: /* write back the updated pointer */ michael@0: pArgs->source=(const char *)source; michael@0: return c; michael@0: } michael@0: michael@0: #if 0 michael@0: /* michael@0: * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus michael@0: * Removal improves code coverage. michael@0: */ michael@0: /** michael@0: * This version of ucnv_MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It does not handle conversion extensions (_extToU()). michael@0: */ michael@0: U_CFUNC UChar32 michael@0: ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, michael@0: uint8_t b, UBool useFallback) { michael@0: int32_t entry; michael@0: uint8_t action; michael@0: michael@0: entry=sharedData->mbcs.stateTable[0][b]; michael@0: /* MBCS_ENTRY_IS_FINAL(entry) */ michael@0: michael@0: if(MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry)) { michael@0: /* output BMP code point */ michael@0: return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: } michael@0: michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_VALID_DIRECT_20) { michael@0: /* output supplementary code point */ michael@0: return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(!TO_U_USE_FALLBACK(useFallback)) { michael@0: return 0xfffe; michael@0: } michael@0: /* output BMP code point */ michael@0: return (UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { michael@0: if(!TO_U_USE_FALLBACK(useFallback)) { michael@0: return 0xfffe; michael@0: } michael@0: /* output supplementary code point */ michael@0: return 0x10000+MBCS_ENTRY_FINAL_VALUE(entry); michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: return 0xfffe; michael@0: } else if(action==MBCS_STATE_ILLEGAL) { michael@0: return 0xffff; michael@0: } else { michael@0: /* reserved, must never occur */ michael@0: return 0xffff; michael@0: } michael@0: } michael@0: #endif michael@0: michael@0: /* michael@0: * This is a simple version of _MBCSGetNextUChar() that is used michael@0: * by other converter implementations. michael@0: * It only returns an "assigned" result if it consumes the entire input. michael@0: * It does not use state from the converter, nor error codes. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It handles conversion extensions but not GB 18030. michael@0: * michael@0: * Return value: michael@0: * U+fffe unassigned michael@0: * U+ffff illegal michael@0: * otherwise the Unicode code point michael@0: */ michael@0: U_CFUNC UChar32 michael@0: ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, michael@0: const char *source, int32_t length, michael@0: UBool useFallback) { michael@0: const int32_t (*stateTable)[256]; michael@0: const uint16_t *unicodeCodeUnits; michael@0: michael@0: uint32_t offset; michael@0: uint8_t state, action; michael@0: michael@0: UChar32 c; michael@0: int32_t i, entry; michael@0: michael@0: if(length<=0) { michael@0: /* no input at all: "illegal" */ michael@0: return 0xffff; michael@0: } michael@0: michael@0: #if 0 michael@0: /* michael@0: * Code disabled 2002dec09 (ICU 2.4) because it is not currently used in ICU. markus michael@0: * TODO In future releases, verify that this function is never called for SBCS michael@0: * conversions, i.e., that sharedData->mbcs.countStates==1 is still true. michael@0: * Removal improves code coverage. michael@0: */ michael@0: /* use optimized function if possible */ michael@0: if(sharedData->mbcs.countStates==1) { michael@0: if(length==1) { michael@0: return ucnv_MBCSSingleSimpleGetNextUChar(sharedData, (uint8_t)*source, useFallback); michael@0: } else { michael@0: return 0xffff; /* illegal: more than a single byte for an SBCS converter */ michael@0: } michael@0: } michael@0: #endif michael@0: michael@0: /* set up the local pointers */ michael@0: stateTable=sharedData->mbcs.stateTable; michael@0: unicodeCodeUnits=sharedData->mbcs.unicodeCodeUnits; michael@0: michael@0: /* converter state */ michael@0: offset=0; michael@0: state=sharedData->mbcs.dbcsOnlyState; michael@0: michael@0: /* conversion loop */ michael@0: for(i=0;;) { michael@0: entry=stateTable[state][(uint8_t)source[i++]]; michael@0: if(MBCS_ENTRY_IS_TRANSITION(entry)) { michael@0: state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); michael@0: offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); michael@0: michael@0: if(i==length) { michael@0: return 0xffff; /* truncated character */ michael@0: } michael@0: } else { michael@0: /* michael@0: * An if-else-if chain provides more reliable performance for michael@0: * the most common cases compared to a switch. michael@0: */ michael@0: action=(uint8_t)(MBCS_ENTRY_FINAL_ACTION(entry)); michael@0: if(action==MBCS_STATE_VALID_16) { michael@0: offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[offset]; michael@0: if(c!=0xfffe) { michael@0: /* done */ michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv)) { michael@0: c=ucnv_MBCSGetFallback(&sharedData->mbcs, offset); michael@0: /* else done with 0xfffe */ michael@0: } michael@0: break; michael@0: } else if(action==MBCS_STATE_VALID_DIRECT_16) { michael@0: /* output BMP code point */ michael@0: c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: break; michael@0: } else if(action==MBCS_STATE_VALID_16_PAIR) { michael@0: offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: c=unicodeCodeUnits[offset++]; michael@0: if(c<0xd800) { michael@0: /* output BMP code point below 0xd800 */ michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? c<=0xdfff : c<=0xdbff) { michael@0: /* output roundtrip or fallback supplementary code point */ michael@0: c=(UChar32)(((c&0x3ff)<<10)+unicodeCodeUnits[offset]+(0x10000-0xdc00)); michael@0: } else if(UCNV_TO_U_USE_FALLBACK(cnv) ? (c&0xfffe)==0xe000 : c==0xe000) { michael@0: /* output roundtrip BMP code point above 0xd800 or fallback BMP code point */ michael@0: c=unicodeCodeUnits[offset]; michael@0: } else if(c==0xffff) { michael@0: return 0xffff; michael@0: } else { michael@0: c=0xfffe; michael@0: } michael@0: break; michael@0: } else if(action==MBCS_STATE_VALID_DIRECT_20) { michael@0: /* output supplementary code point */ michael@0: c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); michael@0: break; michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_16) { michael@0: if(!TO_U_USE_FALLBACK(useFallback)) { michael@0: c=0xfffe; michael@0: break; michael@0: } michael@0: /* output BMP code point */ michael@0: c=(UChar)MBCS_ENTRY_FINAL_VALUE_16(entry); michael@0: break; michael@0: } else if(action==MBCS_STATE_FALLBACK_DIRECT_20) { michael@0: if(!TO_U_USE_FALLBACK(useFallback)) { michael@0: c=0xfffe; michael@0: break; michael@0: } michael@0: /* output supplementary code point */ michael@0: c=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); michael@0: break; michael@0: } else if(action==MBCS_STATE_UNASSIGNED) { michael@0: c=0xfffe; michael@0: break; michael@0: } michael@0: michael@0: /* michael@0: * forbid MBCS_STATE_CHANGE_ONLY for this function, michael@0: * and MBCS_STATE_ILLEGAL and reserved action codes michael@0: */ michael@0: return 0xffff; michael@0: } michael@0: } michael@0: michael@0: if(i!=length) { michael@0: /* illegal for this function: not all input consumed */ michael@0: return 0xffff; michael@0: } michael@0: michael@0: if(c==0xfffe) { michael@0: /* try an extension mapping */ michael@0: const int32_t *cx=sharedData->mbcs.extIndexes; michael@0: if(cx!=NULL) { michael@0: return ucnv_extSimpleMatchToU(cx, source, length, useFallback); michael@0: } michael@0: } michael@0: michael@0: return c; michael@0: } michael@0: michael@0: /* MBCS-from-Unicode conversion functions ----------------------------------- */ michael@0: michael@0: /* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for double-byte codepages. */ michael@0: static void michael@0: ucnv_MBCSDoubleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const UChar *source, *sourceLimit; michael@0: uint8_t *target; michael@0: int32_t targetCapacity; michael@0: int32_t *offsets; michael@0: michael@0: const uint16_t *table; michael@0: const uint16_t *mbcsIndex; michael@0: const uint8_t *bytes; michael@0: michael@0: UChar32 c; michael@0: michael@0: int32_t sourceIndex, nextSourceIndex; michael@0: michael@0: uint32_t stage2Entry; michael@0: uint32_t asciiRoundtrips; michael@0: uint32_t value; michael@0: uint8_t unicodeMask; michael@0: michael@0: /* use optimized function if possible */ michael@0: cnv=pArgs->converter; michael@0: unicodeMask=cnv->sharedData->mbcs.unicodeMask; michael@0: michael@0: /* set up the local pointers */ michael@0: source=pArgs->source; michael@0: sourceLimit=pArgs->sourceLimit; michael@0: target=(uint8_t *)pArgs->target; michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); michael@0: offsets=pArgs->offsets; michael@0: michael@0: table=cnv->sharedData->mbcs.fromUnicodeTable; michael@0: mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; michael@0: } else { michael@0: bytes=cnv->sharedData->mbcs.fromUnicodeBytes; michael@0: } michael@0: asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; michael@0: michael@0: /* get the converter state from UConverter */ michael@0: c=cnv->fromUChar32; michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: sourceIndex= c==0 ? 0 : -1; michael@0: nextSourceIndex=0; michael@0: michael@0: /* conversion loop */ michael@0: if(c!=0 && targetCapacity>0) { michael@0: goto getTrail; michael@0: } michael@0: michael@0: while(source0) { michael@0: /* michael@0: * Get a correct Unicode code point: michael@0: * a single UChar for a BMP code point or michael@0: * a matched surrogate pair for a "supplementary code point". michael@0: */ michael@0: c=*source++; michael@0: ++nextSourceIndex; michael@0: if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { michael@0: *target++=(uint8_t)c; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: sourceIndex=nextSourceIndex; michael@0: } michael@0: --targetCapacity; michael@0: c=0; michael@0: continue; michael@0: } michael@0: /* michael@0: * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX michael@0: * to avoid dealing with surrogates. michael@0: * MBCS_FAST_MAX must be >=0xd7ff. michael@0: */ michael@0: if(c<=0xd7ff) { michael@0: value=DBCS_RESULT_FROM_MOST_BMP(mbcsIndex, (const uint16_t *)bytes, c); michael@0: /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } michael@0: /* output the value */ michael@0: } else { michael@0: /* michael@0: * This also tests if the codepage maps single surrogates. michael@0: * If it does, then surrogates are not paired but mapped separately. michael@0: * Note that in this case unmatched surrogates are not detected. michael@0: */ michael@0: if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { michael@0: if(U16_IS_SURROGATE_LEAD(c)) { michael@0: getTrail: michael@0: if(sourcesource=source; michael@0: c=_extFromU(cnv, cnv->sharedData, michael@0: c, &source, sourceLimit, michael@0: &target, target+targetCapacity, michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: nextSourceIndex+=(int32_t)(source-pArgs->source); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } else { michael@0: /* a mapping was written to the target, continue */ michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: sourceIndex=nextSourceIndex; michael@0: continue; michael@0: } michael@0: } michael@0: } michael@0: michael@0: /* write the output character bytes from value and length */ michael@0: /* from the first if in the loop we know that targetCapacity>0 */ michael@0: if(value<=0xff) { michael@0: /* this is easy because we know that there is enough space */ michael@0: *target++=(uint8_t)value; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: --targetCapacity; michael@0: } else /* length==2 */ { michael@0: *target++=(uint8_t)(value>>8); michael@0: if(2<=targetCapacity) { michael@0: *target++=(uint8_t)value; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: *offsets++=sourceIndex; michael@0: } michael@0: targetCapacity-=2; michael@0: } else { michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: cnv->charErrorBuffer[0]=(char)value; michael@0: cnv->charErrorBufferLength=1; michael@0: michael@0: /* target overflow */ michael@0: targetCapacity=0; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: c=0; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: c=0; michael@0: sourceIndex=nextSourceIndex; michael@0: continue; michael@0: } else { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* set the converter state back into UConverter */ michael@0: cnv->fromUChar32=c; michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=source; michael@0: pArgs->target=(char *)target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: /* This version of ucnv_MBCSFromUnicodeWithOffsets() is optimized for single-byte codepages. */ michael@0: static void michael@0: ucnv_MBCSSingleFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const UChar *source, *sourceLimit; michael@0: uint8_t *target; michael@0: int32_t targetCapacity; michael@0: int32_t *offsets; michael@0: michael@0: const uint16_t *table; michael@0: const uint16_t *results; michael@0: michael@0: UChar32 c; michael@0: michael@0: int32_t sourceIndex, nextSourceIndex; michael@0: michael@0: uint16_t value, minValue; michael@0: UBool hasSupplementary; michael@0: michael@0: /* set up the local pointers */ michael@0: cnv=pArgs->converter; michael@0: source=pArgs->source; michael@0: sourceLimit=pArgs->sourceLimit; michael@0: target=(uint8_t *)pArgs->target; michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); michael@0: offsets=pArgs->offsets; michael@0: michael@0: table=cnv->sharedData->mbcs.fromUnicodeTable; michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; michael@0: } else { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; michael@0: } michael@0: michael@0: if(cnv->useFallback) { michael@0: /* use all roundtrip and fallback results */ michael@0: minValue=0x800; michael@0: } else { michael@0: /* use only roundtrips and fallbacks from private-use characters */ michael@0: minValue=0xc00; michael@0: } michael@0: hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); michael@0: michael@0: /* get the converter state from UConverter */ michael@0: c=cnv->fromUChar32; michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: sourceIndex= c==0 ? 0 : -1; michael@0: nextSourceIndex=0; michael@0: michael@0: /* conversion loop */ michael@0: if(c!=0 && targetCapacity>0) { michael@0: goto getTrail; michael@0: } michael@0: michael@0: while(source0) { michael@0: /* michael@0: * Get a correct Unicode code point: michael@0: * a single UChar for a BMP code point or michael@0: * a matched surrogate pair for a "supplementary code point". michael@0: */ michael@0: c=*source++; michael@0: ++nextSourceIndex; michael@0: if(U16_IS_SURROGATE(c)) { michael@0: if(U16_IS_SURROGATE_LEAD(c)) { michael@0: getTrail: michael@0: if(source=minValue) { michael@0: /* assigned, write the output character bytes from value and length */ michael@0: /* length==1 */ michael@0: /* this is easy because we know that there is enough space */ michael@0: *target++=(uint8_t)value; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: --targetCapacity; michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: c=0; michael@0: sourceIndex=nextSourceIndex; michael@0: } else { /* unassigned */ michael@0: unassigned: michael@0: /* try an extension mapping */ michael@0: pArgs->source=source; michael@0: c=_extFromU(cnv, cnv->sharedData, michael@0: c, &source, sourceLimit, michael@0: &target, target+targetCapacity, michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: nextSourceIndex+=(int32_t)(source-pArgs->source); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } else { michael@0: /* a mapping was written to the target, continue */ michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: sourceIndex=nextSourceIndex; michael@0: } michael@0: } michael@0: } else { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* set the converter state back into UConverter */ michael@0: cnv->fromUChar32=c; michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=source; michael@0: pArgs->target=(char *)target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: /* michael@0: * This version of ucnv_MBCSFromUnicode() is optimized for single-byte codepages michael@0: * that map only to and from the BMP. michael@0: * In addition to single-byte/state optimizations, the offset calculations michael@0: * become much easier. michael@0: * It would be possible to use the sbcsIndex for UTF-8-friendly tables, michael@0: * but measurements have shown that this diminishes performance michael@0: * in more cases than it improves it. michael@0: * See SVN revision 21013 (2007-feb-06) for the last version with #if switches michael@0: * for various MBCS and SBCS optimizations. michael@0: */ michael@0: static void michael@0: ucnv_MBCSSingleFromBMPWithOffsets(UConverterFromUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const UChar *source, *sourceLimit, *lastSource; michael@0: uint8_t *target; michael@0: int32_t targetCapacity, length; michael@0: int32_t *offsets; michael@0: michael@0: const uint16_t *table; michael@0: const uint16_t *results; michael@0: michael@0: UChar32 c; michael@0: michael@0: int32_t sourceIndex; michael@0: michael@0: uint32_t asciiRoundtrips; michael@0: uint16_t value, minValue; michael@0: michael@0: /* set up the local pointers */ michael@0: cnv=pArgs->converter; michael@0: source=pArgs->source; michael@0: sourceLimit=pArgs->sourceLimit; michael@0: target=(uint8_t *)pArgs->target; michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); michael@0: offsets=pArgs->offsets; michael@0: michael@0: table=cnv->sharedData->mbcs.fromUnicodeTable; michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; michael@0: } else { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; michael@0: } michael@0: asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; michael@0: michael@0: if(cnv->useFallback) { michael@0: /* use all roundtrip and fallback results */ michael@0: minValue=0x800; michael@0: } else { michael@0: /* use only roundtrips and fallbacks from private-use characters */ michael@0: minValue=0xc00; michael@0: } michael@0: michael@0: /* get the converter state from UConverter */ michael@0: c=cnv->fromUChar32; michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: sourceIndex= c==0 ? 0 : -1; michael@0: lastSource=source; michael@0: michael@0: /* michael@0: * since the conversion here is 1:1 UChar:uint8_t, we need only one counter michael@0: * for the minimum of the sourceLength and targetCapacity michael@0: */ michael@0: length=(int32_t)(sourceLimit-source); michael@0: if(length0) { michael@0: goto getTrail; michael@0: } michael@0: michael@0: #if MBCS_UNROLL_SINGLE_FROM_BMP michael@0: /* unrolling makes it slower on Pentium III/Windows 2000?! */ michael@0: /* unroll the loop with the most common case */ michael@0: unrolled: michael@0: if(targetCapacity>=4) { michael@0: int32_t count, loops; michael@0: uint16_t andedValues; michael@0: michael@0: loops=count=targetCapacity>>2; michael@0: do { michael@0: c=*source++; michael@0: andedValues=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: *target++=(uint8_t)value; michael@0: c=*source++; michael@0: andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: *target++=(uint8_t)value; michael@0: c=*source++; michael@0: andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: *target++=(uint8_t)value; michael@0: c=*source++; michael@0: andedValues&=value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: *target++=(uint8_t)value; michael@0: michael@0: /* were all 4 entries really valid? */ michael@0: if(andedValues0); michael@0: count=loops-count; michael@0: targetCapacity-=4*count; michael@0: michael@0: if(offsets!=NULL) { michael@0: lastSource+=4*count; michael@0: while(count>0) { michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: *offsets++=sourceIndex++; michael@0: --count; michael@0: } michael@0: } michael@0: michael@0: c=0; michael@0: } michael@0: #endif michael@0: michael@0: while(targetCapacity>0) { michael@0: /* michael@0: * Get a correct Unicode code point: michael@0: * a single UChar for a BMP code point or michael@0: * a matched surrogate pair for a "supplementary code point". michael@0: */ michael@0: c=*source++; michael@0: /* michael@0: * Do not immediately check for single surrogates: michael@0: * Assume that they are unassigned and check for them in that case. michael@0: * This speeds up the conversion of assigned characters. michael@0: */ michael@0: /* convert the Unicode code point in c into codepage bytes */ michael@0: if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { michael@0: *target++=(uint8_t)c; michael@0: --targetCapacity; michael@0: c=0; michael@0: continue; michael@0: } michael@0: value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: /* is this code point assigned, or do we use fallbacks? */ michael@0: if(value>=minValue) { michael@0: /* assigned, write the output character bytes from value and length */ michael@0: /* length==1 */ michael@0: /* this is easy because we know that there is enough space */ michael@0: *target++=(uint8_t)value; michael@0: --targetCapacity; michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: c=0; michael@0: continue; michael@0: } else if(!U16_IS_SURROGATE(c)) { michael@0: /* normal, unassigned BMP character */ michael@0: } else if(U16_IS_SURROGATE_LEAD(c)) { michael@0: getTrail: michael@0: if(sourceflush) { michael@0: *pErrorCode=U_TRUNCATED_CHAR_FOUND; michael@0: } michael@0: break; michael@0: } michael@0: } else { michael@0: /* this is an unmatched trail code unit (2nd surrogate) */ michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: break; michael@0: } michael@0: michael@0: /* c does not have a mapping */ michael@0: michael@0: /* get the number of code units for c to correctly advance sourceIndex */ michael@0: length=U16_LENGTH(c); michael@0: michael@0: /* set offsets since the start or the last extension */ michael@0: if(offsets!=NULL) { michael@0: int32_t count=(int32_t)(source-lastSource); michael@0: michael@0: /* do not set the offset for this character */ michael@0: count-=length; michael@0: michael@0: while(count>0) { michael@0: *offsets++=sourceIndex++; michael@0: --count; michael@0: } michael@0: /* offsets and sourceIndex are now set for the current character */ michael@0: } michael@0: michael@0: /* try an extension mapping */ michael@0: lastSource=source; michael@0: c=_extFromU(cnv, cnv->sharedData, michael@0: c, &source, sourceLimit, michael@0: &target, (const uint8_t *)(pArgs->targetLimit), michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: sourceIndex+=length+(int32_t)(source-lastSource); michael@0: lastSource=source; michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } else { michael@0: /* a mapping was written to the target, continue */ michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); michael@0: length=(int32_t)(sourceLimit-source); michael@0: if(length=(uint8_t *)pArgs->targetLimit) { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: } michael@0: michael@0: /* set offsets since the start or the last callback */ michael@0: if(offsets!=NULL) { michael@0: size_t count=source-lastSource; michael@0: if (count > 0 && *pErrorCode == U_TRUNCATED_CHAR_FOUND) { michael@0: /* michael@0: Caller gave us a partial supplementary character, michael@0: which this function couldn't convert in any case. michael@0: The callback will handle the offset. michael@0: */ michael@0: count--; michael@0: } michael@0: while(count>0) { michael@0: *offsets++=sourceIndex++; michael@0: --count; michael@0: } michael@0: } michael@0: michael@0: /* set the converter state back into UConverter */ michael@0: cnv->fromUChar32=c; michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=source; michael@0: pArgs->target=(char *)target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: U_CFUNC void michael@0: ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv; michael@0: const UChar *source, *sourceLimit; michael@0: uint8_t *target; michael@0: int32_t targetCapacity; michael@0: int32_t *offsets; michael@0: michael@0: const uint16_t *table; michael@0: const uint16_t *mbcsIndex; michael@0: const uint8_t *p, *bytes; michael@0: uint8_t outputType; michael@0: michael@0: UChar32 c; michael@0: michael@0: int32_t prevSourceIndex, sourceIndex, nextSourceIndex; michael@0: michael@0: uint32_t stage2Entry; michael@0: uint32_t asciiRoundtrips; michael@0: uint32_t value; michael@0: /* Shift-In and Shift-Out byte sequences differ by encoding scheme. */ michael@0: uint8_t siBytes[2] = {0, 0}; michael@0: uint8_t soBytes[2] = {0, 0}; michael@0: uint8_t siLength, soLength; michael@0: int32_t length = 0, prevLength; michael@0: uint8_t unicodeMask; michael@0: michael@0: cnv=pArgs->converter; michael@0: michael@0: if(cnv->preFromUFirstCP>=0) { michael@0: /* michael@0: * pass sourceIndex=-1 because we continue from an earlier buffer michael@0: * in the future, this may change with continuous offsets michael@0: */ michael@0: ucnv_extContinueMatchFromU(cnv, pArgs, -1, pErrorCode); michael@0: michael@0: if(U_FAILURE(*pErrorCode) || cnv->preFromULength<0) { michael@0: return; michael@0: } michael@0: } michael@0: michael@0: /* use optimized function if possible */ michael@0: outputType=cnv->sharedData->mbcs.outputType; michael@0: unicodeMask=cnv->sharedData->mbcs.unicodeMask; michael@0: if(outputType==MBCS_OUTPUT_1 && !(unicodeMask&UCNV_HAS_SURROGATES)) { michael@0: if(!(unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { michael@0: ucnv_MBCSSingleFromBMPWithOffsets(pArgs, pErrorCode); michael@0: } else { michael@0: ucnv_MBCSSingleFromUnicodeWithOffsets(pArgs, pErrorCode); michael@0: } michael@0: return; michael@0: } else if(outputType==MBCS_OUTPUT_2 && cnv->sharedData->mbcs.utf8Friendly) { michael@0: ucnv_MBCSDoubleFromUnicodeWithOffsets(pArgs, pErrorCode); michael@0: return; michael@0: } michael@0: michael@0: /* set up the local pointers */ michael@0: source=pArgs->source; michael@0: sourceLimit=pArgs->sourceLimit; michael@0: target=(uint8_t *)pArgs->target; michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-pArgs->target); michael@0: offsets=pArgs->offsets; michael@0: michael@0: table=cnv->sharedData->mbcs.fromUnicodeTable; michael@0: if(cnv->sharedData->mbcs.utf8Friendly) { michael@0: mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; michael@0: } else { michael@0: mbcsIndex=NULL; michael@0: } michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: bytes=cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; michael@0: } else { michael@0: bytes=cnv->sharedData->mbcs.fromUnicodeBytes; michael@0: } michael@0: asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; michael@0: michael@0: /* get the converter state from UConverter */ michael@0: c=cnv->fromUChar32; michael@0: michael@0: if(outputType==MBCS_OUTPUT_2_SISO) { michael@0: prevLength=cnv->fromUnicodeStatus; michael@0: if(prevLength==0) { michael@0: /* set the real value */ michael@0: prevLength=1; michael@0: } michael@0: } else { michael@0: /* prevent fromUnicodeStatus from being set to something non-0 */ michael@0: prevLength=0; michael@0: } michael@0: michael@0: /* sourceIndex=-1 if the current character began in the previous buffer */ michael@0: prevSourceIndex=-1; michael@0: sourceIndex= c==0 ? 0 : -1; michael@0: nextSourceIndex=0; michael@0: michael@0: /* Get the SI/SO character for the converter */ michael@0: siLength = getSISOBytes(SI, cnv->options, siBytes); michael@0: soLength = getSISOBytes(SO, cnv->options, soBytes); michael@0: michael@0: /* conversion loop */ michael@0: /* michael@0: * This is another piece of ugly code: michael@0: * A goto into the loop if the converter state contains a first surrogate michael@0: * from the previous function call. michael@0: * It saves me to check in each loop iteration a check of if(c==0) michael@0: * and duplicating the trail-surrogate-handling code in the else michael@0: * branch of that check. michael@0: * I could not find any other way to get around this other than michael@0: * using a function call for the conversion and callback, which would michael@0: * be even more inefficient. michael@0: * michael@0: * Markus Scherer 2000-jul-19 michael@0: */ michael@0: if(c!=0 && targetCapacity>0) { michael@0: goto getTrail; michael@0: } michael@0: michael@0: while(source0) { michael@0: /* michael@0: * Get a correct Unicode code point: michael@0: * a single UChar for a BMP code point or michael@0: * a matched surrogate pair for a "supplementary code point". michael@0: */ michael@0: c=*source++; michael@0: ++nextSourceIndex; michael@0: if(c<=0x7f && IS_ASCII_ROUNDTRIP(c, asciiRoundtrips)) { michael@0: *target++=(uint8_t)c; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: prevSourceIndex=sourceIndex; michael@0: sourceIndex=nextSourceIndex; michael@0: } michael@0: --targetCapacity; michael@0: c=0; michael@0: continue; michael@0: } michael@0: /* michael@0: * utf8Friendly table: Test for <=0xd7ff rather than <=MBCS_FAST_MAX michael@0: * to avoid dealing with surrogates. michael@0: * MBCS_FAST_MAX must be >=0xd7ff. michael@0: */ michael@0: if(c<=0xd7ff && mbcsIndex!=NULL) { michael@0: value=mbcsIndex[c>>6]; michael@0: michael@0: /* get the bytes and the length for the output (copied from below and adapted for utf8Friendly data) */ michael@0: /* There are only roundtrips (!=0) and no-mapping (==0) entries. */ michael@0: switch(outputType) { michael@0: case MBCS_OUTPUT_2: michael@0: value=((const uint16_t *)bytes)[value +(c&0x3f)]; michael@0: if(value<=0xff) { michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } else { michael@0: length=1; michael@0: } michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_2_SISO: michael@0: /* 1/2-byte stateful with Shift-In/Shift-Out */ michael@0: /* michael@0: * Save the old state in the converter object michael@0: * right here, then change the local prevLength state variable if necessary. michael@0: * Then, if this character turns out to be unassigned or a fallback that michael@0: * is not taken, the callback code must not save the new state in the converter michael@0: * because the new state is for a character that is not output. michael@0: * However, the callback must still restore the state from the converter michael@0: * in case the callback function changed it for its output. michael@0: */ michael@0: cnv->fromUnicodeStatus=prevLength; /* save the old state */ michael@0: value=((const uint16_t *)bytes)[value +(c&0x3f)]; michael@0: if(value<=0xff) { michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } else if(prevLength<=1) { michael@0: length=1; michael@0: } else { michael@0: /* change from double-byte mode to single-byte */ michael@0: if (siLength == 1) { michael@0: value|=(uint32_t)siBytes[0]<<8; michael@0: length = 2; michael@0: } else if (siLength == 2) { michael@0: value|=(uint32_t)siBytes[1]<<8; michael@0: value|=(uint32_t)siBytes[0]<<16; michael@0: length = 3; michael@0: } michael@0: prevLength=1; michael@0: } michael@0: } else { michael@0: if(prevLength==2) { michael@0: length=2; michael@0: } else { michael@0: /* change from single-byte mode to double-byte */ michael@0: if (soLength == 1) { michael@0: value|=(uint32_t)soBytes[0]<<16; michael@0: length = 3; michael@0: } else if (soLength == 2) { michael@0: value|=(uint32_t)soBytes[1]<<16; michael@0: value|=(uint32_t)soBytes[0]<<24; michael@0: length = 4; michael@0: } michael@0: prevLength=2; michael@0: } michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_DBCS_ONLY: michael@0: /* table with single-byte results, but only DBCS mappings used */ michael@0: value=((const uint16_t *)bytes)[value +(c&0x3f)]; michael@0: if(value<=0xff) { michael@0: /* no mapping or SBCS result, not taken for DBCS-only */ michael@0: goto unassigned; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_3: michael@0: p=bytes+(value+(c&0x3f))*3; michael@0: value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; michael@0: if(value<=0xff) { michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } else { michael@0: length=1; michael@0: } michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else { michael@0: length=3; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4: michael@0: value=((const uint32_t *)bytes)[value +(c&0x3f)]; michael@0: if(value<=0xff) { michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } else { michael@0: length=1; michael@0: } michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else if(value<=0xffffff) { michael@0: length=3; michael@0: } else { michael@0: length=4; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_3_EUC: michael@0: value=((const uint16_t *)bytes)[value +(c&0x3f)]; michael@0: /* EUC 16-bit fixed-length representation */ michael@0: if(value<=0xff) { michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } else { michael@0: length=1; michael@0: } michael@0: } else if((value&0x8000)==0) { michael@0: value|=0x8e8000; michael@0: length=3; michael@0: } else if((value&0x80)==0) { michael@0: value|=0x8f0080; michael@0: length=3; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4_EUC: michael@0: p=bytes+(value+(c&0x3f))*3; michael@0: value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; michael@0: /* EUC 16-bit fixed-length representation applied to the first two bytes */ michael@0: if(value<=0xff) { michael@0: if(value==0) { michael@0: goto unassigned; michael@0: } else { michael@0: length=1; michael@0: } michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else if((value&0x800000)==0) { michael@0: value|=0x8e800000; michael@0: length=4; michael@0: } else if((value&0x8000)==0) { michael@0: value|=0x8f008000; michael@0: length=4; michael@0: } else { michael@0: length=3; michael@0: } michael@0: break; michael@0: default: michael@0: /* must not occur */ michael@0: /* michael@0: * To avoid compiler warnings that value & length may be michael@0: * used without having been initialized, we set them here. michael@0: * In reality, this is unreachable code. michael@0: * Not having a default branch also causes warnings with michael@0: * some compilers. michael@0: */ michael@0: value=0; michael@0: length=0; michael@0: break; michael@0: } michael@0: /* output the value */ michael@0: } else { michael@0: /* michael@0: * This also tests if the codepage maps single surrogates. michael@0: * If it does, then surrogates are not paired but mapped separately. michael@0: * Note that in this case unmatched surrogates are not detected. michael@0: */ michael@0: if(U16_IS_SURROGATE(c) && !(unicodeMask&UCNV_HAS_SURROGATES)) { michael@0: if(U16_IS_SURROGATE_LEAD(c)) { michael@0: getTrail: michael@0: if(sourcefromUnicodeStatus=prevLength; /* save the old state */ michael@0: /* callback(unassigned) */ michael@0: goto unassigned; michael@0: } michael@0: /* convert this supplementary code point */ michael@0: /* exit this condition tree */ michael@0: } else { michael@0: /* this is an unmatched lead code unit (1st surrogate) */ michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: break; michael@0: } michael@0: } else { michael@0: /* no more input */ michael@0: break; michael@0: } michael@0: } else { michael@0: /* this is an unmatched trail code unit (2nd surrogate) */ michael@0: /* callback(illegal) */ michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* convert the Unicode code point in c into codepage bytes */ michael@0: michael@0: /* michael@0: * The basic lookup is a triple-stage compact array (trie) lookup. michael@0: * For details see the beginning of this file. michael@0: * michael@0: * Single-byte codepages are handled with a different data structure michael@0: * by _MBCSSingle... functions. michael@0: * michael@0: * The result consists of a 32-bit value from stage 2 and michael@0: * a pointer to as many bytes as are stored per character. michael@0: * The pointer points to the character's bytes in stage 3. michael@0: * Bits 15..0 of the stage 2 entry contain the stage 3 index michael@0: * for that pointer, while bits 31..16 are flags for which of michael@0: * the 16 characters in the block are roundtrip-assigned. michael@0: * michael@0: * For 2-byte and 4-byte codepages, the bytes are stored as uint16_t michael@0: * respectively as uint32_t, in the platform encoding. michael@0: * For 3-byte codepages, the bytes are always stored in big-endian order. michael@0: * michael@0: * For EUC encodings that use only either 0x8e or 0x8f as the first michael@0: * byte of their longest byte sequences, the first two bytes in michael@0: * this third stage indicate with their 7th bits whether these bytes michael@0: * are to be written directly or actually need to be preceeded by michael@0: * one of the two Single-Shift codes. With this, the third stage michael@0: * stores one byte fewer per character than the actual maximum length of michael@0: * EUC byte sequences. michael@0: * michael@0: * Other than that, leading zero bytes are removed and the other michael@0: * bytes output. A single zero byte may be output if the "assigned" michael@0: * bit in stage 2 was on. michael@0: * The data structure does not support zero byte output as a fallback, michael@0: * and also does not allow output of leading zeros. michael@0: */ michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, c); michael@0: michael@0: /* get the bytes and the length for the output */ michael@0: switch(outputType) { michael@0: case MBCS_OUTPUT_2: michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_2_SISO: michael@0: /* 1/2-byte stateful with Shift-In/Shift-Out */ michael@0: /* michael@0: * Save the old state in the converter object michael@0: * right here, then change the local prevLength state variable if necessary. michael@0: * Then, if this character turns out to be unassigned or a fallback that michael@0: * is not taken, the callback code must not save the new state in the converter michael@0: * because the new state is for a character that is not output. michael@0: * However, the callback must still restore the state from the converter michael@0: * in case the callback function changed it for its output. michael@0: */ michael@0: cnv->fromUnicodeStatus=prevLength; /* save the old state */ michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: if(value==0 && MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)==0) { michael@0: /* no mapping, leave value==0 */ michael@0: length=0; michael@0: } else if(prevLength<=1) { michael@0: length=1; michael@0: } else { michael@0: /* change from double-byte mode to single-byte */ michael@0: if (siLength == 1) { michael@0: value|=(uint32_t)siBytes[0]<<8; michael@0: length = 2; michael@0: } else if (siLength == 2) { michael@0: value|=(uint32_t)siBytes[1]<<8; michael@0: value|=(uint32_t)siBytes[0]<<16; michael@0: length = 3; michael@0: } michael@0: prevLength=1; michael@0: } michael@0: } else { michael@0: if(prevLength==2) { michael@0: length=2; michael@0: } else { michael@0: /* change from single-byte mode to double-byte */ michael@0: if (soLength == 1) { michael@0: value|=(uint32_t)soBytes[0]<<16; michael@0: length = 3; michael@0: } else if (soLength == 2) { michael@0: value|=(uint32_t)soBytes[1]<<16; michael@0: value|=(uint32_t)soBytes[0]<<24; michael@0: length = 4; michael@0: } michael@0: prevLength=2; michael@0: } michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_DBCS_ONLY: michael@0: /* table with single-byte results, but only DBCS mappings used */ michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: /* no mapping or SBCS result, not taken for DBCS-only */ michael@0: value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ michael@0: length=0; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_3: michael@0: p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else { michael@0: length=3; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4: michael@0: value=MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else if(value<=0xffffff) { michael@0: length=3; michael@0: } else { michael@0: length=4; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_3_EUC: michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: /* EUC 16-bit fixed-length representation */ michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if((value&0x8000)==0) { michael@0: value|=0x8e8000; michael@0: length=3; michael@0: } else if((value&0x80)==0) { michael@0: value|=0x8f0080; michael@0: length=3; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4_EUC: michael@0: p=MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c); michael@0: value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; michael@0: /* EUC 16-bit fixed-length representation applied to the first two bytes */ michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else if((value&0x800000)==0) { michael@0: value|=0x8e800000; michael@0: length=4; michael@0: } else if((value&0x8000)==0) { michael@0: value|=0x8f008000; michael@0: length=4; michael@0: } else { michael@0: length=3; michael@0: } michael@0: break; michael@0: default: michael@0: /* must not occur */ michael@0: /* michael@0: * To avoid compiler warnings that value & length may be michael@0: * used without having been initialized, we set them here. michael@0: * In reality, this is unreachable code. michael@0: * Not having a default branch also causes warnings with michael@0: * some compilers. michael@0: */ michael@0: value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ michael@0: length=0; michael@0: break; michael@0: } michael@0: michael@0: /* is this code point assigned, or do we use fallbacks? */ michael@0: if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c)!=0 || michael@0: (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) michael@0: ) { michael@0: /* michael@0: * We allow a 0 byte output if the "assigned" bit is set for this entry. michael@0: * There is no way with this data structure for fallback output michael@0: * to be a zero byte. michael@0: */ michael@0: michael@0: unassigned: michael@0: /* try an extension mapping */ michael@0: pArgs->source=source; michael@0: c=_extFromU(cnv, cnv->sharedData, michael@0: c, &source, sourceLimit, michael@0: &target, target+targetCapacity, michael@0: &offsets, sourceIndex, michael@0: pArgs->flush, michael@0: pErrorCode); michael@0: nextSourceIndex+=(int32_t)(source-pArgs->source); michael@0: prevLength=cnv->fromUnicodeStatus; /* restore SISO state */ michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: break; michael@0: } else { michael@0: /* a mapping was written to the target, continue */ michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pArgs->targetLimit-(char *)target); michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: if(offsets!=NULL) { michael@0: prevSourceIndex=sourceIndex; michael@0: sourceIndex=nextSourceIndex; michael@0: } michael@0: continue; michael@0: } michael@0: } michael@0: } michael@0: michael@0: /* write the output character bytes from value and length */ michael@0: /* from the first if in the loop we know that targetCapacity>0 */ michael@0: if(length<=targetCapacity) { michael@0: if(offsets==NULL) { michael@0: switch(length) { michael@0: /* each branch falls through to the next one */ michael@0: case 4: michael@0: *target++=(uint8_t)(value>>24); michael@0: case 3: /*fall through*/ michael@0: *target++=(uint8_t)(value>>16); michael@0: case 2: /*fall through*/ michael@0: *target++=(uint8_t)(value>>8); michael@0: case 1: /*fall through*/ michael@0: *target++=(uint8_t)value; michael@0: default: michael@0: /* will never occur */ michael@0: break; michael@0: } michael@0: } else { michael@0: switch(length) { michael@0: /* each branch falls through to the next one */ michael@0: case 4: michael@0: *target++=(uint8_t)(value>>24); michael@0: *offsets++=sourceIndex; michael@0: case 3: /*fall through*/ michael@0: *target++=(uint8_t)(value>>16); michael@0: *offsets++=sourceIndex; michael@0: case 2: /*fall through*/ michael@0: *target++=(uint8_t)(value>>8); michael@0: *offsets++=sourceIndex; michael@0: case 1: /*fall through*/ michael@0: *target++=(uint8_t)value; michael@0: *offsets++=sourceIndex; michael@0: default: michael@0: /* will never occur */ michael@0: break; michael@0: } michael@0: } michael@0: targetCapacity-=length; michael@0: } else { michael@0: uint8_t *charErrorBuffer; michael@0: michael@0: /* michael@0: * We actually do this backwards here: michael@0: * In order to save an intermediate variable, we output michael@0: * first to the overflow buffer what does not fit into the michael@0: * regular target. michael@0: */ michael@0: /* we know that 1<=targetCapacitycharErrorBuffer; michael@0: switch(length) { michael@0: /* each branch falls through to the next one */ michael@0: case 3: michael@0: *charErrorBuffer++=(uint8_t)(value>>16); michael@0: case 2: /*fall through*/ michael@0: *charErrorBuffer++=(uint8_t)(value>>8); michael@0: case 1: /*fall through*/ michael@0: *charErrorBuffer=(uint8_t)value; michael@0: default: michael@0: /* will never occur */ michael@0: break; michael@0: } michael@0: cnv->charErrorBufferLength=(int8_t)length; michael@0: michael@0: /* now output what fits into the regular target */ michael@0: value>>=8*length; /* length was reduced by targetCapacity */ michael@0: switch(targetCapacity) { michael@0: /* each branch falls through to the next one */ michael@0: case 3: michael@0: *target++=(uint8_t)(value>>16); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: case 2: /*fall through*/ michael@0: *target++=(uint8_t)(value>>8); michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: case 1: /*fall through*/ michael@0: *target++=(uint8_t)value; michael@0: if(offsets!=NULL) { michael@0: *offsets++=sourceIndex; michael@0: } michael@0: default: michael@0: /* will never occur */ michael@0: break; michael@0: } michael@0: michael@0: /* target overflow */ michael@0: targetCapacity=0; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: c=0; michael@0: break; michael@0: } michael@0: michael@0: /* normal end of conversion: prepare for a new character */ michael@0: c=0; michael@0: if(offsets!=NULL) { michael@0: prevSourceIndex=sourceIndex; michael@0: sourceIndex=nextSourceIndex; michael@0: } michael@0: continue; michael@0: } else { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* michael@0: * the end of the input stream and detection of truncated input michael@0: * are handled by the framework, but for EBCDIC_STATEFUL conversion michael@0: * we need to emit an SI at the very end michael@0: * michael@0: * conditions: michael@0: * successful michael@0: * EBCDIC_STATEFUL in DBCS mode michael@0: * end of input and no truncated input michael@0: */ michael@0: if( U_SUCCESS(*pErrorCode) && michael@0: outputType==MBCS_OUTPUT_2_SISO && prevLength==2 && michael@0: pArgs->flush && source>=sourceLimit && c==0 michael@0: ) { michael@0: /* EBCDIC_STATEFUL ending with DBCS: emit an SI to return the output stream to SBCS */ michael@0: if(targetCapacity>0) { michael@0: *target++=(uint8_t)siBytes[0]; michael@0: if (siLength == 2) { michael@0: if (targetCapacity<2) { michael@0: cnv->charErrorBuffer[0]=(uint8_t)siBytes[1]; michael@0: cnv->charErrorBufferLength=1; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: } else { michael@0: *target++=(uint8_t)siBytes[1]; michael@0: } michael@0: } michael@0: if(offsets!=NULL) { michael@0: /* set the last source character's index (sourceIndex points at sourceLimit now) */ michael@0: *offsets++=prevSourceIndex; michael@0: } michael@0: } else { michael@0: /* target is full */ michael@0: cnv->charErrorBuffer[0]=(uint8_t)siBytes[0]; michael@0: if (siLength == 2) { michael@0: cnv->charErrorBuffer[1]=(uint8_t)siBytes[1]; michael@0: } michael@0: cnv->charErrorBufferLength=siLength; michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: } michael@0: prevLength=1; /* we switched into SBCS */ michael@0: } michael@0: michael@0: /* set the converter state back into UConverter */ michael@0: cnv->fromUChar32=c; michael@0: cnv->fromUnicodeStatus=prevLength; michael@0: michael@0: /* write back the updated pointers */ michael@0: pArgs->source=source; michael@0: pArgs->target=(char *)target; michael@0: pArgs->offsets=offsets; michael@0: } michael@0: michael@0: /* michael@0: * This is another simple conversion function for internal use by other michael@0: * conversion implementations. michael@0: * It does not use the converter state nor call callbacks. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It handles conversion extensions but not GB 18030. michael@0: * michael@0: * It converts one single Unicode code point into codepage bytes, encoded michael@0: * as one 32-bit value. The function returns the number of bytes in *pValue: michael@0: * 1..4 the number of bytes in *pValue michael@0: * 0 unassigned (*pValue undefined) michael@0: * -1 illegal (currently not used, *pValue undefined) michael@0: * michael@0: * *pValue will contain the resulting bytes with the last byte in bits 7..0, michael@0: * the second to last byte in bits 15..8, etc. michael@0: * Currently, the function assumes but does not check that 0<=c<=0x10ffff. michael@0: */ michael@0: U_CFUNC int32_t michael@0: ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, michael@0: UChar32 c, uint32_t *pValue, michael@0: UBool useFallback) { michael@0: const int32_t *cx; michael@0: const uint16_t *table; michael@0: #if 0 michael@0: /* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ michael@0: const uint8_t *p; michael@0: #endif michael@0: uint32_t stage2Entry; michael@0: uint32_t value; michael@0: int32_t length; michael@0: michael@0: /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ michael@0: if(c<=0xffff || (sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { michael@0: table=sharedData->mbcs.fromUnicodeTable; michael@0: michael@0: /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ michael@0: if(sharedData->mbcs.outputType==MBCS_OUTPUT_1) { michael@0: value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); michael@0: /* is this code point assigned, or do we use fallbacks? */ michael@0: if(useFallback ? value>=0x800 : value>=0xc00) { michael@0: *pValue=value&0xff; michael@0: return 1; michael@0: } michael@0: } else /* outputType!=MBCS_OUTPUT_1 */ { michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, c); michael@0: michael@0: /* get the bytes and the length for the output */ michael@0: switch(sharedData->mbcs.outputType) { michael@0: case MBCS_OUTPUT_2: michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: #if 0 michael@0: /* #if 0 because this is not currently used in ICU - reduce code, increase code coverage */ michael@0: case MBCS_OUTPUT_DBCS_ONLY: michael@0: /* table with single-byte results, but only DBCS mappings used */ michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: /* no mapping or SBCS result, not taken for DBCS-only */ michael@0: value=stage2Entry=0; /* stage2Entry=0 to reset roundtrip flags */ michael@0: length=0; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_3: michael@0: p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); michael@0: value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else { michael@0: length=3; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4: michael@0: value=MBCS_VALUE_4_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else if(value<=0xffffff) { michael@0: length=3; michael@0: } else { michael@0: length=4; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_3_EUC: michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); michael@0: /* EUC 16-bit fixed-length representation */ michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if((value&0x8000)==0) { michael@0: value|=0x8e8000; michael@0: length=3; michael@0: } else if((value&0x80)==0) { michael@0: value|=0x8f0080; michael@0: length=3; michael@0: } else { michael@0: length=2; michael@0: } michael@0: break; michael@0: case MBCS_OUTPUT_4_EUC: michael@0: p=MBCS_POINTER_3_FROM_STAGE_2(sharedData->mbcs.fromUnicodeBytes, stage2Entry, c); michael@0: value=((uint32_t)*p<<16)|((uint32_t)p[1]<<8)|p[2]; michael@0: /* EUC 16-bit fixed-length representation applied to the first two bytes */ michael@0: if(value<=0xff) { michael@0: length=1; michael@0: } else if(value<=0xffff) { michael@0: length=2; michael@0: } else if((value&0x800000)==0) { michael@0: value|=0x8e800000; michael@0: length=4; michael@0: } else if((value&0x8000)==0) { michael@0: value|=0x8f008000; michael@0: length=4; michael@0: } else { michael@0: length=3; michael@0: } michael@0: break; michael@0: #endif michael@0: default: michael@0: /* must not occur */ michael@0: return -1; michael@0: } michael@0: michael@0: /* is this code point assigned, or do we use fallbacks? */ michael@0: if( MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || michael@0: (FROM_U_USE_FALLBACK(useFallback, c) && value!=0) michael@0: ) { michael@0: /* michael@0: * We allow a 0 byte output if the "assigned" bit is set for this entry. michael@0: * There is no way with this data structure for fallback output michael@0: * to be a zero byte. michael@0: */ michael@0: /* assigned */ michael@0: *pValue=value; michael@0: return length; michael@0: } michael@0: } michael@0: } michael@0: michael@0: cx=sharedData->mbcs.extIndexes; michael@0: if(cx!=NULL) { michael@0: length=ucnv_extSimpleMatchFromU(cx, c, pValue, useFallback); michael@0: return length>=0 ? length : -length; /* return abs(length); */ michael@0: } michael@0: michael@0: /* unassigned */ michael@0: return 0; michael@0: } michael@0: michael@0: michael@0: #if 0 michael@0: /* michael@0: * This function has been moved to ucnv2022.c for inlining. michael@0: * This implementation is here only for documentation purposes michael@0: */ michael@0: michael@0: /** michael@0: * This version of ucnv_MBCSFromUChar32() is optimized for single-byte codepages. michael@0: * It does not handle the EBCDIC swaplfnl option (set in UConverter). michael@0: * It does not handle conversion extensions (_extFromU()). michael@0: * michael@0: * It returns the codepage byte for the code point, or -1 if it is unassigned. michael@0: */ michael@0: U_CFUNC int32_t michael@0: ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, michael@0: UChar32 c, michael@0: UBool useFallback) { michael@0: const uint16_t *table; michael@0: int32_t value; michael@0: michael@0: /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ michael@0: if(c>=0x10000 && !(sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY)) { michael@0: return -1; michael@0: } michael@0: michael@0: /* convert the Unicode code point in c into codepage bytes (same as in _MBCSFromUnicodeWithOffsets) */ michael@0: table=sharedData->mbcs.fromUnicodeTable; michael@0: michael@0: /* get the byte for the output */ michael@0: value=MBCS_SINGLE_RESULT_FROM_U(table, (uint16_t *)sharedData->mbcs.fromUnicodeBytes, c); michael@0: /* is this code point assigned, or do we use fallbacks? */ michael@0: if(useFallback ? value>=0x800 : value>=0xc00) { michael@0: return value&0xff; michael@0: } else { michael@0: return -1; michael@0: } michael@0: } michael@0: #endif michael@0: michael@0: /* MBCS-from-UTF-8 conversion functions ------------------------------------- */ michael@0: michael@0: /* minimum code point values for n-byte UTF-8 sequences, n=0..4 */ michael@0: static const UChar32 michael@0: utf8_minLegal[5]={ 0, 0, 0x80, 0x800, 0x10000 }; michael@0: michael@0: /* offsets for n-byte UTF-8 sequences that were calculated with ((lead<<6)+trail)<<6+trail... */ michael@0: static const UChar32 michael@0: utf8_offsets[7]={ 0, 0, 0x3080, 0xE2080, 0x3C82080 }; michael@0: michael@0: static void michael@0: ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, michael@0: UConverterToUnicodeArgs *pToUArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *utf8, *cnv; michael@0: const uint8_t *source, *sourceLimit; michael@0: uint8_t *target; michael@0: int32_t targetCapacity; michael@0: michael@0: const uint16_t *table, *sbcsIndex; michael@0: const uint16_t *results; michael@0: michael@0: int8_t oldToULength, toULength, toULimit; michael@0: michael@0: UChar32 c; michael@0: uint8_t b, t1, t2; michael@0: michael@0: uint32_t asciiRoundtrips; michael@0: uint16_t value, minValue; michael@0: UBool hasSupplementary; michael@0: michael@0: /* set up the local pointers */ michael@0: utf8=pToUArgs->converter; michael@0: cnv=pFromUArgs->converter; michael@0: source=(uint8_t *)pToUArgs->source; michael@0: sourceLimit=(uint8_t *)pToUArgs->sourceLimit; michael@0: target=(uint8_t *)pFromUArgs->target; michael@0: targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); michael@0: michael@0: table=cnv->sharedData->mbcs.fromUnicodeTable; michael@0: sbcsIndex=cnv->sharedData->mbcs.sbcsIndex; michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; michael@0: } else { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; michael@0: } michael@0: asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; michael@0: michael@0: if(cnv->useFallback) { michael@0: /* use all roundtrip and fallback results */ michael@0: minValue=0x800; michael@0: } else { michael@0: /* use only roundtrips and fallbacks from private-use characters */ michael@0: minValue=0xc00; michael@0: } michael@0: hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); michael@0: michael@0: /* get the converter state from the UTF-8 UConverter */ michael@0: c=(UChar32)utf8->toUnicodeStatus; michael@0: if(c!=0) { michael@0: toULength=oldToULength=utf8->toULength; michael@0: toULimit=(int8_t)utf8->mode; michael@0: } else { michael@0: toULength=oldToULength=toULimit=0; michael@0: } michael@0: michael@0: /* michael@0: * Make sure that the last byte sequence before sourceLimit is complete michael@0: * or runs into a lead byte. michael@0: * Do not go back into the bytes that will be read for finishing a partial michael@0: * sequence from the previous buffer. michael@0: * In the conversion loop compare source with sourceLimit only once michael@0: * per multi-byte character. michael@0: */ michael@0: { michael@0: int32_t i, length; michael@0: michael@0: length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); michael@0: for(i=0; i<3 && i0) { michael@0: utf8->toUnicodeStatus=0; michael@0: utf8->toULength=0; michael@0: goto moreBytes; michael@0: /* michael@0: * Note: We could avoid the goto by duplicating some of the moreBytes michael@0: * code, but only up to the point of collecting a complete UTF-8 michael@0: * sequence; then recurse for the toUBytes[toULength] michael@0: * and then continue with normal conversion. michael@0: * michael@0: * If so, move this code to just after initializing the minimum michael@0: * set of local variables for reading the UTF-8 input michael@0: * (utf8, source, target, limits but not cnv, table, minValue, etc.). michael@0: * michael@0: * Potential advantages: michael@0: * - avoid the goto michael@0: * - oldToULength could become a local variable in just those code blocks michael@0: * that deal with buffer boundaries michael@0: * - possibly faster if the goto prevents some compiler optimizations michael@0: * (this would need measuring to confirm) michael@0: * Disadvantage: michael@0: * - code duplication michael@0: */ michael@0: } michael@0: michael@0: /* conversion loop */ michael@0: while(source0) { michael@0: b=*source++; michael@0: if((int8_t)b>=0) { michael@0: /* convert ASCII */ michael@0: if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { michael@0: *target++=(uint8_t)b; michael@0: --targetCapacity; michael@0: continue; michael@0: } else { michael@0: c=b; michael@0: value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, 0, c); michael@0: } michael@0: } else { michael@0: if(b<0xe0) { michael@0: if( /* handle U+0080..U+07FF inline */ michael@0: b>=0xc2 && michael@0: (t1=(uint8_t)(*source-0x80)) <= 0x3f michael@0: ) { michael@0: c=b&0x1f; michael@0: ++source; michael@0: value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t1); michael@0: if(value>=minValue) { michael@0: *target++=(uint8_t)value; michael@0: --targetCapacity; michael@0: continue; michael@0: } else { michael@0: c=(c<<6)|t1; michael@0: } michael@0: } else { michael@0: c=-1; michael@0: } michael@0: } else if(b==0xe0) { michael@0: if( /* handle U+0800..U+0FFF inline */ michael@0: (t1=(uint8_t)(source[0]-0x80)) <= 0x3f && t1 >= 0x20 && michael@0: (t2=(uint8_t)(source[1]-0x80)) <= 0x3f michael@0: ) { michael@0: c=t1; michael@0: source+=2; michael@0: value=SBCS_RESULT_FROM_UTF8(sbcsIndex, results, c, t2); michael@0: if(value>=minValue) { michael@0: *target++=(uint8_t)value; michael@0: --targetCapacity; michael@0: continue; michael@0: } else { michael@0: c=(c<<6)|t2; michael@0: } michael@0: } else { michael@0: c=-1; michael@0: } michael@0: } else { michael@0: c=-1; michael@0: } michael@0: michael@0: if(c<0) { michael@0: /* handle "complicated" and error cases, and continuing partial characters */ michael@0: oldToULength=0; michael@0: toULength=1; michael@0: toULimit=U8_COUNT_TRAIL_BYTES(b)+1; michael@0: c=b; michael@0: moreBytes: michael@0: while(toULengthsourceLimit) { michael@0: b=*source; michael@0: if(U8_IS_TRAIL(b)) { michael@0: ++source; michael@0: ++toULength; michael@0: c=(c<<6)+b; michael@0: } else { michael@0: break; /* sequence too short, stop with toULengthtoUBytes[oldToULength++]=*source++; michael@0: } michael@0: utf8->toUnicodeStatus=c; michael@0: utf8->toULength=toULength; michael@0: utf8->mode=toULimit; michael@0: pToUArgs->source=(char *)source; michael@0: pFromUArgs->target=(char *)target; michael@0: return; michael@0: } michael@0: } michael@0: michael@0: if( toULength==toULimit && /* consumed all trail bytes */ michael@0: (toULength==3 || toULength==2) && /* BMP */ michael@0: (c-=utf8_offsets[toULength])>=utf8_minLegal[toULength] && michael@0: (c<=0xd7ff || 0xe000<=c) /* not a surrogate */ michael@0: ) { michael@0: value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: } else if( michael@0: toULength==toULimit && toULength==4 && michael@0: (0x10000<=(c-=utf8_offsets[4]) && c<=0x10ffff) michael@0: ) { michael@0: /* supplementary code point */ michael@0: if(!hasSupplementary) { michael@0: /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ michael@0: value=0; michael@0: } else { michael@0: value=MBCS_SINGLE_RESULT_FROM_U(table, results, c); michael@0: } michael@0: } else { michael@0: /* error handling: illegal UTF-8 byte sequence */ michael@0: source-=(toULength-oldToULength); michael@0: while(oldToULengthtoUBytes[oldToULength++]=*source++; michael@0: } michael@0: utf8->toULength=toULength; michael@0: pToUArgs->source=(char *)source; michael@0: pFromUArgs->target=(char *)target; michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: return; michael@0: } michael@0: } michael@0: } michael@0: michael@0: if(value>=minValue) { michael@0: /* output the mapping for c */ michael@0: *target++=(uint8_t)value; michael@0: --targetCapacity; michael@0: } else { michael@0: /* valueUTF-16->charset conversion. michael@0: */ michael@0: static const UChar nul=0; michael@0: const UChar *noSource=&nul; michael@0: c=_extFromU(cnv, cnv->sharedData, michael@0: c, &noSource, noSource, michael@0: &target, target+targetCapacity, michael@0: NULL, -1, michael@0: pFromUArgs->flush, michael@0: pErrorCode); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: cnv->fromUChar32=c; michael@0: break; michael@0: } else if(cnv->preFromUFirstCP>=0) { michael@0: /* michael@0: * Partial match, return and revert to pivoting. michael@0: * In normal from-UTF-16 conversion, we would just continue michael@0: * but then exit the loop because the extension match would michael@0: * have consumed the source. michael@0: */ michael@0: *pErrorCode=U_USING_DEFAULT_WARNING; michael@0: break; michael@0: } else { michael@0: /* a mapping was written to the target, continue */ michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); michael@0: } michael@0: } michael@0: } else { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* michael@0: * The sourceLimit may have been adjusted before the conversion loop michael@0: * to stop before a truncated sequence. michael@0: * If so, then collect the truncated sequence now. michael@0: */ michael@0: if(U_SUCCESS(*pErrorCode) && michael@0: cnv->preFromUFirstCP<0 && michael@0: source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { michael@0: c=utf8->toUBytes[0]=b=*source++; michael@0: toULength=1; michael@0: toULimit=U8_COUNT_TRAIL_BYTES(b)+1; michael@0: while(sourcetoUBytes[toULength++]=b=*source++; michael@0: c=(c<<6)+b; michael@0: } michael@0: utf8->toUnicodeStatus=c; michael@0: utf8->toULength=toULength; michael@0: utf8->mode=toULimit; michael@0: } michael@0: michael@0: /* write back the updated pointers */ michael@0: pToUArgs->source=(char *)source; michael@0: pFromUArgs->target=(char *)target; michael@0: } michael@0: michael@0: static void michael@0: ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs, michael@0: UConverterToUnicodeArgs *pToUArgs, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *utf8, *cnv; michael@0: const uint8_t *source, *sourceLimit; michael@0: uint8_t *target; michael@0: int32_t targetCapacity; michael@0: michael@0: const uint16_t *table, *mbcsIndex; michael@0: const uint16_t *results; michael@0: michael@0: int8_t oldToULength, toULength, toULimit; michael@0: michael@0: UChar32 c; michael@0: uint8_t b, t1, t2; michael@0: michael@0: uint32_t stage2Entry; michael@0: uint32_t asciiRoundtrips; michael@0: uint16_t value; michael@0: UBool hasSupplementary; michael@0: michael@0: /* set up the local pointers */ michael@0: utf8=pToUArgs->converter; michael@0: cnv=pFromUArgs->converter; michael@0: source=(uint8_t *)pToUArgs->source; michael@0: sourceLimit=(uint8_t *)pToUArgs->sourceLimit; michael@0: target=(uint8_t *)pFromUArgs->target; michael@0: targetCapacity=(int32_t)(pFromUArgs->targetLimit-pFromUArgs->target); michael@0: michael@0: table=cnv->sharedData->mbcs.fromUnicodeTable; michael@0: mbcsIndex=cnv->sharedData->mbcs.mbcsIndex; michael@0: if((cnv->options&UCNV_OPTION_SWAP_LFNL)!=0) { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.swapLFNLFromUnicodeBytes; michael@0: } else { michael@0: results=(uint16_t *)cnv->sharedData->mbcs.fromUnicodeBytes; michael@0: } michael@0: asciiRoundtrips=cnv->sharedData->mbcs.asciiRoundtrips; michael@0: michael@0: hasSupplementary=(UBool)(cnv->sharedData->mbcs.unicodeMask&UCNV_HAS_SUPPLEMENTARY); michael@0: michael@0: /* get the converter state from the UTF-8 UConverter */ michael@0: c=(UChar32)utf8->toUnicodeStatus; michael@0: if(c!=0) { michael@0: toULength=oldToULength=utf8->toULength; michael@0: toULimit=(int8_t)utf8->mode; michael@0: } else { michael@0: toULength=oldToULength=toULimit=0; michael@0: } michael@0: michael@0: /* michael@0: * Make sure that the last byte sequence before sourceLimit is complete michael@0: * or runs into a lead byte. michael@0: * Do not go back into the bytes that will be read for finishing a partial michael@0: * sequence from the previous buffer. michael@0: * In the conversion loop compare source with sourceLimit only once michael@0: * per multi-byte character. michael@0: */ michael@0: { michael@0: int32_t i, length; michael@0: michael@0: length=(int32_t)(sourceLimit-source) - (toULimit-oldToULength); michael@0: for(i=0; i<3 && i0) { michael@0: utf8->toUnicodeStatus=0; michael@0: utf8->toULength=0; michael@0: goto moreBytes; michael@0: /* See note in ucnv_SBCSFromUTF8() about this goto. */ michael@0: } michael@0: michael@0: /* conversion loop */ michael@0: while(source0) { michael@0: b=*source++; michael@0: if((int8_t)b>=0) { michael@0: /* convert ASCII */ michael@0: if(IS_ASCII_ROUNDTRIP(b, asciiRoundtrips)) { michael@0: *target++=b; michael@0: --targetCapacity; michael@0: continue; michael@0: } else { michael@0: value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, 0, b); michael@0: if(value==0) { michael@0: c=b; michael@0: goto unassigned; michael@0: } michael@0: } michael@0: } else { michael@0: if(b>0xe0) { michael@0: if( /* handle U+1000..U+D7FF inline */ michael@0: (((t1=(uint8_t)(source[0]-0x80), b<0xed) && (t1 <= 0x3f)) || michael@0: (b==0xed && (t1 <= 0x1f))) && michael@0: (t2=(uint8_t)(source[1]-0x80)) <= 0x3f michael@0: ) { michael@0: c=((b&0xf)<<6)|t1; michael@0: source+=2; michael@0: value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t2); michael@0: if(value==0) { michael@0: c=(c<<6)|t2; michael@0: goto unassigned; michael@0: } michael@0: } else { michael@0: c=-1; michael@0: } michael@0: } else if(b<0xe0) { michael@0: if( /* handle U+0080..U+07FF inline */ michael@0: b>=0xc2 && michael@0: (t1=(uint8_t)(*source-0x80)) <= 0x3f michael@0: ) { michael@0: c=b&0x1f; michael@0: ++source; michael@0: value=DBCS_RESULT_FROM_UTF8(mbcsIndex, results, c, t1); michael@0: if(value==0) { michael@0: c=(c<<6)|t1; michael@0: goto unassigned; michael@0: } michael@0: } else { michael@0: c=-1; michael@0: } michael@0: } else { michael@0: c=-1; michael@0: } michael@0: michael@0: if(c<0) { michael@0: /* handle "complicated" and error cases, and continuing partial characters */ michael@0: oldToULength=0; michael@0: toULength=1; michael@0: toULimit=U8_COUNT_TRAIL_BYTES(b)+1; michael@0: c=b; michael@0: moreBytes: michael@0: while(toULengthsourceLimit) { michael@0: b=*source; michael@0: if(U8_IS_TRAIL(b)) { michael@0: ++source; michael@0: ++toULength; michael@0: c=(c<<6)+b; michael@0: } else { michael@0: break; /* sequence too short, stop with toULengthtoUBytes[oldToULength++]=*source++; michael@0: } michael@0: utf8->toUnicodeStatus=c; michael@0: utf8->toULength=toULength; michael@0: utf8->mode=toULimit; michael@0: pToUArgs->source=(char *)source; michael@0: pFromUArgs->target=(char *)target; michael@0: return; michael@0: } michael@0: } michael@0: michael@0: if( toULength==toULimit && /* consumed all trail bytes */ michael@0: (toULength==3 || toULength==2) && /* BMP */ michael@0: (c-=utf8_offsets[toULength])>=utf8_minLegal[toULength] && michael@0: (c<=0xd7ff || 0xe000<=c) /* not a surrogate */ michael@0: ) { michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, c); michael@0: } else if( michael@0: toULength==toULimit && toULength==4 && michael@0: (0x10000<=(c-=utf8_offsets[4]) && c<=0x10ffff) michael@0: ) { michael@0: /* supplementary code point */ michael@0: if(!hasSupplementary) { michael@0: /* BMP-only codepages are stored without stage 1 entries for supplementary code points */ michael@0: stage2Entry=0; michael@0: } else { michael@0: stage2Entry=MBCS_STAGE_2_FROM_U(table, c); michael@0: } michael@0: } else { michael@0: /* error handling: illegal UTF-8 byte sequence */ michael@0: source-=(toULength-oldToULength); michael@0: while(oldToULengthtoUBytes[oldToULength++]=*source++; michael@0: } michael@0: utf8->toULength=toULength; michael@0: pToUArgs->source=(char *)source; michael@0: pFromUArgs->target=(char *)target; michael@0: *pErrorCode=U_ILLEGAL_CHAR_FOUND; michael@0: return; michael@0: } michael@0: michael@0: /* get the bytes and the length for the output */ michael@0: /* MBCS_OUTPUT_2 */ michael@0: value=MBCS_VALUE_2_FROM_STAGE_2(results, stage2Entry, c); michael@0: michael@0: /* is this code point assigned, or do we use fallbacks? */ michael@0: if(!(MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) || michael@0: (UCNV_FROM_U_USE_FALLBACK(cnv, c) && value!=0)) michael@0: ) { michael@0: goto unassigned; michael@0: } michael@0: } michael@0: } michael@0: michael@0: /* write the output character bytes from value and length */ michael@0: /* from the first if in the loop we know that targetCapacity>0 */ michael@0: if(value<=0xff) { michael@0: /* this is easy because we know that there is enough space */ michael@0: *target++=(uint8_t)value; michael@0: --targetCapacity; michael@0: } else /* length==2 */ { michael@0: *target++=(uint8_t)(value>>8); michael@0: if(2<=targetCapacity) { michael@0: *target++=(uint8_t)value; michael@0: targetCapacity-=2; michael@0: } else { michael@0: cnv->charErrorBuffer[0]=(char)value; michael@0: cnv->charErrorBufferLength=1; michael@0: michael@0: /* target overflow */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: } michael@0: continue; michael@0: michael@0: unassigned: michael@0: { michael@0: /* michael@0: * Try an extension mapping. michael@0: * Pass in no source because we don't have UTF-16 input. michael@0: * If we have a partial match on c, we will return and revert michael@0: * to UTF-8->UTF-16->charset conversion. michael@0: */ michael@0: static const UChar nul=0; michael@0: const UChar *noSource=&nul; michael@0: c=_extFromU(cnv, cnv->sharedData, michael@0: c, &noSource, noSource, michael@0: &target, target+targetCapacity, michael@0: NULL, -1, michael@0: pFromUArgs->flush, michael@0: pErrorCode); michael@0: michael@0: if(U_FAILURE(*pErrorCode)) { michael@0: /* not mappable or buffer overflow */ michael@0: cnv->fromUChar32=c; michael@0: break; michael@0: } else if(cnv->preFromUFirstCP>=0) { michael@0: /* michael@0: * Partial match, return and revert to pivoting. michael@0: * In normal from-UTF-16 conversion, we would just continue michael@0: * but then exit the loop because the extension match would michael@0: * have consumed the source. michael@0: */ michael@0: *pErrorCode=U_USING_DEFAULT_WARNING; michael@0: break; michael@0: } else { michael@0: /* a mapping was written to the target, continue */ michael@0: michael@0: /* recalculate the targetCapacity after an extension mapping */ michael@0: targetCapacity=(int32_t)(pFromUArgs->targetLimit-(char *)target); michael@0: continue; michael@0: } michael@0: } michael@0: } else { michael@0: /* target is full */ michael@0: *pErrorCode=U_BUFFER_OVERFLOW_ERROR; michael@0: break; michael@0: } michael@0: } michael@0: michael@0: /* michael@0: * The sourceLimit may have been adjusted before the conversion loop michael@0: * to stop before a truncated sequence. michael@0: * If so, then collect the truncated sequence now. michael@0: */ michael@0: if(U_SUCCESS(*pErrorCode) && michael@0: cnv->preFromUFirstCP<0 && michael@0: source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) { michael@0: c=utf8->toUBytes[0]=b=*source++; michael@0: toULength=1; michael@0: toULimit=U8_COUNT_TRAIL_BYTES(b)+1; michael@0: while(sourcetoUBytes[toULength++]=b=*source++; michael@0: c=(c<<6)+b; michael@0: } michael@0: utf8->toUnicodeStatus=c; michael@0: utf8->toULength=toULength; michael@0: utf8->mode=toULimit; michael@0: } michael@0: michael@0: /* write back the updated pointers */ michael@0: pToUArgs->source=(char *)source; michael@0: pFromUArgs->target=(char *)target; michael@0: } michael@0: michael@0: /* miscellaneous ------------------------------------------------------------ */ michael@0: michael@0: static void michael@0: ucnv_MBCSGetStarters(const UConverter* cnv, michael@0: UBool starters[256], michael@0: UErrorCode *pErrorCode) { michael@0: const int32_t *state0; michael@0: int i; michael@0: michael@0: state0=cnv->sharedData->mbcs.stateTable[cnv->sharedData->mbcs.dbcsOnlyState]; michael@0: for(i=0; i<256; ++i) { michael@0: /* all bytes that cause a state transition from state 0 are lead bytes */ michael@0: starters[i]= (UBool)MBCS_ENTRY_IS_TRANSITION(state0[i]); michael@0: } michael@0: } michael@0: michael@0: /* michael@0: * This is an internal function that allows other converter implementations michael@0: * to check whether a byte is a lead byte. michael@0: */ michael@0: U_CFUNC UBool michael@0: ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte) { michael@0: return (UBool)MBCS_ENTRY_IS_TRANSITION(sharedData->mbcs.stateTable[0][(uint8_t)byte]); michael@0: } michael@0: michael@0: static void michael@0: ucnv_MBCSWriteSub(UConverterFromUnicodeArgs *pArgs, michael@0: int32_t offsetIndex, michael@0: UErrorCode *pErrorCode) { michael@0: UConverter *cnv=pArgs->converter; michael@0: char *p, *subchar; michael@0: char buffer[4]; michael@0: int32_t length; michael@0: michael@0: /* first, select between subChar and subChar1 */ michael@0: if( cnv->subChar1!=0 && michael@0: (cnv->sharedData->mbcs.extIndexes!=NULL ? michael@0: cnv->useSubChar1 : michael@0: (cnv->invalidUCharBuffer[0]<=0xff)) michael@0: ) { michael@0: /* select subChar1 if it is set (not 0) and the unmappable Unicode code point is up to U+00ff (IBM MBCS behavior) */ michael@0: subchar=(char *)&cnv->subChar1; michael@0: length=1; michael@0: } else { michael@0: /* select subChar in all other cases */ michael@0: subchar=(char *)cnv->subChars; michael@0: length=cnv->subCharLen; michael@0: } michael@0: michael@0: /* reset the selector for the next code point */ michael@0: cnv->useSubChar1=FALSE; michael@0: michael@0: if (cnv->sharedData->mbcs.outputType == MBCS_OUTPUT_2_SISO) { michael@0: p=buffer; michael@0: michael@0: /* fromUnicodeStatus contains prevLength */ michael@0: switch(length) { michael@0: case 1: michael@0: if(cnv->fromUnicodeStatus==2) { michael@0: /* DBCS mode and SBCS sub char: change to SBCS */ michael@0: cnv->fromUnicodeStatus=1; michael@0: *p++=UCNV_SI; michael@0: } michael@0: *p++=subchar[0]; michael@0: break; michael@0: case 2: michael@0: if(cnv->fromUnicodeStatus<=1) { michael@0: /* SBCS mode and DBCS sub char: change to DBCS */ michael@0: cnv->fromUnicodeStatus=2; michael@0: *p++=UCNV_SO; michael@0: } michael@0: *p++=subchar[0]; michael@0: *p++=subchar[1]; michael@0: break; michael@0: default: michael@0: *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; michael@0: return; michael@0: } michael@0: subchar=buffer; michael@0: length=(int32_t)(p-buffer); michael@0: } michael@0: michael@0: ucnv_cbFromUWriteBytes(pArgs, subchar, length, offsetIndex, pErrorCode); michael@0: } michael@0: michael@0: U_CFUNC UConverterType michael@0: ucnv_MBCSGetType(const UConverter* converter) { michael@0: /* SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but here we cheat a little */ michael@0: if(converter->sharedData->mbcs.countStates==1) { michael@0: return (UConverterType)UCNV_SBCS; michael@0: } else if((converter->sharedData->mbcs.outputType&0xff)==MBCS_OUTPUT_2_SISO) { michael@0: return (UConverterType)UCNV_EBCDIC_STATEFUL; michael@0: } else if(converter->sharedData->staticData->minBytesPerChar==2 && converter->sharedData->staticData->maxBytesPerChar==2) { michael@0: return (UConverterType)UCNV_DBCS; michael@0: } michael@0: return (UConverterType)UCNV_MBCS; michael@0: } michael@0: michael@0: static const UConverterImpl _SBCSUTF8Impl={ michael@0: UCNV_MBCS, michael@0: michael@0: ucnv_MBCSLoad, michael@0: ucnv_MBCSUnload, michael@0: michael@0: ucnv_MBCSOpen, michael@0: NULL, michael@0: NULL, michael@0: michael@0: ucnv_MBCSToUnicodeWithOffsets, michael@0: ucnv_MBCSToUnicodeWithOffsets, michael@0: ucnv_MBCSFromUnicodeWithOffsets, michael@0: ucnv_MBCSFromUnicodeWithOffsets, michael@0: ucnv_MBCSGetNextUChar, michael@0: michael@0: ucnv_MBCSGetStarters, michael@0: ucnv_MBCSGetName, michael@0: ucnv_MBCSWriteSub, michael@0: NULL, michael@0: ucnv_MBCSGetUnicodeSet, michael@0: michael@0: NULL, michael@0: ucnv_SBCSFromUTF8 michael@0: }; michael@0: michael@0: static const UConverterImpl _DBCSUTF8Impl={ michael@0: UCNV_MBCS, michael@0: michael@0: ucnv_MBCSLoad, michael@0: ucnv_MBCSUnload, michael@0: michael@0: ucnv_MBCSOpen, michael@0: NULL, michael@0: NULL, michael@0: michael@0: ucnv_MBCSToUnicodeWithOffsets, michael@0: ucnv_MBCSToUnicodeWithOffsets, michael@0: ucnv_MBCSFromUnicodeWithOffsets, michael@0: ucnv_MBCSFromUnicodeWithOffsets, michael@0: ucnv_MBCSGetNextUChar, michael@0: michael@0: ucnv_MBCSGetStarters, michael@0: ucnv_MBCSGetName, michael@0: ucnv_MBCSWriteSub, michael@0: NULL, michael@0: ucnv_MBCSGetUnicodeSet, michael@0: michael@0: NULL, michael@0: ucnv_DBCSFromUTF8 michael@0: }; michael@0: michael@0: static const UConverterImpl _MBCSImpl={ michael@0: UCNV_MBCS, michael@0: michael@0: ucnv_MBCSLoad, michael@0: ucnv_MBCSUnload, michael@0: michael@0: ucnv_MBCSOpen, michael@0: NULL, michael@0: NULL, michael@0: michael@0: ucnv_MBCSToUnicodeWithOffsets, michael@0: ucnv_MBCSToUnicodeWithOffsets, michael@0: ucnv_MBCSFromUnicodeWithOffsets, michael@0: ucnv_MBCSFromUnicodeWithOffsets, michael@0: ucnv_MBCSGetNextUChar, michael@0: michael@0: ucnv_MBCSGetStarters, michael@0: ucnv_MBCSGetName, michael@0: ucnv_MBCSWriteSub, michael@0: NULL, michael@0: ucnv_MBCSGetUnicodeSet michael@0: }; michael@0: michael@0: michael@0: /* Static data is in tools/makeconv/ucnvstat.c for data-based michael@0: * converters. Be sure to update it as well. michael@0: */ michael@0: michael@0: const UConverterSharedData _MBCSData={ michael@0: sizeof(UConverterSharedData), 1, michael@0: NULL, NULL, NULL, FALSE, &_MBCSImpl, michael@0: 0 michael@0: }; michael@0: michael@0: #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */