1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/common/ucnvmbcs.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,603 @@
1.4 +/*
1.5 +******************************************************************************
1.6 +*
1.7 +* Copyright (C) 2000-2013, International Business Machines
1.8 +* Corporation and others. All Rights Reserved.
1.9 +*
1.10 +******************************************************************************
1.11 +* file name: ucnvmbcs.h
1.12 +* encoding: US-ASCII
1.13 +* tab size: 8 (not used)
1.14 +* indentation:4
1.15 +*
1.16 +* created on: 2000jul07
1.17 +* created by: Markus W. Scherer
1.18 +*/
1.19 +
1.20 +#ifndef __UCNVMBCS_H__
1.21 +#define __UCNVMBCS_H__
1.22 +
1.23 +#include "unicode/utypes.h"
1.24 +
1.25 +#if !UCONFIG_NO_CONVERSION
1.26 +
1.27 +#include "unicode/ucnv.h"
1.28 +#include "ucnv_cnv.h"
1.29 +#include "ucnv_ext.h"
1.30 +
1.31 +/**
1.32 + * ICU conversion (.cnv) data file structure, following the usual UDataInfo
1.33 + * header.
1.34 + *
1.35 + * Format version: 6.2
1.36 + *
1.37 + * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
1.38 + * min/max bytes per character, etc.
1.39 + * see ucnv_bld.h
1.40 + *
1.41 + * --------------------
1.42 + *
1.43 + * The static data is followed by conversionType-specific data structures.
1.44 + * At the moment, there are only variations of MBCS converters. They all have
1.45 + * the same toUnicode structures, while the fromUnicode structures for SBCS
1.46 + * differ from those for other MBCS-style converters.
1.47 + *
1.48 + * _MBCSHeader.version 5 is optional and not backward-compatible
1.49 + * (as usual for changes in the major version field).
1.50 + *
1.51 + * Versions 5.m work like versions 4.m except:
1.52 + * - The _MBCSHeader has variable length (and is always longer than in version 4).
1.53 + * See the struct _MBCSHeader further description below.
1.54 + * - There is a set of flags which indicate further incompatible changes.
1.55 + * (Reader code must reject the file if it does not recognize them all.)
1.56 + * - In particular, one of these flags indicates that most of the fromUnicode
1.57 + * data is missing and must be reconstituted from the toUnicode data
1.58 + * and from the utf8Friendly mbcsIndex at load time.
1.59 + * (This only works with a utf8Friendly table.)
1.60 + * In this case, makeconv may increase maxFastUChar automatically to U+FFFF.
1.61 + *
1.62 + * The first of these versions is 5.3, which is like 4.3 except for the differences above.
1.63 + *
1.64 + * When possible, makeconv continues to generate version 4.m files.
1.65 + *
1.66 + * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4)
1.67 + * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details.
1.68 + *
1.69 + * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures
1.70 + * slightly and optionally adds a table for conversion to MBCS (non-SBCS)
1.71 + * charsets.
1.72 + *
1.73 + * The modifications are to make the data utf8Friendly. Not every 4.3 file
1.74 + * file contains utf8Friendly data.
1.75 + * It is utf8Friendly if _MBCSHeader.version[2]!=0.
1.76 + * In this case, the data structures are utf8Friendly up to the code point
1.77 + * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff)
1.78 + *
1.79 + * A utf8Friendly file has fromUnicode stage 3 entries for code points up to
1.80 + * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from
1.81 + * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries.
1.82 + *
1.83 + * In addition, a utf8Friendly MBCS file contains an additional
1.84 + * uint16_t mbcsIndex[(maxFastUChar+1)>>6];
1.85 + * which replaces the stage 1 and 2 tables for indexing with bits from the
1.86 + * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table,
1.87 + * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks
1.88 + * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to
1.89 + * the extension data structure. This also allows for faster roundtrip
1.90 + * conversion from UTF-16.
1.91 + *
1.92 + * SBCS files do not contain an additional sbcsIndex[] array because the
1.93 + * proportional size increase would be noticeable, but the runtime
1.94 + * code builds one for the code point range for which the runtime conversion
1.95 + * code is optimized.
1.96 + *
1.97 + * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv
1.98 + * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in
1.99 + * the file, a larger maxFastUChar only affects stage 3 block allocation size
1.100 + * and is free in empty blocks. (Larger blocks with sparse contents cause larger
1.101 + * files.) U+1FFF includes almost all of the small scripts.
1.102 + * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with
1.103 + * 0xe0. This includes most scripts with legacy SBCS charsets.
1.104 + * The initial runtime implementation using 4.3 files only builds an sbcsIndex
1.105 + * for code points up to U+0FFF.
1.106 + *
1.107 + * For MBCS, maxFastUChar should be at least U+D7FF (=initial value).
1.108 + * This boundary is convenient because practically all of the commonly used
1.109 + * characters are below it, and because it is the boundary to surrogate
1.110 + * code points, above which special handling is necessary anyway.
1.111 + * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.)
1.112 + *
1.113 + * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be
1.114 + * useful especially for conversion from UTF-8 when the input can be assumed
1.115 + * to be valid, because the surrogate range would then not have to be
1.116 + * checked.
1.117 + * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value
1.118 + * overflow because with the all-unassigned block 0 and nearly full mappings
1.119 + * from the BMP it is theoretically possible that an index into stage 3
1.120 + * exceeds 16 bits.)
1.121 + *
1.122 + * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
1.123 + * If it is present, then an ICU version reading header versions 4.0 or 4.1
1.124 + * will be able to use the base table and ignore the extension.
1.125 + *
1.126 + * The unicodeMask in the static data is part of the base table data structure.
1.127 + * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
1.128 + * fromUnicode stage 1 array.
1.129 + * The static data unicodeMask refers only to the base table's properties if
1.130 + * a base table is included.
1.131 + * In an extension-only file, the static data unicodeMask is 0.
1.132 + * The extension data indexes have a separate field with the unicodeMask flags.
1.133 + *
1.134 + * MBCS-style data structure following the static data.
1.135 + * Offsets are counted in bytes from the beginning of the MBCS header structure.
1.136 + * Details about usage in comments in ucnvmbcs.c.
1.137 + *
1.138 + * struct _MBCSHeader (see the definition in this header file below)
1.139 + * contains 32-bit fields as follows:
1.140 + * 8 values:
1.141 + * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0)
1.142 + * 1 uint32_t countStates
1.143 + * 2 uint32_t countToUFallbacks
1.144 + * 3 uint32_t offsetToUCodeUnits
1.145 + * 4 uint32_t offsetFromUTable
1.146 + * 5 uint32_t offsetFromUBytes
1.147 + * 6 uint32_t flags, bits:
1.148 + * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
1.149 + * 0 for older versions and if
1.150 + * there is not extension structure
1.151 + * 7.. 0 outputType
1.152 + * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
1.153 + * counts bytes in fromUBytes[]
1.154 + *
1.155 + * New and required in version 5:
1.156 + * 8 uint32_t options, bits:
1.157 + * 31..16 reserved for flags that can be added without breaking
1.158 + * backward compatibility
1.159 + * 15.. 6 reserved for flags whose addition will break
1.160 + * backward compatibility
1.161 + * 6 MBCS_OPT_FROM_U -- if set,
1.162 + * then most of the fromUnicode data is omitted;
1.163 + * fullStage2Length is present and the missing
1.164 + * bottom part of stage 2 must be reconstituted from
1.165 + * the toUnicode data;
1.166 + * stage 3 is missing completely as well;
1.167 + * not used for SBCS tables
1.168 + * 5.. 0 length of the _MBCSHeader (number of uint32_t)
1.169 + *
1.170 + * New and optional in version 5:
1.171 + * 9 uint32_t fullStage2Length: used if MBCS_OPT_FROM_U is set
1.172 + * specifies the full length of stage 2
1.173 + * including the omitted part
1.174 + *
1.175 + * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
1.176 + * -- base table name for extension-only table
1.177 + * char baseTableName[variable]; -- with NUL plus padding for 4-alignment
1.178 + *
1.179 + * -- all _MBCSHeader fields except for version and flags are 0
1.180 + * } else {
1.181 + * -- normal base table with optional extension
1.182 + *
1.183 + * int32_t stateTable[countStates][256];
1.184 + *
1.185 + * struct _MBCSToUFallback { (fallbacks are sorted by offset)
1.186 + * uint32_t offset;
1.187 + * UChar32 codePoint;
1.188 + * } toUFallbacks[countToUFallbacks];
1.189 + *
1.190 + * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
1.191 + * (padded to an even number of units)
1.192 + *
1.193 + * -- stage 1 tables
1.194 + * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
1.195 + * -- stage 1 table for all of Unicode
1.196 + * uint16_t fromUTable[0x440]; (32-bit-aligned)
1.197 + * } else {
1.198 + * -- BMP-only tables have a smaller stage 1 table
1.199 + * uint16_t fromUTable[0x40]; (32-bit-aligned)
1.200 + * }
1.201 + *
1.202 + * -- stage 2 tables
1.203 + * length determined by top of stage 1 and bottom of stage 3 tables
1.204 + * if(outputType==MBCS_OUTPUT_1) {
1.205 + * -- SBCS: pure indexes
1.206 + * uint16_t stage 2 indexes[?];
1.207 + * } else {
1.208 + * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
1.209 + * uint32_t stage 2 flags and indexes[?];
1.210 + * if(options&MBCS_OPT_NO_FROM_U) {
1.211 + * stage 2 really has length fullStage2Length
1.212 + * and the omitted lower part must be reconstituted from
1.213 + * the toUnicode data
1.214 + * }
1.215 + * }
1.216 + *
1.217 + * -- stage 3 tables with byte results
1.218 + * if(outputType==MBCS_OUTPUT_1) {
1.219 + * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
1.220 + * uint16_t fromUBytes[fromUBytesLength/2];
1.221 + * } else if(!(options&MBCS_OPT_NO_FROM_U)) {
1.222 + * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
1.223 + * uint8_t fromUBytes[fromUBytesLength]; or
1.224 + * uint16_t fromUBytes[fromUBytesLength/2]; or
1.225 + * uint32_t fromUBytes[fromUBytesLength/4];
1.226 + * } else {
1.227 + * fromUBytes[] must be reconstituted from the toUnicode data
1.228 + * }
1.229 + *
1.230 + * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher
1.231 + * if(outputType!=MBCS_OUTPUT_1 &&
1.232 + * _MBCSHeader.version[1]>=3 &&
1.233 + * (maxFastUChar=_MBCSHeader.version[2])!=0
1.234 + * ) {
1.235 + * maxFastUChar=(maxFastUChar<<8)|0xff;
1.236 + * uint16_t mbcsIndex[(maxFastUChar+1)>>6];
1.237 + * }
1.238 + * }
1.239 + *
1.240 + * -- extension table, details see ucnv_ext.h
1.241 + * int32_t indexes[>=32]; ...
1.242 + */
1.243 +
1.244 +/* MBCS converter data and state -------------------------------------------- */
1.245 +
1.246 +enum {
1.247 + MBCS_MAX_STATE_COUNT=128
1.248 +};
1.249 +
1.250 +/**
1.251 + * MBCS action codes for conversions to Unicode.
1.252 + * These values are in bits 23..20 of the state table entries.
1.253 + */
1.254 +enum {
1.255 + MBCS_STATE_VALID_DIRECT_16,
1.256 + MBCS_STATE_VALID_DIRECT_20,
1.257 +
1.258 + MBCS_STATE_FALLBACK_DIRECT_16,
1.259 + MBCS_STATE_FALLBACK_DIRECT_20,
1.260 +
1.261 + MBCS_STATE_VALID_16,
1.262 + MBCS_STATE_VALID_16_PAIR,
1.263 +
1.264 + MBCS_STATE_UNASSIGNED,
1.265 + MBCS_STATE_ILLEGAL,
1.266 +
1.267 + MBCS_STATE_CHANGE_ONLY
1.268 +};
1.269 +
1.270 +/* Macros for state table entries */
1.271 +#define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset))
1.272 +#define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset))
1.273 +#define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))
1.274 +
1.275 +#define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value))
1.276 +#define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000)
1.277 +#define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L))
1.278 +#define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value))
1.279 +#define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value))
1.280 +
1.281 +#define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L))
1.282 +
1.283 +#define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
1.284 +
1.285 +#define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
1.286 +#define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)
1.287 +
1.288 +#define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24)
1.289 +#define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)
1.290 +
1.291 +#define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
1.292 +#define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
1.293 +#define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf)
1.294 +#define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
1.295 +#define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)
1.296 +
1.297 +#define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0)
1.298 +
1.299 +/* single-byte fromUnicode: get the 16-bit result word */
1.300 +#define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]
1.301 +
1.302 +/* single-byte fromUnicode using the sbcsIndex */
1.303 +#define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
1.304 +
1.305 +/* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */
1.306 +#define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ]
1.307 +
1.308 +/* multi-byte fromUnicode: get the 32-bit stage 2 entry */
1.309 +#define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
1.310 +#define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)
1.311 +
1.312 +#define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
1.313 +#define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
1.314 +
1.315 +#define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3)
1.316 +
1.317 +/* double-byte fromUnicode using the mbcsIndex */
1.318 +#define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
1.319 +
1.320 +/* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */
1.321 +#define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ]
1.322 +
1.323 +
1.324 +/**
1.325 + * MBCS output types for conversions from Unicode.
1.326 + * These per-converter types determine the storage method in stage 3 of the lookup table,
1.327 + * mostly how many bytes are stored per entry.
1.328 + */
1.329 +enum {
1.330 + MBCS_OUTPUT_1, /* 0 */
1.331 + MBCS_OUTPUT_2, /* 1 */
1.332 + MBCS_OUTPUT_3, /* 2 */
1.333 + MBCS_OUTPUT_4, /* 3 */
1.334 +
1.335 + MBCS_OUTPUT_3_EUC=8, /* 8 */
1.336 + MBCS_OUTPUT_4_EUC, /* 9 */
1.337 +
1.338 + MBCS_OUTPUT_2_SISO=12, /* c */
1.339 + MBCS_OUTPUT_2_HZ, /* d */
1.340 +
1.341 + MBCS_OUTPUT_EXT_ONLY, /* e */
1.342 +
1.343 + MBCS_OUTPUT_COUNT,
1.344 +
1.345 + MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */
1.346 +};
1.347 +
1.348 +/**
1.349 + * Fallbacks to Unicode are stored outside the normal state table and code point structures
1.350 + * in a vector of items of this type. They are sorted by offset.
1.351 + */
1.352 +typedef struct {
1.353 + uint32_t offset;
1.354 + UChar32 codePoint;
1.355 +} _MBCSToUFallback;
1.356 +
1.357 +/** Constants for fast and UTF-8-friendly conversion. */
1.358 +enum {
1.359 + SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */
1.360 + SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */
1.361 + MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */
1.362 + MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */
1.363 +};
1.364 +
1.365 +/**
1.366 + * This is the MBCS part of the UConverterTable union (a runtime data structure).
1.367 + * It keeps all the per-converter data and points into the loaded mapping tables.
1.368 + *
1.369 + * utf8Friendly data structures added with _MBCSHeader.version 4.3
1.370 + */
1.371 +typedef struct UConverterMBCSTable {
1.372 + /* toUnicode */
1.373 + uint8_t countStates, dbcsOnlyState, stateTableOwned;
1.374 + uint32_t countToUFallbacks;
1.375 +
1.376 + const int32_t (*stateTable)/*[countStates]*/[256];
1.377 + int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */
1.378 + const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/;
1.379 + const _MBCSToUFallback *toUFallbacks;
1.380 +
1.381 + /* fromUnicode */
1.382 + const uint16_t *fromUnicodeTable;
1.383 + const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */
1.384 + uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */
1.385 + const uint8_t *fromUnicodeBytes;
1.386 + uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */
1.387 + uint32_t fromUBytesLength;
1.388 + uint8_t outputType, unicodeMask;
1.389 + UBool utf8Friendly; /* for utf8Friendly data */
1.390 + UChar maxFastUChar; /* for utf8Friendly data */
1.391 +
1.392 + /* roundtrips */
1.393 + uint32_t asciiRoundtrips;
1.394 +
1.395 + /* reconstituted data that was omitted from the .cnv file */
1.396 + uint8_t *reconstitutedData;
1.397 +
1.398 + /* converter name for swaplfnl */
1.399 + char *swapLFNLName;
1.400 +
1.401 + /* extension data */
1.402 + struct UConverterSharedData *baseSharedData;
1.403 + const int32_t *extIndexes;
1.404 +} UConverterMBCSTable;
1.405 +
1.406 +#define UCNV_MBCS_TABLE_INITIALIZER { \
1.407 + /* toUnicode */ \
1.408 + 0, 0, 0, \
1.409 + 0, \
1.410 + \
1.411 + NULL, \
1.412 + NULL, \
1.413 + NULL, \
1.414 + NULL, \
1.415 + \
1.416 + /* fromUnicode */ \
1.417 + NULL, \
1.418 + NULL, \
1.419 + { 0 }, \
1.420 + NULL, \
1.421 + NULL, \
1.422 + 0, \
1.423 + 0, 0, \
1.424 + FALSE, \
1.425 + 0, \
1.426 + \
1.427 + /* roundtrips */ \
1.428 + 0, \
1.429 + \
1.430 + /* reconstituted data that was omitted from the .cnv file */ \
1.431 + NULL, \
1.432 + \
1.433 + /* converter name for swaplfnl */ \
1.434 + NULL, \
1.435 + \
1.436 + /* extension data */ \
1.437 + NULL, \
1.438 + NULL \
1.439 +}
1.440 +
1.441 +enum {
1.442 + MBCS_OPT_LENGTH_MASK=0x3f,
1.443 + MBCS_OPT_NO_FROM_U=0x40,
1.444 + /*
1.445 + * If any of the following options bits are set,
1.446 + * then the file must be rejected.
1.447 + */
1.448 + MBCS_OPT_INCOMPATIBLE_MASK=0xffc0,
1.449 + /*
1.450 + * Remove bits from this mask as more options are recognized
1.451 + * by all implementations that use this constant.
1.452 + */
1.453 + MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80
1.454 +};
1.455 +
1.456 +enum {
1.457 + MBCS_HEADER_V4_LENGTH=8,
1.458 + MBCS_HEADER_V5_MIN_LENGTH=9
1.459 +};
1.460 +
1.461 +/**
1.462 + * MBCS data header. See data format description above.
1.463 + */
1.464 +typedef struct {
1.465 + UVersionInfo version;
1.466 + uint32_t countStates,
1.467 + countToUFallbacks,
1.468 + offsetToUCodeUnits,
1.469 + offsetFromUTable,
1.470 + offsetFromUBytes,
1.471 + flags,
1.472 + fromUBytesLength;
1.473 +
1.474 + /* new and required in version 5 */
1.475 + uint32_t options;
1.476 +
1.477 + /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */
1.478 + uint32_t fullStage2Length; /* number of 32-bit units */
1.479 +} _MBCSHeader;
1.480 +
1.481 +#define UCNV_MBCS_HEADER_INITIALIZER { { 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
1.482 +
1.483 +/*
1.484 + * This is a simple version of _MBCSGetNextUChar() that is used
1.485 + * by other converter implementations.
1.486 + * It only returns an "assigned" result if it consumes the entire input.
1.487 + * It does not use state from the converter, nor error codes.
1.488 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.489 + * It handles conversion extensions but not GB 18030.
1.490 + *
1.491 + * Return value:
1.492 + * U+fffe unassigned
1.493 + * U+ffff illegal
1.494 + * otherwise the Unicode code point
1.495 + */
1.496 +U_CFUNC UChar32
1.497 +ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
1.498 + const char *source, int32_t length,
1.499 + UBool useFallback);
1.500 +
1.501 +/**
1.502 + * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
1.503 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.504 + * It does not handle conversion extensions (_extToU()).
1.505 + */
1.506 +U_CFUNC UChar32
1.507 +ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
1.508 + uint8_t b, UBool useFallback);
1.509 +
1.510 +/**
1.511 + * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
1.512 + * It works for single-byte, single-state codepages that only map
1.513 + * to and from BMP code points, and it always
1.514 + * returns fallback values.
1.515 + */
1.516 +#define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
1.517 + (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])
1.518 +
1.519 +/**
1.520 + * This is an internal function that allows other converter implementations
1.521 + * to check whether a byte is a lead byte.
1.522 + */
1.523 +U_CFUNC UBool
1.524 +ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);
1.525 +
1.526 +/** This is a macro version of _MBCSIsLeadByte(). */
1.527 +#define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
1.528 + (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])
1.529 +
1.530 +/*
1.531 + * This is another simple conversion function for internal use by other
1.532 + * conversion implementations.
1.533 + * It does not use the converter state nor call callbacks.
1.534 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.535 + * It handles conversion extensions but not GB 18030.
1.536 + *
1.537 + * It converts one single Unicode code point into codepage bytes, encoded
1.538 + * as one 32-bit value. The function returns the number of bytes in *pValue:
1.539 + * 1..4 the number of bytes in *pValue
1.540 + * 0 unassigned (*pValue undefined)
1.541 + * -1 illegal (currently not used, *pValue undefined)
1.542 + *
1.543 + * *pValue will contain the resulting bytes with the last byte in bits 7..0,
1.544 + * the second to last byte in bits 15..8, etc.
1.545 + * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
1.546 + */
1.547 +U_CFUNC int32_t
1.548 +ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
1.549 + UChar32 c, uint32_t *pValue,
1.550 + UBool useFallback);
1.551 +
1.552 +/**
1.553 + * This version of _MBCSFromUChar32() is optimized for single-byte codepages.
1.554 + * It does not handle the EBCDIC swaplfnl option (set in UConverter).
1.555 + *
1.556 + * It returns the codepage byte for the code point, or -1 if it is unassigned.
1.557 + */
1.558 +U_CFUNC int32_t
1.559 +ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
1.560 + UChar32 c,
1.561 + UBool useFallback);
1.562 +
1.563 +/**
1.564 + * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
1.565 + * we cheat a little about the type, returning the old types if appropriate.
1.566 + */
1.567 +U_CFUNC UConverterType
1.568 +ucnv_MBCSGetType(const UConverter* converter);
1.569 +
1.570 +U_CFUNC void
1.571 +ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
1.572 + UErrorCode *pErrorCode);
1.573 +U_CFUNC void
1.574 +ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
1.575 + UErrorCode *pErrorCode);
1.576 +
1.577 +/*
1.578 + * Internal function returning a UnicodeSet for toUnicode() conversion.
1.579 + * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
1.580 + * In the future, if we add support for fallback sets, this function
1.581 + * needs to be updated.
1.582 + * Handles extensions.
1.583 + * Does not empty the set first.
1.584 + */
1.585 +U_CFUNC void
1.586 +ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
1.587 + const USetAdder *sa,
1.588 + UConverterUnicodeSet which,
1.589 + UErrorCode *pErrorCode);
1.590 +
1.591 +/*
1.592 + * Same as ucnv_MBCSGetUnicodeSetForUnicode() but
1.593 + * the set can be filtered by encoding scheme.
1.594 + * Used by stateful converters which share regular conversion tables
1.595 + * but only use a subset of their mappings.
1.596 + */
1.597 +U_CFUNC void
1.598 +ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData,
1.599 + const USetAdder *sa,
1.600 + UConverterUnicodeSet which,
1.601 + UConverterSetFilter filter,
1.602 + UErrorCode *pErrorCode);
1.603 +
1.604 +#endif
1.605 +
1.606 +#endif
