The Tor Browser / file comparison

comparison: intl/icu/source/common/ucnvmbcs.h

intl/icu/source/common/ucnvmbcs.h

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:95a45e74cbe9
1 /*
2 ******************************************************************************
3 *
4 * Copyright (C) 2000-2013, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 ******************************************************************************
8 * file name: ucnvmbcs.h
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2000jul07
14 * created by: Markus W. Scherer
15 */
16
17 #ifndef __UCNVMBCS_H__
18 #define __UCNVMBCS_H__
19
20 #include "unicode/utypes.h"
21
22 #if !UCONFIG_NO_CONVERSION
23
24 #include "unicode/ucnv.h"
25 #include "ucnv_cnv.h"
26 #include "ucnv_ext.h"
27
28 /**
29 * ICU conversion (.cnv) data file structure, following the usual UDataInfo
30 * header.
31 *
32 * Format version: 6.2
33 *
34 * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
35 * min/max bytes per character, etc.
36 * see ucnv_bld.h
37 *
38 * --------------------
39 *
40 * The static data is followed by conversionType-specific data structures.
41 * At the moment, there are only variations of MBCS converters. They all have
42 * the same toUnicode structures, while the fromUnicode structures for SBCS
43 * differ from those for other MBCS-style converters.
44 *
45 * _MBCSHeader.version 5 is optional and not backward-compatible
46 * (as usual for changes in the major version field).
47 *
48 * Versions 5.m work like versions 4.m except:
49 * - The _MBCSHeader has variable length (and is always longer than in version 4).
50 * See the struct _MBCSHeader further description below.
51 * - There is a set of flags which indicate further incompatible changes.
52 * (Reader code must reject the file if it does not recognize them all.)
53 * - In particular, one of these flags indicates that most of the fromUnicode
54 * data is missing and must be reconstituted from the toUnicode data
55 * and from the utf8Friendly mbcsIndex at load time.
56 * (This only works with a utf8Friendly table.)
57 * In this case, makeconv may increase maxFastUChar automatically to U+FFFF.
58 *
59 * The first of these versions is 5.3, which is like 4.3 except for the differences above.
60 *
61 * When possible, makeconv continues to generate version 4.m files.
62 *
63 * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4)
64 * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details.
65 *
66 * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures
67 * slightly and optionally adds a table for conversion to MBCS (non-SBCS)
68 * charsets.
69 *
70 * The modifications are to make the data utf8Friendly. Not every 4.3 file
71 * file contains utf8Friendly data.
72 * It is utf8Friendly if _MBCSHeader.version[2]!=0.
73 * In this case, the data structures are utf8Friendly up to the code point
74 * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff)
75 *
76 * A utf8Friendly file has fromUnicode stage 3 entries for code points up to
77 * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from
78 * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries.
79 *
80 * In addition, a utf8Friendly MBCS file contains an additional
81 * uint16_t mbcsIndex[(maxFastUChar+1)>>6];
82 * which replaces the stage 1 and 2 tables for indexing with bits from the
83 * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table,
84 * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks
85 * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to
86 * the extension data structure. This also allows for faster roundtrip
87 * conversion from UTF-16.
88 *
89 * SBCS files do not contain an additional sbcsIndex[] array because the
90 * proportional size increase would be noticeable, but the runtime
91 * code builds one for the code point range for which the runtime conversion
92 * code is optimized.
93 *
94 * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv
95 * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in
96 * the file, a larger maxFastUChar only affects stage 3 block allocation size
97 * and is free in empty blocks. (Larger blocks with sparse contents cause larger
98 * files.) U+1FFF includes almost all of the small scripts.
99 * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with
100 * 0xe0. This includes most scripts with legacy SBCS charsets.
101 * The initial runtime implementation using 4.3 files only builds an sbcsIndex
102 * for code points up to U+0FFF.
103 *
104 * For MBCS, maxFastUChar should be at least U+D7FF (=initial value).
105 * This boundary is convenient because practically all of the commonly used
106 * characters are below it, and because it is the boundary to surrogate
107 * code points, above which special handling is necessary anyway.
108 * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.)
109 *
110 * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be
111 * useful especially for conversion from UTF-8 when the input can be assumed
112 * to be valid, because the surrogate range would then not have to be
113 * checked.
114 * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value
115 * overflow because with the all-unassigned block 0 and nearly full mappings
116 * from the BMP it is theoretically possible that an index into stage 3
117 * exceeds 16 bits.)
118 *
119 * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
120 * If it is present, then an ICU version reading header versions 4.0 or 4.1
121 * will be able to use the base table and ignore the extension.
122 *
123 * The unicodeMask in the static data is part of the base table data structure.
124 * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
125 * fromUnicode stage 1 array.
126 * The static data unicodeMask refers only to the base table's properties if
127 * a base table is included.
128 * In an extension-only file, the static data unicodeMask is 0.
129 * The extension data indexes have a separate field with the unicodeMask flags.
130 *
131 * MBCS-style data structure following the static data.
132 * Offsets are counted in bytes from the beginning of the MBCS header structure.
133 * Details about usage in comments in ucnvmbcs.c.
134 *
135 * struct _MBCSHeader (see the definition in this header file below)
136 * contains 32-bit fields as follows:
137 * 8 values:
138 * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0)
139 * 1 uint32_t countStates
140 * 2 uint32_t countToUFallbacks
141 * 3 uint32_t offsetToUCodeUnits
142 * 4 uint32_t offsetFromUTable
143 * 5 uint32_t offsetFromUBytes
144 * 6 uint32_t flags, bits:
145 * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
146 * 0 for older versions and if
147 * there is not extension structure
148 * 7.. 0 outputType
149 * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
150 * counts bytes in fromUBytes[]
151 *
152 * New and required in version 5:
153 * 8 uint32_t options, bits:
154 * 31..16 reserved for flags that can be added without breaking
155 * backward compatibility
156 * 15.. 6 reserved for flags whose addition will break
157 * backward compatibility
158 * 6 MBCS_OPT_FROM_U -- if set,
159 * then most of the fromUnicode data is omitted;
160 * fullStage2Length is present and the missing
161 * bottom part of stage 2 must be reconstituted from
162 * the toUnicode data;
163 * stage 3 is missing completely as well;
164 * not used for SBCS tables
165 * 5.. 0 length of the _MBCSHeader (number of uint32_t)
166 *
167 * New and optional in version 5:
168 * 9 uint32_t fullStage2Length: used if MBCS_OPT_FROM_U is set
169 * specifies the full length of stage 2
170 * including the omitted part
171 *
172 * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
173 * -- base table name for extension-only table
174 * char baseTableName[variable]; -- with NUL plus padding for 4-alignment
175 *
176 * -- all _MBCSHeader fields except for version and flags are 0
177 * } else {
178 * -- normal base table with optional extension
179 *
180 * int32_t stateTable[countStates][256];
181 *
182 * struct _MBCSToUFallback { (fallbacks are sorted by offset)
183 * uint32_t offset;
184 * UChar32 codePoint;
185 * } toUFallbacks[countToUFallbacks];
186 *
187 * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
188 * (padded to an even number of units)
189 *
190 * -- stage 1 tables
191 * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
192 * -- stage 1 table for all of Unicode
193 * uint16_t fromUTable[0x440]; (32-bit-aligned)
194 * } else {
195 * -- BMP-only tables have a smaller stage 1 table
196 * uint16_t fromUTable[0x40]; (32-bit-aligned)
197 * }
198 *
199 * -- stage 2 tables
200 * length determined by top of stage 1 and bottom of stage 3 tables
201 * if(outputType==MBCS_OUTPUT_1) {
202 * -- SBCS: pure indexes
203 * uint16_t stage 2 indexes[?];
204 * } else {
205 * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
206 * uint32_t stage 2 flags and indexes[?];
207 * if(options&MBCS_OPT_NO_FROM_U) {
208 * stage 2 really has length fullStage2Length
209 * and the omitted lower part must be reconstituted from
210 * the toUnicode data
211 * }
212 * }
213 *
214 * -- stage 3 tables with byte results
215 * if(outputType==MBCS_OUTPUT_1) {
216 * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
217 * uint16_t fromUBytes[fromUBytesLength/2];
218 * } else if(!(options&MBCS_OPT_NO_FROM_U)) {
219 * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
220 * uint8_t fromUBytes[fromUBytesLength]; or
221 * uint16_t fromUBytes[fromUBytesLength/2]; or
222 * uint32_t fromUBytes[fromUBytesLength/4];
223 * } else {
224 * fromUBytes[] must be reconstituted from the toUnicode data
225 * }
226 *
227 * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher
228 * if(outputType!=MBCS_OUTPUT_1 &&
229 * _MBCSHeader.version[1]>=3 &&
230 * (maxFastUChar=_MBCSHeader.version[2])!=0
231 * ) {
232 * maxFastUChar=(maxFastUChar<<8)|0xff;
233 * uint16_t mbcsIndex[(maxFastUChar+1)>>6];
234 * }
235 * }
236 *
237 * -- extension table, details see ucnv_ext.h
238 * int32_t indexes[>=32]; ...
239 */
240
241 /* MBCS converter data and state -------------------------------------------- */
242
243 enum {
244 MBCS_MAX_STATE_COUNT=128
245 };
246
247 /**
248 * MBCS action codes for conversions to Unicode.
249 * These values are in bits 23..20 of the state table entries.
250 */
251 enum {
252 MBCS_STATE_VALID_DIRECT_16,
253 MBCS_STATE_VALID_DIRECT_20,
254
255 MBCS_STATE_FALLBACK_DIRECT_16,
256 MBCS_STATE_FALLBACK_DIRECT_20,
257
258 MBCS_STATE_VALID_16,
259 MBCS_STATE_VALID_16_PAIR,
260
261 MBCS_STATE_UNASSIGNED,
262 MBCS_STATE_ILLEGAL,
263
264 MBCS_STATE_CHANGE_ONLY
265 };
266
267 /* Macros for state table entries */
268 #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset))
269 #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset))
270 #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))
271
272 #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value))
273 #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000)
274 #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L))
275 #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value))
276 #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value))
277
278 #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L))
279
280 #define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
281
282 #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
283 #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)
284
285 #define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24)
286 #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)
287
288 #define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
289 #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
290 #define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf)
291 #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
292 #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)
293
294 #define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0)
295
296 /* single-byte fromUnicode: get the 16-bit result word */
297 #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]
298
299 /* single-byte fromUnicode using the sbcsIndex */
300 #define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
301
302 /* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */
303 #define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ]
304
305 /* multi-byte fromUnicode: get the 32-bit stage 2 entry */
306 #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
307 #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)
308
309 #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
310 #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
311
312 #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3)
313
314 /* double-byte fromUnicode using the mbcsIndex */
315 #define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
316
317 /* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */
318 #define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ]
319
320
321 /**
322 * MBCS output types for conversions from Unicode.
323 * These per-converter types determine the storage method in stage 3 of the lookup table,
324 * mostly how many bytes are stored per entry.
325 */
326 enum {
327 MBCS_OUTPUT_1, /* 0 */
328 MBCS_OUTPUT_2, /* 1 */
329 MBCS_OUTPUT_3, /* 2 */
330 MBCS_OUTPUT_4, /* 3 */
331
332 MBCS_OUTPUT_3_EUC=8, /* 8 */
333 MBCS_OUTPUT_4_EUC, /* 9 */
334
335 MBCS_OUTPUT_2_SISO=12, /* c */
336 MBCS_OUTPUT_2_HZ, /* d */
337
338 MBCS_OUTPUT_EXT_ONLY, /* e */
339
340 MBCS_OUTPUT_COUNT,
341
342 MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */
343 };
344
345 /**
346 * Fallbacks to Unicode are stored outside the normal state table and code point structures
347 * in a vector of items of this type. They are sorted by offset.
348 */
349 typedef struct {
350 uint32_t offset;
351 UChar32 codePoint;
352 } _MBCSToUFallback;
353
354 /** Constants for fast and UTF-8-friendly conversion. */
355 enum {
356 SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */
357 SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */
358 MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */
359 MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */
360 };
361
362 /**
363 * This is the MBCS part of the UConverterTable union (a runtime data structure).
364 * It keeps all the per-converter data and points into the loaded mapping tables.
365 *
366 * utf8Friendly data structures added with _MBCSHeader.version 4.3
367 */
368 typedef struct UConverterMBCSTable {
369 /* toUnicode */
370 uint8_t countStates, dbcsOnlyState, stateTableOwned;
371 uint32_t countToUFallbacks;
372
373 const int32_t (*stateTable)/*[countStates]*/[256];
374 int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */
375 const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/;
376 const _MBCSToUFallback *toUFallbacks;
377
378 /* fromUnicode */
379 const uint16_t *fromUnicodeTable;
380 const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */
381 uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */
382 const uint8_t *fromUnicodeBytes;
383 uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */
384 uint32_t fromUBytesLength;
385 uint8_t outputType, unicodeMask;
386 UBool utf8Friendly; /* for utf8Friendly data */
387 UChar maxFastUChar; /* for utf8Friendly data */
388
389 /* roundtrips */
390 uint32_t asciiRoundtrips;
391
392 /* reconstituted data that was omitted from the .cnv file */
393 uint8_t *reconstitutedData;
394
395 /* converter name for swaplfnl */
396 char *swapLFNLName;
397
398 /* extension data */
399 struct UConverterSharedData *baseSharedData;
400 const int32_t *extIndexes;
401 } UConverterMBCSTable;
402
403 #define UCNV_MBCS_TABLE_INITIALIZER { \
404 /* toUnicode */ \
405 0, 0, 0, \
406 0, \
407 \
408 NULL, \
409 NULL, \
410 NULL, \
411 NULL, \
412 \
413 /* fromUnicode */ \
414 NULL, \
415 NULL, \
416 { 0 }, \
417 NULL, \
418 NULL, \
419 0, \
420 0, 0, \
421 FALSE, \
422 0, \
423 \
424 /* roundtrips */ \
425 0, \
426 \
427 /* reconstituted data that was omitted from the .cnv file */ \
428 NULL, \
429 \
430 /* converter name for swaplfnl */ \
431 NULL, \
432 \
433 /* extension data */ \
434 NULL, \
435 NULL \
436 }
437
438 enum {
439 MBCS_OPT_LENGTH_MASK=0x3f,
440 MBCS_OPT_NO_FROM_U=0x40,
441 /*
442 * If any of the following options bits are set,
443 * then the file must be rejected.
444 */
445 MBCS_OPT_INCOMPATIBLE_MASK=0xffc0,
446 /*
447 * Remove bits from this mask as more options are recognized
448 * by all implementations that use this constant.
449 */
450 MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80
451 };
452
453 enum {
454 MBCS_HEADER_V4_LENGTH=8,
455 MBCS_HEADER_V5_MIN_LENGTH=9
456 };
457
458 /**
459 * MBCS data header. See data format description above.
460 */
461 typedef struct {
462 UVersionInfo version;
463 uint32_t countStates,
464 countToUFallbacks,
465 offsetToUCodeUnits,
466 offsetFromUTable,
467 offsetFromUBytes,
468 flags,
469 fromUBytesLength;
470
471 /* new and required in version 5 */
472 uint32_t options;
473
474 /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */
475 uint32_t fullStage2Length; /* number of 32-bit units */
476 } _MBCSHeader;
477
478 #define UCNV_MBCS_HEADER_INITIALIZER { { 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
479
480 /*
481 * This is a simple version of _MBCSGetNextUChar() that is used
482 * by other converter implementations.
483 * It only returns an "assigned" result if it consumes the entire input.
484 * It does not use state from the converter, nor error codes.
485 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
486 * It handles conversion extensions but not GB 18030.
487 *
488 * Return value:
489 * U+fffe unassigned
490 * U+ffff illegal
491 * otherwise the Unicode code point
492 */
493 U_CFUNC UChar32
494 ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
495 const char *source, int32_t length,
496 UBool useFallback);
497
498 /**
499 * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
500 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
501 * It does not handle conversion extensions (_extToU()).
502 */
503 U_CFUNC UChar32
504 ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
505 uint8_t b, UBool useFallback);
506
507 /**
508 * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
509 * It works for single-byte, single-state codepages that only map
510 * to and from BMP code points, and it always
511 * returns fallback values.
512 */
513 #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
514 (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])
515
516 /**
517 * This is an internal function that allows other converter implementations
518 * to check whether a byte is a lead byte.
519 */
520 U_CFUNC UBool
521 ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);
522
523 /** This is a macro version of _MBCSIsLeadByte(). */
524 #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
525 (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])
526
527 /*
528 * This is another simple conversion function for internal use by other
529 * conversion implementations.
530 * It does not use the converter state nor call callbacks.
531 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
532 * It handles conversion extensions but not GB 18030.
533 *
534 * It converts one single Unicode code point into codepage bytes, encoded
535 * as one 32-bit value. The function returns the number of bytes in *pValue:
536 * 1..4 the number of bytes in *pValue
537 * 0 unassigned (*pValue undefined)
538 * -1 illegal (currently not used, *pValue undefined)
539 *
540 * *pValue will contain the resulting bytes with the last byte in bits 7..0,
541 * the second to last byte in bits 15..8, etc.
542 * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
543 */
544 U_CFUNC int32_t
545 ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
546 UChar32 c, uint32_t *pValue,
547 UBool useFallback);
548
549 /**
550 * This version of _MBCSFromUChar32() is optimized for single-byte codepages.
551 * It does not handle the EBCDIC swaplfnl option (set in UConverter).
552 *
553 * It returns the codepage byte for the code point, or -1 if it is unassigned.
554 */
555 U_CFUNC int32_t
556 ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
557 UChar32 c,
558 UBool useFallback);
559
560 /**
561 * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
562 * we cheat a little about the type, returning the old types if appropriate.
563 */
564 U_CFUNC UConverterType
565 ucnv_MBCSGetType(const UConverter* converter);
566
567 U_CFUNC void
568 ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
569 UErrorCode *pErrorCode);
570 U_CFUNC void
571 ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
572 UErrorCode *pErrorCode);
573
574 /*
575 * Internal function returning a UnicodeSet for toUnicode() conversion.
576 * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
577 * In the future, if we add support for fallback sets, this function
578 * needs to be updated.
579 * Handles extensions.
580 * Does not empty the set first.
581 */
582 U_CFUNC void
583 ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
584 const USetAdder *sa,
585 UConverterUnicodeSet which,
586 UErrorCode *pErrorCode);
587
588 /*
589 * Same as ucnv_MBCSGetUnicodeSetForUnicode() but
590 * the set can be filtered by encoding scheme.
591 * Used by stateful converters which share regular conversion tables
592 * but only use a subset of their mappings.
593 */
594 U_CFUNC void
595 ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData,
596 const USetAdder *sa,
597 UConverterUnicodeSet which,
598 UConverterSetFilter filter,
599 UErrorCode *pErrorCode);
600
601 #endif
602
603 #endif

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial