The Tor Browser / file comparison
comparison: intl/icu/source/common/ucnv_lmb.c
intl/icu/source/common/ucnv_lmb.c
- branch
- TOR_BUG_3246
- changeset 4
- fc2d59ddac77
equal
deleted
inserted
replaced
| -1:000000000000 | 0:adad3a754e3f |
|---|---|
| 1 /* | |
| 2 ********************************************************************** | |
| 3 * Copyright (C) 2000-2011, International Business Machines | |
| 4 * Corporation and others. All Rights Reserved. | |
| 5 ********************************************************************** | |
| 6 * file name: ucnv_lmb.cpp | |
| 7 * encoding: US-ASCII | |
| 8 * tab size: 4 (not used) | |
| 9 * indentation:4 | |
| 10 * | |
| 11 * created on: 2000feb09 | |
| 12 * created by: Brendan Murray | |
| 13 * extensively hacked up by: Jim Snyder-Grant | |
| 14 * | |
| 15 * Modification History: | |
| 16 * | |
| 17 * Date Name Description | |
| 18 * | |
| 19 * 06/20/2000 helena OS/400 port changes; mostly typecast. | |
| 20 * 06/27/2000 Jim Snyder-Grant Deal with partial characters and small buffers. | |
| 21 * Add comments to document LMBCS format and implementation | |
| 22 * restructured order & breakdown of functions | |
| 23 * 06/28/2000 helena Major rewrite for the callback API changes. | |
| 24 */ | |
| 25 | |
| 26 #include "unicode/utypes.h" | |
| 27 | |
| 28 #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION | |
| 29 | |
| 30 #include "unicode/ucnv_err.h" | |
| 31 #include "unicode/ucnv.h" | |
| 32 #include "unicode/uset.h" | |
| 33 #include "cmemory.h" | |
| 34 #include "cstring.h" | |
| 35 #include "uassert.h" | |
| 36 #include "ucnv_imp.h" | |
| 37 #include "ucnv_bld.h" | |
| 38 #include "ucnv_cnv.h" | |
| 39 | |
| 40 #ifdef EBCDIC_RTL | |
| 41 #include "ascii_a.h" | |
| 42 #endif | |
| 43 | |
| 44 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) | |
| 45 | |
| 46 /* | |
| 47 LMBCS | |
| 48 | |
| 49 (Lotus Multi-Byte Character Set) | |
| 50 | |
| 51 LMBCS was invented in the late 1980's and is primarily used in Lotus Notes | |
| 52 databases and in Lotus 1-2-3 files. Programmers who work with the APIs | |
| 53 into these products will sometimes need to deal with strings in this format. | |
| 54 | |
| 55 The code in this file provides an implementation for an ICU converter of | |
| 56 LMBCS to and from Unicode. | |
| 57 | |
| 58 Since the LMBCS character set is only sparsely documented in existing | |
| 59 printed or online material, we have added extensive annotation to this | |
| 60 file to serve as a guide to understanding LMBCS. | |
| 61 | |
| 62 LMBCS was originally designed with these four sometimes-competing design goals: | |
| 63 | |
| 64 -Provide encodings for the characters in 12 existing national standards | |
| 65 (plus a few other characters) | |
| 66 -Minimal memory footprint | |
| 67 -Maximal speed of conversion into the existing national character sets | |
| 68 -No need to track a changing state as you interpret a string. | |
| 69 | |
| 70 | |
| 71 All of the national character sets LMBCS was trying to encode are 'ANSI' | |
| 72 based, in that the bytes from 0x20 - 0x7F are almost exactly the | |
| 73 same common Latin unaccented characters and symbols in all character sets. | |
| 74 | |
| 75 So, in order to help meet the speed & memory design goals, the common ANSI | |
| 76 bytes from 0x20-0x7F are represented by the same single-byte values in LMBCS. | |
| 77 | |
| 78 The general LMBCS code unit is from 1-3 bytes. We can describe the 3 bytes as | |
| 79 follows: | |
| 80 | |
| 81 [G] D1 [D2] | |
| 82 | |
| 83 That is, a sometimes-optional 'group' byte, followed by 1 and sometimes 2 | |
| 84 data bytes. The maximum size of a LMBCS chjaracter is 3 bytes: | |
| 85 */ | |
| 86 #define ULMBCS_CHARSIZE_MAX 3 | |
| 87 /* | |
| 88 The single-byte values from 0x20 to 0x7F are examples of single D1 bytes. | |
| 89 We often have to figure out if byte values are below or above this, so we | |
| 90 use the ANSI nomenclature 'C0' and 'C1' to refer to the range of control | |
| 91 characters just above & below the common lower-ANSI range */ | |
| 92 #define ULMBCS_C0END 0x1F | |
| 93 #define ULMBCS_C1START 0x80 | |
| 94 /* | |
| 95 Since LMBCS is always dealing in byte units. we create a local type here for | |
| 96 dealing with these units of LMBCS code units: | |
| 97 | |
| 98 */ | |
| 99 typedef uint8_t ulmbcs_byte_t; | |
| 100 | |
| 101 /* | |
| 102 Most of the values less than 0x20 are reserved in LMBCS to announce | |
| 103 which national character standard is being used for the 'D' bytes. | |
| 104 In the comments we show the common name and the IBM character-set ID | |
| 105 for these character-set announcers: | |
| 106 */ | |
| 107 | |
| 108 #define ULMBCS_GRP_L1 0x01 /* Latin-1 :ibm-850 */ | |
| 109 #define ULMBCS_GRP_GR 0x02 /* Greek :ibm-851 */ | |
| 110 #define ULMBCS_GRP_HE 0x03 /* Hebrew :ibm-1255 */ | |
| 111 #define ULMBCS_GRP_AR 0x04 /* Arabic :ibm-1256 */ | |
| 112 #define ULMBCS_GRP_RU 0x05 /* Cyrillic :ibm-1251 */ | |
| 113 #define ULMBCS_GRP_L2 0x06 /* Latin-2 :ibm-852 */ | |
| 114 #define ULMBCS_GRP_TR 0x08 /* Turkish :ibm-1254 */ | |
| 115 #define ULMBCS_GRP_TH 0x0B /* Thai :ibm-874 */ | |
| 116 #define ULMBCS_GRP_JA 0x10 /* Japanese :ibm-943 */ | |
| 117 #define ULMBCS_GRP_KO 0x11 /* Korean :ibm-1261 */ | |
| 118 #define ULMBCS_GRP_TW 0x12 /* Chinese SC :ibm-950 */ | |
| 119 #define ULMBCS_GRP_CN 0x13 /* Chinese TC :ibm-1386 */ | |
| 120 | |
| 121 /* | |
| 122 So, the beginning of understanding LMBCS is that IF the first byte of a LMBCS | |
| 123 character is one of those 12 values, you can interpret the remaining bytes of | |
| 124 that character as coming from one of those character sets. Since the lower | |
| 125 ANSI bytes already are represented in single bytes, using one of the character | |
| 126 set announcers is used to announce a character that starts with a byte of | |
| 127 0x80 or greater. | |
| 128 | |
| 129 The character sets are arranged so that the single byte sets all appear | |
| 130 before the multi-byte character sets. When we need to tell whether a | |
| 131 group byte is for a single byte char set or not we use this define: */ | |
| 132 | |
| 133 #define ULMBCS_DOUBLEOPTGROUP_START 0x10 | |
| 134 | |
| 135 /* | |
| 136 However, to fully understand LMBCS, you must also understand a series of | |
| 137 exceptions & optimizations made in service of the design goals. | |
| 138 | |
| 139 First, those of you who are character set mavens may have noticed that | |
| 140 the 'double-byte' character sets are actually multi-byte character sets | |
| 141 that can have 1 or two bytes, even in the upper-ascii range. To force | |
| 142 each group byte to introduce a fixed-width encoding (to make it faster to | |
| 143 count characters), we use a convention of doubling up on the group byte | |
| 144 to introduce any single-byte character > 0x80 in an otherwise double-byte | |
| 145 character set. So, for example, the LMBCS sequence x10 x10 xAE is the | |
| 146 same as '0xAE' in the Japanese code page 943. | |
| 147 | |
| 148 Next, you will notice that the list of group bytes has some gaps. | |
| 149 These are used in various ways. | |
| 150 | |
| 151 We reserve a few special single byte values for common control | |
| 152 characters. These are in the same place as their ANSI eqivalents for speed. | |
| 153 */ | |
| 154 | |
| 155 #define ULMBCS_HT 0x09 /* Fixed control char - Horizontal Tab */ | |
| 156 #define ULMBCS_LF 0x0A /* Fixed control char - Line Feed */ | |
| 157 #define ULMBCS_CR 0x0D /* Fixed control char - Carriage Return */ | |
| 158 | |
| 159 /* Then, 1-2-3 reserved a special single-byte character to put at the | |
| 160 beginning of internal 'system' range names: */ | |
| 161 | |
| 162 #define ULMBCS_123SYSTEMRANGE 0x19 | |
| 163 | |
| 164 /* Then we needed a place to put all the other ansi control characters | |
| 165 that must be moved to different values because LMBCS reserves those | |
| 166 values for other purposes. To represent the control characters, we start | |
| 167 with a first byte of 0xF & add the control chaarcter value as the | |
| 168 second byte */ | |
| 169 #define ULMBCS_GRP_CTRL 0x0F | |
| 170 | |
| 171 /* For the C0 controls (less than 0x20), we add 0x20 to preserve the | |
| 172 useful doctrine that any byte less than 0x20 in a LMBCS char must be | |
| 173 the first byte of a character:*/ | |
| 174 #define ULMBCS_CTRLOFFSET 0x20 | |
| 175 | |
| 176 /* | |
| 177 Where to put the characters that aren't part of any of the 12 national | |
| 178 character sets? The first thing that was done, in the earlier years of | |
| 179 LMBCS, was to use up the spaces of the form | |
| 180 | |
| 181 [G] D1, | |
| 182 | |
| 183 where 'G' was one of the single-byte character groups, and | |
| 184 D1 was less than 0x80. These sequences are gathered together | |
| 185 into a Lotus-invented doublebyte character set to represent a | |
| 186 lot of stray values. Internally, in this implementation, we track this | |
| 187 as group '0', as a place to tuck this exceptions list.*/ | |
| 188 | |
| 189 #define ULMBCS_GRP_EXCEPT 0x00 | |
| 190 /* | |
| 191 Finally, as the durability and usefulness of UNICODE became clear, | |
| 192 LOTUS added a new group 0x14 to hold Unicode values not otherwise | |
| 193 represented in LMBCS: */ | |
| 194 #define ULMBCS_GRP_UNICODE 0x14 | |
| 195 /* The two bytes appearing after a 0x14 are intrepreted as UFT-16 BE | |
| 196 (Big-Endian) characters. The exception comes when the UTF16 | |
| 197 representation would have a zero as the second byte. In that case, | |
| 198 'F6' is used in its place, and the bytes are swapped. (This prevents | |
| 199 LMBCS from encoding any Unicode values of the form U+F6xx, but that's OK: | |
| 200 0xF6xx is in the middle of the Private Use Area.)*/ | |
| 201 #define ULMBCS_UNICOMPATZERO 0xF6 | |
| 202 | |
| 203 /* It is also useful in our code to have a constant for the size of | |
| 204 a LMBCS char that holds a literal Unicode value */ | |
| 205 #define ULMBCS_UNICODE_SIZE 3 | |
| 206 | |
| 207 /* | |
| 208 To squish the LMBCS representations down even further, and to make | |
| 209 translations even faster,sometimes the optimization group byte can be dropped | |
| 210 from a LMBCS character. This is decided on a process-by-process basis. The | |
| 211 group byte that is dropped is called the 'optimization group'. | |
| 212 | |
| 213 For Notes, the optimzation group is always 0x1.*/ | |
| 214 #define ULMBCS_DEFAULTOPTGROUP 0x1 | |
| 215 /* For 1-2-3 files, the optimzation group is stored in the header of the 1-2-3 | |
| 216 file. | |
| 217 | |
| 218 In any case, when using ICU, you either pass in the | |
| 219 optimization group as part of the name of the converter (LMBCS-1, LMBCS-2, | |
| 220 etc.). Using plain 'LMBCS' as the name of the converter will give you | |
| 221 LMBCS-1. | |
| 222 | |
| 223 | |
| 224 *** Implementation strategy *** | |
| 225 | |
| 226 | |
| 227 Because of the extensive use of other character sets, the LMBCS converter | |
| 228 keeps a mapping between optimization groups and IBM character sets, so that | |
| 229 ICU converters can be created and used as needed. */ | |
| 230 | |
| 231 /* As you can see, even though any byte below 0x20 could be an optimization | |
| 232 byte, only those at 0x13 or below can map to an actual converter. To limit | |
| 233 some loops and searches, we define a value for that last group converter:*/ | |
| 234 | |
| 235 #define ULMBCS_GRP_LAST 0x13 /* last LMBCS group that has a converter */ | |
| 236 | |
| 237 static const char * const OptGroupByteToCPName[ULMBCS_GRP_LAST + 1] = { | |
| 238 /* 0x0000 */ "lmb-excp", /* internal home for the LOTUS exceptions list */ | |
| 239 /* 0x0001 */ "ibm-850", | |
| 240 /* 0x0002 */ "ibm-851", | |
| 241 /* 0x0003 */ "windows-1255", | |
| 242 /* 0x0004 */ "windows-1256", | |
| 243 /* 0x0005 */ "windows-1251", | |
| 244 /* 0x0006 */ "ibm-852", | |
| 245 /* 0x0007 */ NULL, /* Unused */ | |
| 246 /* 0x0008 */ "windows-1254", | |
| 247 /* 0x0009 */ NULL, /* Control char HT */ | |
| 248 /* 0x000A */ NULL, /* Control char LF */ | |
| 249 /* 0x000B */ "windows-874", | |
| 250 /* 0x000C */ NULL, /* Unused */ | |
| 251 /* 0x000D */ NULL, /* Control char CR */ | |
| 252 /* 0x000E */ NULL, /* Unused */ | |
| 253 /* 0x000F */ NULL, /* Control chars: 0x0F20 + C0/C1 character: algorithmic */ | |
| 254 /* 0x0010 */ "windows-932", | |
| 255 /* 0x0011 */ "windows-949", | |
| 256 /* 0x0012 */ "windows-950", | |
| 257 /* 0x0013 */ "windows-936" | |
| 258 | |
| 259 /* The rest are null, including the 0x0014 Unicode compatibility region | |
| 260 and 0x0019, the 1-2-3 system range control char */ | |
| 261 }; | |
| 262 | |
| 263 | |
| 264 /* That's approximately all the data that's needed for translating | |
| 265 LMBCS to Unicode. | |
| 266 | |
| 267 | |
| 268 However, to translate Unicode to LMBCS, we need some more support. | |
| 269 | |
| 270 That's because there are often more than one possible mappings from a Unicode | |
| 271 code point back into LMBCS. The first thing we do is look up into a table | |
| 272 to figure out if there are more than one possible mappings. This table, | |
| 273 arranged by Unicode values (including ranges) either lists which group | |
| 274 to use, or says that it could go into one or more of the SBCS sets, or | |
| 275 into one or more of the DBCS sets. (If the character exists in both DBCS & | |
| 276 SBCS, the table will place it in the SBCS sets, to make the LMBCS code point | |
| 277 length as small as possible. Here's the two special markers we use to indicate | |
| 278 ambiguous mappings: */ | |
| 279 | |
| 280 #define ULMBCS_AMBIGUOUS_SBCS 0x80 /* could fit in more than one | |
| 281 LMBCS sbcs native encoding | |
| 282 (example: most accented latin) */ | |
| 283 #define ULMBCS_AMBIGUOUS_MBCS 0x81 /* could fit in more than one | |
| 284 LMBCS mbcs native encoding | |
| 285 (example: Unihan) */ | |
| 286 #define ULMBCS_AMBIGUOUS_ALL 0x82 | |
| 287 /* And here's a simple way to see if a group falls in an appropriate range */ | |
| 288 #define ULMBCS_AMBIGUOUS_MATCH(agroup, xgroup) \ | |
| 289 ((((agroup) == ULMBCS_AMBIGUOUS_SBCS) && \ | |
| 290 (xgroup) < ULMBCS_DOUBLEOPTGROUP_START) || \ | |
| 291 (((agroup) == ULMBCS_AMBIGUOUS_MBCS) && \ | |
| 292 (xgroup) >= ULMBCS_DOUBLEOPTGROUP_START)) || \ | |
| 293 ((agroup) == ULMBCS_AMBIGUOUS_ALL) | |
| 294 | |
| 295 | |
| 296 /* The table & some code to use it: */ | |
| 297 | |
| 298 | |
| 299 static const struct _UniLMBCSGrpMap | |
| 300 { | |
| 301 const UChar uniStartRange; | |
| 302 const UChar uniEndRange; | |
| 303 const ulmbcs_byte_t GrpType; | |
| 304 } UniLMBCSGrpMap[] | |
| 305 = | |
| 306 { | |
| 307 | |
| 308 {0x0001, 0x001F, ULMBCS_GRP_CTRL}, | |
| 309 {0x0080, 0x009F, ULMBCS_GRP_CTRL}, | |
| 310 {0x00A0, 0x00A6, ULMBCS_AMBIGUOUS_SBCS}, | |
| 311 {0x00A7, 0x00A8, ULMBCS_AMBIGUOUS_ALL}, | |
| 312 {0x00A9, 0x00AF, ULMBCS_AMBIGUOUS_SBCS}, | |
| 313 {0x00B0, 0x00B1, ULMBCS_AMBIGUOUS_ALL}, | |
| 314 {0x00B2, 0x00B3, ULMBCS_AMBIGUOUS_SBCS}, | |
| 315 {0x00B4, 0x00B4, ULMBCS_AMBIGUOUS_ALL}, | |
| 316 {0x00B5, 0x00B5, ULMBCS_AMBIGUOUS_SBCS}, | |
| 317 {0x00B6, 0x00B6, ULMBCS_AMBIGUOUS_ALL}, | |
| 318 {0x00B7, 0x00D6, ULMBCS_AMBIGUOUS_SBCS}, | |
| 319 {0x00D7, 0x00D7, ULMBCS_AMBIGUOUS_ALL}, | |
| 320 {0x00D8, 0x00F6, ULMBCS_AMBIGUOUS_SBCS}, | |
| 321 {0x00F7, 0x00F7, ULMBCS_AMBIGUOUS_ALL}, | |
| 322 {0x00F8, 0x01CD, ULMBCS_AMBIGUOUS_SBCS}, | |
| 323 {0x01CE, 0x01CE, ULMBCS_GRP_TW }, | |
| 324 {0x01CF, 0x02B9, ULMBCS_AMBIGUOUS_SBCS}, | |
| 325 {0x02BA, 0x02BA, ULMBCS_GRP_CN}, | |
| 326 {0x02BC, 0x02C8, ULMBCS_AMBIGUOUS_SBCS}, | |
| 327 {0x02C9, 0x02D0, ULMBCS_AMBIGUOUS_MBCS}, | |
| 328 {0x02D8, 0x02DD, ULMBCS_AMBIGUOUS_SBCS}, | |
| 329 {0x0384, 0x0390, ULMBCS_AMBIGUOUS_SBCS}, | |
| 330 {0x0391, 0x03A9, ULMBCS_AMBIGUOUS_ALL}, | |
| 331 {0x03AA, 0x03B0, ULMBCS_AMBIGUOUS_SBCS}, | |
| 332 {0x03B1, 0x03C9, ULMBCS_AMBIGUOUS_ALL}, | |
| 333 {0x03CA, 0x03CE, ULMBCS_AMBIGUOUS_SBCS}, | |
| 334 {0x0400, 0x0400, ULMBCS_GRP_RU}, | |
| 335 {0x0401, 0x0401, ULMBCS_AMBIGUOUS_ALL}, | |
| 336 {0x0402, 0x040F, ULMBCS_GRP_RU}, | |
| 337 {0x0410, 0x0431, ULMBCS_AMBIGUOUS_ALL}, | |
| 338 {0x0432, 0x044E, ULMBCS_GRP_RU}, | |
| 339 {0x044F, 0x044F, ULMBCS_AMBIGUOUS_ALL}, | |
| 340 {0x0450, 0x0491, ULMBCS_GRP_RU}, | |
| 341 {0x05B0, 0x05F2, ULMBCS_GRP_HE}, | |
| 342 {0x060C, 0x06AF, ULMBCS_GRP_AR}, | |
| 343 {0x0E01, 0x0E5B, ULMBCS_GRP_TH}, | |
| 344 {0x200C, 0x200F, ULMBCS_AMBIGUOUS_SBCS}, | |
| 345 {0x2010, 0x2010, ULMBCS_AMBIGUOUS_MBCS}, | |
| 346 {0x2013, 0x2014, ULMBCS_AMBIGUOUS_SBCS}, | |
| 347 {0x2015, 0x2015, ULMBCS_AMBIGUOUS_MBCS}, | |
| 348 {0x2016, 0x2016, ULMBCS_AMBIGUOUS_MBCS}, | |
| 349 {0x2017, 0x2017, ULMBCS_AMBIGUOUS_SBCS}, | |
| 350 {0x2018, 0x2019, ULMBCS_AMBIGUOUS_ALL}, | |
| 351 {0x201A, 0x201B, ULMBCS_AMBIGUOUS_SBCS}, | |
| 352 {0x201C, 0x201D, ULMBCS_AMBIGUOUS_ALL}, | |
| 353 {0x201E, 0x201F, ULMBCS_AMBIGUOUS_SBCS}, | |
| 354 {0x2020, 0x2021, ULMBCS_AMBIGUOUS_ALL}, | |
| 355 {0x2022, 0x2024, ULMBCS_AMBIGUOUS_SBCS}, | |
| 356 {0x2025, 0x2025, ULMBCS_AMBIGUOUS_MBCS}, | |
| 357 {0x2026, 0x2026, ULMBCS_AMBIGUOUS_ALL}, | |
| 358 {0x2027, 0x2027, ULMBCS_GRP_TW}, | |
| 359 {0x2030, 0x2030, ULMBCS_AMBIGUOUS_ALL}, | |
| 360 {0x2031, 0x2031, ULMBCS_AMBIGUOUS_SBCS}, | |
| 361 {0x2032, 0x2033, ULMBCS_AMBIGUOUS_MBCS}, | |
| 362 {0x2035, 0x2035, ULMBCS_AMBIGUOUS_MBCS}, | |
| 363 {0x2039, 0x203A, ULMBCS_AMBIGUOUS_SBCS}, | |
| 364 {0x203B, 0x203B, ULMBCS_AMBIGUOUS_MBCS}, | |
| 365 {0x203C, 0x203C, ULMBCS_GRP_EXCEPT}, | |
| 366 {0x2074, 0x2074, ULMBCS_GRP_KO}, | |
| 367 {0x207F, 0x207F, ULMBCS_GRP_EXCEPT}, | |
| 368 {0x2081, 0x2084, ULMBCS_GRP_KO}, | |
| 369 {0x20A4, 0x20AC, ULMBCS_AMBIGUOUS_SBCS}, | |
| 370 {0x2103, 0x2109, ULMBCS_AMBIGUOUS_MBCS}, | |
| 371 {0x2111, 0x2120, ULMBCS_AMBIGUOUS_SBCS}, | |
| 372 /*zhujin: upgrade, for regressiont test, spr HKIA4YHTSU*/ | |
| 373 {0x2121, 0x2121, ULMBCS_AMBIGUOUS_MBCS}, | |
| 374 {0x2122, 0x2126, ULMBCS_AMBIGUOUS_SBCS}, | |
| 375 {0x212B, 0x212B, ULMBCS_AMBIGUOUS_MBCS}, | |
| 376 {0x2135, 0x2135, ULMBCS_AMBIGUOUS_SBCS}, | |
| 377 {0x2153, 0x2154, ULMBCS_GRP_KO}, | |
| 378 {0x215B, 0x215E, ULMBCS_GRP_EXCEPT}, | |
| 379 {0x2160, 0x2179, ULMBCS_AMBIGUOUS_MBCS}, | |
| 380 {0x2190, 0x2193, ULMBCS_AMBIGUOUS_ALL}, | |
| 381 {0x2194, 0x2195, ULMBCS_GRP_EXCEPT}, | |
| 382 {0x2196, 0x2199, ULMBCS_AMBIGUOUS_MBCS}, | |
| 383 {0x21A8, 0x21A8, ULMBCS_GRP_EXCEPT}, | |
| 384 {0x21B8, 0x21B9, ULMBCS_GRP_CN}, | |
| 385 {0x21D0, 0x21D1, ULMBCS_GRP_EXCEPT}, | |
| 386 {0x21D2, 0x21D2, ULMBCS_AMBIGUOUS_MBCS}, | |
| 387 {0x21D3, 0x21D3, ULMBCS_GRP_EXCEPT}, | |
| 388 {0x21D4, 0x21D4, ULMBCS_AMBIGUOUS_MBCS}, | |
| 389 {0x21D5, 0x21D5, ULMBCS_GRP_EXCEPT}, | |
| 390 {0x21E7, 0x21E7, ULMBCS_GRP_CN}, | |
| 391 {0x2200, 0x2200, ULMBCS_AMBIGUOUS_MBCS}, | |
| 392 {0x2201, 0x2201, ULMBCS_GRP_EXCEPT}, | |
| 393 {0x2202, 0x2202, ULMBCS_AMBIGUOUS_MBCS}, | |
| 394 {0x2203, 0x2203, ULMBCS_AMBIGUOUS_MBCS}, | |
| 395 {0x2204, 0x2206, ULMBCS_GRP_EXCEPT}, | |
| 396 {0x2207, 0x2208, ULMBCS_AMBIGUOUS_MBCS}, | |
| 397 {0x2209, 0x220A, ULMBCS_GRP_EXCEPT}, | |
| 398 {0x220B, 0x220B, ULMBCS_AMBIGUOUS_MBCS}, | |
| 399 {0x220F, 0x2215, ULMBCS_AMBIGUOUS_MBCS}, | |
| 400 {0x2219, 0x2219, ULMBCS_GRP_EXCEPT}, | |
| 401 {0x221A, 0x221A, ULMBCS_AMBIGUOUS_MBCS}, | |
| 402 {0x221B, 0x221C, ULMBCS_GRP_EXCEPT}, | |
| 403 {0x221D, 0x221E, ULMBCS_AMBIGUOUS_MBCS}, | |
| 404 {0x221F, 0x221F, ULMBCS_GRP_EXCEPT}, | |
| 405 {0x2220, 0x2220, ULMBCS_AMBIGUOUS_MBCS}, | |
| 406 {0x2223, 0x222A, ULMBCS_AMBIGUOUS_MBCS}, | |
| 407 {0x222B, 0x223D, ULMBCS_AMBIGUOUS_MBCS}, | |
| 408 {0x2245, 0x2248, ULMBCS_GRP_EXCEPT}, | |
| 409 {0x224C, 0x224C, ULMBCS_GRP_TW}, | |
| 410 {0x2252, 0x2252, ULMBCS_AMBIGUOUS_MBCS}, | |
| 411 {0x2260, 0x2261, ULMBCS_AMBIGUOUS_MBCS}, | |
| 412 {0x2262, 0x2265, ULMBCS_GRP_EXCEPT}, | |
| 413 {0x2266, 0x226F, ULMBCS_AMBIGUOUS_MBCS}, | |
| 414 {0x2282, 0x2283, ULMBCS_AMBIGUOUS_MBCS}, | |
| 415 {0x2284, 0x2285, ULMBCS_GRP_EXCEPT}, | |
| 416 {0x2286, 0x2287, ULMBCS_AMBIGUOUS_MBCS}, | |
| 417 {0x2288, 0x2297, ULMBCS_GRP_EXCEPT}, | |
| 418 {0x2299, 0x22BF, ULMBCS_AMBIGUOUS_MBCS}, | |
| 419 {0x22C0, 0x22C0, ULMBCS_GRP_EXCEPT}, | |
| 420 {0x2310, 0x2310, ULMBCS_GRP_EXCEPT}, | |
| 421 {0x2312, 0x2312, ULMBCS_AMBIGUOUS_MBCS}, | |
| 422 {0x2318, 0x2321, ULMBCS_GRP_EXCEPT}, | |
| 423 {0x2318, 0x2321, ULMBCS_GRP_CN}, | |
| 424 {0x2460, 0x24E9, ULMBCS_AMBIGUOUS_MBCS}, | |
| 425 {0x2500, 0x2500, ULMBCS_AMBIGUOUS_SBCS}, | |
| 426 {0x2501, 0x2501, ULMBCS_AMBIGUOUS_MBCS}, | |
| 427 {0x2502, 0x2502, ULMBCS_AMBIGUOUS_ALL}, | |
| 428 {0x2503, 0x2503, ULMBCS_AMBIGUOUS_MBCS}, | |
| 429 {0x2504, 0x2505, ULMBCS_GRP_TW}, | |
| 430 {0x2506, 0x2665, ULMBCS_AMBIGUOUS_ALL}, | |
| 431 {0x2666, 0x2666, ULMBCS_GRP_EXCEPT}, | |
| 432 {0x2667, 0x2669, ULMBCS_AMBIGUOUS_SBCS}, | |
| 433 {0x266A, 0x266A, ULMBCS_AMBIGUOUS_ALL}, | |
| 434 {0x266B, 0x266C, ULMBCS_AMBIGUOUS_SBCS}, | |
| 435 {0x266D, 0x266D, ULMBCS_AMBIGUOUS_MBCS}, | |
| 436 {0x266E, 0x266E, ULMBCS_AMBIGUOUS_SBCS}, | |
| 437 {0x266F, 0x266F, ULMBCS_GRP_JA}, | |
| 438 {0x2670, 0x2E7F, ULMBCS_AMBIGUOUS_SBCS}, | |
| 439 {0x2E80, 0xF861, ULMBCS_AMBIGUOUS_MBCS}, | |
| 440 {0xF862, 0xF8FF, ULMBCS_GRP_EXCEPT}, | |
| 441 {0xF900, 0xFA2D, ULMBCS_AMBIGUOUS_MBCS}, | |
| 442 {0xFB00, 0xFEFF, ULMBCS_AMBIGUOUS_SBCS}, | |
| 443 {0xFF01, 0xFFEE, ULMBCS_AMBIGUOUS_MBCS}, | |
| 444 {0xFFFF, 0xFFFF, ULMBCS_GRP_UNICODE} | |
| 445 }; | |
| 446 | |
| 447 static ulmbcs_byte_t | |
| 448 FindLMBCSUniRange(UChar uniChar) | |
| 449 { | |
| 450 const struct _UniLMBCSGrpMap * pTable = UniLMBCSGrpMap; | |
| 451 | |
| 452 while (uniChar > pTable->uniEndRange) | |
| 453 { | |
| 454 pTable++; | |
| 455 } | |
| 456 | |
| 457 if (uniChar >= pTable->uniStartRange) | |
| 458 { | |
| 459 return pTable->GrpType; | |
| 460 } | |
| 461 return ULMBCS_GRP_UNICODE; | |
| 462 } | |
| 463 | |
| 464 /* | |
| 465 We also ask the creator of a converter to send in a preferred locale | |
| 466 that we can use in resolving ambiguous mappings. They send the locale | |
| 467 in as a string, and we map it, if possible, to one of the | |
| 468 LMBCS groups. We use this table, and the associated code, to | |
| 469 do the lookup: */ | |
| 470 | |
| 471 /************************************************** | |
| 472 This table maps locale ID's to LMBCS opt groups. | |
| 473 The default return is group 0x01. Note that for | |
| 474 performance reasons, the table is sorted in | |
| 475 increasing alphabetic order, with the notable | |
| 476 exception of zhTW. This is to force the check | |
| 477 for Traditonal Chinese before dropping back to | |
| 478 Simplified. | |
| 479 | |
| 480 Note too that the Latin-1 groups have been | |
| 481 commented out because it's the default, and | |
| 482 this shortens the table, allowing a serial | |
| 483 search to go quickly. | |
| 484 *************************************************/ | |
| 485 | |
| 486 static const struct _LocaleLMBCSGrpMap | |
| 487 { | |
| 488 const char *LocaleID; | |
| 489 const ulmbcs_byte_t OptGroup; | |
| 490 } LocaleLMBCSGrpMap[] = | |
| 491 { | |
| 492 {"ar", ULMBCS_GRP_AR}, | |
| 493 {"be", ULMBCS_GRP_RU}, | |
| 494 {"bg", ULMBCS_GRP_L2}, | |
| 495 /* {"ca", ULMBCS_GRP_L1}, */ | |
| 496 {"cs", ULMBCS_GRP_L2}, | |
| 497 /* {"da", ULMBCS_GRP_L1}, */ | |
| 498 /* {"de", ULMBCS_GRP_L1}, */ | |
| 499 {"el", ULMBCS_GRP_GR}, | |
| 500 /* {"en", ULMBCS_GRP_L1}, */ | |
| 501 /* {"es", ULMBCS_GRP_L1}, */ | |
| 502 /* {"et", ULMBCS_GRP_L1}, */ | |
| 503 /* {"fi", ULMBCS_GRP_L1}, */ | |
| 504 /* {"fr", ULMBCS_GRP_L1}, */ | |
| 505 {"he", ULMBCS_GRP_HE}, | |
| 506 {"hu", ULMBCS_GRP_L2}, | |
| 507 /* {"is", ULMBCS_GRP_L1}, */ | |
| 508 /* {"it", ULMBCS_GRP_L1}, */ | |
| 509 {"iw", ULMBCS_GRP_HE}, | |
| 510 {"ja", ULMBCS_GRP_JA}, | |
| 511 {"ko", ULMBCS_GRP_KO}, | |
| 512 /* {"lt", ULMBCS_GRP_L1}, */ | |
| 513 /* {"lv", ULMBCS_GRP_L1}, */ | |
| 514 {"mk", ULMBCS_GRP_RU}, | |
| 515 /* {"nl", ULMBCS_GRP_L1}, */ | |
| 516 /* {"no", ULMBCS_GRP_L1}, */ | |
| 517 {"pl", ULMBCS_GRP_L2}, | |
| 518 /* {"pt", ULMBCS_GRP_L1}, */ | |
| 519 {"ro", ULMBCS_GRP_L2}, | |
| 520 {"ru", ULMBCS_GRP_RU}, | |
| 521 {"sh", ULMBCS_GRP_L2}, | |
| 522 {"sk", ULMBCS_GRP_L2}, | |
| 523 {"sl", ULMBCS_GRP_L2}, | |
| 524 {"sq", ULMBCS_GRP_L2}, | |
| 525 {"sr", ULMBCS_GRP_RU}, | |
| 526 /* {"sv", ULMBCS_GRP_L1}, */ | |
| 527 {"th", ULMBCS_GRP_TH}, | |
| 528 {"tr", ULMBCS_GRP_TR}, | |
| 529 {"uk", ULMBCS_GRP_RU}, | |
| 530 /* {"vi", ULMBCS_GRP_L1}, */ | |
| 531 {"zhTW", ULMBCS_GRP_TW}, | |
| 532 {"zh", ULMBCS_GRP_CN}, | |
| 533 {NULL, ULMBCS_GRP_L1} | |
| 534 }; | |
| 535 | |
| 536 | |
| 537 static ulmbcs_byte_t | |
| 538 FindLMBCSLocale(const char *LocaleID) | |
| 539 { | |
| 540 const struct _LocaleLMBCSGrpMap *pTable = LocaleLMBCSGrpMap; | |
| 541 | |
| 542 if ((!LocaleID) || (!*LocaleID)) | |
| 543 { | |
| 544 return 0; | |
| 545 } | |
| 546 | |
| 547 while (pTable->LocaleID) | |
| 548 { | |
| 549 if (*pTable->LocaleID == *LocaleID) /* Check only first char for speed */ | |
| 550 { | |
| 551 /* First char matches - check whole name, for entry-length */ | |
| 552 if (uprv_strncmp(pTable->LocaleID, LocaleID, strlen(pTable->LocaleID)) == 0) | |
| 553 return pTable->OptGroup; | |
| 554 } | |
| 555 else | |
| 556 if (*pTable->LocaleID > *LocaleID) /* Sorted alphabetically - exit */ | |
| 557 break; | |
| 558 pTable++; | |
| 559 } | |
| 560 return ULMBCS_GRP_L1; | |
| 561 } | |
| 562 | |
| 563 | |
| 564 /* | |
| 565 Before we get to the main body of code, here's how we hook up to the rest | |
| 566 of ICU. ICU converters are required to define a structure that includes | |
| 567 some function pointers, and some common data, in the style of a C++ | |
| 568 vtable. There is also room in there for converter-specific data. LMBCS | |
| 569 uses that converter-specific data to keep track of the 12 subconverters | |
| 570 we use, the optimization group, and the group (if any) that matches the | |
| 571 locale. We have one structure instantiated for each of the 12 possible | |
| 572 optimization groups. To avoid typos & to avoid boring the reader, we | |
| 573 put the declarations of these structures and functions into macros. To see | |
| 574 the definitions of these structures, see unicode\ucnv_bld.h | |
| 575 */ | |
| 576 | |
| 577 typedef struct | |
| 578 { | |
| 579 UConverterSharedData *OptGrpConverter[ULMBCS_GRP_LAST+1]; /* Converter per Opt. grp. */ | |
| 580 uint8_t OptGroup; /* default Opt. grp. for this LMBCS session */ | |
| 581 uint8_t localeConverterIndex; /* reasonable locale match for index */ | |
| 582 } | |
| 583 UConverterDataLMBCS; | |
| 584 | |
| 585 static void _LMBCSClose(UConverter * _this); | |
| 586 | |
| 587 #define DECLARE_LMBCS_DATA(n) \ | |
| 588 static const UConverterImpl _LMBCSImpl##n={\ | |
| 589 UCNV_LMBCS_##n,\ | |
| 590 NULL,NULL,\ | |
| 591 _LMBCSOpen##n,\ | |
| 592 _LMBCSClose,\ | |
| 593 NULL,\ | |
| 594 _LMBCSToUnicodeWithOffsets,\ | |
| 595 _LMBCSToUnicodeWithOffsets,\ | |
| 596 _LMBCSFromUnicode,\ | |
| 597 _LMBCSFromUnicode,\ | |
| 598 NULL,\ | |
| 599 NULL,\ | |
| 600 NULL,\ | |
| 601 NULL,\ | |
| 602 _LMBCSSafeClone,\ | |
| 603 ucnv_getCompleteUnicodeSet\ | |
| 604 };\ | |
| 605 static const UConverterStaticData _LMBCSStaticData##n={\ | |
| 606 sizeof(UConverterStaticData),\ | |
| 607 "LMBCS-" #n,\ | |
| 608 0, UCNV_IBM, UCNV_LMBCS_##n, 1, 3,\ | |
| 609 { 0x3f, 0, 0, 0 },1,FALSE,FALSE,0,0,{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} \ | |
| 610 };\ | |
| 611 const UConverterSharedData _LMBCSData##n={\ | |
| 612 sizeof(UConverterSharedData), ~((uint32_t) 0),\ | |
| 613 NULL, NULL, &_LMBCSStaticData##n, FALSE, &_LMBCSImpl##n, \ | |
| 614 0 \ | |
| 615 }; | |
| 616 | |
| 617 /* The only function we needed to duplicate 12 times was the 'open' | |
| 618 function, which will do basically the same thing except set a different | |
| 619 optimization group. So, we put the common stuff into a worker function, | |
| 620 and set up another macro to stamp out the 12 open functions:*/ | |
| 621 #define DEFINE_LMBCS_OPEN(n) \ | |
| 622 static void \ | |
| 623 _LMBCSOpen##n(UConverter* _this, UConverterLoadArgs* pArgs, UErrorCode* err) \ | |
| 624 { _LMBCSOpenWorker(_this, pArgs, err, n); } | |
| 625 | |
| 626 | |
| 627 | |
| 628 /* Here's the open worker & the common close function */ | |
| 629 static void | |
| 630 _LMBCSOpenWorker(UConverter* _this, | |
| 631 UConverterLoadArgs *pArgs, | |
| 632 UErrorCode* err, | |
| 633 ulmbcs_byte_t OptGroup) | |
| 634 { | |
| 635 UConverterDataLMBCS * extraInfo = _this->extraInfo = | |
| 636 (UConverterDataLMBCS*)uprv_malloc (sizeof (UConverterDataLMBCS)); | |
| 637 if(extraInfo != NULL) | |
| 638 { | |
| 639 UConverterNamePieces stackPieces; | |
| 640 UConverterLoadArgs stackArgs={ (int32_t)sizeof(UConverterLoadArgs) }; | |
| 641 ulmbcs_byte_t i; | |
| 642 | |
| 643 uprv_memset(extraInfo, 0, sizeof(UConverterDataLMBCS)); | |
| 644 | |
| 645 stackArgs.onlyTestIsLoadable = pArgs->onlyTestIsLoadable; | |
| 646 | |
| 647 for (i=0; i <= ULMBCS_GRP_LAST && U_SUCCESS(*err); i++) | |
| 648 { | |
| 649 if(OptGroupByteToCPName[i] != NULL) { | |
| 650 extraInfo->OptGrpConverter[i] = ucnv_loadSharedData(OptGroupByteToCPName[i], &stackPieces, &stackArgs, err); | |
| 651 } | |
| 652 } | |
| 653 | |
| 654 if(U_FAILURE(*err) || pArgs->onlyTestIsLoadable) { | |
| 655 _LMBCSClose(_this); | |
| 656 return; | |
| 657 } | |
| 658 extraInfo->OptGroup = OptGroup; | |
| 659 extraInfo->localeConverterIndex = FindLMBCSLocale(pArgs->locale); | |
| 660 } | |
| 661 else | |
| 662 { | |
| 663 *err = U_MEMORY_ALLOCATION_ERROR; | |
| 664 } | |
| 665 } | |
| 666 | |
| 667 static void | |
| 668 _LMBCSClose(UConverter * _this) | |
| 669 { | |
| 670 if (_this->extraInfo != NULL) | |
| 671 { | |
| 672 ulmbcs_byte_t Ix; | |
| 673 UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) _this->extraInfo; | |
| 674 | |
| 675 for (Ix=0; Ix <= ULMBCS_GRP_LAST; Ix++) | |
| 676 { | |
| 677 if (extraInfo->OptGrpConverter[Ix] != NULL) | |
| 678 ucnv_unloadSharedDataIfReady(extraInfo->OptGrpConverter[Ix]); | |
| 679 } | |
| 680 if (!_this->isExtraLocal) { | |
| 681 uprv_free (_this->extraInfo); | |
| 682 _this->extraInfo = NULL; | |
| 683 } | |
| 684 } | |
| 685 } | |
| 686 | |
| 687 typedef struct LMBCSClone { | |
| 688 UConverter cnv; | |
| 689 UConverterDataLMBCS lmbcs; | |
| 690 } LMBCSClone; | |
| 691 | |
| 692 static UConverter * | |
| 693 _LMBCSSafeClone(const UConverter *cnv, | |
| 694 void *stackBuffer, | |
| 695 int32_t *pBufferSize, | |
| 696 UErrorCode *status) { | |
| 697 LMBCSClone *newLMBCS; | |
| 698 UConverterDataLMBCS *extraInfo; | |
| 699 int32_t i; | |
| 700 | |
| 701 if(*pBufferSize<=0) { | |
| 702 *pBufferSize=(int32_t)sizeof(LMBCSClone); | |
| 703 return NULL; | |
| 704 } | |
| 705 | |
| 706 extraInfo=(UConverterDataLMBCS *)cnv->extraInfo; | |
| 707 newLMBCS=(LMBCSClone *)stackBuffer; | |
| 708 | |
| 709 /* ucnv.c/ucnv_safeClone() copied the main UConverter already */ | |
| 710 | |
| 711 uprv_memcpy(&newLMBCS->lmbcs, extraInfo, sizeof(UConverterDataLMBCS)); | |
| 712 | |
| 713 /* share the subconverters */ | |
| 714 for(i = 0; i <= ULMBCS_GRP_LAST; ++i) { | |
| 715 if(extraInfo->OptGrpConverter[i] != NULL) { | |
| 716 ucnv_incrementRefCount(extraInfo->OptGrpConverter[i]); | |
| 717 } | |
| 718 } | |
| 719 | |
| 720 newLMBCS->cnv.extraInfo = &newLMBCS->lmbcs; | |
| 721 newLMBCS->cnv.isExtraLocal = TRUE; | |
| 722 return &newLMBCS->cnv; | |
| 723 } | |
| 724 | |
| 725 /* | |
| 726 * There used to be a _LMBCSGetUnicodeSet() function here (up to svn revision 20117) | |
| 727 * which added all code points except for U+F6xx | |
| 728 * because those cannot be represented in the Unicode group. | |
| 729 * However, it turns out that windows-950 has roundtrips for all of U+F6xx | |
| 730 * which means that LMBCS can convert all Unicode code points after all. | |
| 731 * We now simply use ucnv_getCompleteUnicodeSet(). | |
| 732 * | |
| 733 * This may need to be looked at again as Lotus uses _LMBCSGetUnicodeSet(). (091216) | |
| 734 */ | |
| 735 | |
| 736 /* | |
| 737 Here's the basic helper function that we use when converting from | |
| 738 Unicode to LMBCS, and we suspect that a Unicode character will fit into | |
| 739 one of the 12 groups. The return value is the number of bytes written | |
| 740 starting at pStartLMBCS (if any). | |
| 741 */ | |
| 742 | |
| 743 static size_t | |
| 744 LMBCSConversionWorker ( | |
| 745 UConverterDataLMBCS * extraInfo, /* subconverters, opt & locale groups */ | |
| 746 ulmbcs_byte_t group, /* The group to try */ | |
| 747 ulmbcs_byte_t * pStartLMBCS, /* where to put the results */ | |
| 748 UChar * pUniChar, /* The input unicode character */ | |
| 749 ulmbcs_byte_t * lastConverterIndex, /* output: track last successful group used */ | |
| 750 UBool * groups_tried /* output: track any unsuccessful groups */ | |
| 751 ) | |
| 752 { | |
| 753 ulmbcs_byte_t * pLMBCS = pStartLMBCS; | |
| 754 UConverterSharedData * xcnv = extraInfo->OptGrpConverter[group]; | |
| 755 | |
| 756 int bytesConverted; | |
| 757 uint32_t value; | |
| 758 ulmbcs_byte_t firstByte; | |
| 759 | |
| 760 U_ASSERT(xcnv); | |
| 761 U_ASSERT(group<ULMBCS_GRP_UNICODE); | |
| 762 | |
| 763 bytesConverted = ucnv_MBCSFromUChar32(xcnv, *pUniChar, &value, FALSE); | |
| 764 | |
| 765 /* get the first result byte */ | |
| 766 if(bytesConverted > 0) { | |
| 767 firstByte = (ulmbcs_byte_t)(value >> ((bytesConverted - 1) * 8)); | |
| 768 } else { | |
| 769 /* most common failure mode is an unassigned character */ | |
| 770 groups_tried[group] = TRUE; | |
| 771 return 0; | |
| 772 } | |
| 773 | |
| 774 *lastConverterIndex = group; | |
| 775 | |
| 776 /* All initial byte values in lower ascii range should have been caught by now, | |
| 777 except with the exception group. | |
| 778 */ | |
| 779 U_ASSERT((firstByte <= ULMBCS_C0END) || (firstByte >= ULMBCS_C1START) || (group == ULMBCS_GRP_EXCEPT)); | |
| 780 | |
| 781 /* use converted data: first write 0, 1 or two group bytes */ | |
| 782 if (group != ULMBCS_GRP_EXCEPT && extraInfo->OptGroup != group) | |
| 783 { | |
| 784 *pLMBCS++ = group; | |
| 785 if (bytesConverted == 1 && group >= ULMBCS_DOUBLEOPTGROUP_START) | |
| 786 { | |
| 787 *pLMBCS++ = group; | |
| 788 } | |
| 789 } | |
| 790 | |
| 791 /* don't emit control chars */ | |
| 792 if ( bytesConverted == 1 && firstByte < 0x20 ) | |
| 793 return 0; | |
| 794 | |
| 795 | |
| 796 /* then move over the converted data */ | |
| 797 switch(bytesConverted) | |
| 798 { | |
| 799 case 4: | |
| 800 *pLMBCS++ = (ulmbcs_byte_t)(value >> 24); | |
| 801 case 3: /*fall through*/ | |
| 802 *pLMBCS++ = (ulmbcs_byte_t)(value >> 16); | |
| 803 case 2: /*fall through*/ | |
| 804 *pLMBCS++ = (ulmbcs_byte_t)(value >> 8); | |
| 805 case 1: /*fall through*/ | |
| 806 *pLMBCS++ = (ulmbcs_byte_t)value; | |
| 807 default: | |
| 808 /* will never occur */ | |
| 809 break; | |
| 810 } | |
| 811 | |
| 812 return (pLMBCS - pStartLMBCS); | |
| 813 } | |
| 814 | |
| 815 | |
| 816 /* This is a much simpler version of above, when we | |
| 817 know we are writing LMBCS using the Unicode group | |
| 818 */ | |
| 819 static size_t | |
| 820 LMBCSConvertUni(ulmbcs_byte_t * pLMBCS, UChar uniChar) | |
| 821 { | |
| 822 /* encode into LMBCS Unicode range */ | |
| 823 uint8_t LowCh = (uint8_t)(uniChar & 0x00FF); | |
| 824 uint8_t HighCh = (uint8_t)(uniChar >> 8); | |
| 825 | |
| 826 *pLMBCS++ = ULMBCS_GRP_UNICODE; | |
| 827 | |
| 828 if (LowCh == 0) | |
| 829 { | |
| 830 *pLMBCS++ = ULMBCS_UNICOMPATZERO; | |
| 831 *pLMBCS++ = HighCh; | |
| 832 } | |
| 833 else | |
| 834 { | |
| 835 *pLMBCS++ = HighCh; | |
| 836 *pLMBCS++ = LowCh; | |
| 837 } | |
| 838 return ULMBCS_UNICODE_SIZE; | |
| 839 } | |
| 840 | |
| 841 | |
| 842 | |
| 843 /* The main Unicode to LMBCS conversion function */ | |
| 844 static void | |
| 845 _LMBCSFromUnicode(UConverterFromUnicodeArgs* args, | |
| 846 UErrorCode* err) | |
| 847 { | |
| 848 ulmbcs_byte_t lastConverterIndex = 0; | |
| 849 UChar uniChar; | |
| 850 ulmbcs_byte_t LMBCS[ULMBCS_CHARSIZE_MAX]; | |
| 851 ulmbcs_byte_t * pLMBCS; | |
| 852 int32_t bytes_written; | |
| 853 UBool groups_tried[ULMBCS_GRP_LAST+1]; | |
| 854 UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; | |
| 855 int sourceIndex = 0; | |
| 856 | |
| 857 /* Basic strategy: attempt to fill in local LMBCS 1-char buffer.(LMBCS) | |
| 858 If that succeeds, see if it will all fit into the target & copy it over | |
| 859 if it does. | |
| 860 | |
| 861 We try conversions in the following order: | |
| 862 | |
| 863 1. Single-byte ascii & special fixed control chars (&null) | |
| 864 2. Look up group in table & try that (could be | |
| 865 A) Unicode group | |
| 866 B) control group, | |
| 867 C) national encoding, | |
| 868 or ambiguous SBCS or MBCS group (on to step 4...) | |
| 869 | |
| 870 3. If its ambiguous, try this order: | |
| 871 A) The optimization group | |
| 872 B) The locale group | |
| 873 C) The last group that succeeded with this string. | |
| 874 D) every other group that's relevent (single or double) | |
| 875 E) If its single-byte ambiguous, try the exceptions group | |
| 876 | |
| 877 4. And as a grand fallback: Unicode | |
| 878 */ | |
| 879 | |
| 880 /*Fix for SPR#DJOE66JFN3 (Lotus)*/ | |
| 881 ulmbcs_byte_t OldConverterIndex = 0; | |
| 882 | |
| 883 while (args->source < args->sourceLimit && !U_FAILURE(*err)) | |
| 884 { | |
| 885 /*Fix for SPR#DJOE66JFN3 (Lotus)*/ | |
| 886 OldConverterIndex = extraInfo->localeConverterIndex; | |
| 887 | |
| 888 if (args->target >= args->targetLimit) | |
| 889 { | |
| 890 *err = U_BUFFER_OVERFLOW_ERROR; | |
| 891 break; | |
| 892 } | |
| 893 uniChar = *(args->source); | |
| 894 bytes_written = 0; | |
| 895 pLMBCS = LMBCS; | |
| 896 | |
| 897 /* check cases in rough order of how common they are, for speed */ | |
| 898 | |
| 899 /* single byte matches: strategy 1 */ | |
| 900 /*Fix for SPR#DJOE66JFN3 (Lotus)*/ | |
| 901 if((uniChar>=0x80) && (uniChar<=0xff) | |
| 902 /*Fix for SPR#JUYA6XAERU and TSAO7GL5NK (Lotus)*/ &&(uniChar!=0xB1) &&(uniChar!=0xD7) &&(uniChar!=0xF7) | |
| 903 &&(uniChar!=0xB0) &&(uniChar!=0xB4) &&(uniChar!=0xB6) &&(uniChar!=0xA7) &&(uniChar!=0xA8)) | |
| 904 { | |
| 905 extraInfo->localeConverterIndex = ULMBCS_GRP_L1; | |
| 906 } | |
| 907 if (((uniChar > ULMBCS_C0END) && (uniChar < ULMBCS_C1START)) || | |
| 908 uniChar == 0 || uniChar == ULMBCS_HT || uniChar == ULMBCS_CR || | |
| 909 uniChar == ULMBCS_LF || uniChar == ULMBCS_123SYSTEMRANGE | |
| 910 ) | |
| 911 { | |
| 912 *pLMBCS++ = (ulmbcs_byte_t ) uniChar; | |
| 913 bytes_written = 1; | |
| 914 } | |
| 915 | |
| 916 | |
| 917 if (!bytes_written) | |
| 918 { | |
| 919 /* Check by UNICODE range (Strategy 2) */ | |
| 920 ulmbcs_byte_t group = FindLMBCSUniRange(uniChar); | |
| 921 | |
| 922 if (group == ULMBCS_GRP_UNICODE) /* (Strategy 2A) */ | |
| 923 { | |
| 924 pLMBCS += LMBCSConvertUni(pLMBCS,uniChar); | |
| 925 | |
| 926 bytes_written = (int32_t)(pLMBCS - LMBCS); | |
| 927 } | |
| 928 else if (group == ULMBCS_GRP_CTRL) /* (Strategy 2B) */ | |
| 929 { | |
| 930 /* Handle control characters here */ | |
| 931 if (uniChar <= ULMBCS_C0END) | |
| 932 { | |
| 933 *pLMBCS++ = ULMBCS_GRP_CTRL; | |
| 934 *pLMBCS++ = (ulmbcs_byte_t)(ULMBCS_CTRLOFFSET + uniChar); | |
| 935 } | |
| 936 else if (uniChar >= ULMBCS_C1START && uniChar <= ULMBCS_C1START + ULMBCS_CTRLOFFSET) | |
| 937 { | |
| 938 *pLMBCS++ = ULMBCS_GRP_CTRL; | |
| 939 *pLMBCS++ = (ulmbcs_byte_t ) (uniChar & 0x00FF); | |
| 940 } | |
| 941 bytes_written = (int32_t)(pLMBCS - LMBCS); | |
| 942 } | |
| 943 else if (group < ULMBCS_GRP_UNICODE) /* (Strategy 2C) */ | |
| 944 { | |
| 945 /* a specific converter has been identified - use it */ | |
| 946 bytes_written = (int32_t)LMBCSConversionWorker ( | |
| 947 extraInfo, group, pLMBCS, &uniChar, | |
| 948 &lastConverterIndex, groups_tried); | |
| 949 } | |
| 950 if (!bytes_written) /* the ambiguous group cases (Strategy 3) */ | |
| 951 { | |
| 952 uprv_memset(groups_tried, 0, sizeof(groups_tried)); | |
| 953 | |
| 954 /* check for non-default optimization group (Strategy 3A )*/ | |
| 955 if ((extraInfo->OptGroup != 1) && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->OptGroup))) | |
| 956 { | |
| 957 /*zhujin: upgrade, merge #39299 here (Lotus) */ | |
| 958 /*To make R5 compatible translation, look for exceptional group first for non-DBCS*/ | |
| 959 | |
| 960 if(extraInfo->localeConverterIndex < ULMBCS_DOUBLEOPTGROUP_START) | |
| 961 { | |
| 962 bytes_written = LMBCSConversionWorker (extraInfo, | |
| 963 ULMBCS_GRP_L1, pLMBCS, &uniChar, | |
| 964 &lastConverterIndex, groups_tried); | |
| 965 | |
| 966 if(!bytes_written) | |
| 967 { | |
| 968 bytes_written = LMBCSConversionWorker (extraInfo, | |
| 969 ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar, | |
| 970 &lastConverterIndex, groups_tried); | |
| 971 } | |
| 972 if(!bytes_written) | |
| 973 { | |
| 974 bytes_written = LMBCSConversionWorker (extraInfo, | |
| 975 extraInfo->localeConverterIndex, pLMBCS, &uniChar, | |
| 976 &lastConverterIndex, groups_tried); | |
| 977 } | |
| 978 } | |
| 979 else | |
| 980 { | |
| 981 bytes_written = LMBCSConversionWorker (extraInfo, | |
| 982 extraInfo->localeConverterIndex, pLMBCS, &uniChar, | |
| 983 &lastConverterIndex, groups_tried); | |
| 984 } | |
| 985 } | |
| 986 /* check for locale optimization group (Strategy 3B) */ | |
| 987 if (!bytes_written && (extraInfo->localeConverterIndex) && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->localeConverterIndex))) | |
| 988 { | |
| 989 bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, | |
| 990 extraInfo->localeConverterIndex, pLMBCS, &uniChar, &lastConverterIndex, groups_tried); | |
| 991 } | |
| 992 /* check for last optimization group used for this string (Strategy 3C) */ | |
| 993 if (!bytes_written && (lastConverterIndex) && (ULMBCS_AMBIGUOUS_MATCH(group, lastConverterIndex))) | |
| 994 { | |
| 995 bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, | |
| 996 lastConverterIndex, pLMBCS, &uniChar, &lastConverterIndex, groups_tried); | |
| 997 } | |
| 998 if (!bytes_written) | |
| 999 { | |
| 1000 /* just check every possible matching converter (Strategy 3D) */ | |
| 1001 ulmbcs_byte_t grp_start; | |
| 1002 ulmbcs_byte_t grp_end; | |
| 1003 ulmbcs_byte_t grp_ix; | |
| 1004 grp_start = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS) | |
| 1005 ? ULMBCS_DOUBLEOPTGROUP_START | |
| 1006 : ULMBCS_GRP_L1); | |
| 1007 grp_end = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS) | |
| 1008 ? ULMBCS_GRP_LAST | |
| 1009 : ULMBCS_GRP_TH); | |
| 1010 if(group == ULMBCS_AMBIGUOUS_ALL) | |
| 1011 { | |
| 1012 grp_start = ULMBCS_GRP_L1; | |
| 1013 grp_end = ULMBCS_GRP_LAST; | |
| 1014 } | |
| 1015 for (grp_ix = grp_start; | |
| 1016 grp_ix <= grp_end && !bytes_written; | |
| 1017 grp_ix++) | |
| 1018 { | |
| 1019 if (extraInfo->OptGrpConverter [grp_ix] && !groups_tried [grp_ix]) | |
| 1020 { | |
| 1021 bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, | |
| 1022 grp_ix, pLMBCS, &uniChar, | |
| 1023 &lastConverterIndex, groups_tried); | |
| 1024 } | |
| 1025 } | |
| 1026 /* a final conversion fallback to the exceptions group if its likely | |
| 1027 to be single byte (Strategy 3E) */ | |
| 1028 if (!bytes_written && grp_start == ULMBCS_GRP_L1) | |
| 1029 { | |
| 1030 bytes_written = (int32_t)LMBCSConversionWorker (extraInfo, | |
| 1031 ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar, | |
| 1032 &lastConverterIndex, groups_tried); | |
| 1033 } | |
| 1034 } | |
| 1035 /* all of our other strategies failed. Fallback to Unicode. (Strategy 4)*/ | |
| 1036 if (!bytes_written) | |
| 1037 { | |
| 1038 | |
| 1039 pLMBCS += LMBCSConvertUni(pLMBCS, uniChar); | |
| 1040 bytes_written = (int32_t)(pLMBCS - LMBCS); | |
| 1041 } | |
| 1042 } | |
| 1043 } | |
| 1044 | |
| 1045 /* we have a translation. increment source and write as much as posible to target */ | |
| 1046 args->source++; | |
| 1047 pLMBCS = LMBCS; | |
| 1048 while (args->target < args->targetLimit && bytes_written--) | |
| 1049 { | |
| 1050 *(args->target)++ = *pLMBCS++; | |
| 1051 if (args->offsets) | |
| 1052 { | |
| 1053 *(args->offsets)++ = sourceIndex; | |
| 1054 } | |
| 1055 } | |
| 1056 sourceIndex++; | |
| 1057 if (bytes_written > 0) | |
| 1058 { | |
| 1059 /* write any bytes that didn't fit in target to the error buffer, | |
| 1060 common code will move this to target if we get called back with | |
| 1061 enough target room | |
| 1062 */ | |
| 1063 uint8_t * pErrorBuffer = args->converter->charErrorBuffer; | |
| 1064 *err = U_BUFFER_OVERFLOW_ERROR; | |
| 1065 args->converter->charErrorBufferLength = (int8_t)bytes_written; | |
| 1066 while (bytes_written--) | |
| 1067 { | |
| 1068 *pErrorBuffer++ = *pLMBCS++; | |
| 1069 } | |
| 1070 } | |
| 1071 /*Fix for SPR#DJOE66JFN3 (Lotus)*/ | |
| 1072 extraInfo->localeConverterIndex = OldConverterIndex; | |
| 1073 } | |
| 1074 } | |
| 1075 | |
| 1076 | |
| 1077 /* Now, the Unicode from LMBCS section */ | |
| 1078 | |
| 1079 | |
| 1080 /* A function to call when we are looking at the Unicode group byte in LMBCS */ | |
| 1081 static UChar | |
| 1082 GetUniFromLMBCSUni(char const ** ppLMBCSin) /* Called with LMBCS-style Unicode byte stream */ | |
| 1083 { | |
| 1084 uint8_t HighCh = *(*ppLMBCSin)++; /* Big-endian Unicode in LMBCS compatibility group*/ | |
| 1085 uint8_t LowCh = *(*ppLMBCSin)++; | |
| 1086 | |
| 1087 if (HighCh == ULMBCS_UNICOMPATZERO ) | |
| 1088 { | |
| 1089 HighCh = LowCh; | |
| 1090 LowCh = 0; /* zero-byte in LSB special character */ | |
| 1091 } | |
| 1092 return (UChar)((HighCh << 8) | LowCh); | |
| 1093 } | |
| 1094 | |
| 1095 | |
| 1096 | |
| 1097 /* CHECK_SOURCE_LIMIT: Helper macro to verify that there are at least'index' | |
| 1098 bytes left in source up to sourceLimit.Errors appropriately if not. | |
| 1099 If we reach the limit, then update the source pointer to there to consume | |
| 1100 all input as required by ICU converter semantics. | |
| 1101 */ | |
| 1102 | |
| 1103 #define CHECK_SOURCE_LIMIT(index) \ | |
| 1104 if (args->source+index > args->sourceLimit){\ | |
| 1105 *err = U_TRUNCATED_CHAR_FOUND;\ | |
| 1106 args->source = args->sourceLimit;\ | |
| 1107 return 0xffff;} | |
| 1108 | |
| 1109 /* Return the Unicode representation for the current LMBCS character */ | |
| 1110 | |
| 1111 static UChar32 | |
| 1112 _LMBCSGetNextUCharWorker(UConverterToUnicodeArgs* args, | |
| 1113 UErrorCode* err) | |
| 1114 { | |
| 1115 UChar32 uniChar = 0; /* an output UNICODE char */ | |
| 1116 ulmbcs_byte_t CurByte; /* A byte from the input stream */ | |
| 1117 | |
| 1118 /* error check */ | |
| 1119 if (args->source >= args->sourceLimit) | |
| 1120 { | |
| 1121 *err = U_ILLEGAL_ARGUMENT_ERROR; | |
| 1122 return 0xffff; | |
| 1123 } | |
| 1124 /* Grab first byte & save address for error recovery */ | |
| 1125 CurByte = *((ulmbcs_byte_t *) (args->source++)); | |
| 1126 | |
| 1127 /* | |
| 1128 * at entry of each if clause: | |
| 1129 * 1. 'CurByte' points at the first byte of a LMBCS character | |
| 1130 * 2. '*source'points to the next byte of the source stream after 'CurByte' | |
| 1131 * | |
| 1132 * the job of each if clause is: | |
| 1133 * 1. set '*source' to point at the beginning of next char (nop if LMBCS char is only 1 byte) | |
| 1134 * 2. set 'uniChar' up with the right Unicode value, or set 'err' appropriately | |
| 1135 */ | |
| 1136 | |
| 1137 /* First lets check the simple fixed values. */ | |
| 1138 | |
| 1139 if(((CurByte > ULMBCS_C0END) && (CurByte < ULMBCS_C1START)) /* ascii range */ | |
| 1140 || (CurByte == 0) | |
| 1141 || CurByte == ULMBCS_HT || CurByte == ULMBCS_CR | |
| 1142 || CurByte == ULMBCS_LF || CurByte == ULMBCS_123SYSTEMRANGE) | |
| 1143 { | |
| 1144 uniChar = CurByte; | |
| 1145 } | |
| 1146 else | |
| 1147 { | |
| 1148 UConverterDataLMBCS * extraInfo; | |
| 1149 ulmbcs_byte_t group; | |
| 1150 UConverterSharedData *cnv; | |
| 1151 | |
| 1152 if (CurByte == ULMBCS_GRP_CTRL) /* Control character group - no opt group update */ | |
| 1153 { | |
| 1154 ulmbcs_byte_t C0C1byte; | |
| 1155 CHECK_SOURCE_LIMIT(1); | |
| 1156 C0C1byte = *(args->source)++; | |
| 1157 uniChar = (C0C1byte < ULMBCS_C1START) ? C0C1byte - ULMBCS_CTRLOFFSET : C0C1byte; | |
| 1158 } | |
| 1159 else | |
| 1160 if (CurByte == ULMBCS_GRP_UNICODE) /* Unicode compatibility group: BigEndian UTF16 */ | |
| 1161 { | |
| 1162 CHECK_SOURCE_LIMIT(2); | |
| 1163 | |
| 1164 /* don't check for error indicators fffe/ffff below */ | |
| 1165 return GetUniFromLMBCSUni(&(args->source)); | |
| 1166 } | |
| 1167 else if (CurByte <= ULMBCS_CTRLOFFSET) | |
| 1168 { | |
| 1169 group = CurByte; /* group byte is in the source */ | |
| 1170 extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; | |
| 1171 if (group > ULMBCS_GRP_LAST || (cnv = extraInfo->OptGrpConverter[group]) == NULL) | |
| 1172 { | |
| 1173 /* this is not a valid group byte - no converter*/ | |
| 1174 *err = U_INVALID_CHAR_FOUND; | |
| 1175 } | |
| 1176 else if (group >= ULMBCS_DOUBLEOPTGROUP_START) /* double byte conversion */ | |
| 1177 { | |
| 1178 | |
| 1179 CHECK_SOURCE_LIMIT(2); | |
| 1180 | |
| 1181 /* check for LMBCS doubled-group-byte case */ | |
| 1182 if (*args->source == group) { | |
| 1183 /* single byte */ | |
| 1184 ++args->source; | |
| 1185 uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 1, FALSE); | |
| 1186 ++args->source; | |
| 1187 } else { | |
| 1188 /* double byte */ | |
| 1189 uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 2, FALSE); | |
| 1190 args->source += 2; | |
| 1191 } | |
| 1192 } | |
| 1193 else { /* single byte conversion */ | |
| 1194 CHECK_SOURCE_LIMIT(1); | |
| 1195 CurByte = *(args->source)++; | |
| 1196 | |
| 1197 if (CurByte >= ULMBCS_C1START) | |
| 1198 { | |
| 1199 uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte); | |
| 1200 } | |
| 1201 else | |
| 1202 { | |
| 1203 /* The non-optimizable oddballs where there is an explicit byte | |
| 1204 * AND the second byte is not in the upper ascii range | |
| 1205 */ | |
| 1206 char bytes[2]; | |
| 1207 | |
| 1208 extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; | |
| 1209 cnv = extraInfo->OptGrpConverter [ULMBCS_GRP_EXCEPT]; | |
| 1210 | |
| 1211 /* Lookup value must include opt group */ | |
| 1212 bytes[0] = group; | |
| 1213 bytes[1] = CurByte; | |
| 1214 uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, bytes, 2, FALSE); | |
| 1215 } | |
| 1216 } | |
| 1217 } | |
| 1218 else if (CurByte >= ULMBCS_C1START) /* group byte is implicit */ | |
| 1219 { | |
| 1220 extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo; | |
| 1221 group = extraInfo->OptGroup; | |
| 1222 cnv = extraInfo->OptGrpConverter[group]; | |
| 1223 if (group >= ULMBCS_DOUBLEOPTGROUP_START) /* double byte conversion */ | |
| 1224 { | |
| 1225 if (!ucnv_MBCSIsLeadByte(cnv, CurByte)) | |
| 1226 { | |
| 1227 CHECK_SOURCE_LIMIT(0); | |
| 1228 | |
| 1229 /* let the MBCS conversion consume CurByte again */ | |
| 1230 uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 1, FALSE); | |
| 1231 } | |
| 1232 else | |
| 1233 { | |
| 1234 CHECK_SOURCE_LIMIT(1); | |
| 1235 /* let the MBCS conversion consume CurByte again */ | |
| 1236 uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 2, FALSE); | |
| 1237 ++args->source; | |
| 1238 } | |
| 1239 } | |
| 1240 else /* single byte conversion */ | |
| 1241 { | |
| 1242 uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte); | |
| 1243 } | |
| 1244 } | |
| 1245 } | |
| 1246 return uniChar; | |
| 1247 } | |
| 1248 | |
| 1249 | |
| 1250 /* The exported function that converts lmbcs to one or more | |
| 1251 UChars - currently UTF-16 | |
| 1252 */ | |
| 1253 static void | |
| 1254 _LMBCSToUnicodeWithOffsets(UConverterToUnicodeArgs* args, | |
| 1255 UErrorCode* err) | |
| 1256 { | |
| 1257 char LMBCS [ULMBCS_CHARSIZE_MAX]; | |
| 1258 UChar uniChar; /* one output UNICODE char */ | |
| 1259 const char * saveSource; /* beginning of current code point */ | |
| 1260 const char * pStartLMBCS = args->source; /* beginning of whole string */ | |
| 1261 const char * errSource = NULL; /* pointer to actual input in case an error occurs */ | |
| 1262 int8_t savebytes = 0; | |
| 1263 | |
| 1264 /* Process from source to limit, or until error */ | |
| 1265 while (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit > args->target) | |
| 1266 { | |
| 1267 saveSource = args->source; /* beginning of current code point */ | |
| 1268 | |
| 1269 if (args->converter->toULength) /* reassemble char from previous call */ | |
| 1270 { | |
| 1271 const char *saveSourceLimit; | |
| 1272 size_t size_old = args->converter->toULength; | |
| 1273 | |
| 1274 /* limit from source is either remainder of temp buffer, or user limit on source */ | |
| 1275 size_t size_new_maybe_1 = sizeof(LMBCS) - size_old; | |
| 1276 size_t size_new_maybe_2 = args->sourceLimit - args->source; | |
| 1277 size_t size_new = (size_new_maybe_1 < size_new_maybe_2) ? size_new_maybe_1 : size_new_maybe_2; | |
| 1278 | |
| 1279 | |
| 1280 uprv_memcpy(LMBCS, args->converter->toUBytes, size_old); | |
| 1281 uprv_memcpy(LMBCS + size_old, args->source, size_new); | |
| 1282 saveSourceLimit = args->sourceLimit; | |
| 1283 args->source = errSource = LMBCS; | |
| 1284 args->sourceLimit = LMBCS+size_old+size_new; | |
| 1285 savebytes = (int8_t)(size_old+size_new); | |
| 1286 uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err); | |
| 1287 args->source = saveSource + ((args->source - LMBCS) - size_old); | |
| 1288 args->sourceLimit = saveSourceLimit; | |
| 1289 | |
| 1290 if (*err == U_TRUNCATED_CHAR_FOUND) | |
| 1291 { | |
| 1292 /* evil special case: source buffers so small a char spans more than 2 buffers */ | |
| 1293 args->converter->toULength = savebytes; | |
| 1294 uprv_memcpy(args->converter->toUBytes, LMBCS, savebytes); | |
| 1295 args->source = args->sourceLimit; | |
| 1296 *err = U_ZERO_ERROR; | |
| 1297 return; | |
| 1298 } | |
| 1299 else | |
| 1300 { | |
| 1301 /* clear the partial-char marker */ | |
| 1302 args->converter->toULength = 0; | |
| 1303 } | |
| 1304 } | |
| 1305 else | |
| 1306 { | |
| 1307 errSource = saveSource; | |
| 1308 uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err); | |
| 1309 savebytes = (int8_t)(args->source - saveSource); | |
| 1310 } | |
| 1311 if (U_SUCCESS(*err)) | |
| 1312 { | |
| 1313 if (uniChar < 0xfffe) | |
| 1314 { | |
| 1315 *(args->target)++ = uniChar; | |
| 1316 if(args->offsets) | |
| 1317 { | |
| 1318 *(args->offsets)++ = (int32_t)(saveSource - pStartLMBCS); | |
| 1319 } | |
| 1320 } | |
| 1321 else if (uniChar == 0xfffe) | |
| 1322 { | |
| 1323 *err = U_INVALID_CHAR_FOUND; | |
| 1324 } | |
| 1325 else /* if (uniChar == 0xffff) */ | |
| 1326 { | |
| 1327 *err = U_ILLEGAL_CHAR_FOUND; | |
| 1328 } | |
| 1329 } | |
| 1330 } | |
| 1331 /* if target ran out before source, return U_BUFFER_OVERFLOW_ERROR */ | |
| 1332 if (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit <= args->target) | |
| 1333 { | |
| 1334 *err = U_BUFFER_OVERFLOW_ERROR; | |
| 1335 } | |
| 1336 else if (U_FAILURE(*err)) | |
| 1337 { | |
| 1338 /* If character incomplete or unmappable/illegal, store it in toUBytes[] */ | |
| 1339 args->converter->toULength = savebytes; | |
| 1340 if (savebytes > 0) { | |
| 1341 uprv_memcpy(args->converter->toUBytes, errSource, savebytes); | |
| 1342 } | |
| 1343 if (*err == U_TRUNCATED_CHAR_FOUND) { | |
| 1344 *err = U_ZERO_ERROR; | |
| 1345 } | |
| 1346 } | |
| 1347 } | |
| 1348 | |
| 1349 /* And now, the macroized declarations of data & functions: */ | |
| 1350 DEFINE_LMBCS_OPEN(1) | |
| 1351 DEFINE_LMBCS_OPEN(2) | |
| 1352 DEFINE_LMBCS_OPEN(3) | |
| 1353 DEFINE_LMBCS_OPEN(4) | |
| 1354 DEFINE_LMBCS_OPEN(5) | |
| 1355 DEFINE_LMBCS_OPEN(6) | |
| 1356 DEFINE_LMBCS_OPEN(8) | |
| 1357 DEFINE_LMBCS_OPEN(11) | |
| 1358 DEFINE_LMBCS_OPEN(16) | |
| 1359 DEFINE_LMBCS_OPEN(17) | |
| 1360 DEFINE_LMBCS_OPEN(18) | |
| 1361 DEFINE_LMBCS_OPEN(19) | |
| 1362 | |
| 1363 | |
| 1364 DECLARE_LMBCS_DATA(1) | |
| 1365 DECLARE_LMBCS_DATA(2) | |
| 1366 DECLARE_LMBCS_DATA(3) | |
| 1367 DECLARE_LMBCS_DATA(4) | |
| 1368 DECLARE_LMBCS_DATA(5) | |
| 1369 DECLARE_LMBCS_DATA(6) | |
| 1370 DECLARE_LMBCS_DATA(8) | |
| 1371 DECLARE_LMBCS_DATA(11) | |
| 1372 DECLARE_LMBCS_DATA(16) | |
| 1373 DECLARE_LMBCS_DATA(17) | |
| 1374 DECLARE_LMBCS_DATA(18) | |
| 1375 DECLARE_LMBCS_DATA(19) | |
| 1376 | |
| 1377 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ |
