The Tor Browser / file comparison

comparison: intl/icu/source/common/ucnv_u32.c

intl/icu/source/common/ucnv_u32.c

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:55b01d951aa7
1 /*
2 **********************************************************************
3 * Copyright (C) 2002-2011, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 * file name: ucnv_u32.c
7 * encoding: US-ASCII
8 * tab size: 8 (not used)
9 * indentation:4
10 *
11 * created on: 2002jul01
12 * created by: Markus W. Scherer
13 *
14 * UTF-32 converter implementation. Used to be in ucnv_utf.c.
15 */
16
17 #include "unicode/utypes.h"
18
19 #if !UCONFIG_NO_CONVERSION
20
21 #include "unicode/ucnv.h"
22 #include "unicode/utf.h"
23 #include "ucnv_bld.h"
24 #include "ucnv_cnv.h"
25 #include "cmemory.h"
26
27 #define MAXIMUM_UCS2 0x0000FFFF
28 #define MAXIMUM_UTF 0x0010FFFF
29 #define HALF_SHIFT 10
30 #define HALF_BASE 0x0010000
31 #define HALF_MASK 0x3FF
32 #define SURROGATE_HIGH_START 0xD800
33 #define SURROGATE_LOW_START 0xDC00
34
35 /* -SURROGATE_LOW_START + HALF_BASE */
36 #define SURROGATE_LOW_BASE 9216
37
38 enum {
39 UCNV_NEED_TO_WRITE_BOM=1
40 };
41
42 /* UTF-32BE ----------------------------------------------------------------- */
43
44 static void
45 T_UConverter_toUnicode_UTF32_BE(UConverterToUnicodeArgs * args,
46 UErrorCode * err)
47 {
48 const unsigned char *mySource = (unsigned char *) args->source;
49 UChar *myTarget = args->target;
50 const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit;
51 const UChar *targetLimit = args->targetLimit;
52 unsigned char *toUBytes = args->converter->toUBytes;
53 uint32_t ch, i;
54
55 /* Restore state of current sequence */
56 if (args->converter->toUnicodeStatus && myTarget < targetLimit) {
57 i = args->converter->toULength; /* restore # of bytes consumed */
58 args->converter->toULength = 0;
59
60 ch = args->converter->toUnicodeStatus - 1;/*Stores the previously calculated ch from a previous call*/
61 args->converter->toUnicodeStatus = 0;
62 goto morebytes;
63 }
64
65 while (mySource < sourceLimit && myTarget < targetLimit) {
66 i = 0;
67 ch = 0;
68 morebytes:
69 while (i < sizeof(uint32_t)) {
70 if (mySource < sourceLimit) {
71 ch = (ch << 8) | (uint8_t)(*mySource);
72 toUBytes[i++] = (char) *(mySource++);
73 }
74 else {
75 /* stores a partially calculated target*/
76 /* + 1 to make 0 a valid character */
77 args->converter->toUnicodeStatus = ch + 1;
78 args->converter->toULength = (int8_t) i;
79 goto donefornow;
80 }
81 }
82
83 if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) {
84 /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */
85 if (ch <= MAXIMUM_UCS2)
86 {
87 /* fits in 16 bits */
88 *(myTarget++) = (UChar) ch;
89 }
90 else {
91 /* write out the surrogates */
92 *(myTarget++) = U16_LEAD(ch);
93 ch = U16_TRAIL(ch);
94 if (myTarget < targetLimit) {
95 *(myTarget++) = (UChar)ch;
96 }
97 else {
98 /* Put in overflow buffer (not handled here) */
99 args->converter->UCharErrorBuffer[0] = (UChar) ch;
100 args->converter->UCharErrorBufferLength = 1;
101 *err = U_BUFFER_OVERFLOW_ERROR;
102 break;
103 }
104 }
105 }
106 else {
107 args->converter->toULength = (int8_t)i;
108 *err = U_ILLEGAL_CHAR_FOUND;
109 break;
110 }
111 }
112
113 donefornow:
114 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) {
115 /* End of target buffer */
116 *err = U_BUFFER_OVERFLOW_ERROR;
117 }
118
119 args->target = myTarget;
120 args->source = (const char *) mySource;
121 }
122
123 static void
124 T_UConverter_toUnicode_UTF32_BE_OFFSET_LOGIC(UConverterToUnicodeArgs * args,
125 UErrorCode * err)
126 {
127 const unsigned char *mySource = (unsigned char *) args->source;
128 UChar *myTarget = args->target;
129 int32_t *myOffsets = args->offsets;
130 const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit;
131 const UChar *targetLimit = args->targetLimit;
132 unsigned char *toUBytes = args->converter->toUBytes;
133 uint32_t ch, i;
134 int32_t offsetNum = 0;
135
136 /* Restore state of current sequence */
137 if (args->converter->toUnicodeStatus && myTarget < targetLimit) {
138 i = args->converter->toULength; /* restore # of bytes consumed */
139 args->converter->toULength = 0;
140
141 ch = args->converter->toUnicodeStatus - 1;/*Stores the previously calculated ch from a previous call*/
142 args->converter->toUnicodeStatus = 0;
143 goto morebytes;
144 }
145
146 while (mySource < sourceLimit && myTarget < targetLimit) {
147 i = 0;
148 ch = 0;
149 morebytes:
150 while (i < sizeof(uint32_t)) {
151 if (mySource < sourceLimit) {
152 ch = (ch << 8) | (uint8_t)(*mySource);
153 toUBytes[i++] = (char) *(mySource++);
154 }
155 else {
156 /* stores a partially calculated target*/
157 /* + 1 to make 0 a valid character */
158 args->converter->toUnicodeStatus = ch + 1;
159 args->converter->toULength = (int8_t) i;
160 goto donefornow;
161 }
162 }
163
164 if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) {
165 /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */
166 if (ch <= MAXIMUM_UCS2) {
167 /* fits in 16 bits */
168 *(myTarget++) = (UChar) ch;
169 *(myOffsets++) = offsetNum;
170 }
171 else {
172 /* write out the surrogates */
173 *(myTarget++) = U16_LEAD(ch);
174 *myOffsets++ = offsetNum;
175 ch = U16_TRAIL(ch);
176 if (myTarget < targetLimit)
177 {
178 *(myTarget++) = (UChar)ch;
179 *(myOffsets++) = offsetNum;
180 }
181 else {
182 /* Put in overflow buffer (not handled here) */
183 args->converter->UCharErrorBuffer[0] = (UChar) ch;
184 args->converter->UCharErrorBufferLength = 1;
185 *err = U_BUFFER_OVERFLOW_ERROR;
186 break;
187 }
188 }
189 }
190 else {
191 args->converter->toULength = (int8_t)i;
192 *err = U_ILLEGAL_CHAR_FOUND;
193 break;
194 }
195 offsetNum += i;
196 }
197
198 donefornow:
199 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
200 {
201 /* End of target buffer */
202 *err = U_BUFFER_OVERFLOW_ERROR;
203 }
204
205 args->target = myTarget;
206 args->source = (const char *) mySource;
207 args->offsets = myOffsets;
208 }
209
210 static void
211 T_UConverter_fromUnicode_UTF32_BE(UConverterFromUnicodeArgs * args,
212 UErrorCode * err)
213 {
214 const UChar *mySource = args->source;
215 unsigned char *myTarget;
216 const UChar *sourceLimit = args->sourceLimit;
217 const unsigned char *targetLimit = (unsigned char *) args->targetLimit;
218 UChar32 ch, ch2;
219 unsigned int indexToWrite;
220 unsigned char temp[sizeof(uint32_t)];
221
222 if(mySource >= sourceLimit) {
223 /* no input, nothing to do */
224 return;
225 }
226
227 /* write the BOM if necessary */
228 if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) {
229 static const char bom[]={ 0, 0, (char)0xfe, (char)0xff };
230 ucnv_fromUWriteBytes(args->converter,
231 bom, 4,
232 &args->target, args->targetLimit,
233 &args->offsets, -1,
234 err);
235 args->converter->fromUnicodeStatus=0;
236 }
237
238 myTarget = (unsigned char *) args->target;
239 temp[0] = 0;
240
241 if (args->converter->fromUChar32) {
242 ch = args->converter->fromUChar32;
243 args->converter->fromUChar32 = 0;
244 goto lowsurogate;
245 }
246
247 while (mySource < sourceLimit && myTarget < targetLimit) {
248 ch = *(mySource++);
249
250 if (U_IS_SURROGATE(ch)) {
251 if (U_IS_LEAD(ch)) {
252 lowsurogate:
253 if (mySource < sourceLimit) {
254 ch2 = *mySource;
255 if (U_IS_TRAIL(ch2)) {
256 ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE;
257 mySource++;
258 }
259 else {
260 /* this is an unmatched trail code unit (2nd surrogate) */
261 /* callback(illegal) */
262 args->converter->fromUChar32 = ch;
263 *err = U_ILLEGAL_CHAR_FOUND;
264 break;
265 }
266 }
267 else {
268 /* ran out of source */
269 args->converter->fromUChar32 = ch;
270 if (args->flush) {
271 /* this is an unmatched trail code unit (2nd surrogate) */
272 /* callback(illegal) */
273 *err = U_ILLEGAL_CHAR_FOUND;
274 }
275 break;
276 }
277 }
278 else {
279 /* this is an unmatched trail code unit (2nd surrogate) */
280 /* callback(illegal) */
281 args->converter->fromUChar32 = ch;
282 *err = U_ILLEGAL_CHAR_FOUND;
283 break;
284 }
285 }
286
287 /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */
288 temp[1] = (uint8_t) (ch >> 16 & 0x1F);
289 temp[2] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */
290 temp[3] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */
291
292 for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++) {
293 if (myTarget < targetLimit) {
294 *(myTarget++) = temp[indexToWrite];
295 }
296 else {
297 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite];
298 *err = U_BUFFER_OVERFLOW_ERROR;
299 }
300 }
301 }
302
303 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) {
304 *err = U_BUFFER_OVERFLOW_ERROR;
305 }
306
307 args->target = (char *) myTarget;
308 args->source = mySource;
309 }
310
311 static void
312 T_UConverter_fromUnicode_UTF32_BE_OFFSET_LOGIC(UConverterFromUnicodeArgs * args,
313 UErrorCode * err)
314 {
315 const UChar *mySource = args->source;
316 unsigned char *myTarget;
317 int32_t *myOffsets;
318 const UChar *sourceLimit = args->sourceLimit;
319 const unsigned char *targetLimit = (unsigned char *) args->targetLimit;
320 UChar32 ch, ch2;
321 int32_t offsetNum = 0;
322 unsigned int indexToWrite;
323 unsigned char temp[sizeof(uint32_t)];
324
325 if(mySource >= sourceLimit) {
326 /* no input, nothing to do */
327 return;
328 }
329
330 /* write the BOM if necessary */
331 if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) {
332 static const char bom[]={ 0, 0, (char)0xfe, (char)0xff };
333 ucnv_fromUWriteBytes(args->converter,
334 bom, 4,
335 &args->target, args->targetLimit,
336 &args->offsets, -1,
337 err);
338 args->converter->fromUnicodeStatus=0;
339 }
340
341 myTarget = (unsigned char *) args->target;
342 myOffsets = args->offsets;
343 temp[0] = 0;
344
345 if (args->converter->fromUChar32) {
346 ch = args->converter->fromUChar32;
347 args->converter->fromUChar32 = 0;
348 goto lowsurogate;
349 }
350
351 while (mySource < sourceLimit && myTarget < targetLimit) {
352 ch = *(mySource++);
353
354 if (U_IS_SURROGATE(ch)) {
355 if (U_IS_LEAD(ch)) {
356 lowsurogate:
357 if (mySource < sourceLimit) {
358 ch2 = *mySource;
359 if (U_IS_TRAIL(ch2)) {
360 ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE;
361 mySource++;
362 }
363 else {
364 /* this is an unmatched trail code unit (2nd surrogate) */
365 /* callback(illegal) */
366 args->converter->fromUChar32 = ch;
367 *err = U_ILLEGAL_CHAR_FOUND;
368 break;
369 }
370 }
371 else {
372 /* ran out of source */
373 args->converter->fromUChar32 = ch;
374 if (args->flush) {
375 /* this is an unmatched trail code unit (2nd surrogate) */
376 /* callback(illegal) */
377 *err = U_ILLEGAL_CHAR_FOUND;
378 }
379 break;
380 }
381 }
382 else {
383 /* this is an unmatched trail code unit (2nd surrogate) */
384 /* callback(illegal) */
385 args->converter->fromUChar32 = ch;
386 *err = U_ILLEGAL_CHAR_FOUND;
387 break;
388 }
389 }
390
391 /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */
392 temp[1] = (uint8_t) (ch >> 16 & 0x1F);
393 temp[2] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */
394 temp[3] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */
395
396 for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++) {
397 if (myTarget < targetLimit) {
398 *(myTarget++) = temp[indexToWrite];
399 *(myOffsets++) = offsetNum;
400 }
401 else {
402 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite];
403 *err = U_BUFFER_OVERFLOW_ERROR;
404 }
405 }
406 offsetNum = offsetNum + 1 + (temp[1] != 0);
407 }
408
409 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err)) {
410 *err = U_BUFFER_OVERFLOW_ERROR;
411 }
412
413 args->target = (char *) myTarget;
414 args->source = mySource;
415 args->offsets = myOffsets;
416 }
417
418 static UChar32
419 T_UConverter_getNextUChar_UTF32_BE(UConverterToUnicodeArgs* args,
420 UErrorCode* err)
421 {
422 const uint8_t *mySource;
423 UChar32 myUChar;
424 int32_t length;
425
426 mySource = (const uint8_t *)args->source;
427 if (mySource >= (const uint8_t *)args->sourceLimit)
428 {
429 /* no input */
430 *err = U_INDEX_OUTOFBOUNDS_ERROR;
431 return 0xffff;
432 }
433
434 length = (int32_t)((const uint8_t *)args->sourceLimit - mySource);
435 if (length < 4)
436 {
437 /* got a partial character */
438 uprv_memcpy(args->converter->toUBytes, mySource, length);
439 args->converter->toULength = (int8_t)length;
440 args->source = (const char *)(mySource + length);
441 *err = U_TRUNCATED_CHAR_FOUND;
442 return 0xffff;
443 }
444
445 /* Don't even try to do a direct cast because the value may be on an odd address. */
446 myUChar = ((UChar32)mySource[0] << 24)
447 | ((UChar32)mySource[1] << 16)
448 | ((UChar32)mySource[2] << 8)
449 | ((UChar32)mySource[3]);
450
451 args->source = (const char *)(mySource + 4);
452 if ((uint32_t)myUChar <= MAXIMUM_UTF && !U_IS_SURROGATE(myUChar)) {
453 return myUChar;
454 }
455
456 uprv_memcpy(args->converter->toUBytes, mySource, 4);
457 args->converter->toULength = 4;
458
459 *err = U_ILLEGAL_CHAR_FOUND;
460 return 0xffff;
461 }
462
463 static const UConverterImpl _UTF32BEImpl = {
464 UCNV_UTF32_BigEndian,
465
466 NULL,
467 NULL,
468
469 NULL,
470 NULL,
471 NULL,
472
473 T_UConverter_toUnicode_UTF32_BE,
474 T_UConverter_toUnicode_UTF32_BE_OFFSET_LOGIC,
475 T_UConverter_fromUnicode_UTF32_BE,
476 T_UConverter_fromUnicode_UTF32_BE_OFFSET_LOGIC,
477 T_UConverter_getNextUChar_UTF32_BE,
478
479 NULL,
480 NULL,
481 NULL,
482 NULL,
483 ucnv_getNonSurrogateUnicodeSet
484 };
485
486 /* The 1232 CCSID refers to any version of Unicode with any endianess of UTF-32 */
487 static const UConverterStaticData _UTF32BEStaticData = {
488 sizeof(UConverterStaticData),
489 "UTF-32BE",
490 1232,
491 UCNV_IBM, UCNV_UTF32_BigEndian, 4, 4,
492 { 0, 0, 0xff, 0xfd }, 4, FALSE, FALSE,
493 0,
494 0,
495 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
496 };
497
498 const UConverterSharedData _UTF32BEData = {
499 sizeof(UConverterSharedData), ~((uint32_t) 0),
500 NULL, NULL, &_UTF32BEStaticData, FALSE, &_UTF32BEImpl,
501 0
502 };
503
504 /* UTF-32LE ---------------------------------------------------------- */
505
506 static void
507 T_UConverter_toUnicode_UTF32_LE(UConverterToUnicodeArgs * args,
508 UErrorCode * err)
509 {
510 const unsigned char *mySource = (unsigned char *) args->source;
511 UChar *myTarget = args->target;
512 const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit;
513 const UChar *targetLimit = args->targetLimit;
514 unsigned char *toUBytes = args->converter->toUBytes;
515 uint32_t ch, i;
516
517 /* Restore state of current sequence */
518 if (args->converter->toUnicodeStatus && myTarget < targetLimit)
519 {
520 i = args->converter->toULength; /* restore # of bytes consumed */
521 args->converter->toULength = 0;
522
523 /* Stores the previously calculated ch from a previous call*/
524 ch = args->converter->toUnicodeStatus - 1;
525 args->converter->toUnicodeStatus = 0;
526 goto morebytes;
527 }
528
529 while (mySource < sourceLimit && myTarget < targetLimit)
530 {
531 i = 0;
532 ch = 0;
533 morebytes:
534 while (i < sizeof(uint32_t))
535 {
536 if (mySource < sourceLimit)
537 {
538 ch |= ((uint8_t)(*mySource)) << (i * 8);
539 toUBytes[i++] = (char) *(mySource++);
540 }
541 else
542 {
543 /* stores a partially calculated target*/
544 /* + 1 to make 0 a valid character */
545 args->converter->toUnicodeStatus = ch + 1;
546 args->converter->toULength = (int8_t) i;
547 goto donefornow;
548 }
549 }
550
551 if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch)) {
552 /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */
553 if (ch <= MAXIMUM_UCS2) {
554 /* fits in 16 bits */
555 *(myTarget++) = (UChar) ch;
556 }
557 else {
558 /* write out the surrogates */
559 *(myTarget++) = U16_LEAD(ch);
560 ch = U16_TRAIL(ch);
561 if (myTarget < targetLimit) {
562 *(myTarget++) = (UChar)ch;
563 }
564 else {
565 /* Put in overflow buffer (not handled here) */
566 args->converter->UCharErrorBuffer[0] = (UChar) ch;
567 args->converter->UCharErrorBufferLength = 1;
568 *err = U_BUFFER_OVERFLOW_ERROR;
569 break;
570 }
571 }
572 }
573 else {
574 args->converter->toULength = (int8_t)i;
575 *err = U_ILLEGAL_CHAR_FOUND;
576 break;
577 }
578 }
579
580 donefornow:
581 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
582 {
583 /* End of target buffer */
584 *err = U_BUFFER_OVERFLOW_ERROR;
585 }
586
587 args->target = myTarget;
588 args->source = (const char *) mySource;
589 }
590
591 static void
592 T_UConverter_toUnicode_UTF32_LE_OFFSET_LOGIC(UConverterToUnicodeArgs * args,
593 UErrorCode * err)
594 {
595 const unsigned char *mySource = (unsigned char *) args->source;
596 UChar *myTarget = args->target;
597 int32_t *myOffsets = args->offsets;
598 const unsigned char *sourceLimit = (unsigned char *) args->sourceLimit;
599 const UChar *targetLimit = args->targetLimit;
600 unsigned char *toUBytes = args->converter->toUBytes;
601 uint32_t ch, i;
602 int32_t offsetNum = 0;
603
604 /* Restore state of current sequence */
605 if (args->converter->toUnicodeStatus && myTarget < targetLimit)
606 {
607 i = args->converter->toULength; /* restore # of bytes consumed */
608 args->converter->toULength = 0;
609
610 /* Stores the previously calculated ch from a previous call*/
611 ch = args->converter->toUnicodeStatus - 1;
612 args->converter->toUnicodeStatus = 0;
613 goto morebytes;
614 }
615
616 while (mySource < sourceLimit && myTarget < targetLimit)
617 {
618 i = 0;
619 ch = 0;
620 morebytes:
621 while (i < sizeof(uint32_t))
622 {
623 if (mySource < sourceLimit)
624 {
625 ch |= ((uint8_t)(*mySource)) << (i * 8);
626 toUBytes[i++] = (char) *(mySource++);
627 }
628 else
629 {
630 /* stores a partially calculated target*/
631 /* + 1 to make 0 a valid character */
632 args->converter->toUnicodeStatus = ch + 1;
633 args->converter->toULength = (int8_t) i;
634 goto donefornow;
635 }
636 }
637
638 if (ch <= MAXIMUM_UTF && !U_IS_SURROGATE(ch))
639 {
640 /* Normal valid byte when the loop has not prematurely terminated (i < inBytes) */
641 if (ch <= MAXIMUM_UCS2)
642 {
643 /* fits in 16 bits */
644 *(myTarget++) = (UChar) ch;
645 *(myOffsets++) = offsetNum;
646 }
647 else {
648 /* write out the surrogates */
649 *(myTarget++) = U16_LEAD(ch);
650 *(myOffsets++) = offsetNum;
651 ch = U16_TRAIL(ch);
652 if (myTarget < targetLimit)
653 {
654 *(myTarget++) = (UChar)ch;
655 *(myOffsets++) = offsetNum;
656 }
657 else
658 {
659 /* Put in overflow buffer (not handled here) */
660 args->converter->UCharErrorBuffer[0] = (UChar) ch;
661 args->converter->UCharErrorBufferLength = 1;
662 *err = U_BUFFER_OVERFLOW_ERROR;
663 break;
664 }
665 }
666 }
667 else
668 {
669 args->converter->toULength = (int8_t)i;
670 *err = U_ILLEGAL_CHAR_FOUND;
671 break;
672 }
673 offsetNum += i;
674 }
675
676 donefornow:
677 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
678 {
679 /* End of target buffer */
680 *err = U_BUFFER_OVERFLOW_ERROR;
681 }
682
683 args->target = myTarget;
684 args->source = (const char *) mySource;
685 args->offsets = myOffsets;
686 }
687
688 static void
689 T_UConverter_fromUnicode_UTF32_LE(UConverterFromUnicodeArgs * args,
690 UErrorCode * err)
691 {
692 const UChar *mySource = args->source;
693 unsigned char *myTarget;
694 const UChar *sourceLimit = args->sourceLimit;
695 const unsigned char *targetLimit = (unsigned char *) args->targetLimit;
696 UChar32 ch, ch2;
697 unsigned int indexToWrite;
698 unsigned char temp[sizeof(uint32_t)];
699
700 if(mySource >= sourceLimit) {
701 /* no input, nothing to do */
702 return;
703 }
704
705 /* write the BOM if necessary */
706 if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) {
707 static const char bom[]={ (char)0xff, (char)0xfe, 0, 0 };
708 ucnv_fromUWriteBytes(args->converter,
709 bom, 4,
710 &args->target, args->targetLimit,
711 &args->offsets, -1,
712 err);
713 args->converter->fromUnicodeStatus=0;
714 }
715
716 myTarget = (unsigned char *) args->target;
717 temp[3] = 0;
718
719 if (args->converter->fromUChar32)
720 {
721 ch = args->converter->fromUChar32;
722 args->converter->fromUChar32 = 0;
723 goto lowsurogate;
724 }
725
726 while (mySource < sourceLimit && myTarget < targetLimit)
727 {
728 ch = *(mySource++);
729
730 if (U16_IS_SURROGATE(ch)) {
731 if (U16_IS_LEAD(ch))
732 {
733 lowsurogate:
734 if (mySource < sourceLimit)
735 {
736 ch2 = *mySource;
737 if (U16_IS_TRAIL(ch2)) {
738 ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE;
739 mySource++;
740 }
741 else {
742 /* this is an unmatched trail code unit (2nd surrogate) */
743 /* callback(illegal) */
744 args->converter->fromUChar32 = ch;
745 *err = U_ILLEGAL_CHAR_FOUND;
746 break;
747 }
748 }
749 else {
750 /* ran out of source */
751 args->converter->fromUChar32 = ch;
752 if (args->flush) {
753 /* this is an unmatched trail code unit (2nd surrogate) */
754 /* callback(illegal) */
755 *err = U_ILLEGAL_CHAR_FOUND;
756 }
757 break;
758 }
759 }
760 else {
761 /* this is an unmatched trail code unit (2nd surrogate) */
762 /* callback(illegal) */
763 args->converter->fromUChar32 = ch;
764 *err = U_ILLEGAL_CHAR_FOUND;
765 break;
766 }
767 }
768
769 /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */
770 temp[2] = (uint8_t) (ch >> 16 & 0x1F);
771 temp[1] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */
772 temp[0] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */
773
774 for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++)
775 {
776 if (myTarget < targetLimit)
777 {
778 *(myTarget++) = temp[indexToWrite];
779 }
780 else
781 {
782 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite];
783 *err = U_BUFFER_OVERFLOW_ERROR;
784 }
785 }
786 }
787
788 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
789 {
790 *err = U_BUFFER_OVERFLOW_ERROR;
791 }
792
793 args->target = (char *) myTarget;
794 args->source = mySource;
795 }
796
797 static void
798 T_UConverter_fromUnicode_UTF32_LE_OFFSET_LOGIC(UConverterFromUnicodeArgs * args,
799 UErrorCode * err)
800 {
801 const UChar *mySource = args->source;
802 unsigned char *myTarget;
803 int32_t *myOffsets;
804 const UChar *sourceLimit = args->sourceLimit;
805 const unsigned char *targetLimit = (unsigned char *) args->targetLimit;
806 UChar32 ch, ch2;
807 unsigned int indexToWrite;
808 unsigned char temp[sizeof(uint32_t)];
809 int32_t offsetNum = 0;
810
811 if(mySource >= sourceLimit) {
812 /* no input, nothing to do */
813 return;
814 }
815
816 /* write the BOM if necessary */
817 if(args->converter->fromUnicodeStatus==UCNV_NEED_TO_WRITE_BOM) {
818 static const char bom[]={ (char)0xff, (char)0xfe, 0, 0 };
819 ucnv_fromUWriteBytes(args->converter,
820 bom, 4,
821 &args->target, args->targetLimit,
822 &args->offsets, -1,
823 err);
824 args->converter->fromUnicodeStatus=0;
825 }
826
827 myTarget = (unsigned char *) args->target;
828 myOffsets = args->offsets;
829 temp[3] = 0;
830
831 if (args->converter->fromUChar32)
832 {
833 ch = args->converter->fromUChar32;
834 args->converter->fromUChar32 = 0;
835 goto lowsurogate;
836 }
837
838 while (mySource < sourceLimit && myTarget < targetLimit)
839 {
840 ch = *(mySource++);
841
842 if (U16_IS_SURROGATE(ch)) {
843 if (U16_IS_LEAD(ch))
844 {
845 lowsurogate:
846 if (mySource < sourceLimit)
847 {
848 ch2 = *mySource;
849 if (U16_IS_TRAIL(ch2))
850 {
851 ch = ((ch - SURROGATE_HIGH_START) << HALF_SHIFT) + ch2 + SURROGATE_LOW_BASE;
852 mySource++;
853 }
854 else {
855 /* this is an unmatched trail code unit (2nd surrogate) */
856 /* callback(illegal) */
857 args->converter->fromUChar32 = ch;
858 *err = U_ILLEGAL_CHAR_FOUND;
859 break;
860 }
861 }
862 else {
863 /* ran out of source */
864 args->converter->fromUChar32 = ch;
865 if (args->flush) {
866 /* this is an unmatched trail code unit (2nd surrogate) */
867 /* callback(illegal) */
868 *err = U_ILLEGAL_CHAR_FOUND;
869 }
870 break;
871 }
872 }
873 else {
874 /* this is an unmatched trail code unit (2nd surrogate) */
875 /* callback(illegal) */
876 args->converter->fromUChar32 = ch;
877 *err = U_ILLEGAL_CHAR_FOUND;
878 break;
879 }
880 }
881
882 /* We cannot get any larger than 10FFFF because we are coming from UTF-16 */
883 temp[2] = (uint8_t) (ch >> 16 & 0x1F);
884 temp[1] = (uint8_t) (ch >> 8); /* unsigned cast implicitly does (ch & FF) */
885 temp[0] = (uint8_t) (ch); /* unsigned cast implicitly does (ch & FF) */
886
887 for (indexToWrite = 0; indexToWrite <= sizeof(uint32_t) - 1; indexToWrite++)
888 {
889 if (myTarget < targetLimit)
890 {
891 *(myTarget++) = temp[indexToWrite];
892 *(myOffsets++) = offsetNum;
893 }
894 else
895 {
896 args->converter->charErrorBuffer[args->converter->charErrorBufferLength++] = temp[indexToWrite];
897 *err = U_BUFFER_OVERFLOW_ERROR;
898 }
899 }
900 offsetNum = offsetNum + 1 + (temp[2] != 0);
901 }
902
903 if (mySource < sourceLimit && myTarget >= targetLimit && U_SUCCESS(*err))
904 {
905 *err = U_BUFFER_OVERFLOW_ERROR;
906 }
907
908 args->target = (char *) myTarget;
909 args->source = mySource;
910 args->offsets = myOffsets;
911 }
912
913 static UChar32
914 T_UConverter_getNextUChar_UTF32_LE(UConverterToUnicodeArgs* args,
915 UErrorCode* err)
916 {
917 const uint8_t *mySource;
918 UChar32 myUChar;
919 int32_t length;
920
921 mySource = (const uint8_t *)args->source;
922 if (mySource >= (const uint8_t *)args->sourceLimit)
923 {
924 /* no input */
925 *err = U_INDEX_OUTOFBOUNDS_ERROR;
926 return 0xffff;
927 }
928
929 length = (int32_t)((const uint8_t *)args->sourceLimit - mySource);
930 if (length < 4)
931 {
932 /* got a partial character */
933 uprv_memcpy(args->converter->toUBytes, mySource, length);
934 args->converter->toULength = (int8_t)length;
935 args->source = (const char *)(mySource + length);
936 *err = U_TRUNCATED_CHAR_FOUND;
937 return 0xffff;
938 }
939
940 /* Don't even try to do a direct cast because the value may be on an odd address. */
941 myUChar = ((UChar32)mySource[3] << 24)
942 | ((UChar32)mySource[2] << 16)
943 | ((UChar32)mySource[1] << 8)
944 | ((UChar32)mySource[0]);
945
946 args->source = (const char *)(mySource + 4);
947 if ((uint32_t)myUChar <= MAXIMUM_UTF && !U_IS_SURROGATE(myUChar)) {
948 return myUChar;
949 }
950
951 uprv_memcpy(args->converter->toUBytes, mySource, 4);
952 args->converter->toULength = 4;
953
954 *err = U_ILLEGAL_CHAR_FOUND;
955 return 0xffff;
956 }
957
958 static const UConverterImpl _UTF32LEImpl = {
959 UCNV_UTF32_LittleEndian,
960
961 NULL,
962 NULL,
963
964 NULL,
965 NULL,
966 NULL,
967
968 T_UConverter_toUnicode_UTF32_LE,
969 T_UConverter_toUnicode_UTF32_LE_OFFSET_LOGIC,
970 T_UConverter_fromUnicode_UTF32_LE,
971 T_UConverter_fromUnicode_UTF32_LE_OFFSET_LOGIC,
972 T_UConverter_getNextUChar_UTF32_LE,
973
974 NULL,
975 NULL,
976 NULL,
977 NULL,
978 ucnv_getNonSurrogateUnicodeSet
979 };
980
981 /* The 1232 CCSID refers to any version of Unicode with any endianess of UTF-32 */
982 static const UConverterStaticData _UTF32LEStaticData = {
983 sizeof(UConverterStaticData),
984 "UTF-32LE",
985 1234,
986 UCNV_IBM, UCNV_UTF32_LittleEndian, 4, 4,
987 { 0xfd, 0xff, 0, 0 }, 4, FALSE, FALSE,
988 0,
989 0,
990 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
991 };
992
993
994 const UConverterSharedData _UTF32LEData = {
995 sizeof(UConverterSharedData), ~((uint32_t) 0),
996 NULL, NULL, &_UTF32LEStaticData, FALSE, &_UTF32LEImpl,
997 0
998 };
999
1000 /* UTF-32 (Detect BOM) ------------------------------------------------------ */
1001
1002 /*
1003 * Detect a BOM at the beginning of the stream and select UTF-32BE or UTF-32LE
1004 * accordingly.
1005 *
1006 * State values:
1007 * 0 initial state
1008 * 1 saw 00
1009 * 2 saw 00 00
1010 * 3 saw 00 00 FE
1011 * 4 -
1012 * 5 saw FF
1013 * 6 saw FF FE
1014 * 7 saw FF FE 00
1015 * 8 UTF-32BE mode
1016 * 9 UTF-32LE mode
1017 *
1018 * During detection: state&3==number of matching bytes so far.
1019 *
1020 * On output, emit U+FEFF as the first code point.
1021 */
1022
1023 static void
1024 _UTF32Reset(UConverter *cnv, UConverterResetChoice choice) {
1025 if(choice<=UCNV_RESET_TO_UNICODE) {
1026 /* reset toUnicode: state=0 */
1027 cnv->mode=0;
1028 }
1029 if(choice!=UCNV_RESET_TO_UNICODE) {
1030 /* reset fromUnicode: prepare to output the UTF-32PE BOM */
1031 cnv->fromUnicodeStatus=UCNV_NEED_TO_WRITE_BOM;
1032 }
1033 }
1034
1035 static void
1036 _UTF32Open(UConverter *cnv,
1037 UConverterLoadArgs *pArgs,
1038 UErrorCode *pErrorCode) {
1039 _UTF32Reset(cnv, UCNV_RESET_BOTH);
1040 }
1041
1042 static const char utf32BOM[8]={ 0, 0, (char)0xfe, (char)0xff, (char)0xff, (char)0xfe, 0, 0 };
1043
1044 static void
1045 _UTF32ToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
1046 UErrorCode *pErrorCode) {
1047 UConverter *cnv=pArgs->converter;
1048 const char *source=pArgs->source;
1049 const char *sourceLimit=pArgs->sourceLimit;
1050 int32_t *offsets=pArgs->offsets;
1051
1052 int32_t state, offsetDelta;
1053 char b;
1054
1055 state=cnv->mode;
1056
1057 /*
1058 * If we detect a BOM in this buffer, then we must add the BOM size to the
1059 * offsets because the actual converter function will not see and count the BOM.
1060 * offsetDelta will have the number of the BOM bytes that are in the current buffer.
1061 */
1062 offsetDelta=0;
1063
1064 while(source<sourceLimit && U_SUCCESS(*pErrorCode)) {
1065 switch(state) {
1066 case 0:
1067 b=*source;
1068 if(b==0) {
1069 state=1; /* could be 00 00 FE FF */
1070 } else if(b==(char)0xff) {
1071 state=5; /* could be FF FE 00 00 */
1072 } else {
1073 state=8; /* default to UTF-32BE */
1074 continue;
1075 }
1076 ++source;
1077 break;
1078 case 1:
1079 case 2:
1080 case 3:
1081 case 5:
1082 case 6:
1083 case 7:
1084 if(*source==utf32BOM[state]) {
1085 ++state;
1086 ++source;
1087 if(state==4) {
1088 state=8; /* detect UTF-32BE */
1089 offsetDelta=(int32_t)(source-pArgs->source);
1090 } else if(state==8) {
1091 state=9; /* detect UTF-32LE */
1092 offsetDelta=(int32_t)(source-pArgs->source);
1093 }
1094 } else {
1095 /* switch to UTF-32BE and pass the previous bytes */
1096 int32_t count=(int32_t)(source-pArgs->source); /* number of bytes from this buffer */
1097
1098 /* reset the source */
1099 source=pArgs->source;
1100
1101 if(count==(state&3)) {
1102 /* simple: all in the same buffer, just reset source */
1103 } else {
1104 UBool oldFlush=pArgs->flush;
1105
1106 /* some of the bytes are from a previous buffer, replay those first */
1107 pArgs->source=utf32BOM+(state&4); /* select the correct BOM */
1108 pArgs->sourceLimit=pArgs->source+((state&3)-count); /* replay previous bytes */
1109 pArgs->flush=FALSE; /* this sourceLimit is not the real source stream limit */
1110
1111 /* no offsets: bytes from previous buffer, and not enough for output */
1112 T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode);
1113
1114 /* restore real pointers; pArgs->source will be set in case 8/9 */
1115 pArgs->sourceLimit=sourceLimit;
1116 pArgs->flush=oldFlush;
1117 }
1118 state=8;
1119 continue;
1120 }
1121 break;
1122 case 8:
1123 /* call UTF-32BE */
1124 pArgs->source=source;
1125 if(offsets==NULL) {
1126 T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode);
1127 } else {
1128 T_UConverter_toUnicode_UTF32_BE_OFFSET_LOGIC(pArgs, pErrorCode);
1129 }
1130 source=pArgs->source;
1131 break;
1132 case 9:
1133 /* call UTF-32LE */
1134 pArgs->source=source;
1135 if(offsets==NULL) {
1136 T_UConverter_toUnicode_UTF32_LE(pArgs, pErrorCode);
1137 } else {
1138 T_UConverter_toUnicode_UTF32_LE_OFFSET_LOGIC(pArgs, pErrorCode);
1139 }
1140 source=pArgs->source;
1141 break;
1142 default:
1143 break; /* does not occur */
1144 }
1145 }
1146
1147 /* add BOM size to offsets - see comment at offsetDelta declaration */
1148 if(offsets!=NULL && offsetDelta!=0) {
1149 int32_t *offsetsLimit=pArgs->offsets;
1150 while(offsets<offsetsLimit) {
1151 *offsets++ += offsetDelta;
1152 }
1153 }
1154
1155 pArgs->source=source;
1156
1157 if(source==sourceLimit && pArgs->flush) {
1158 /* handle truncated input */
1159 switch(state) {
1160 case 0:
1161 break; /* no input at all, nothing to do */
1162 case 8:
1163 T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode);
1164 break;
1165 case 9:
1166 T_UConverter_toUnicode_UTF32_LE(pArgs, pErrorCode);
1167 break;
1168 default:
1169 /* handle 0<state<8: call UTF-32BE with too-short input */
1170 pArgs->source=utf32BOM+(state&4); /* select the correct BOM */
1171 pArgs->sourceLimit=pArgs->source+(state&3); /* replay bytes */
1172
1173 /* no offsets: not enough for output */
1174 T_UConverter_toUnicode_UTF32_BE(pArgs, pErrorCode);
1175 pArgs->source=source;
1176 pArgs->sourceLimit=sourceLimit;
1177 state=8;
1178 break;
1179 }
1180 }
1181
1182 cnv->mode=state;
1183 }
1184
1185 static UChar32
1186 _UTF32GetNextUChar(UConverterToUnicodeArgs *pArgs,
1187 UErrorCode *pErrorCode) {
1188 switch(pArgs->converter->mode) {
1189 case 8:
1190 return T_UConverter_getNextUChar_UTF32_BE(pArgs, pErrorCode);
1191 case 9:
1192 return T_UConverter_getNextUChar_UTF32_LE(pArgs, pErrorCode);
1193 default:
1194 return UCNV_GET_NEXT_UCHAR_USE_TO_U;
1195 }
1196 }
1197
1198 static const UConverterImpl _UTF32Impl = {
1199 UCNV_UTF32,
1200
1201 NULL,
1202 NULL,
1203
1204 _UTF32Open,
1205 NULL,
1206 _UTF32Reset,
1207
1208 _UTF32ToUnicodeWithOffsets,
1209 _UTF32ToUnicodeWithOffsets,
1210 #if U_IS_BIG_ENDIAN
1211 T_UConverter_fromUnicode_UTF32_BE,
1212 T_UConverter_fromUnicode_UTF32_BE_OFFSET_LOGIC,
1213 #else
1214 T_UConverter_fromUnicode_UTF32_LE,
1215 T_UConverter_fromUnicode_UTF32_LE_OFFSET_LOGIC,
1216 #endif
1217 _UTF32GetNextUChar,
1218
1219 NULL, /* ### TODO implement getStarters for all Unicode encodings?! */
1220 NULL,
1221 NULL,
1222 NULL,
1223 ucnv_getNonSurrogateUnicodeSet
1224 };
1225
1226 /* The 1236 CCSID refers to any version of Unicode with a BOM sensitive endianess of UTF-32 */
1227 static const UConverterStaticData _UTF32StaticData = {
1228 sizeof(UConverterStaticData),
1229 "UTF-32",
1230 1236,
1231 UCNV_IBM, UCNV_UTF32, 4, 4,
1232 #if U_IS_BIG_ENDIAN
1233 { 0, 0, 0xff, 0xfd }, 4,
1234 #else
1235 { 0xfd, 0xff, 0, 0 }, 4,
1236 #endif
1237 FALSE, FALSE,
1238 0,
1239 0,
1240 { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } /* reserved */
1241 };
1242
1243 const UConverterSharedData _UTF32Data = {
1244 sizeof(UConverterSharedData), ~((uint32_t) 0),
1245 NULL, NULL, &_UTF32StaticData, FALSE, &_UTF32Impl,
1246 0
1247 };
1248
1249 #endif

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial