The Tor Browser / file comparison
comparison: intl/icu/source/i18n/bocsu.cpp
intl/icu/source/i18n/bocsu.cpp
- branch
- TOR_BUG_3246
- changeset 4
- fc2d59ddac77
equal
deleted
inserted
replaced
| -1:000000000000 | 0:fe4e8ac33969 |
|---|---|
| 1 /* | |
| 2 ******************************************************************************* | |
| 3 * Copyright (C) 2001-2011, International Business Machines | |
| 4 * Corporation and others. All Rights Reserved. | |
| 5 ******************************************************************************* | |
| 6 * file name: bocsu.cpp | |
| 7 * encoding: US-ASCII | |
| 8 * tab size: 8 (not used) | |
| 9 * indentation:4 | |
| 10 * | |
| 11 * Author: Markus W. Scherer | |
| 12 * | |
| 13 * Modification history: | |
| 14 * 05/18/2001 weiv Made into separate module | |
| 15 */ | |
| 16 | |
| 17 | |
| 18 #include "unicode/utypes.h" | |
| 19 | |
| 20 #if !UCONFIG_NO_COLLATION | |
| 21 | |
| 22 #include "unicode/bytestream.h" | |
| 23 #include "unicode/utf16.h" | |
| 24 #include "bocsu.h" | |
| 25 | |
| 26 /* | |
| 27 * encode one difference value -0x10ffff..+0x10ffff in 1..3 bytes, | |
| 28 * preserving lexical order | |
| 29 */ | |
| 30 U_CFUNC uint8_t * | |
| 31 u_writeDiff(int32_t diff, uint8_t *p) { | |
| 32 if(diff>=SLOPE_REACH_NEG_1) { | |
| 33 if(diff<=SLOPE_REACH_POS_1) { | |
| 34 *p++=(uint8_t)(SLOPE_MIDDLE+diff); | |
| 35 } else if(diff<=SLOPE_REACH_POS_2) { | |
| 36 *p++=(uint8_t)(SLOPE_START_POS_2+(diff/SLOPE_TAIL_COUNT)); | |
| 37 *p++=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); | |
| 38 } else if(diff<=SLOPE_REACH_POS_3) { | |
| 39 p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); | |
| 40 diff/=SLOPE_TAIL_COUNT; | |
| 41 p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); | |
| 42 *p=(uint8_t)(SLOPE_START_POS_3+(diff/SLOPE_TAIL_COUNT)); | |
| 43 p+=3; | |
| 44 } else { | |
| 45 p[3]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); | |
| 46 diff/=SLOPE_TAIL_COUNT; | |
| 47 p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); | |
| 48 diff/=SLOPE_TAIL_COUNT; | |
| 49 p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); | |
| 50 *p=SLOPE_MAX; | |
| 51 p+=4; | |
| 52 } | |
| 53 } else { | |
| 54 int32_t m; | |
| 55 | |
| 56 if(diff>=SLOPE_REACH_NEG_2) { | |
| 57 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); | |
| 58 *p++=(uint8_t)(SLOPE_START_NEG_2+diff); | |
| 59 *p++=(uint8_t)(SLOPE_MIN+m); | |
| 60 } else if(diff>=SLOPE_REACH_NEG_3) { | |
| 61 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); | |
| 62 p[2]=(uint8_t)(SLOPE_MIN+m); | |
| 63 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); | |
| 64 p[1]=(uint8_t)(SLOPE_MIN+m); | |
| 65 *p=(uint8_t)(SLOPE_START_NEG_3+diff); | |
| 66 p+=3; | |
| 67 } else { | |
| 68 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); | |
| 69 p[3]=(uint8_t)(SLOPE_MIN+m); | |
| 70 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); | |
| 71 p[2]=(uint8_t)(SLOPE_MIN+m); | |
| 72 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); | |
| 73 p[1]=(uint8_t)(SLOPE_MIN+m); | |
| 74 *p=SLOPE_MIN; | |
| 75 p+=4; | |
| 76 } | |
| 77 } | |
| 78 return p; | |
| 79 } | |
| 80 | |
| 81 /* | |
| 82 * Encode the code points of a string as | |
| 83 * a sequence of byte-encoded differences (slope detection), | |
| 84 * preserving lexical order. | |
| 85 * | |
| 86 * Optimize the difference-taking for runs of Unicode text within | |
| 87 * small scripts: | |
| 88 * | |
| 89 * Most small scripts are allocated within aligned 128-blocks of Unicode | |
| 90 * code points. Lexical order is preserved if "prev" is always moved | |
| 91 * into the middle of such a block. | |
| 92 * | |
| 93 * Additionally, "prev" is moved from anywhere in the Unihan | |
| 94 * area into the middle of that area. | |
| 95 * Note that the identical-level run in a sort key is generated from | |
| 96 * NFD text - there are never Hangul characters included. | |
| 97 */ | |
| 98 U_CFUNC void | |
| 99 u_writeIdenticalLevelRun(const UChar *s, int32_t length, icu::ByteSink &sink) { | |
| 100 char scratch[64]; | |
| 101 int32_t capacity; | |
| 102 | |
| 103 UChar32 prev=0; | |
| 104 int32_t i=0; | |
| 105 while(i<length) { | |
| 106 char *buffer=sink.GetAppendBuffer(1, length*2, scratch, (int32_t)sizeof(scratch), &capacity); | |
| 107 uint8_t *p; | |
| 108 // We must have capacity>=SLOPE_MAX_BYTES in case u_writeDiff() writes that much, | |
| 109 // but we do not want to force the sink.GetAppendBuffer() to allocate | |
| 110 // for a large min_capacity because we might actually only write one byte. | |
| 111 if(capacity<16) { | |
| 112 buffer=scratch; | |
| 113 capacity=(int32_t)sizeof(scratch); | |
| 114 } | |
| 115 p=reinterpret_cast<uint8_t *>(buffer); | |
| 116 uint8_t *lastSafe=p+capacity-SLOPE_MAX_BYTES; | |
| 117 while(i<length && p<=lastSafe) { | |
| 118 if(prev<0x4e00 || prev>=0xa000) { | |
| 119 prev=(prev&~0x7f)-SLOPE_REACH_NEG_1; | |
| 120 } else { | |
| 121 /* | |
| 122 * Unihan U+4e00..U+9fa5: | |
| 123 * double-bytes down from the upper end | |
| 124 */ | |
| 125 prev=0x9fff-SLOPE_REACH_POS_2; | |
| 126 } | |
| 127 | |
| 128 UChar32 c; | |
| 129 U16_NEXT(s, i, length, c); | |
| 130 p=u_writeDiff(c-prev, p); | |
| 131 prev=c; | |
| 132 } | |
| 133 sink.Append(buffer, (int32_t)(p-reinterpret_cast<uint8_t *>(buffer))); | |
| 134 } | |
| 135 } | |
| 136 | |
| 137 U_CFUNC int32_t | |
| 138 u_writeIdenticalLevelRunTwoChars(UChar32 first, UChar32 second, uint8_t *p) { | |
| 139 uint8_t *p0 = p; | |
| 140 if(first<0x4e00 || first>=0xa000) { | |
| 141 first=(first&~0x7f)-SLOPE_REACH_NEG_1; | |
| 142 } else { | |
| 143 /* | |
| 144 * Unihan U+4e00..U+9fa5: | |
| 145 * double-bytes down from the upper end | |
| 146 */ | |
| 147 first=0x9fff-SLOPE_REACH_POS_2; | |
| 148 } | |
| 149 | |
| 150 p=u_writeDiff(second-first, p); | |
| 151 return (int32_t)(p-p0); | |
| 152 } | |
| 153 | |
| 154 #endif /* #if !UCONFIG_NO_COLLATION */ |
