intl/icu/source/i18n/bocsu.cpp@fc2d59ddac77 (annotated)
intl/icu/source/i18n/bocsu.cpp
Wed, 31 Dec 2014 07:22:50 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 07:22:50 +0100
- branch
- TOR_BUG_3246
- changeset 4
- fc2d59ddac77
- permissions
- -rw-r--r--
Correct previous dual key logic pending first delivery installment.
| michael@0 | 1 | /* |
| michael@0 | 2 | ******************************************************************************* |
| michael@0 | 3 | * Copyright (C) 2001-2011, International Business Machines |
| michael@0 | 4 | * Corporation and others. All Rights Reserved. |
| michael@0 | 5 | ******************************************************************************* |
| michael@0 | 6 | * file name: bocsu.cpp |
| michael@0 | 7 | * encoding: US-ASCII |
| michael@0 | 8 | * tab size: 8 (not used) |
| michael@0 | 9 | * indentation:4 |
| michael@0 | 10 | * |
| michael@0 | 11 | * Author: Markus W. Scherer |
| michael@0 | 12 | * |
| michael@0 | 13 | * Modification history: |
| michael@0 | 14 | * 05/18/2001 weiv Made into separate module |
| michael@0 | 15 | */ |
| michael@0 | 16 | |
| michael@0 | 17 | |
| michael@0 | 18 | #include "unicode/utypes.h" |
| michael@0 | 19 | |
| michael@0 | 20 | #if !UCONFIG_NO_COLLATION |
| michael@0 | 21 | |
| michael@0 | 22 | #include "unicode/bytestream.h" |
| michael@0 | 23 | #include "unicode/utf16.h" |
| michael@0 | 24 | #include "bocsu.h" |
| michael@0 | 25 | |
| michael@0 | 26 | /* |
| michael@0 | 27 | * encode one difference value -0x10ffff..+0x10ffff in 1..3 bytes, |
| michael@0 | 28 | * preserving lexical order |
| michael@0 | 29 | */ |
| michael@0 | 30 | U_CFUNC uint8_t * |
| michael@0 | 31 | u_writeDiff(int32_t diff, uint8_t *p) { |
| michael@0 | 32 | if(diff>=SLOPE_REACH_NEG_1) { |
| michael@0 | 33 | if(diff<=SLOPE_REACH_POS_1) { |
| michael@0 | 34 | *p++=(uint8_t)(SLOPE_MIDDLE+diff); |
| michael@0 | 35 | } else if(diff<=SLOPE_REACH_POS_2) { |
| michael@0 | 36 | *p++=(uint8_t)(SLOPE_START_POS_2+(diff/SLOPE_TAIL_COUNT)); |
| michael@0 | 37 | *p++=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); |
| michael@0 | 38 | } else if(diff<=SLOPE_REACH_POS_3) { |
| michael@0 | 39 | p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); |
| michael@0 | 40 | diff/=SLOPE_TAIL_COUNT; |
| michael@0 | 41 | p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); |
| michael@0 | 42 | *p=(uint8_t)(SLOPE_START_POS_3+(diff/SLOPE_TAIL_COUNT)); |
| michael@0 | 43 | p+=3; |
| michael@0 | 44 | } else { |
| michael@0 | 45 | p[3]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); |
| michael@0 | 46 | diff/=SLOPE_TAIL_COUNT; |
| michael@0 | 47 | p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); |
| michael@0 | 48 | diff/=SLOPE_TAIL_COUNT; |
| michael@0 | 49 | p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT); |
| michael@0 | 50 | *p=SLOPE_MAX; |
| michael@0 | 51 | p+=4; |
| michael@0 | 52 | } |
| michael@0 | 53 | } else { |
| michael@0 | 54 | int32_t m; |
| michael@0 | 55 | |
| michael@0 | 56 | if(diff>=SLOPE_REACH_NEG_2) { |
| michael@0 | 57 | NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); |
| michael@0 | 58 | *p++=(uint8_t)(SLOPE_START_NEG_2+diff); |
| michael@0 | 59 | *p++=(uint8_t)(SLOPE_MIN+m); |
| michael@0 | 60 | } else if(diff>=SLOPE_REACH_NEG_3) { |
| michael@0 | 61 | NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); |
| michael@0 | 62 | p[2]=(uint8_t)(SLOPE_MIN+m); |
| michael@0 | 63 | NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); |
| michael@0 | 64 | p[1]=(uint8_t)(SLOPE_MIN+m); |
| michael@0 | 65 | *p=(uint8_t)(SLOPE_START_NEG_3+diff); |
| michael@0 | 66 | p+=3; |
| michael@0 | 67 | } else { |
| michael@0 | 68 | NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); |
| michael@0 | 69 | p[3]=(uint8_t)(SLOPE_MIN+m); |
| michael@0 | 70 | NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); |
| michael@0 | 71 | p[2]=(uint8_t)(SLOPE_MIN+m); |
| michael@0 | 72 | NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); |
| michael@0 | 73 | p[1]=(uint8_t)(SLOPE_MIN+m); |
| michael@0 | 74 | *p=SLOPE_MIN; |
| michael@0 | 75 | p+=4; |
| michael@0 | 76 | } |
| michael@0 | 77 | } |
| michael@0 | 78 | return p; |
| michael@0 | 79 | } |
| michael@0 | 80 | |
| michael@0 | 81 | /* |
| michael@0 | 82 | * Encode the code points of a string as |
| michael@0 | 83 | * a sequence of byte-encoded differences (slope detection), |
| michael@0 | 84 | * preserving lexical order. |
| michael@0 | 85 | * |
| michael@0 | 86 | * Optimize the difference-taking for runs of Unicode text within |
| michael@0 | 87 | * small scripts: |
| michael@0 | 88 | * |
| michael@0 | 89 | * Most small scripts are allocated within aligned 128-blocks of Unicode |
| michael@0 | 90 | * code points. Lexical order is preserved if "prev" is always moved |
| michael@0 | 91 | * into the middle of such a block. |
| michael@0 | 92 | * |
| michael@0 | 93 | * Additionally, "prev" is moved from anywhere in the Unihan |
| michael@0 | 94 | * area into the middle of that area. |
| michael@0 | 95 | * Note that the identical-level run in a sort key is generated from |
| michael@0 | 96 | * NFD text - there are never Hangul characters included. |
| michael@0 | 97 | */ |
| michael@0 | 98 | U_CFUNC void |
| michael@0 | 99 | u_writeIdenticalLevelRun(const UChar *s, int32_t length, icu::ByteSink &sink) { |
| michael@0 | 100 | char scratch[64]; |
| michael@0 | 101 | int32_t capacity; |
| michael@0 | 102 | |
| michael@0 | 103 | UChar32 prev=0; |
| michael@0 | 104 | int32_t i=0; |
| michael@0 | 105 | while(i<length) { |
| michael@0 | 106 | char *buffer=sink.GetAppendBuffer(1, length*2, scratch, (int32_t)sizeof(scratch), &capacity); |
| michael@0 | 107 | uint8_t *p; |
| michael@0 | 108 | // We must have capacity>=SLOPE_MAX_BYTES in case u_writeDiff() writes that much, |
| michael@0 | 109 | // but we do not want to force the sink.GetAppendBuffer() to allocate |
| michael@0 | 110 | // for a large min_capacity because we might actually only write one byte. |
| michael@0 | 111 | if(capacity<16) { |
| michael@0 | 112 | buffer=scratch; |
| michael@0 | 113 | capacity=(int32_t)sizeof(scratch); |
| michael@0 | 114 | } |
| michael@0 | 115 | p=reinterpret_cast<uint8_t *>(buffer); |
| michael@0 | 116 | uint8_t *lastSafe=p+capacity-SLOPE_MAX_BYTES; |
| michael@0 | 117 | while(i<length && p<=lastSafe) { |
| michael@0 | 118 | if(prev<0x4e00 || prev>=0xa000) { |
| michael@0 | 119 | prev=(prev&~0x7f)-SLOPE_REACH_NEG_1; |
| michael@0 | 120 | } else { |
| michael@0 | 121 | /* |
| michael@0 | 122 | * Unihan U+4e00..U+9fa5: |
| michael@0 | 123 | * double-bytes down from the upper end |
| michael@0 | 124 | */ |
| michael@0 | 125 | prev=0x9fff-SLOPE_REACH_POS_2; |
| michael@0 | 126 | } |
| michael@0 | 127 | |
| michael@0 | 128 | UChar32 c; |
| michael@0 | 129 | U16_NEXT(s, i, length, c); |
| michael@0 | 130 | p=u_writeDiff(c-prev, p); |
| michael@0 | 131 | prev=c; |
| michael@0 | 132 | } |
| michael@0 | 133 | sink.Append(buffer, (int32_t)(p-reinterpret_cast<uint8_t *>(buffer))); |
| michael@0 | 134 | } |
| michael@0 | 135 | } |
| michael@0 | 136 | |
| michael@0 | 137 | U_CFUNC int32_t |
| michael@0 | 138 | u_writeIdenticalLevelRunTwoChars(UChar32 first, UChar32 second, uint8_t *p) { |
| michael@0 | 139 | uint8_t *p0 = p; |
| michael@0 | 140 | if(first<0x4e00 || first>=0xa000) { |
| michael@0 | 141 | first=(first&~0x7f)-SLOPE_REACH_NEG_1; |
| michael@0 | 142 | } else { |
| michael@0 | 143 | /* |
| michael@0 | 144 | * Unihan U+4e00..U+9fa5: |
| michael@0 | 145 | * double-bytes down from the upper end |
| michael@0 | 146 | */ |
| michael@0 | 147 | first=0x9fff-SLOPE_REACH_POS_2; |
| michael@0 | 148 | } |
| michael@0 | 149 | |
| michael@0 | 150 | p=u_writeDiff(second-first, p); |
| michael@0 | 151 | return (int32_t)(p-p0); |
| michael@0 | 152 | } |
| michael@0 | 153 | |
| michael@0 | 154 | #endif /* #if !UCONFIG_NO_COLLATION */ |
