The Tor Browser / annotate

intl/icu/source/common/unorm.cpp@b8a032363ba2 (annotated)

intl/icu/source/common/unorm.cpp

Thu, 22 Jan 2015 13:21:57 +0100

author
Michael Schloh von Bennewitz <michael@schloh.com>
date
Thu, 22 Jan 2015 13:21:57 +0100
branch
TOR_BUG_9701
changeset 15
b8a032363ba2
permissions
-rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

michael@0 1 /*
michael@0 2 ******************************************************************************
michael@0 3 * Copyright (c) 1996-2011, International Business Machines
michael@0 4 * Corporation and others. All Rights Reserved.
michael@0 5 ******************************************************************************
michael@0 6 * File unorm.cpp
michael@0 7 *
michael@0 8 * Created by: Vladimir Weinstein 12052000
michael@0 9 *
michael@0 10 * Modification history :
michael@0 11 *
michael@0 12 * Date Name Description
michael@0 13 * 02/01/01 synwee Added normalization quickcheck enum and method.
michael@0 14 * 02/12/01 synwee Commented out quickcheck util api has been approved
michael@0 15 * Added private method for doing FCD checks
michael@0 16 * 02/23/01 synwee Modified quickcheck and checkFCE to run through
michael@0 17 * string for codepoints < 0x300 for the normalization
michael@0 18 * mode NFC.
michael@0 19 * 05/25/01+ Markus Scherer total rewrite, implement all normalization here
michael@0 20 * instead of just wrappers around normlzr.cpp,
michael@0 21 * load unorm.dat, support Unicode 3.1 with
michael@0 22 * supplementary code points, etc.
michael@0 23 * 2009-nov..2010-jan Markus Scherer total rewrite, new Normalizer2 API & code
michael@0 24 */
michael@0 25
michael@0 26 #include "unicode/utypes.h"
michael@0 27
michael@0 28 #if !UCONFIG_NO_NORMALIZATION
michael@0 29
michael@0 30 #include "unicode/udata.h"
michael@0 31 #include "unicode/ustring.h"
michael@0 32 #include "unicode/uiter.h"
michael@0 33 #include "unicode/unorm.h"
michael@0 34 #include "unicode/unorm2.h"
michael@0 35 #include "normalizer2impl.h"
michael@0 36 #include "unormimp.h"
michael@0 37 #include "uprops.h"
michael@0 38 #include "ustr_imp.h"
michael@0 39
michael@0 40 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
michael@0 41
michael@0 42 U_NAMESPACE_USE
michael@0 43
michael@0 44 /* quick check functions ---------------------------------------------------- */
michael@0 45
michael@0 46 U_CAPI UNormalizationCheckResult U_EXPORT2
michael@0 47 unorm_quickCheck(const UChar *src,
michael@0 48 int32_t srcLength,
michael@0 49 UNormalizationMode mode,
michael@0 50 UErrorCode *pErrorCode) {
michael@0 51 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 52 return unorm2_quickCheck((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
michael@0 53 }
michael@0 54
michael@0 55 U_CAPI UNormalizationCheckResult U_EXPORT2
michael@0 56 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
michael@0 57 UNormalizationMode mode, int32_t options,
michael@0 58 UErrorCode *pErrorCode) {
michael@0 59 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 60 if(options&UNORM_UNICODE_3_2) {
michael@0 61 FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*pErrorCode));
michael@0 62 return unorm2_quickCheck(
michael@0 63 reinterpret_cast<const UNormalizer2 *>(static_cast<Normalizer2 *>(&fn2)),
michael@0 64 src, srcLength, pErrorCode);
michael@0 65 } else {
michael@0 66 return unorm2_quickCheck((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
michael@0 67 }
michael@0 68 }
michael@0 69
michael@0 70 U_CAPI UBool U_EXPORT2
michael@0 71 unorm_isNormalized(const UChar *src, int32_t srcLength,
michael@0 72 UNormalizationMode mode,
michael@0 73 UErrorCode *pErrorCode) {
michael@0 74 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 75 return unorm2_isNormalized((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
michael@0 76 }
michael@0 77
michael@0 78 U_CAPI UBool U_EXPORT2
michael@0 79 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
michael@0 80 UNormalizationMode mode, int32_t options,
michael@0 81 UErrorCode *pErrorCode) {
michael@0 82 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 83 if(options&UNORM_UNICODE_3_2) {
michael@0 84 FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*pErrorCode));
michael@0 85 return unorm2_isNormalized(
michael@0 86 reinterpret_cast<const UNormalizer2 *>(static_cast<Normalizer2 *>(&fn2)),
michael@0 87 src, srcLength, pErrorCode);
michael@0 88 } else {
michael@0 89 return unorm2_isNormalized((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
michael@0 90 }
michael@0 91 }
michael@0 92
michael@0 93 /* normalize() API ---------------------------------------------------------- */
michael@0 94
michael@0 95 /** Public API for normalizing. */
michael@0 96 U_CAPI int32_t U_EXPORT2
michael@0 97 unorm_normalize(const UChar *src, int32_t srcLength,
michael@0 98 UNormalizationMode mode, int32_t options,
michael@0 99 UChar *dest, int32_t destCapacity,
michael@0 100 UErrorCode *pErrorCode) {
michael@0 101 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 102 if(options&UNORM_UNICODE_3_2) {
michael@0 103 FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*pErrorCode));
michael@0 104 return unorm2_normalize(
michael@0 105 reinterpret_cast<const UNormalizer2 *>(static_cast<Normalizer2 *>(&fn2)),
michael@0 106 src, srcLength, dest, destCapacity, pErrorCode);
michael@0 107 } else {
michael@0 108 return unorm2_normalize((const UNormalizer2 *)n2,
michael@0 109 src, srcLength, dest, destCapacity, pErrorCode);
michael@0 110 }
michael@0 111 }
michael@0 112
michael@0 113
michael@0 114 /* iteration functions ------------------------------------------------------ */
michael@0 115
michael@0 116 static int32_t
michael@0 117 _iterate(UCharIterator *src, UBool forward,
michael@0 118 UChar *dest, int32_t destCapacity,
michael@0 119 const Normalizer2 *n2,
michael@0 120 UBool doNormalize, UBool *pNeededToNormalize,
michael@0 121 UErrorCode *pErrorCode) {
michael@0 122 if(U_FAILURE(*pErrorCode)) {
michael@0 123 return 0;
michael@0 124 }
michael@0 125 if(destCapacity<0 || (dest==NULL && destCapacity>0) || src==NULL) {
michael@0 126 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
michael@0 127 return 0;
michael@0 128 }
michael@0 129
michael@0 130 if(pNeededToNormalize!=NULL) {
michael@0 131 *pNeededToNormalize=FALSE;
michael@0 132 }
michael@0 133 if(!(forward ? src->hasNext(src) : src->hasPrevious(src))) {
michael@0 134 return u_terminateUChars(dest, destCapacity, 0, pErrorCode);
michael@0 135 }
michael@0 136
michael@0 137 UnicodeString buffer;
michael@0 138 UChar32 c;
michael@0 139 if(forward) {
michael@0 140 /* get one character and ignore its properties */
michael@0 141 buffer.append(uiter_next32(src));
michael@0 142 /* get all following characters until we see a boundary */
michael@0 143 while((c=uiter_next32(src))>=0) {
michael@0 144 if(n2->hasBoundaryBefore(c)) {
michael@0 145 /* back out the latest movement to stop at the boundary */
michael@0 146 src->move(src, -U16_LENGTH(c), UITER_CURRENT);
michael@0 147 break;
michael@0 148 } else {
michael@0 149 buffer.append(c);
michael@0 150 }
michael@0 151 }
michael@0 152 } else {
michael@0 153 while((c=uiter_previous32(src))>=0) {
michael@0 154 /* always write this character to the front of the buffer */
michael@0 155 buffer.insert(0, c);
michael@0 156 /* stop if this just-copied character is a boundary */
michael@0 157 if(n2->hasBoundaryBefore(c)) {
michael@0 158 break;
michael@0 159 }
michael@0 160 }
michael@0 161 }
michael@0 162
michael@0 163 UnicodeString destString(dest, 0, destCapacity);
michael@0 164 if(buffer.length()>0 && doNormalize) {
michael@0 165 n2->normalize(buffer, destString, *pErrorCode).extract(dest, destCapacity, *pErrorCode);
michael@0 166 if(pNeededToNormalize!=NULL && U_SUCCESS(*pErrorCode)) {
michael@0 167 *pNeededToNormalize= destString!=buffer;
michael@0 168 }
michael@0 169 return destString.length();
michael@0 170 } else {
michael@0 171 /* just copy the source characters */
michael@0 172 return buffer.extract(dest, destCapacity, *pErrorCode);
michael@0 173 }
michael@0 174 }
michael@0 175
michael@0 176 static int32_t
michael@0 177 unorm_iterate(UCharIterator *src, UBool forward,
michael@0 178 UChar *dest, int32_t destCapacity,
michael@0 179 UNormalizationMode mode, int32_t options,
michael@0 180 UBool doNormalize, UBool *pNeededToNormalize,
michael@0 181 UErrorCode *pErrorCode) {
michael@0 182 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 183 if(options&UNORM_UNICODE_3_2) {
michael@0 184 const UnicodeSet *uni32 = uniset_getUnicode32Instance(*pErrorCode);
michael@0 185 if(U_FAILURE(*pErrorCode)) {
michael@0 186 return 0;
michael@0 187 }
michael@0 188 FilteredNormalizer2 fn2(*n2, *uni32);
michael@0 189 return _iterate(src, forward, dest, destCapacity,
michael@0 190 &fn2, doNormalize, pNeededToNormalize, pErrorCode);
michael@0 191 }
michael@0 192 return _iterate(src, forward, dest, destCapacity,
michael@0 193 n2, doNormalize, pNeededToNormalize, pErrorCode);
michael@0 194 }
michael@0 195
michael@0 196 U_CAPI int32_t U_EXPORT2
michael@0 197 unorm_previous(UCharIterator *src,
michael@0 198 UChar *dest, int32_t destCapacity,
michael@0 199 UNormalizationMode mode, int32_t options,
michael@0 200 UBool doNormalize, UBool *pNeededToNormalize,
michael@0 201 UErrorCode *pErrorCode) {
michael@0 202 return unorm_iterate(src, FALSE,
michael@0 203 dest, destCapacity,
michael@0 204 mode, options,
michael@0 205 doNormalize, pNeededToNormalize,
michael@0 206 pErrorCode);
michael@0 207 }
michael@0 208
michael@0 209 U_CAPI int32_t U_EXPORT2
michael@0 210 unorm_next(UCharIterator *src,
michael@0 211 UChar *dest, int32_t destCapacity,
michael@0 212 UNormalizationMode mode, int32_t options,
michael@0 213 UBool doNormalize, UBool *pNeededToNormalize,
michael@0 214 UErrorCode *pErrorCode) {
michael@0 215 return unorm_iterate(src, TRUE,
michael@0 216 dest, destCapacity,
michael@0 217 mode, options,
michael@0 218 doNormalize, pNeededToNormalize,
michael@0 219 pErrorCode);
michael@0 220 }
michael@0 221
michael@0 222 /* Concatenation of normalized strings -------------------------------------- */
michael@0 223
michael@0 224 static int32_t
michael@0 225 _concatenate(const UChar *left, int32_t leftLength,
michael@0 226 const UChar *right, int32_t rightLength,
michael@0 227 UChar *dest, int32_t destCapacity,
michael@0 228 const Normalizer2 *n2,
michael@0 229 UErrorCode *pErrorCode) {
michael@0 230 if(U_FAILURE(*pErrorCode)) {
michael@0 231 return 0;
michael@0 232 }
michael@0 233 if(destCapacity<0 || (dest==NULL && destCapacity>0) ||
michael@0 234 left==NULL || leftLength<-1 || right==NULL || rightLength<-1) {
michael@0 235 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
michael@0 236 return 0;
michael@0 237 }
michael@0 238
michael@0 239 /* check for overlapping right and destination */
michael@0 240 if( dest!=NULL &&
michael@0 241 ((right>=dest && right<(dest+destCapacity)) ||
michael@0 242 (rightLength>0 && dest>=right && dest<(right+rightLength)))
michael@0 243 ) {
michael@0 244 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
michael@0 245 return 0;
michael@0 246 }
michael@0 247
michael@0 248 /* allow left==dest */
michael@0 249 UnicodeString destString;
michael@0 250 if(left==dest) {
michael@0 251 destString.setTo(dest, leftLength, destCapacity);
michael@0 252 } else {
michael@0 253 destString.setTo(dest, 0, destCapacity);
michael@0 254 destString.append(left, leftLength);
michael@0 255 }
michael@0 256 return n2->append(destString, UnicodeString(rightLength<0, right, rightLength), *pErrorCode).
michael@0 257 extract(dest, destCapacity, *pErrorCode);
michael@0 258 }
michael@0 259
michael@0 260 U_CAPI int32_t U_EXPORT2
michael@0 261 unorm_concatenate(const UChar *left, int32_t leftLength,
michael@0 262 const UChar *right, int32_t rightLength,
michael@0 263 UChar *dest, int32_t destCapacity,
michael@0 264 UNormalizationMode mode, int32_t options,
michael@0 265 UErrorCode *pErrorCode) {
michael@0 266 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
michael@0 267 if(options&UNORM_UNICODE_3_2) {
michael@0 268 const UnicodeSet *uni32 = uniset_getUnicode32Instance(*pErrorCode);
michael@0 269 if(U_FAILURE(*pErrorCode)) {
michael@0 270 return 0;
michael@0 271 }
michael@0 272 FilteredNormalizer2 fn2(*n2, *uni32);
michael@0 273 return _concatenate(left, leftLength, right, rightLength,
michael@0 274 dest, destCapacity, &fn2, pErrorCode);
michael@0 275 }
michael@0 276 return _concatenate(left, leftLength, right, rightLength,
michael@0 277 dest, destCapacity, n2, pErrorCode);
michael@0 278 }
michael@0 279
michael@0 280 #endif /* #if !UCONFIG_NO_NORMALIZATION */

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial