The Tor Browser / file comparison

comparison: intl/icu/source/common/unorm.cpp

intl/icu/source/common/unorm.cpp

changeset 0
6474c204b198
equal deleted inserted replaced
-1:000000000000 0:67ebaadfcbea
1 /*
2 ******************************************************************************
3 * Copyright (c) 1996-2011, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 ******************************************************************************
6 * File unorm.cpp
7 *
8 * Created by: Vladimir Weinstein 12052000
9 *
10 * Modification history :
11 *
12 * Date Name Description
13 * 02/01/01 synwee Added normalization quickcheck enum and method.
14 * 02/12/01 synwee Commented out quickcheck util api has been approved
15 * Added private method for doing FCD checks
16 * 02/23/01 synwee Modified quickcheck and checkFCE to run through
17 * string for codepoints < 0x300 for the normalization
18 * mode NFC.
19 * 05/25/01+ Markus Scherer total rewrite, implement all normalization here
20 * instead of just wrappers around normlzr.cpp,
21 * load unorm.dat, support Unicode 3.1 with
22 * supplementary code points, etc.
23 * 2009-nov..2010-jan Markus Scherer total rewrite, new Normalizer2 API & code
24 */
25
26 #include "unicode/utypes.h"
27
28 #if !UCONFIG_NO_NORMALIZATION
29
30 #include "unicode/udata.h"
31 #include "unicode/ustring.h"
32 #include "unicode/uiter.h"
33 #include "unicode/unorm.h"
34 #include "unicode/unorm2.h"
35 #include "normalizer2impl.h"
36 #include "unormimp.h"
37 #include "uprops.h"
38 #include "ustr_imp.h"
39
40 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
41
42 U_NAMESPACE_USE
43
44 /* quick check functions ---------------------------------------------------- */
45
46 U_CAPI UNormalizationCheckResult U_EXPORT2
47 unorm_quickCheck(const UChar *src,
48 int32_t srcLength,
49 UNormalizationMode mode,
50 UErrorCode *pErrorCode) {
51 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
52 return unorm2_quickCheck((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
53 }
54
55 U_CAPI UNormalizationCheckResult U_EXPORT2
56 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
57 UNormalizationMode mode, int32_t options,
58 UErrorCode *pErrorCode) {
59 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
60 if(options&UNORM_UNICODE_3_2) {
61 FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*pErrorCode));
62 return unorm2_quickCheck(
63 reinterpret_cast<const UNormalizer2 *>(static_cast<Normalizer2 *>(&fn2)),
64 src, srcLength, pErrorCode);
65 } else {
66 return unorm2_quickCheck((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
67 }
68 }
69
70 U_CAPI UBool U_EXPORT2
71 unorm_isNormalized(const UChar *src, int32_t srcLength,
72 UNormalizationMode mode,
73 UErrorCode *pErrorCode) {
74 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
75 return unorm2_isNormalized((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
76 }
77
78 U_CAPI UBool U_EXPORT2
79 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
80 UNormalizationMode mode, int32_t options,
81 UErrorCode *pErrorCode) {
82 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
83 if(options&UNORM_UNICODE_3_2) {
84 FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*pErrorCode));
85 return unorm2_isNormalized(
86 reinterpret_cast<const UNormalizer2 *>(static_cast<Normalizer2 *>(&fn2)),
87 src, srcLength, pErrorCode);
88 } else {
89 return unorm2_isNormalized((const UNormalizer2 *)n2, src, srcLength, pErrorCode);
90 }
91 }
92
93 /* normalize() API ---------------------------------------------------------- */
94
95 /** Public API for normalizing. */
96 U_CAPI int32_t U_EXPORT2
97 unorm_normalize(const UChar *src, int32_t srcLength,
98 UNormalizationMode mode, int32_t options,
99 UChar *dest, int32_t destCapacity,
100 UErrorCode *pErrorCode) {
101 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
102 if(options&UNORM_UNICODE_3_2) {
103 FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*pErrorCode));
104 return unorm2_normalize(
105 reinterpret_cast<const UNormalizer2 *>(static_cast<Normalizer2 *>(&fn2)),
106 src, srcLength, dest, destCapacity, pErrorCode);
107 } else {
108 return unorm2_normalize((const UNormalizer2 *)n2,
109 src, srcLength, dest, destCapacity, pErrorCode);
110 }
111 }
112
113
114 /* iteration functions ------------------------------------------------------ */
115
116 static int32_t
117 _iterate(UCharIterator *src, UBool forward,
118 UChar *dest, int32_t destCapacity,
119 const Normalizer2 *n2,
120 UBool doNormalize, UBool *pNeededToNormalize,
121 UErrorCode *pErrorCode) {
122 if(U_FAILURE(*pErrorCode)) {
123 return 0;
124 }
125 if(destCapacity<0 || (dest==NULL && destCapacity>0) || src==NULL) {
126 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
127 return 0;
128 }
129
130 if(pNeededToNormalize!=NULL) {
131 *pNeededToNormalize=FALSE;
132 }
133 if(!(forward ? src->hasNext(src) : src->hasPrevious(src))) {
134 return u_terminateUChars(dest, destCapacity, 0, pErrorCode);
135 }
136
137 UnicodeString buffer;
138 UChar32 c;
139 if(forward) {
140 /* get one character and ignore its properties */
141 buffer.append(uiter_next32(src));
142 /* get all following characters until we see a boundary */
143 while((c=uiter_next32(src))>=0) {
144 if(n2->hasBoundaryBefore(c)) {
145 /* back out the latest movement to stop at the boundary */
146 src->move(src, -U16_LENGTH(c), UITER_CURRENT);
147 break;
148 } else {
149 buffer.append(c);
150 }
151 }
152 } else {
153 while((c=uiter_previous32(src))>=0) {
154 /* always write this character to the front of the buffer */
155 buffer.insert(0, c);
156 /* stop if this just-copied character is a boundary */
157 if(n2->hasBoundaryBefore(c)) {
158 break;
159 }
160 }
161 }
162
163 UnicodeString destString(dest, 0, destCapacity);
164 if(buffer.length()>0 && doNormalize) {
165 n2->normalize(buffer, destString, *pErrorCode).extract(dest, destCapacity, *pErrorCode);
166 if(pNeededToNormalize!=NULL && U_SUCCESS(*pErrorCode)) {
167 *pNeededToNormalize= destString!=buffer;
168 }
169 return destString.length();
170 } else {
171 /* just copy the source characters */
172 return buffer.extract(dest, destCapacity, *pErrorCode);
173 }
174 }
175
176 static int32_t
177 unorm_iterate(UCharIterator *src, UBool forward,
178 UChar *dest, int32_t destCapacity,
179 UNormalizationMode mode, int32_t options,
180 UBool doNormalize, UBool *pNeededToNormalize,
181 UErrorCode *pErrorCode) {
182 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
183 if(options&UNORM_UNICODE_3_2) {
184 const UnicodeSet *uni32 = uniset_getUnicode32Instance(*pErrorCode);
185 if(U_FAILURE(*pErrorCode)) {
186 return 0;
187 }
188 FilteredNormalizer2 fn2(*n2, *uni32);
189 return _iterate(src, forward, dest, destCapacity,
190 &fn2, doNormalize, pNeededToNormalize, pErrorCode);
191 }
192 return _iterate(src, forward, dest, destCapacity,
193 n2, doNormalize, pNeededToNormalize, pErrorCode);
194 }
195
196 U_CAPI int32_t U_EXPORT2
197 unorm_previous(UCharIterator *src,
198 UChar *dest, int32_t destCapacity,
199 UNormalizationMode mode, int32_t options,
200 UBool doNormalize, UBool *pNeededToNormalize,
201 UErrorCode *pErrorCode) {
202 return unorm_iterate(src, FALSE,
203 dest, destCapacity,
204 mode, options,
205 doNormalize, pNeededToNormalize,
206 pErrorCode);
207 }
208
209 U_CAPI int32_t U_EXPORT2
210 unorm_next(UCharIterator *src,
211 UChar *dest, int32_t destCapacity,
212 UNormalizationMode mode, int32_t options,
213 UBool doNormalize, UBool *pNeededToNormalize,
214 UErrorCode *pErrorCode) {
215 return unorm_iterate(src, TRUE,
216 dest, destCapacity,
217 mode, options,
218 doNormalize, pNeededToNormalize,
219 pErrorCode);
220 }
221
222 /* Concatenation of normalized strings -------------------------------------- */
223
224 static int32_t
225 _concatenate(const UChar *left, int32_t leftLength,
226 const UChar *right, int32_t rightLength,
227 UChar *dest, int32_t destCapacity,
228 const Normalizer2 *n2,
229 UErrorCode *pErrorCode) {
230 if(U_FAILURE(*pErrorCode)) {
231 return 0;
232 }
233 if(destCapacity<0 || (dest==NULL && destCapacity>0) ||
234 left==NULL || leftLength<-1 || right==NULL || rightLength<-1) {
235 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
236 return 0;
237 }
238
239 /* check for overlapping right and destination */
240 if( dest!=NULL &&
241 ((right>=dest && right<(dest+destCapacity)) ||
242 (rightLength>0 && dest>=right && dest<(right+rightLength)))
243 ) {
244 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
245 return 0;
246 }
247
248 /* allow left==dest */
249 UnicodeString destString;
250 if(left==dest) {
251 destString.setTo(dest, leftLength, destCapacity);
252 } else {
253 destString.setTo(dest, 0, destCapacity);
254 destString.append(left, leftLength);
255 }
256 return n2->append(destString, UnicodeString(rightLength<0, right, rightLength), *pErrorCode).
257 extract(dest, destCapacity, *pErrorCode);
258 }
259
260 U_CAPI int32_t U_EXPORT2
261 unorm_concatenate(const UChar *left, int32_t leftLength,
262 const UChar *right, int32_t rightLength,
263 UChar *dest, int32_t destCapacity,
264 UNormalizationMode mode, int32_t options,
265 UErrorCode *pErrorCode) {
266 const Normalizer2 *n2=Normalizer2Factory::getInstance(mode, *pErrorCode);
267 if(options&UNORM_UNICODE_3_2) {
268 const UnicodeSet *uni32 = uniset_getUnicode32Instance(*pErrorCode);
269 if(U_FAILURE(*pErrorCode)) {
270 return 0;
271 }
272 FilteredNormalizer2 fn2(*n2, *uni32);
273 return _concatenate(left, leftLength, right, rightLength,
274 dest, destCapacity, &fn2, pErrorCode);
275 }
276 return _concatenate(left, leftLength, right, rightLength,
277 dest, destCapacity, n2, pErrorCode);
278 }
279
280 #endif /* #if !UCONFIG_NO_NORMALIZATION */

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial