The Tor Browser / file comparison

comparison: intl/icu/source/common/unicode/unorm2.h

intl/icu/source/common/unicode/unorm2.h

branch
TOR_BUG_3246
changeset 4
fc2d59ddac77
equal deleted inserted replaced
-1:000000000000 0:c683ce641c73
1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2009-2013, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: unorm2.h
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2009dec15
14 * created by: Markus W. Scherer
15 */
16
17 #ifndef __UNORM2_H__
18 #define __UNORM2_H__
19
20 /**
21 * \file
22 * \brief C API: New API for Unicode Normalization.
23 *
24 * Unicode normalization functionality for standard Unicode normalization or
25 * for using custom mapping tables.
26 * All instances of UNormalizer2 are unmodifiable/immutable.
27 * Instances returned by unorm2_getInstance() are singletons that must not be deleted by the caller.
28 * For more details see the Normalizer2 C++ class.
29 */
30
31 #include "unicode/utypes.h"
32 #include "unicode/localpointer.h"
33 #include "unicode/uset.h"
34
35 /**
36 * Constants for normalization modes.
37 * For details about standard Unicode normalization forms
38 * and about the algorithms which are also used with custom mapping tables
39 * see http://www.unicode.org/unicode/reports/tr15/
40 * @stable ICU 4.4
41 */
42 typedef enum {
43 /**
44 * Decomposition followed by composition.
45 * Same as standard NFC when using an "nfc" instance.
46 * Same as standard NFKC when using an "nfkc" instance.
47 * For details about standard Unicode normalization forms
48 * see http://www.unicode.org/unicode/reports/tr15/
49 * @stable ICU 4.4
50 */
51 UNORM2_COMPOSE,
52 /**
53 * Map, and reorder canonically.
54 * Same as standard NFD when using an "nfc" instance.
55 * Same as standard NFKD when using an "nfkc" instance.
56 * For details about standard Unicode normalization forms
57 * see http://www.unicode.org/unicode/reports/tr15/
58 * @stable ICU 4.4
59 */
60 UNORM2_DECOMPOSE,
61 /**
62 * "Fast C or D" form.
63 * If a string is in this form, then further decomposition <i>without reordering</i>
64 * would yield the same form as DECOMPOSE.
65 * Text in "Fast C or D" form can be processed efficiently with data tables
66 * that are "canonically closed", that is, that provide equivalent data for
67 * equivalent text, without having to be fully normalized.
68 * Not a standard Unicode normalization form.
69 * Not a unique form: Different FCD strings can be canonically equivalent.
70 * For details see http://www.unicode.org/notes/tn5/#FCD
71 * @stable ICU 4.4
72 */
73 UNORM2_FCD,
74 /**
75 * Compose only contiguously.
76 * Also known as "FCC" or "Fast C Contiguous".
77 * The result will often but not always be in NFC.
78 * The result will conform to FCD which is useful for processing.
79 * Not a standard Unicode normalization form.
80 * For details see http://www.unicode.org/notes/tn5/#FCC
81 * @stable ICU 4.4
82 */
83 UNORM2_COMPOSE_CONTIGUOUS
84 } UNormalization2Mode;
85
86 /**
87 * Result values for normalization quick check functions.
88 * For details see http://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
89 * @stable ICU 2.0
90 */
91 typedef enum UNormalizationCheckResult {
92 /**
93 * The input string is not in the normalization form.
94 * @stable ICU 2.0
95 */
96 UNORM_NO,
97 /**
98 * The input string is in the normalization form.
99 * @stable ICU 2.0
100 */
101 UNORM_YES,
102 /**
103 * The input string may or may not be in the normalization form.
104 * This value is only returned for composition forms like NFC and FCC,
105 * when a backward-combining character is found for which the surrounding text
106 * would have to be analyzed further.
107 * @stable ICU 2.0
108 */
109 UNORM_MAYBE
110 } UNormalizationCheckResult;
111
112 /**
113 * Opaque C service object type for the new normalization API.
114 * @stable ICU 4.4
115 */
116 struct UNormalizer2;
117 typedef struct UNormalizer2 UNormalizer2; /**< C typedef for struct UNormalizer2. @stable ICU 4.4 */
118
119 #if !UCONFIG_NO_NORMALIZATION
120
121 /**
122 * Returns a UNormalizer2 instance for Unicode NFC normalization.
123 * Same as unorm2_getInstance(NULL, "nfc", UNORM2_COMPOSE, pErrorCode).
124 * Returns an unmodifiable singleton instance. Do not delete it.
125 * @param pErrorCode Standard ICU error code. Its input value must
126 * pass the U_SUCCESS() test, or else the function returns
127 * immediately. Check for U_FAILURE() on output or use with
128 * function chaining. (See User Guide for details.)
129 * @return the requested Normalizer2, if successful
130 * @stable ICU 49
131 */
132 U_STABLE const UNormalizer2 * U_EXPORT2
133 unorm2_getNFCInstance(UErrorCode *pErrorCode);
134
135 /**
136 * Returns a UNormalizer2 instance for Unicode NFD normalization.
137 * Same as unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, pErrorCode).
138 * Returns an unmodifiable singleton instance. Do not delete it.
139 * @param pErrorCode Standard ICU error code. Its input value must
140 * pass the U_SUCCESS() test, or else the function returns
141 * immediately. Check for U_FAILURE() on output or use with
142 * function chaining. (See User Guide for details.)
143 * @return the requested Normalizer2, if successful
144 * @stable ICU 49
145 */
146 U_STABLE const UNormalizer2 * U_EXPORT2
147 unorm2_getNFDInstance(UErrorCode *pErrorCode);
148
149 /**
150 * Returns a UNormalizer2 instance for Unicode NFKC normalization.
151 * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_COMPOSE, pErrorCode).
152 * Returns an unmodifiable singleton instance. Do not delete it.
153 * @param pErrorCode Standard ICU error code. Its input value must
154 * pass the U_SUCCESS() test, or else the function returns
155 * immediately. Check for U_FAILURE() on output or use with
156 * function chaining. (See User Guide for details.)
157 * @return the requested Normalizer2, if successful
158 * @stable ICU 49
159 */
160 U_STABLE const UNormalizer2 * U_EXPORT2
161 unorm2_getNFKCInstance(UErrorCode *pErrorCode);
162
163 /**
164 * Returns a UNormalizer2 instance for Unicode NFKD normalization.
165 * Same as unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, pErrorCode).
166 * Returns an unmodifiable singleton instance. Do not delete it.
167 * @param pErrorCode Standard ICU error code. Its input value must
168 * pass the U_SUCCESS() test, or else the function returns
169 * immediately. Check for U_FAILURE() on output or use with
170 * function chaining. (See User Guide for details.)
171 * @return the requested Normalizer2, if successful
172 * @stable ICU 49
173 */
174 U_STABLE const UNormalizer2 * U_EXPORT2
175 unorm2_getNFKDInstance(UErrorCode *pErrorCode);
176
177 /**
178 * Returns a UNormalizer2 instance for Unicode NFKC_Casefold normalization.
179 * Same as unorm2_getInstance(NULL, "nfkc_cf", UNORM2_COMPOSE, pErrorCode).
180 * Returns an unmodifiable singleton instance. Do not delete it.
181 * @param pErrorCode Standard ICU error code. Its input value must
182 * pass the U_SUCCESS() test, or else the function returns
183 * immediately. Check for U_FAILURE() on output or use with
184 * function chaining. (See User Guide for details.)
185 * @return the requested Normalizer2, if successful
186 * @stable ICU 49
187 */
188 U_STABLE const UNormalizer2 * U_EXPORT2
189 unorm2_getNFKCCasefoldInstance(UErrorCode *pErrorCode);
190
191 /**
192 * Returns a UNormalizer2 instance which uses the specified data file
193 * (packageName/name similar to ucnv_openPackage() and ures_open()/ResourceBundle)
194 * and which composes or decomposes text according to the specified mode.
195 * Returns an unmodifiable singleton instance. Do not delete it.
196 *
197 * Use packageName=NULL for data files that are part of ICU's own data.
198 * Use name="nfc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFC/NFD.
199 * Use name="nfkc" and UNORM2_COMPOSE/UNORM2_DECOMPOSE for Unicode standard NFKC/NFKD.
200 * Use name="nfkc_cf" and UNORM2_COMPOSE for Unicode standard NFKC_CF=NFKC_Casefold.
201 *
202 * @param packageName NULL for ICU built-in data, otherwise application data package name
203 * @param name "nfc" or "nfkc" or "nfkc_cf" or name of custom data file
204 * @param mode normalization mode (compose or decompose etc.)
205 * @param pErrorCode Standard ICU error code. Its input value must
206 * pass the U_SUCCESS() test, or else the function returns
207 * immediately. Check for U_FAILURE() on output or use with
208 * function chaining. (See User Guide for details.)
209 * @return the requested UNormalizer2, if successful
210 * @stable ICU 4.4
211 */
212 U_STABLE const UNormalizer2 * U_EXPORT2
213 unorm2_getInstance(const char *packageName,
214 const char *name,
215 UNormalization2Mode mode,
216 UErrorCode *pErrorCode);
217
218 /**
219 * Constructs a filtered normalizer wrapping any UNormalizer2 instance
220 * and a filter set.
221 * Both are aliased and must not be modified or deleted while this object
222 * is used.
223 * The filter set should be frozen; otherwise the performance will suffer greatly.
224 * @param norm2 wrapped UNormalizer2 instance
225 * @param filterSet USet which determines the characters to be normalized
226 * @param pErrorCode Standard ICU error code. Its input value must
227 * pass the U_SUCCESS() test, or else the function returns
228 * immediately. Check for U_FAILURE() on output or use with
229 * function chaining. (See User Guide for details.)
230 * @return the requested UNormalizer2, if successful
231 * @stable ICU 4.4
232 */
233 U_STABLE UNormalizer2 * U_EXPORT2
234 unorm2_openFiltered(const UNormalizer2 *norm2, const USet *filterSet, UErrorCode *pErrorCode);
235
236 /**
237 * Closes a UNormalizer2 instance from unorm2_openFiltered().
238 * Do not close instances from unorm2_getInstance()!
239 * @param norm2 UNormalizer2 instance to be closed
240 * @stable ICU 4.4
241 */
242 U_STABLE void U_EXPORT2
243 unorm2_close(UNormalizer2 *norm2);
244
245 #if U_SHOW_CPLUSPLUS_API
246
247 U_NAMESPACE_BEGIN
248
249 /**
250 * \class LocalUNormalizer2Pointer
251 * "Smart pointer" class, closes a UNormalizer2 via unorm2_close().
252 * For most methods see the LocalPointerBase base class.
253 *
254 * @see LocalPointerBase
255 * @see LocalPointer
256 * @stable ICU 4.4
257 */
258 U_DEFINE_LOCAL_OPEN_POINTER(LocalUNormalizer2Pointer, UNormalizer2, unorm2_close);
259
260 U_NAMESPACE_END
261
262 #endif
263
264 /**
265 * Writes the normalized form of the source string to the destination string
266 * (replacing its contents) and returns the length of the destination string.
267 * The source and destination strings must be different buffers.
268 * @param norm2 UNormalizer2 instance
269 * @param src source string
270 * @param length length of the source string, or -1 if NUL-terminated
271 * @param dest destination string; its contents is replaced with normalized src
272 * @param capacity number of UChars that can be written to dest
273 * @param pErrorCode Standard ICU error code. Its input value must
274 * pass the U_SUCCESS() test, or else the function returns
275 * immediately. Check for U_FAILURE() on output or use with
276 * function chaining. (See User Guide for details.)
277 * @return dest
278 * @stable ICU 4.4
279 */
280 U_STABLE int32_t U_EXPORT2
281 unorm2_normalize(const UNormalizer2 *norm2,
282 const UChar *src, int32_t length,
283 UChar *dest, int32_t capacity,
284 UErrorCode *pErrorCode);
285 /**
286 * Appends the normalized form of the second string to the first string
287 * (merging them at the boundary) and returns the length of the first string.
288 * The result is normalized if the first string was normalized.
289 * The first and second strings must be different buffers.
290 * @param norm2 UNormalizer2 instance
291 * @param first string, should be normalized
292 * @param firstLength length of the first string, or -1 if NUL-terminated
293 * @param firstCapacity number of UChars that can be written to first
294 * @param second string, will be normalized
295 * @param secondLength length of the source string, or -1 if NUL-terminated
296 * @param pErrorCode Standard ICU error code. Its input value must
297 * pass the U_SUCCESS() test, or else the function returns
298 * immediately. Check for U_FAILURE() on output or use with
299 * function chaining. (See User Guide for details.)
300 * @return first
301 * @stable ICU 4.4
302 */
303 U_STABLE int32_t U_EXPORT2
304 unorm2_normalizeSecondAndAppend(const UNormalizer2 *norm2,
305 UChar *first, int32_t firstLength, int32_t firstCapacity,
306 const UChar *second, int32_t secondLength,
307 UErrorCode *pErrorCode);
308 /**
309 * Appends the second string to the first string
310 * (merging them at the boundary) and returns the length of the first string.
311 * The result is normalized if both the strings were normalized.
312 * The first and second strings must be different buffers.
313 * @param norm2 UNormalizer2 instance
314 * @param first string, should be normalized
315 * @param firstLength length of the first string, or -1 if NUL-terminated
316 * @param firstCapacity number of UChars that can be written to first
317 * @param second string, should be normalized
318 * @param secondLength length of the source string, or -1 if NUL-terminated
319 * @param pErrorCode Standard ICU error code. Its input value must
320 * pass the U_SUCCESS() test, or else the function returns
321 * immediately. Check for U_FAILURE() on output or use with
322 * function chaining. (See User Guide for details.)
323 * @return first
324 * @stable ICU 4.4
325 */
326 U_STABLE int32_t U_EXPORT2
327 unorm2_append(const UNormalizer2 *norm2,
328 UChar *first, int32_t firstLength, int32_t firstCapacity,
329 const UChar *second, int32_t secondLength,
330 UErrorCode *pErrorCode);
331
332 /**
333 * Gets the decomposition mapping of c.
334 * Roughly equivalent to normalizing the String form of c
335 * on a UNORM2_DECOMPOSE UNormalizer2 instance, but much faster, and except that this function
336 * returns a negative value and does not write a string
337 * if c does not have a decomposition mapping in this instance's data.
338 * This function is independent of the mode of the UNormalizer2.
339 * @param norm2 UNormalizer2 instance
340 * @param c code point
341 * @param decomposition String buffer which will be set to c's
342 * decomposition mapping, if there is one.
343 * @param capacity number of UChars that can be written to decomposition
344 * @param pErrorCode Standard ICU error code. Its input value must
345 * pass the U_SUCCESS() test, or else the function returns
346 * immediately. Check for U_FAILURE() on output or use with
347 * function chaining. (See User Guide for details.)
348 * @return the non-negative length of c's decomposition, if there is one; otherwise a negative value
349 * @stable ICU 4.6
350 */
351 U_STABLE int32_t U_EXPORT2
352 unorm2_getDecomposition(const UNormalizer2 *norm2,
353 UChar32 c, UChar *decomposition, int32_t capacity,
354 UErrorCode *pErrorCode);
355
356 /**
357 * Gets the raw decomposition mapping of c.
358 *
359 * This is similar to the unorm2_getDecomposition() function but returns the
360 * raw decomposition mapping as specified in UnicodeData.txt or
361 * (for custom data) in the mapping files processed by the gennorm2 tool.
362 * By contrast, unorm2_getDecomposition() returns the processed,
363 * recursively-decomposed version of this mapping.
364 *
365 * When used on a standard NFKC Normalizer2 instance,
366 * unorm2_getRawDecomposition() returns the Unicode Decomposition_Mapping (dm) property.
367 *
368 * When used on a standard NFC Normalizer2 instance,
369 * it returns the Decomposition_Mapping only if the Decomposition_Type (dt) is Canonical (Can);
370 * in this case, the result contains either one or two code points (=1..4 UChars).
371 *
372 * This function is independent of the mode of the UNormalizer2.
373 * @param norm2 UNormalizer2 instance
374 * @param c code point
375 * @param decomposition String buffer which will be set to c's
376 * raw decomposition mapping, if there is one.
377 * @param capacity number of UChars that can be written to decomposition
378 * @param pErrorCode Standard ICU error code. Its input value must
379 * pass the U_SUCCESS() test, or else the function returns
380 * immediately. Check for U_FAILURE() on output or use with
381 * function chaining. (See User Guide for details.)
382 * @return the non-negative length of c's raw decomposition, if there is one; otherwise a negative value
383 * @stable ICU 49
384 */
385 U_STABLE int32_t U_EXPORT2
386 unorm2_getRawDecomposition(const UNormalizer2 *norm2,
387 UChar32 c, UChar *decomposition, int32_t capacity,
388 UErrorCode *pErrorCode);
389
390 /**
391 * Performs pairwise composition of a & b and returns the composite if there is one.
392 *
393 * Returns a composite code point c only if c has a two-way mapping to a+b.
394 * In standard Unicode normalization, this means that
395 * c has a canonical decomposition to a+b
396 * and c does not have the Full_Composition_Exclusion property.
397 *
398 * This function is independent of the mode of the UNormalizer2.
399 * @param norm2 UNormalizer2 instance
400 * @param a A (normalization starter) code point.
401 * @param b Another code point.
402 * @return The non-negative composite code point if there is one; otherwise a negative value.
403 * @stable ICU 49
404 */
405 U_STABLE UChar32 U_EXPORT2
406 unorm2_composePair(const UNormalizer2 *norm2, UChar32 a, UChar32 b);
407
408 /**
409 * Gets the combining class of c.
410 * The default implementation returns 0
411 * but all standard implementations return the Unicode Canonical_Combining_Class value.
412 * @param norm2 UNormalizer2 instance
413 * @param c code point
414 * @return c's combining class
415 * @stable ICU 49
416 */
417 U_STABLE uint8_t U_EXPORT2
418 unorm2_getCombiningClass(const UNormalizer2 *norm2, UChar32 c);
419
420 /**
421 * Tests if the string is normalized.
422 * Internally, in cases where the quickCheck() method would return "maybe"
423 * (which is only possible for the two COMPOSE modes) this method
424 * resolves to "yes" or "no" to provide a definitive result,
425 * at the cost of doing more work in those cases.
426 * @param norm2 UNormalizer2 instance
427 * @param s input string
428 * @param length length of the string, or -1 if NUL-terminated
429 * @param pErrorCode Standard ICU error code. Its input value must
430 * pass the U_SUCCESS() test, or else the function returns
431 * immediately. Check for U_FAILURE() on output or use with
432 * function chaining. (See User Guide for details.)
433 * @return TRUE if s is normalized
434 * @stable ICU 4.4
435 */
436 U_STABLE UBool U_EXPORT2
437 unorm2_isNormalized(const UNormalizer2 *norm2,
438 const UChar *s, int32_t length,
439 UErrorCode *pErrorCode);
440
441 /**
442 * Tests if the string is normalized.
443 * For the two COMPOSE modes, the result could be "maybe" in cases that
444 * would take a little more work to resolve definitively.
445 * Use spanQuickCheckYes() and normalizeSecondAndAppend() for a faster
446 * combination of quick check + normalization, to avoid
447 * re-checking the "yes" prefix.
448 * @param norm2 UNormalizer2 instance
449 * @param s input string
450 * @param length length of the string, or -1 if NUL-terminated
451 * @param pErrorCode Standard ICU error code. Its input value must
452 * pass the U_SUCCESS() test, or else the function returns
453 * immediately. Check for U_FAILURE() on output or use with
454 * function chaining. (See User Guide for details.)
455 * @return UNormalizationCheckResult
456 * @stable ICU 4.4
457 */
458 U_STABLE UNormalizationCheckResult U_EXPORT2
459 unorm2_quickCheck(const UNormalizer2 *norm2,
460 const UChar *s, int32_t length,
461 UErrorCode *pErrorCode);
462
463 /**
464 * Returns the end of the normalized substring of the input string.
465 * In other words, with <code>end=spanQuickCheckYes(s, ec);</code>
466 * the substring <code>UnicodeString(s, 0, end)</code>
467 * will pass the quick check with a "yes" result.
468 *
469 * The returned end index is usually one or more characters before the
470 * "no" or "maybe" character: The end index is at a normalization boundary.
471 * (See the class documentation for more about normalization boundaries.)
472 *
473 * When the goal is a normalized string and most input strings are expected
474 * to be normalized already, then call this method,
475 * and if it returns a prefix shorter than the input string,
476 * copy that prefix and use normalizeSecondAndAppend() for the remainder.
477 * @param norm2 UNormalizer2 instance
478 * @param s input string
479 * @param length length of the string, or -1 if NUL-terminated
480 * @param pErrorCode Standard ICU error code. Its input value must
481 * pass the U_SUCCESS() test, or else the function returns
482 * immediately. Check for U_FAILURE() on output or use with
483 * function chaining. (See User Guide for details.)
484 * @return "yes" span end index
485 * @stable ICU 4.4
486 */
487 U_STABLE int32_t U_EXPORT2
488 unorm2_spanQuickCheckYes(const UNormalizer2 *norm2,
489 const UChar *s, int32_t length,
490 UErrorCode *pErrorCode);
491
492 /**
493 * Tests if the character always has a normalization boundary before it,
494 * regardless of context.
495 * For details see the Normalizer2 base class documentation.
496 * @param norm2 UNormalizer2 instance
497 * @param c character to test
498 * @return TRUE if c has a normalization boundary before it
499 * @stable ICU 4.4
500 */
501 U_STABLE UBool U_EXPORT2
502 unorm2_hasBoundaryBefore(const UNormalizer2 *norm2, UChar32 c);
503
504 /**
505 * Tests if the character always has a normalization boundary after it,
506 * regardless of context.
507 * For details see the Normalizer2 base class documentation.
508 * @param norm2 UNormalizer2 instance
509 * @param c character to test
510 * @return TRUE if c has a normalization boundary after it
511 * @stable ICU 4.4
512 */
513 U_STABLE UBool U_EXPORT2
514 unorm2_hasBoundaryAfter(const UNormalizer2 *norm2, UChar32 c);
515
516 /**
517 * Tests if the character is normalization-inert.
518 * For details see the Normalizer2 base class documentation.
519 * @param norm2 UNormalizer2 instance
520 * @param c character to test
521 * @return TRUE if c is normalization-inert
522 * @stable ICU 4.4
523 */
524 U_STABLE UBool U_EXPORT2
525 unorm2_isInert(const UNormalizer2 *norm2, UChar32 c);
526
527 #endif /* !UCONFIG_NO_NORMALIZATION */
528 #endif /* __UNORM2_H__ */

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial