The Tor Browser / file comparison

comparison: intl/icu/source/i18n/tblcoll.cpp

intl/icu/source/i18n/tblcoll.cpp

changeset 0
6474c204b198
equal deleted inserted replaced
-1:000000000000 0:0d6b3bf7e0cc
1 /*
2 ******************************************************************************
3 * Copyright (C) 1996-2013, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ******************************************************************************
6 */
7
8 /**
9 * File tblcoll.cpp
10 *
11 * Created by: Helena Shih
12 *
13 * Modification History:
14 *
15 * Date Name Description
16 * 2/5/97 aliu Added streamIn and streamOut methods. Added
17 * constructor which reads RuleBasedCollator object from
18 * a binary file. Added writeToFile method which streams
19 * RuleBasedCollator out to a binary file. The streamIn
20 * and streamOut methods use istream and ostream objects
21 * in binary mode.
22 * 2/11/97 aliu Moved declarations out of for loop initializer.
23 * Added Mac compatibility #ifdef for ios::nocreate.
24 * 2/12/97 aliu Modified to use TableCollationData sub-object to
25 * hold invariant data.
26 * 2/13/97 aliu Moved several methods into this class from Collation.
27 * Added a private RuleBasedCollator(Locale&) constructor,
28 * to be used by Collator::getInstance(). General
29 * clean up. Made use of UErrorCode variables consistent.
30 * 2/20/97 helena Added clone, operator==, operator!=, operator=, and copy
31 * constructor and getDynamicClassID.
32 * 3/5/97 aliu Changed compaction cycle to improve performance. We
33 * use the maximum allowable value which is kBlockCount.
34 * Modified getRules() to load rules dynamically. Changed
35 * constructFromFile() call to accomodate this (added
36 * parameter to specify whether binary loading is to
37 * take place).
38 * 05/06/97 helena Added memory allocation error check.
39 * 6/20/97 helena Java class name change.
40 * 6/23/97 helena Adding comments to make code more readable.
41 * 09/03/97 helena Added createCollationKeyValues().
42 * 06/26/98 erm Changes for CollationKeys using byte arrays.
43 * 08/10/98 erm Synched with 1.2 version of RuleBasedCollator.java
44 * 04/23/99 stephen Removed EDecompositionMode, merged with
45 * Normalizer::EMode
46 * 06/14/99 stephen Removed kResourceBundleSuffix
47 * 06/22/99 stephen Fixed logic in constructFromFile() since .ctx
48 * files are no longer used.
49 * 11/02/99 helena Collator performance enhancements. Special case
50 * for NO_OP situations.
51 * 11/17/99 srl More performance enhancements. Inlined some internal functions.
52 * 12/15/99 aliu Update to support Thai collation. Move NormalizerIterator
53 * to implementation file.
54 * 01/29/01 synwee Modified into a C++ wrapper calling C APIs (ucol.h)
55 */
56
57 #include "unicode/utypes.h"
58
59 #if !UCONFIG_NO_COLLATION
60
61 #include "unicode/tblcoll.h"
62 #include "unicode/coleitr.h"
63 #include "unicode/ures.h"
64 #include "unicode/uset.h"
65 #include "ucol_imp.h"
66 #include "uresimp.h"
67 #include "uhash.h"
68 #include "cmemory.h"
69 #include "cstring.h"
70 #include "putilimp.h"
71 #include "ustr_imp.h"
72
73 /* public RuleBasedCollator constructor ---------------------------------- */
74
75 U_NAMESPACE_BEGIN
76
77 /**
78 * Copy constructor, aliasing, not write-through
79 */
80 RuleBasedCollator::RuleBasedCollator(const RuleBasedCollator& that)
81 : Collator(that)
82 , dataIsOwned(FALSE)
83 , isWriteThroughAlias(FALSE)
84 , ucollator(NULL)
85 {
86 RuleBasedCollator::operator=(that);
87 }
88
89 RuleBasedCollator::RuleBasedCollator(const UnicodeString& rules,
90 UErrorCode& status) :
91 dataIsOwned(FALSE)
92 {
93 construct(rules,
94 UCOL_DEFAULT_STRENGTH,
95 UCOL_DEFAULT,
96 status);
97 }
98
99 RuleBasedCollator::RuleBasedCollator(const UnicodeString& rules,
100 ECollationStrength collationStrength,
101 UErrorCode& status) : dataIsOwned(FALSE)
102 {
103 construct(rules,
104 (UColAttributeValue)collationStrength,
105 UCOL_DEFAULT,
106 status);
107 }
108
109 RuleBasedCollator::RuleBasedCollator(const UnicodeString& rules,
110 UColAttributeValue decompositionMode,
111 UErrorCode& status) :
112 dataIsOwned(FALSE)
113 {
114 construct(rules,
115 UCOL_DEFAULT_STRENGTH,
116 decompositionMode,
117 status);
118 }
119
120 RuleBasedCollator::RuleBasedCollator(const UnicodeString& rules,
121 ECollationStrength collationStrength,
122 UColAttributeValue decompositionMode,
123 UErrorCode& status) : dataIsOwned(FALSE)
124 {
125 construct(rules,
126 (UColAttributeValue)collationStrength,
127 decompositionMode,
128 status);
129 }
130 RuleBasedCollator::RuleBasedCollator(const uint8_t *bin, int32_t length,
131 const RuleBasedCollator *base,
132 UErrorCode &status) :
133 dataIsOwned(TRUE),
134 isWriteThroughAlias(FALSE)
135 {
136 ucollator = ucol_openBinary(bin, length, base->ucollator, &status);
137 }
138
139 void
140 RuleBasedCollator::setRuleStringFromCollator()
141 {
142 int32_t length;
143 const UChar *r = ucol_getRules(ucollator, &length);
144
145 if (r && length > 0) {
146 // alias the rules string
147 urulestring.setTo(TRUE, r, length);
148 }
149 else {
150 urulestring.truncate(0); // Clear string.
151 }
152 }
153
154 // not aliasing, not write-through
155 void
156 RuleBasedCollator::construct(const UnicodeString& rules,
157 UColAttributeValue collationStrength,
158 UColAttributeValue decompositionMode,
159 UErrorCode& status)
160 {
161 ucollator = ucol_openRules(rules.getBuffer(), rules.length(),
162 decompositionMode, collationStrength,
163 NULL, &status);
164
165 dataIsOwned = TRUE; // since we own a collator now, we need to get rid of it
166 isWriteThroughAlias = FALSE;
167
168 if(ucollator == NULL) {
169 if(U_SUCCESS(status)) {
170 status = U_MEMORY_ALLOCATION_ERROR;
171 }
172 return; // Failure
173 }
174
175 setRuleStringFromCollator();
176 }
177
178 /* RuleBasedCollator public destructor ----------------------------------- */
179
180 RuleBasedCollator::~RuleBasedCollator()
181 {
182 if (dataIsOwned)
183 {
184 ucol_close(ucollator);
185 }
186 ucollator = 0;
187 }
188
189 /* RuleBaseCollator public methods --------------------------------------- */
190
191 UBool RuleBasedCollator::operator==(const Collator& that) const
192 {
193 /* only checks for address equals here */
194 if (this == &that) {
195 return TRUE;
196 }
197 if (!Collator::operator==(that)) {
198 return FALSE; /* not the same class */
199 }
200
201 RuleBasedCollator& thatAlias = (RuleBasedCollator&)that;
202
203 return ucol_equals(this->ucollator, thatAlias.ucollator);
204 }
205
206 // aliasing, not write-through
207 RuleBasedCollator& RuleBasedCollator::operator=(const RuleBasedCollator& that)
208 {
209 if (this == &that) { return *this; }
210
211 UErrorCode intStatus = U_ZERO_ERROR;
212 UCollator *ucol = ucol_safeClone(that.ucollator, NULL, NULL, &intStatus);
213 if (U_FAILURE(intStatus)) { return *this; }
214
215 if (dataIsOwned) {
216 ucol_close(ucollator);
217 }
218 ucollator = ucol;
219 dataIsOwned = TRUE;
220 isWriteThroughAlias = FALSE;
221 setRuleStringFromCollator();
222 return *this;
223 }
224
225 // aliasing, not write-through
226 Collator* RuleBasedCollator::clone() const
227 {
228 RuleBasedCollator* coll = new RuleBasedCollator(*this);
229 // There is a small chance that the internal ucol_safeClone() call fails.
230 if (coll != NULL && coll->ucollator == NULL) {
231 delete coll;
232 return NULL;
233 }
234 return coll;
235 }
236
237
238 CollationElementIterator* RuleBasedCollator::createCollationElementIterator
239 (const UnicodeString& source) const
240 {
241 UErrorCode status = U_ZERO_ERROR;
242 CollationElementIterator *result = new CollationElementIterator(source, this,
243 status);
244 if (U_FAILURE(status)) {
245 delete result;
246 return NULL;
247 }
248
249 return result;
250 }
251
252 /**
253 * Create a CollationElementIterator object that will iterate over the
254 * elements in a string, using the collation rules defined in this
255 * RuleBasedCollator
256 */
257 CollationElementIterator* RuleBasedCollator::createCollationElementIterator
258 (const CharacterIterator& source) const
259 {
260 UErrorCode status = U_ZERO_ERROR;
261 CollationElementIterator *result = new CollationElementIterator(source, this,
262 status);
263
264 if (U_FAILURE(status)) {
265 delete result;
266 return NULL;
267 }
268
269 return result;
270 }
271
272 /**
273 * Return a string representation of this collator's rules. The string can
274 * later be passed to the constructor that takes a UnicodeString argument,
275 * which will construct a collator that's functionally identical to this one.
276 * You can also allow users to edit the string in order to change the collation
277 * data, or you can print it out for inspection, or whatever.
278 */
279 const UnicodeString& RuleBasedCollator::getRules() const
280 {
281 return urulestring;
282 }
283
284 void RuleBasedCollator::getRules(UColRuleOption delta, UnicodeString &buffer)
285 {
286 int32_t rulesize = ucol_getRulesEx(ucollator, delta, NULL, -1);
287
288 if (rulesize > 0) {
289 UChar *rules = (UChar*) uprv_malloc( sizeof(UChar) * (rulesize) );
290 if(rules != NULL) {
291 ucol_getRulesEx(ucollator, delta, rules, rulesize);
292 buffer.setTo(rules, rulesize);
293 uprv_free(rules);
294 } else { // couldn't allocate
295 buffer.remove();
296 }
297 }
298 else {
299 buffer.remove();
300 }
301 }
302
303 UnicodeSet *
304 RuleBasedCollator::getTailoredSet(UErrorCode &status) const
305 {
306 if(U_FAILURE(status)) {
307 return NULL;
308 }
309 return (UnicodeSet *)ucol_getTailoredSet(this->ucollator, &status);
310 }
311
312
313 void RuleBasedCollator::getVersion(UVersionInfo versionInfo) const
314 {
315 if (versionInfo!=NULL){
316 ucol_getVersion(ucollator, versionInfo);
317 }
318 }
319
320 /**
321 * Compare two strings using this collator
322 */
323 UCollationResult RuleBasedCollator::compare(
324 const UnicodeString& source,
325 const UnicodeString& target,
326 int32_t length,
327 UErrorCode &status) const
328 {
329 return compare(source.getBuffer(), uprv_min(length,source.length()), target.getBuffer(), uprv_min(length,target.length()), status);
330 }
331
332 UCollationResult RuleBasedCollator::compare(const UChar* source,
333 int32_t sourceLength,
334 const UChar* target,
335 int32_t targetLength,
336 UErrorCode &status) const
337 {
338 if(U_SUCCESS(status)) {
339 return ucol_strcoll(ucollator, source, sourceLength, target, targetLength);
340 } else {
341 return UCOL_EQUAL;
342 }
343 }
344
345 UCollationResult RuleBasedCollator::compare(
346 const UnicodeString& source,
347 const UnicodeString& target,
348 UErrorCode &status) const
349 {
350 if(U_SUCCESS(status)) {
351 return ucol_strcoll(ucollator, source.getBuffer(), source.length(),
352 target.getBuffer(), target.length());
353 } else {
354 return UCOL_EQUAL;
355 }
356 }
357
358 UCollationResult RuleBasedCollator::compare(UCharIterator &sIter,
359 UCharIterator &tIter,
360 UErrorCode &status) const {
361 if(U_SUCCESS(status)) {
362 return ucol_strcollIter(ucollator, &sIter, &tIter, &status);
363 } else {
364 return UCOL_EQUAL;
365 }
366 }
367
368 /**
369 * Retrieve a collation key for the specified string. The key can be compared
370 * with other collation keys using a bitwise comparison (e.g. memcmp) to find
371 * the ordering of their respective source strings. This is handy when doing a
372 * sort, where each sort key must be compared many times.
373 *
374 * The basic algorithm here is to find all of the collation elements for each
375 * character in the source string, convert them to an ASCII representation, and
376 * put them into the collation key. But it's trickier than that. Each
377 * collation element in a string has three components: primary ('A' vs 'B'),
378 * secondary ('u' vs '\u00FC'), and tertiary ('A' vs 'a'), and a primary difference
379 * at the end of a string takes precedence over a secondary or tertiary
380 * difference earlier in the string.
381 *
382 * To account for this, we put all of the primary orders at the beginning of
383 * the string, followed by the secondary and tertiary orders. Each set of
384 * orders is terminated by nulls so that a key for a string which is a initial
385 * substring of another key will compare less without any special case.
386 *
387 * Here's a hypothetical example, with the collation element represented as a
388 * three-digit number, one digit for primary, one for secondary, etc.
389 *
390 * String: A a B \u00C9
391 * Collation Elements: 101 100 201 511
392 * Collation Key: 1125<null>0001<null>1011<null>
393 *
394 * To make things even trickier, secondary differences (accent marks) are
395 * compared starting at the *end* of the string in languages with French
396 * secondary ordering. But when comparing the accent marks on a single base
397 * character, they are compared from the beginning. To handle this, we reverse
398 * all of the accents that belong to each base character, then we reverse the
399 * entire string of secondary orderings at the end.
400 */
401 CollationKey& RuleBasedCollator::getCollationKey(
402 const UnicodeString& source,
403 CollationKey& sortkey,
404 UErrorCode& status) const
405 {
406 return getCollationKey(source.getBuffer(), source.length(), sortkey, status);
407 }
408
409 CollationKey& RuleBasedCollator::getCollationKey(const UChar* source,
410 int32_t sourceLen,
411 CollationKey& sortkey,
412 UErrorCode& status) const
413 {
414 if (U_FAILURE(status)) {
415 return sortkey.setToBogus();
416 }
417 if (sourceLen < -1 || (source == NULL && sourceLen != 0)) {
418 status = U_ILLEGAL_ARGUMENT_ERROR;
419 return sortkey.setToBogus();
420 }
421
422 if (sourceLen < 0) {
423 sourceLen = u_strlen(source);
424 }
425 if (sourceLen == 0) {
426 return sortkey.reset();
427 }
428
429 int32_t resultLen = ucol_getCollationKey(ucollator, source, sourceLen, sortkey, status);
430
431 if (U_SUCCESS(status)) {
432 sortkey.setLength(resultLen);
433 } else {
434 sortkey.setToBogus();
435 }
436 return sortkey;
437 }
438
439 /**
440 * Return the maximum length of any expansion sequences that end with the
441 * specified comparison order.
442 * @param order a collation order returned by previous or next.
443 * @return the maximum length of any expansion seuences ending with the
444 * specified order or 1 if collation order does not occur at the end of any
445 * expansion sequence.
446 * @see CollationElementIterator#getMaxExpansion
447 */
448 int32_t RuleBasedCollator::getMaxExpansion(int32_t order) const
449 {
450 uint8_t result;
451 UCOL_GETMAXEXPANSION(ucollator, (uint32_t)order, result);
452 return result;
453 }
454
455 uint8_t* RuleBasedCollator::cloneRuleData(int32_t &length,
456 UErrorCode &status)
457 {
458 if (U_FAILURE(status)) { return NULL; }
459 LocalMemory<uint8_t> buffer((uint8_t *)uprv_malloc(20000));
460 if (buffer.isNull()) {
461 status = U_MEMORY_ALLOCATION_ERROR;
462 return NULL;
463 }
464 length = cloneBinary(buffer.getAlias(), 20000, status);
465 if (status == U_BUFFER_OVERFLOW_ERROR) {
466 if (buffer.allocateInsteadAndCopy(length, 0) == NULL) {
467 status = U_MEMORY_ALLOCATION_ERROR;
468 return NULL;
469 }
470 status = U_ZERO_ERROR;
471 length = cloneBinary(buffer.getAlias(), length, status);
472 }
473 if (U_FAILURE(status)) { return NULL; }
474 return buffer.orphan();
475 }
476
477
478 int32_t RuleBasedCollator::cloneBinary(uint8_t *buffer, int32_t capacity, UErrorCode &status)
479 {
480 return ucol_cloneBinary(ucollator, buffer, capacity, &status);
481 }
482
483 void RuleBasedCollator::setAttribute(UColAttribute attr,
484 UColAttributeValue value,
485 UErrorCode &status)
486 {
487 if (U_FAILURE(status))
488 return;
489 checkOwned();
490 ucol_setAttribute(ucollator, attr, value, &status);
491 }
492
493 UColAttributeValue RuleBasedCollator::getAttribute(UColAttribute attr,
494 UErrorCode &status) const
495 {
496 if (U_FAILURE(status))
497 return UCOL_DEFAULT;
498 return ucol_getAttribute(ucollator, attr, &status);
499 }
500
501 uint32_t RuleBasedCollator::setVariableTop(const UChar *varTop, int32_t len, UErrorCode &status) {
502 checkOwned();
503 return ucol_setVariableTop(ucollator, varTop, len, &status);
504 }
505
506 uint32_t RuleBasedCollator::setVariableTop(const UnicodeString &varTop, UErrorCode &status) {
507 checkOwned();
508 return ucol_setVariableTop(ucollator, varTop.getBuffer(), varTop.length(), &status);
509 }
510
511 void RuleBasedCollator::setVariableTop(uint32_t varTop, UErrorCode &status) {
512 checkOwned();
513 ucol_restoreVariableTop(ucollator, varTop, &status);
514 }
515
516 uint32_t RuleBasedCollator::getVariableTop(UErrorCode &status) const {
517 return ucol_getVariableTop(ucollator, &status);
518 }
519
520 int32_t RuleBasedCollator::getSortKey(const UnicodeString& source,
521 uint8_t *result, int32_t resultLength)
522 const
523 {
524 return ucol_getSortKey(ucollator, source.getBuffer(), source.length(), result, resultLength);
525 }
526
527 int32_t RuleBasedCollator::getSortKey(const UChar *source,
528 int32_t sourceLength, uint8_t *result,
529 int32_t resultLength) const
530 {
531 return ucol_getSortKey(ucollator, source, sourceLength, result, resultLength);
532 }
533
534 int32_t RuleBasedCollator::getReorderCodes(int32_t *dest,
535 int32_t destCapacity,
536 UErrorCode& status) const
537 {
538 return ucol_getReorderCodes(ucollator, dest, destCapacity, &status);
539 }
540
541 void RuleBasedCollator::setReorderCodes(const int32_t *reorderCodes,
542 int32_t reorderCodesLength,
543 UErrorCode& status)
544 {
545 checkOwned();
546 ucol_setReorderCodes(ucollator, reorderCodes, reorderCodesLength, &status);
547 }
548
549 int32_t RuleBasedCollator::getEquivalentReorderCodes(int32_t reorderCode,
550 int32_t* dest,
551 int32_t destCapacity,
552 UErrorCode& status)
553 {
554 return ucol_getEquivalentReorderCodes(reorderCode, dest, destCapacity, &status);
555 }
556
557 /**
558 * Create a hash code for this collation. Just hash the main rule table -- that
559 * should be good enough for almost any use.
560 */
561 int32_t RuleBasedCollator::hashCode() const
562 {
563 int32_t length;
564 const UChar *rules = ucol_getRules(ucollator, &length);
565 return ustr_hashUCharsN(rules, length);
566 }
567
568 /**
569 * return the locale of this collator
570 */
571 Locale RuleBasedCollator::getLocale(ULocDataLocaleType type, UErrorCode &status) const {
572 const char *result = ucol_getLocaleByType(ucollator, type, &status);
573 if(result == NULL) {
574 Locale res("");
575 res.setToBogus();
576 return res;
577 } else {
578 return Locale(result);
579 }
580 }
581
582 void
583 RuleBasedCollator::setLocales(const Locale& requestedLocale, const Locale& validLocale, const Locale& actualLocale) {
584 checkOwned();
585 char* rloc = uprv_strdup(requestedLocale.getName());
586 if (rloc) {
587 char* vloc = uprv_strdup(validLocale.getName());
588 if (vloc) {
589 char* aloc = uprv_strdup(actualLocale.getName());
590 if (aloc) {
591 ucol_setReqValidLocales(ucollator, rloc, vloc, aloc);
592 return;
593 }
594 uprv_free(vloc);
595 }
596 uprv_free(rloc);
597 }
598 }
599
600 // RuleBaseCollatorNew private constructor ----------------------------------
601
602 RuleBasedCollator::RuleBasedCollator()
603 : dataIsOwned(FALSE), isWriteThroughAlias(FALSE), ucollator(NULL)
604 {
605 }
606
607 RuleBasedCollator::RuleBasedCollator(const Locale& desiredLocale,
608 UErrorCode& status)
609 : dataIsOwned(FALSE), isWriteThroughAlias(FALSE), ucollator(NULL)
610 {
611 if (U_FAILURE(status))
612 return;
613
614 /*
615 Try to load, in order:
616 1. The desired locale's collation.
617 2. A fallback of the desired locale.
618 3. The default locale's collation.
619 4. A fallback of the default locale.
620 5. The default collation rules, which contains en_US collation rules.
621
622 To reiterate, we try:
623 Specific:
624 language+country+variant
625 language+country
626 language
627 Default:
628 language+country+variant
629 language+country
630 language
631 Root: (aka DEFAULTRULES)
632 steps 1-5 are handled by resource bundle fallback mechanism.
633 however, in a very unprobable situation that no resource bundle
634 data exists, step 5 is repeated with hardcoded default rules.
635 */
636
637 setUCollator(desiredLocale, status);
638
639 if (U_FAILURE(status))
640 {
641 status = U_ZERO_ERROR;
642
643 setUCollator(kRootLocaleName, status);
644 if (status == U_ZERO_ERROR) {
645 status = U_USING_DEFAULT_WARNING;
646 }
647 }
648
649 if (U_SUCCESS(status))
650 {
651 setRuleStringFromCollator();
652 }
653 }
654
655 void
656 RuleBasedCollator::setUCollator(const char *locale,
657 UErrorCode &status)
658 {
659 if (U_FAILURE(status)) {
660 return;
661 }
662 if (ucollator && dataIsOwned)
663 ucol_close(ucollator);
664 ucollator = ucol_open_internal(locale, &status);
665 dataIsOwned = TRUE;
666 isWriteThroughAlias = FALSE;
667 }
668
669
670 void
671 RuleBasedCollator::checkOwned() {
672 if (!(dataIsOwned || isWriteThroughAlias)) {
673 UErrorCode status = U_ZERO_ERROR;
674 ucollator = ucol_safeClone(ucollator, NULL, NULL, &status);
675 setRuleStringFromCollator();
676 dataIsOwned = TRUE;
677 isWriteThroughAlias = FALSE;
678 }
679 }
680
681
682 int32_t RuleBasedCollator::internalGetShortDefinitionString(const char *locale,
683 char *buffer,
684 int32_t capacity,
685 UErrorCode &status) const {
686 /* simply delegate */
687 return ucol_getShortDefinitionString(ucollator, locale, buffer, capacity, &status);
688 }
689
690
691 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedCollator)
692
693 U_NAMESPACE_END
694
695 #endif /* #if !UCONFIG_NO_COLLATION */

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial