The Tor Browser: intl/icu/source/i18n/ucol_tok.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/ucol_tok.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/ucol_tok.cpp

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 /*
 *******************************************************************************
 *
 *   Copyright (C) 2001-2012, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************
 *   file name:  ucol_tok.cpp
 *   encoding:   US-ASCII
 *   tab size:   8 (not used)
 *   indentation:4
 *
 *   created 02/22/2001
 *   created by: Vladimir Weinstein
 *
 * This module reads a tailoring rule string and produces a list of
 * tokens that will be turned into collation elements
 *
 */
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_COLLATION
 #include "unicode/uscript.h"
 #include "unicode/ustring.h"
 #include "unicode/uchar.h"
 #include "unicode/uniset.h"
 #include "cmemory.h"
 #include "cstring.h"
 #include "patternprops.h"
 #include "ucol_bld.h"
 #include "ucol_tok.h"
 #include "ulocimp.h"
 #include "uresimp.h"
 // Define this only for debugging.
 // #define DEBUG_FOR_COLL_RULES 1
 #ifdef DEBUG_FOR_COLL_RULES
 #include <iostream>
 #endif
 U_NAMESPACE_USE
 U_CDECL_BEGIN
 static int32_t U_CALLCONV
 uhash_hashTokens(const UHashTok k)
 {
     int32_t hash = 0;
     //uint32_t key = (uint32_t)k.integer;
     UColToken *key = (UColToken *)k.pointer;
     if (key != 0) {
         int32_t len = (key->source & 0xFF000000)>>24;
         int32_t inc = ((len - 32) / 32) + 1;
         const UChar *p = (key->source & 0x00FFFFFF) + *(key->rulesToParseHdl);
         const UChar *limit = p + len;
         while (p<limit) {
             hash = (hash * 37) + *p;
             p += inc;
         }
     }
     return hash;
 }
 static UBool U_CALLCONV
 uhash_compareTokens(const UHashTok key1, const UHashTok key2)
 {
     //uint32_t p1 = (uint32_t) key1.integer;
     //uint32_t p2 = (uint32_t) key2.integer;
     UColToken *p1 = (UColToken *)key1.pointer;
     UColToken *p2 = (UColToken *)key2.pointer;
     const UChar *s1 = (p1->source & 0x00FFFFFF) + *(p1->rulesToParseHdl);
     const UChar *s2 = (p2->source & 0x00FFFFFF) + *(p2->rulesToParseHdl);
     uint32_t s1L = ((p1->source & 0xFF000000) >> 24);
     uint32_t s2L = ((p2->source & 0xFF000000) >> 24);
     const UChar *end = s1+s1L-1;
     if (p1 == p2) {
         return TRUE;
     }
     if (p1->source == 0 || p2->source == 0) {
         return FALSE;
     }
     if(s1L != s2L) {
         return FALSE;
     }
     if(p1->source == p2->source) {
         return TRUE;
     }
     while((s1 < end) && *s1 == *s2) {
         ++s1;
         ++s2;
     }
     if(*s1 == *s2) {
         return TRUE;
     } else {
         return FALSE;
     }
 }
 U_CDECL_END
 /*
  * Debug messages used to pinpoint where a format error occurred.
  * A better way is to include context-sensitive information in syntaxError() function.
  *
  * To turn this debugging on, either uncomment the following line, or define use -DDEBUG_FOR_FORMAT_ERROR
  * in the compile line.
  */
 /* #define DEBUG_FOR_FORMAT_ERROR 1 */
 #ifdef DEBUG_FOR_FORMAT_ERROR
 #define DBG_FORMAT_ERROR { printf("U_INVALID_FORMAT_ERROR at line %d", __LINE__);}
 #else
 #define DBG_FORMAT_ERROR
 #endif
 /*
  * Controls debug messages so that the output can be compared before and after a
  * big change.  Prints the information of every code point that comes out of the
  * collation parser and its strength into a file.  When a big change in format
  * happens, the files before and after the change should be identical.
  *
  * To turn this debugging on, either uncomment the following line, or define use -DDEBUG_FOR_CODE_POINTS
  * in the compile line.
  */
 // #define DEBUG_FOR_CODE_POINTS 1
 #ifdef DEBUG_FOR_CODE_POINTS
     FILE* dfcp_fp = NULL;
 #endif
 typedef struct {
     uint32_t startCE;
     uint32_t startContCE;
     uint32_t limitCE;
     uint32_t limitContCE;
 } indirectBoundaries;
 /* these values are used for finding CE values for indirect positioning. */
 /* Indirect positioning is a mechanism for allowing resets on symbolic   */
 /* values. It only works for resets and you cannot tailor indirect names */
 /* An indirect name can define either an anchor point or a range. An     */
 /* anchor point behaves in exactly the same way as a code point in reset */
 /* would, except that it cannot be tailored. A range (we currently only  */
 /* know for the [top] range will explicitly set the upper bound for      */
 /* generated CEs, thus allowing for better control over how many CEs can */
 /* be squeezed between in the range without performance penalty.         */
 /* In that respect, we use [top] for tailoring of locales that use CJK   */
 /* characters. Other indirect values are currently a pure convenience,   */
 /* they can be used to assure that the CEs will be always positioned in  */
 /* the same place relative to a point with known properties (e.g. first  */
 /* primary ignorable). */
 static indirectBoundaries ucolIndirectBoundaries[15];
 /*
 static indirectBoundaries ucolIndirectBoundaries[11] = {
 { UCOL_RESET_TOP_VALUE,               0,
 UCOL_NEXT_TOP_VALUE,                0 },
 { UCOL_FIRST_PRIMARY_IGNORABLE,       0,
 ,                                  0 },
 { UCOL_LAST_PRIMARY_IGNORABLE,        UCOL_LAST_PRIMARY_IGNORABLE_CONT,
 ,                                  0 },
 { UCOL_FIRST_SECONDARY_IGNORABLE,     0,
 ,                                  0 },
 { UCOL_LAST_SECONDARY_IGNORABLE,      0,
 ,                                  0 },
 { UCOL_FIRST_TERTIARY_IGNORABLE,      0,
 ,                                  0 },
 { UCOL_LAST_TERTIARY_IGNORABLE,       0,
 ,                                  0 },
 { UCOL_FIRST_VARIABLE,                0,
 ,                                  0 },
 { UCOL_LAST_VARIABLE,                 0,
 ,                                  0 },
 { UCOL_FIRST_NON_VARIABLE,            0,
 ,                                  0 },
 { UCOL_LAST_NON_VARIABLE,             0,
 ,                                  0 },
 };
 */
 static void setIndirectBoundaries(uint32_t indexR, uint32_t *start, uint32_t *end) {
     // Set values for the top - TODO: once we have values for all the indirects, we are going
     // to initalize here.
     ucolIndirectBoundaries[indexR].startCE = start[0];
     ucolIndirectBoundaries[indexR].startContCE = start[1];
     if(end) {
         ucolIndirectBoundaries[indexR].limitCE = end[0];
         ucolIndirectBoundaries[indexR].limitContCE = end[1];
     } else {
         ucolIndirectBoundaries[indexR].limitCE = 0;
         ucolIndirectBoundaries[indexR].limitContCE = 0;
     }
 }
 static inline
 void syntaxError(const UChar* rules,
                  int32_t pos,
                  int32_t rulesLen,
                  UParseError* parseError)
 {
     parseError->offset = pos;
     parseError->line = 0 ; /* we are not using line numbers */
     // for pre-context
     int32_t start = (pos < U_PARSE_CONTEXT_LEN)? 0 : (pos - (U_PARSE_CONTEXT_LEN-1));
     int32_t stop  = pos;
     u_memcpy(parseError->preContext,rules+start,stop-start);
     //null terminate the buffer
     parseError->preContext[stop-start] = 0;
     //for post-context
     start = pos+1;
     stop  = ((pos+U_PARSE_CONTEXT_LEN)<= rulesLen )? (pos+(U_PARSE_CONTEXT_LEN-1)) :
     rulesLen;
     if(start < stop) {
         u_memcpy(parseError->postContext,rules+start,stop-start);
         //null terminate the buffer
         parseError->postContext[stop-start]= 0;
     } else {
         parseError->postContext[0] = 0;
     }
 }
 static
 void ucol_uprv_tok_setOptionInImage(UColOptionSet *opts, UColAttribute attrib, UColAttributeValue value) {
     switch(attrib) {
     case UCOL_HIRAGANA_QUATERNARY_MODE:
         opts->hiraganaQ = value;
         break;
     case UCOL_FRENCH_COLLATION:
         opts->frenchCollation = value;
         break;
     case UCOL_ALTERNATE_HANDLING:
         opts->alternateHandling = value;
         break;
     case UCOL_CASE_FIRST:
         opts->caseFirst = value;
         break;
     case UCOL_CASE_LEVEL:
         opts->caseLevel = value;
         break;
     case UCOL_NORMALIZATION_MODE:
         opts->normalizationMode = value;
         break;
     case UCOL_STRENGTH:
         opts->strength = value;
         break;
     case UCOL_NUMERIC_COLLATION:
         opts->numericCollation = value;
         break;
     case UCOL_ATTRIBUTE_COUNT:
     default:
         break;
     }
 }
 #define UTOK_OPTION_COUNT 22
 static UBool didInit = FALSE;
 /* we can be strict, or we can be lenient */
 /* I'd surely be lenient with the option arguments */
 /* maybe even with options */
 U_STRING_DECL(suboption_00, "non-ignorable", 13);
 U_STRING_DECL(suboption_01, "shifted",        7);
 U_STRING_DECL(suboption_02, "lower",          5);
 U_STRING_DECL(suboption_03, "upper",          5);
 U_STRING_DECL(suboption_04, "off",            3);
 U_STRING_DECL(suboption_05, "on",             2);
 U_STRING_DECL(suboption_06, "1",              1);
 U_STRING_DECL(suboption_07, "2",              1);
 U_STRING_DECL(suboption_08, "3",              1);
 U_STRING_DECL(suboption_09, "4",              1);
 U_STRING_DECL(suboption_10, "I",              1);
 U_STRING_DECL(suboption_11, "primary",        7);
 U_STRING_DECL(suboption_12, "secondary",      9);
 U_STRING_DECL(suboption_13, "tertiary",       8);
 U_STRING_DECL(suboption_14, "variable",       8);
 U_STRING_DECL(suboption_15, "regular",        7);
 U_STRING_DECL(suboption_16, "implicit",       8);
 U_STRING_DECL(suboption_17, "trailing",       8);
 U_STRING_DECL(option_00,    "undefined",      9);
 U_STRING_DECL(option_01,    "rearrange",      9);
 U_STRING_DECL(option_02,    "alternate",      9);
 U_STRING_DECL(option_03,    "backwards",      9);
 U_STRING_DECL(option_04,    "variable top",  12);
 U_STRING_DECL(option_05,    "top",            3);
 U_STRING_DECL(option_06,    "normalization", 13);
 U_STRING_DECL(option_07,    "caseLevel",      9);
 U_STRING_DECL(option_08,    "caseFirst",      9);
 U_STRING_DECL(option_09,    "scriptOrder",   11);
 U_STRING_DECL(option_10,    "charsetname",   11);
 U_STRING_DECL(option_11,    "charset",        7);
 U_STRING_DECL(option_12,    "before",         6);
 U_STRING_DECL(option_13,    "hiraganaQ",      9);
 U_STRING_DECL(option_14,    "strength",       8);
 U_STRING_DECL(option_15,    "first",          5);
 U_STRING_DECL(option_16,    "last",           4);
 U_STRING_DECL(option_17,    "optimize",       8);
 U_STRING_DECL(option_18,    "suppressContractions",         20);
 U_STRING_DECL(option_19,    "numericOrdering",              15);
 U_STRING_DECL(option_20,    "import",         6);
 U_STRING_DECL(option_21,    "reorder",         7);
 /*
 [last variable] last variable value
 [last primary ignorable] largest CE for primary ignorable
 [last secondary ignorable] largest CE for secondary ignorable
 [last tertiary ignorable] largest CE for tertiary ignorable
 [top] guaranteed to be above all implicit CEs, for now and in the future (in 1.8)
 */
 static const ucolTokSuboption alternateSub[2] = {
     {suboption_00, 13, UCOL_NON_IGNORABLE},
     {suboption_01,  7, UCOL_SHIFTED}
 };
 static const ucolTokSuboption caseFirstSub[3] = {
     {suboption_02, 5, UCOL_LOWER_FIRST},
     {suboption_03,  5, UCOL_UPPER_FIRST},
     {suboption_04,  3, UCOL_OFF},
 };
 static const ucolTokSuboption onOffSub[2] = {
     {suboption_04, 3, UCOL_OFF},
     {suboption_05, 2, UCOL_ON}
 };
 static const ucolTokSuboption frenchSub[1] = {
     {suboption_07, 1, UCOL_ON}
 };
 static const ucolTokSuboption beforeSub[3] = {
     {suboption_06, 1, UCOL_PRIMARY},
     {suboption_07, 1, UCOL_SECONDARY},
     {suboption_08, 1, UCOL_TERTIARY}
 };
 static const ucolTokSuboption strengthSub[5] = {
     {suboption_06, 1, UCOL_PRIMARY},
     {suboption_07, 1, UCOL_SECONDARY},
     {suboption_08, 1, UCOL_TERTIARY},
     {suboption_09, 1, UCOL_QUATERNARY},
     {suboption_10, 1, UCOL_IDENTICAL},
 };
 static const ucolTokSuboption firstLastSub[7] = {
     {suboption_11, 7, UCOL_PRIMARY},
     {suboption_12, 9, UCOL_PRIMARY},
     {suboption_13, 8, UCOL_PRIMARY},
     {suboption_14, 8, UCOL_PRIMARY},
     {suboption_15, 7, UCOL_PRIMARY},
     {suboption_16, 8, UCOL_PRIMARY},
     {suboption_17, 8, UCOL_PRIMARY},
 };
 enum OptionNumber {
     OPTION_ALTERNATE_HANDLING = 0,
     OPTION_FRENCH_COLLATION,
     OPTION_CASE_LEVEL,
     OPTION_CASE_FIRST,
     OPTION_NORMALIZATION_MODE,
     OPTION_HIRAGANA_QUATERNARY,
     OPTION_STRENGTH,
     OPTION_NUMERIC_COLLATION,
     OPTION_NORMAL_OPTIONS_LIMIT = OPTION_NUMERIC_COLLATION,
     OPTION_VARIABLE_TOP,
     OPTION_REARRANGE,
     OPTION_BEFORE,
     OPTION_TOP,
     OPTION_FIRST,
     OPTION_LAST,
     OPTION_OPTIMIZE,
     OPTION_SUPPRESS_CONTRACTIONS,
     OPTION_UNDEFINED,
     OPTION_SCRIPT_ORDER,
     OPTION_CHARSET_NAME,
     OPTION_CHARSET,
     OPTION_IMPORT,
     OPTION_SCRIPTREORDER
 } ;
 static const ucolTokOption rulesOptions[UTOK_OPTION_COUNT] = {
     /*00*/ {option_02,  9, alternateSub, 2, UCOL_ALTERNATE_HANDLING}, /*"alternate" */
     /*01*/ {option_03,  9, frenchSub, 1, UCOL_FRENCH_COLLATION}, /*"backwards"      */
     /*02*/ {option_07,  9, onOffSub, 2, UCOL_CASE_LEVEL},  /*"caseLevel"      */
     /*03*/ {option_08,  9, caseFirstSub, 3, UCOL_CASE_FIRST}, /*"caseFirst"   */
     /*04*/ {option_06, 13, onOffSub, 2, UCOL_NORMALIZATION_MODE}, /*"normalization" */
     /*05*/ {option_13, 9, onOffSub, 2, UCOL_HIRAGANA_QUATERNARY_MODE}, /*"hiraganaQ" */
     /*06*/ {option_14, 8, strengthSub, 5, UCOL_STRENGTH}, /*"strength" */
     /*07*/ {option_19, 15, onOffSub, 2, UCOL_NUMERIC_COLLATION},  /*"numericOrdering"*/
     /*08*/ {option_04, 12, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"variable top"   */
     /*09*/ {option_01,  9, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"rearrange"      */
     /*10*/ {option_12,  6, beforeSub, 3, UCOL_ATTRIBUTE_COUNT}, /*"before"    */
     /*11*/ {option_05,  3, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"top"            */
     /*12*/ {option_15,  5, firstLastSub, 7, UCOL_ATTRIBUTE_COUNT}, /*"first" */
     /*13*/ {option_16,  4, firstLastSub, 7, UCOL_ATTRIBUTE_COUNT}, /*"last" */
     /*14*/ {option_17,  8, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"optimize"      */
     /*15*/ {option_18, 20, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"suppressContractions"      */
     /*16*/ {option_00,  9, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"undefined"      */
     /*17*/ {option_09, 11, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"scriptOrder"    */
     /*18*/ {option_10, 11, NULL, 0, UCOL_ATTRIBUTE_COUNT}, /*"charsetname"    */
     /*19*/ {option_11,  7, NULL, 0, UCOL_ATTRIBUTE_COUNT},  /*"charset"        */
     /*20*/ {option_20,  6, NULL, 0, UCOL_ATTRIBUTE_COUNT},  /*"import"        */
     /*21*/ {option_21,  7, NULL, 0, UCOL_ATTRIBUTE_COUNT}  /*"reorder"        */
 };
 static
 int32_t u_strncmpNoCase(const UChar     *s1,
                         const UChar     *s2,
                         int32_t     n)
 {
     if(n > 0) {
         int32_t rc;
         for(;;) {
             rc = (int32_t)u_tolower(*s1) - (int32_t)u_tolower(*s2);
             if(rc != 0 || *s1 == 0 || --n == 0) {
                 return rc;
             }
             ++s1;
             ++s2;
         }
     }
     return 0;
 }
 static
 void ucol_uprv_tok_initData() {
     if(!didInit) {
         U_STRING_INIT(suboption_00, "non-ignorable", 13);
         U_STRING_INIT(suboption_01, "shifted",        7);
         U_STRING_INIT(suboption_02, "lower",          5);
         U_STRING_INIT(suboption_03, "upper",          5);
         U_STRING_INIT(suboption_04, "off",            3);
         U_STRING_INIT(suboption_05, "on",             2);
         U_STRING_INIT(suboption_06, "1",              1);
         U_STRING_INIT(suboption_07, "2",              1);
         U_STRING_INIT(suboption_08, "3",              1);
         U_STRING_INIT(suboption_09, "4",              1);
         U_STRING_INIT(suboption_10, "I",              1);
         U_STRING_INIT(suboption_11, "primary",        7);
         U_STRING_INIT(suboption_12, "secondary",      9);
         U_STRING_INIT(suboption_13, "tertiary",       8);
         U_STRING_INIT(suboption_14, "variable",       8);
         U_STRING_INIT(suboption_15, "regular",        7);
         U_STRING_INIT(suboption_16, "implicit",       8);
         U_STRING_INIT(suboption_17, "trailing",       8);
         U_STRING_INIT(option_00, "undefined",      9);
         U_STRING_INIT(option_01, "rearrange",      9);
         U_STRING_INIT(option_02, "alternate",      9);
         U_STRING_INIT(option_03, "backwards",      9);
         U_STRING_INIT(option_04, "variable top",  12);
         U_STRING_INIT(option_05, "top",            3);
         U_STRING_INIT(option_06, "normalization", 13);
         U_STRING_INIT(option_07, "caseLevel",      9);
         U_STRING_INIT(option_08, "caseFirst",      9);
         U_STRING_INIT(option_09, "scriptOrder",   11);
         U_STRING_INIT(option_10, "charsetname",   11);
         U_STRING_INIT(option_11, "charset",        7);
         U_STRING_INIT(option_12, "before",         6);
         U_STRING_INIT(option_13, "hiraganaQ",      9);
         U_STRING_INIT(option_14, "strength",       8);
         U_STRING_INIT(option_15, "first",          5);
         U_STRING_INIT(option_16, "last",           4);
         U_STRING_INIT(option_17, "optimize",       8);
         U_STRING_INIT(option_18, "suppressContractions",         20);
         U_STRING_INIT(option_19, "numericOrdering",      15);
         U_STRING_INIT(option_20, "import ",        6);
         U_STRING_INIT(option_21, "reorder",        7);
         didInit = TRUE;
     }
 }
 // This function reads basic options to set in the runtime collator
 // used by data driven tests. Should not support build time options
 U_CAPI const UChar * U_EXPORT2
 ucol_tok_getNextArgument(const UChar *start, const UChar *end,
                          UColAttribute *attrib, UColAttributeValue *value,
                          UErrorCode *status)
 {
     uint32_t i = 0;
     int32_t j=0;
     UBool foundOption = FALSE;
     const UChar *optionArg = NULL;
     ucol_uprv_tok_initData();
     while(start < end && PatternProps::isWhiteSpace(*start)) { /* eat whitespace */
         start++;
     }
     if(start >= end) {
         return NULL;
     }
     /* skip opening '[' */
     if(*start == 0x005b) {
         start++;
     } else {
         *status = U_ILLEGAL_ARGUMENT_ERROR; // no opening '['
         return NULL;
     }
     while(i < UTOK_OPTION_COUNT) {
         if(u_strncmpNoCase(start, rulesOptions[i].optionName, rulesOptions[i].optionLen) == 0) {
             foundOption = TRUE;
             if(end - start > rulesOptions[i].optionLen) {
                 optionArg = start+rulesOptions[i].optionLen+1; /* start of the options, skip space */
                 while(PatternProps::isWhiteSpace(*optionArg)) { /* eat whitespace */
                     optionArg++;
                 }
             }
             break;
         }
         i++;
     }
     if(!foundOption) {
         *status = U_ILLEGAL_ARGUMENT_ERROR;
         return NULL;
     }
     if(optionArg) {
         for(j = 0; j<rulesOptions[i].subSize; j++) {
             if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
                 //ucol_uprv_tok_setOptionInImage(src->opts, rulesOptions[i].attr, rulesOptions[i].subopts[j].attrVal);
                 *attrib = rulesOptions[i].attr;
                 *value = rulesOptions[i].subopts[j].attrVal;
                 optionArg += rulesOptions[i].subopts[j].subLen;
                 while(PatternProps::isWhiteSpace(*optionArg)) { /* eat whitespace */
                     optionArg++;
                 }
                 if(*optionArg == 0x005d) {
                     optionArg++;
                     return optionArg;
                 } else {
                     *status = U_ILLEGAL_ARGUMENT_ERROR;
                     return NULL;
                 }
             }
         }
     }
     *status = U_ILLEGAL_ARGUMENT_ERROR;
     return NULL;
 }
 static
 USet *ucol_uprv_tok_readAndSetUnicodeSet(const UChar *start, const UChar *end, UErrorCode *status) {
     while(*start != 0x005b) { /* advance while we find the first '[' */
         start++;
     }
     // now we need to get a balanced set of '[]'. The problem is that a set can have
     // many, and *end point to the first closing '['
     int32_t noOpenBraces = 1;
     int32_t current = 1; // skip the opening brace
     while(start+current < end && noOpenBraces != 0) {
         if(start[current] == 0x005b) {
             noOpenBraces++;
         } else if(start[current] == 0x005D) { // closing brace
             noOpenBraces--;
         }
         current++;
     }
     if(noOpenBraces != 0 || u_strchr(start+current, 0x005d /*']'*/) == NULL) {
         *status = U_ILLEGAL_ARGUMENT_ERROR;
         return NULL;
     }
     return uset_openPattern(start, current, status);
 }
 /**
  * Reads an option and matches the option name with the predefined options. (Case-insensitive.)
  * @param start Pointer to the start UChar.
  * @param end Pointer to the last valid pointer beyond which the option will not extend.
  * @param optionArg Address of the pointer at which the options start (after the option name)
  * @return The index of the option, or -1 if the option is not valid.
  */
 static
 int32_t ucol_uprv_tok_readOption(const UChar *start, const UChar *end, const UChar **optionArg) {
     int32_t i = 0;
     ucol_uprv_tok_initData();
     while(PatternProps::isWhiteSpace(*start)) { /* eat whitespace */
         start++;
     }
     while(i < UTOK_OPTION_COUNT) {
         if(u_strncmpNoCase(start, rulesOptions[i].optionName, rulesOptions[i].optionLen) == 0) {
             if(end - start > rulesOptions[i].optionLen) {
                 *optionArg = start+rulesOptions[i].optionLen; /* End of option name; start of the options */
                 while(PatternProps::isWhiteSpace(**optionArg)) { /* eat whitespace */
                     (*optionArg)++;
                 }
             }
             break;
         }
         i++;
     }
     if(i == UTOK_OPTION_COUNT) {
         i = -1; // didn't find an option
     }
     return i;
 }
 static
 void ucol_tok_parseScriptReorder(UColTokenParser *src, UErrorCode *status) {
     int32_t codeCount = 0;
     int32_t codeIndex = 0;
     char conversion[64];
     int32_t tokenLength = 0;
     const UChar* space;
     const UChar* current = src->current;
     const UChar* end = u_memchr(src->current, 0x005d, src->end - src->current);
     // eat leading whitespace
     while(current < end && u_isWhitespace(*current)) {
         current++;
     }
     while(current < end) {
         space = u_memchr(current, 0x0020, end - current);
         space = space == 0 ? end : space;
         tokenLength = space - current;
         if (tokenLength < 4) {
             *status = U_INVALID_FORMAT_ERROR;
             return;
         }
         codeCount++;
         current += tokenLength;
         while(current < end && u_isWhitespace(*current)) { /* eat whitespace */
             ++current;
         }
     }
     if (codeCount == 0) {
         *status = U_INVALID_FORMAT_ERROR;
     }
     src->reorderCodesLength = codeCount;
     src->reorderCodes = (int32_t*)uprv_malloc(codeCount * sizeof(int32_t));
     current = src->current;
     // eat leading whitespace
     while(current < end && u_isWhitespace(*current)) {
         current++;
     }
     while(current < end) {
         space = u_memchr(current, 0x0020, end - current);
         space = space == 0 ? end : space;
         tokenLength = space - current;
         if (tokenLength < 4) {
             *status = U_ILLEGAL_ARGUMENT_ERROR;
             return;
         } else {
             u_UCharsToChars(current, conversion, tokenLength);
             conversion[tokenLength] = '\0';
             src->reorderCodes[codeIndex] = ucol_findReorderingEntry(conversion);
             if (src->reorderCodes[codeIndex] == USCRIPT_INVALID_CODE) {
                 src->reorderCodes[codeIndex] = u_getPropertyValueEnum(UCHAR_SCRIPT, conversion);
             }
             if (src->reorderCodes[codeIndex] == USCRIPT_INVALID_CODE) {
                 *status = U_ILLEGAL_ARGUMENT_ERROR;
             }
         }
         codeIndex++;
         current += tokenLength;
         while(current < end && u_isWhitespace(*current)) { /* eat whitespace */
             ++current;
         }
     }
 }
 // reads and conforms to various options in rules
 // end is the position of the first closing ']'
 // However, some of the options take an UnicodeSet definition
 // which needs to duplicate the closing ']'
 // for example: '[copy [\uAC00-\uD7FF]]'
 // These options will move end to the second ']' and the
 // caller will set the current to it.
 static
 uint8_t ucol_uprv_tok_readAndSetOption(UColTokenParser *src, UErrorCode *status) {
     const UChar* start = src->current;
     int32_t i = 0;
     int32_t j=0;
     const UChar *optionArg = NULL;
     uint8_t result = 0;
     start++; /*skip opening '['*/
     i = ucol_uprv_tok_readOption(start, src->end, &optionArg);
     if(optionArg) {
         src->current = optionArg;
     }
     if(i < 0) {
         *status = U_ILLEGAL_ARGUMENT_ERROR;
     } else {
         int32_t noOpenBraces = 1;
         switch(i) {
     case OPTION_ALTERNATE_HANDLING:
     case OPTION_FRENCH_COLLATION:
     case OPTION_CASE_LEVEL:
     case OPTION_CASE_FIRST:
     case OPTION_NORMALIZATION_MODE:
     case OPTION_HIRAGANA_QUATERNARY:
     case OPTION_STRENGTH:
     case OPTION_NUMERIC_COLLATION:
         if(optionArg) {
             for(j = 0; j<rulesOptions[i].subSize; j++) {
                 if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
                     ucol_uprv_tok_setOptionInImage(src->opts, rulesOptions[i].attr, rulesOptions[i].subopts[j].attrVal);
                     result =  UCOL_TOK_SUCCESS;
                 }
             }
         }
         if(result == 0) {
             *status = U_ILLEGAL_ARGUMENT_ERROR;
         }
         break;
     case OPTION_VARIABLE_TOP:
         result = UCOL_TOK_SUCCESS | UCOL_TOK_VARIABLE_TOP;
         break;
     case OPTION_REARRANGE:
         result = UCOL_TOK_SUCCESS;
         break;
     case OPTION_BEFORE:
         if(optionArg) {
             for(j = 0; j<rulesOptions[i].subSize; j++) {
                 if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
                     result = UCOL_TOK_SUCCESS | (rulesOptions[i].subopts[j].attrVal + 1);
                 }
             }
         }
         if(result == 0) {
             *status = U_ILLEGAL_ARGUMENT_ERROR;
         }
         break;
     case OPTION_TOP: /* we are going to have an array with structures of limit CEs */
         /* index to this array will be src->parsedToken.indirectIndex*/
         src->parsedToken.indirectIndex = 0;
         result = UCOL_TOK_SUCCESS | UCOL_TOK_TOP;
         break;
     case OPTION_FIRST:
     case OPTION_LAST: /* first, last */
         for(j = 0; j<rulesOptions[i].subSize; j++) {
             if(u_strncmpNoCase(optionArg, rulesOptions[i].subopts[j].subName, rulesOptions[i].subopts[j].subLen) == 0) {
                 // the calculation below assumes that OPTION_FIRST and OPTION_LAST are at i and i+1 and that the first
                 // element of indirect boundaries is reserved for top.
                 src->parsedToken.indirectIndex = (uint16_t)(i-OPTION_FIRST+1+j*2);
                 result =  UCOL_TOK_SUCCESS | UCOL_TOK_TOP;;
             }
         }
         if(result == 0) {
             *status = U_ILLEGAL_ARGUMENT_ERROR;
         }
         break;
     case OPTION_OPTIMIZE:
     case OPTION_SUPPRESS_CONTRACTIONS:  // copy and remove are handled before normalization
         // we need to move end here
         src->current++; // skip opening brace
         while(src->current < src->end && noOpenBraces != 0) {
             if(*src->current == 0x005b) {
                 noOpenBraces++;
             } else if(*src->current == 0x005D) { // closing brace
                 noOpenBraces--;
             }
             src->current++;
         }
         result = UCOL_TOK_SUCCESS;
         break;
     case OPTION_SCRIPTREORDER:
         ucol_tok_parseScriptReorder(src, status);
         break;
     default:
         *status = U_UNSUPPORTED_ERROR;
         break;
         }
     }
     src->current = u_memchr(src->current, 0x005d, (int32_t)(src->end-src->current));
     return result;
 }
 inline void ucol_tok_addToExtraCurrent(UColTokenParser *src, const UChar *stuff, int32_t len, UErrorCode *status) {
     if (stuff == NULL || len <= 0) {
         return;
     }
     UnicodeString tempStuff(FALSE, stuff, len);
     if(src->extraCurrent+len >= src->extraEnd) {
         /* reallocate */
         if (stuff >= src->source && stuff <= src->end) {
             // Copy the "stuff" contents into tempStuff's own buffer.
             // UnicodeString is copy-on-write.
             if (len > 0) {
                 tempStuff.setCharAt(0, tempStuff[0]);
             } else {
                 tempStuff.remove();
             }
         }
         UChar *newSrc = (UChar *)uprv_realloc(src->source, (src->extraEnd-src->source)*2*sizeof(UChar));
         if(newSrc != NULL) {
             src->current = newSrc + (src->current - src->source);
             src->extraCurrent = newSrc + (src->extraCurrent - src->source);
             src->end = newSrc + (src->end - src->source);
             src->extraEnd = newSrc + (src->extraEnd-src->source)*2;
             src->sourceCurrent = newSrc + (src->sourceCurrent-src->source);
             src->source = newSrc;
         } else {
             *status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
     }
     if(len == 1) {
         *src->extraCurrent++ = tempStuff[0];
     } else {
         u_memcpy(src->extraCurrent, tempStuff.getBuffer(), len);
         src->extraCurrent += len;
     }
 }
 inline UBool ucol_tok_doSetTop(UColTokenParser *src, UErrorCode *status) {
     /*
     top = TRUE;
     */
     UChar buff[5];
     src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
     buff[0] = 0xFFFE;
     buff[1] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE >> 16);
     buff[2] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE & 0xFFFF);
     if(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE == 0) {
         src->parsedToken.charsLen = 3;
         ucol_tok_addToExtraCurrent(src, buff, 3, status);
     } else {
         buff[3] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE >> 16);
         buff[4] = (UChar)(ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE & 0xFFFF);
         src->parsedToken.charsLen = 5;
         ucol_tok_addToExtraCurrent(src, buff, 5, status);
     }
     return TRUE;
 }
 static UBool isCharNewLine(UChar c){
     switch(c){
     case 0x000A: /* LF  */
     case 0x000D: /* CR  */
     case 0x000C: /* FF  */
     case 0x0085: /* NEL */
     case 0x2028: /* LS  */
     case 0x2029: /* PS  */
         return TRUE;
     default:
         return FALSE;
     }
 }
 /*
  * This function is called several times when a range is processed.  Each time, the next code point
  * is processed.
  * The following variables must be set before calling this function:
  *   src->currentRangeCp:  The current code point to process.
  *   src->lastRangeCp: The last code point in the range.
  * Pre-requisite: src->currentRangeCp <= src->lastRangeCp.
  */
 static const UChar*
 ucol_tok_processNextCodePointInRange(UColTokenParser *src,
                                      UErrorCode *status)
 {
   // Append current code point to source
   UChar buff[U16_MAX_LENGTH];
   uint32_t i = 0;
   uint32_t nChars = U16_LENGTH(src->currentRangeCp);
   src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
   src->parsedToken.charsLen = nChars;
   U16_APPEND_UNSAFE(buff, i, src->currentRangeCp);
   ucol_tok_addToExtraCurrent(src, buff, nChars, status);
   ++src->currentRangeCp;
   if (src->currentRangeCp > src->lastRangeCp) {
     src->inRange = FALSE;
     if (src->currentStarredCharIndex > src->lastStarredCharIndex) {
       src->isStarred = FALSE;
     }
   } else {
     src->previousCp = src->currentRangeCp;
   }
   return src->current;
 }
 /*
  * This function is called several times when a starred list is processed.  Each time, the next code point
  * in the list is processed.
  * The following variables must be set before calling this function:
  *   src->currentStarredCharIndex:  Index (in src->source) of the first char of the current code point.
  *   src->lastStarredCharIndex: Index to the last character in the list.
  * Pre-requisite: src->currentStarredCharIndex <= src->lastStarredCharIndex.
  */
 static const UChar*
 ucol_tok_processNextTokenInStarredList(UColTokenParser *src)
 {
   // Extract the characters corresponding to the next code point.
   UChar32 cp;
   src->parsedToken.charsOffset = src->currentStarredCharIndex;
   int32_t prev = src->currentStarredCharIndex;
   U16_NEXT(src->source, src->currentStarredCharIndex, (uint32_t)(src->end - src->source), cp);
   src->parsedToken.charsLen = src->currentStarredCharIndex - prev;
   // When we are done parsing the starred string, turn the flag off so that
   // the normal processing is restored.
   if (src->currentStarredCharIndex > src->lastStarredCharIndex) {
     src->isStarred = FALSE;
   }
   src->previousCp = cp;
   return src->current;
 }
 /*
  * Partially parses the next token, keeps the indices in src->parsedToken, and updates the counters.
  *
  * This routine parses and separates almost all tokens. The following are the syntax characters recognized.
  *  # : Comment character
  *  & : Reset operator
  *  = : Equality
  *  < : Primary collation
  *  << : Secondary collation
  *  <<< : Tertiary collation
  *  ; : Secondary collation
  *  , : Tertiary collation
  *  / : Expansions
  *  | : Prefix
  *  - : Range
  *  ! : Java Thai modifier, ignored
  *  @ : French only
  * [] : Options
  * '' : Quotes
  *
  *  Along with operators =, <, <<, <<<, the operator * is supported to indicate a list.  For example, &a<*bcdexyz
  *  is equivalent to &a<b<c<d<e<x<y<z.  In lists, ranges also can be given, so &a*b-ex-z is equivalent to the above.
  *  This function do not separate the tokens in a list.  Instead, &a<*b-ex-z is parsed as three tokens - "&a",
  *  "<*b", "-ex", "-z".  The strength (< in this case), whether in a list, whether in a range and the previous
  *  character returned as cached so that the calling program can do further splitting.
  */
 static const UChar*
 ucol_tok_parseNextTokenInternal(UColTokenParser *src,
                                 UBool startOfRules,
                                 UParseError *parseError,
                                 UErrorCode *status)
 {
     UBool variableTop = FALSE;
     UBool top = FALSE;
     UBool inChars = TRUE;
     UBool inQuote = FALSE;
     UBool wasInQuote = FALSE;
     uint8_t before = 0;
     UBool isEscaped = FALSE;
     // TODO: replace these variables with src->parsedToken counterparts
     // no need to use them anymore since we have src->parsedToken.
     // Ideally, token parser would be a nice class... Once, when I have
     // more time (around 2020 probably).
     uint32_t newExtensionLen = 0;
     uint32_t extensionOffset = 0;
     uint32_t newStrength = UCOL_TOK_UNSET;
     UChar buff[10];
     src->parsedToken.charsOffset = 0;  src->parsedToken.charsLen = 0;
     src->parsedToken.prefixOffset = 0; src->parsedToken.prefixLen = 0;
     src->parsedToken.indirectIndex = 0;
     while (src->current < src->end) {
         UChar ch = *(src->current);
         if (inQuote) {
             if (ch == 0x0027/*'\''*/) {
                 inQuote = FALSE;
             } else {
                 if ((src->parsedToken.charsLen == 0) || inChars) {
                     if(src->parsedToken.charsLen == 0) {
                         src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
                     }
                     src->parsedToken.charsLen++;
                 } else {
                     if(newExtensionLen == 0) {
                         extensionOffset = (uint32_t)(src->extraCurrent - src->source);
                     }
                     newExtensionLen++;
                 }
             }
         }else if(isEscaped){
             isEscaped =FALSE;
             if (newStrength == UCOL_TOK_UNSET) {
                 *status = U_INVALID_FORMAT_ERROR;
                 syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                 DBG_FORMAT_ERROR
                 return NULL;
                 // enabling rules to start with non-tokens a < b
                 // newStrength = UCOL_TOK_RESET;
             }
             if(ch != 0x0000  && src->current != src->end) {
                 if (inChars) {
                     if(src->parsedToken.charsLen == 0) {
                         src->parsedToken.charsOffset = (uint32_t)(src->current - src->source);
                     }
                     src->parsedToken.charsLen++;
                 } else {
                     if(newExtensionLen == 0) {
                         extensionOffset = (uint32_t)(src->current - src->source);
                     }
                     newExtensionLen++;
                 }
             }
         }else {
             if(!PatternProps::isWhiteSpace(ch)) {
                 /* Sets the strength for this entry */
                 switch (ch) {
                 case 0x003D/*'='*/ :
                     if (newStrength != UCOL_TOK_UNSET) {
                         goto EndOfLoop;
                     }
                     /* if we start with strength, we'll reset to top */
                     if(startOfRules == TRUE) {
                         src->parsedToken.indirectIndex = 5;
                         top = ucol_tok_doSetTop(src, status);
                         newStrength = UCOL_TOK_RESET;
                         goto EndOfLoop;
                     }
                     newStrength = UCOL_IDENTICAL;
                     if(*(src->current+1) == 0x002A) {/*'*'*/
                         src->current++;
                         src->isStarred = TRUE;
                     }
                     break;
                 case 0x002C/*','*/:
                     if (newStrength != UCOL_TOK_UNSET) {
                         goto EndOfLoop;
                     }
                     /* if we start with strength, we'll reset to top */
                     if(startOfRules == TRUE) {
                         src->parsedToken.indirectIndex = 5;
                         top = ucol_tok_doSetTop(src, status);
                         newStrength = UCOL_TOK_RESET;
                         goto EndOfLoop;
                     }
                     newStrength = UCOL_TERTIARY;
                     break;
                 case  0x003B/*';'*/:
                     if (newStrength != UCOL_TOK_UNSET) {
                         goto EndOfLoop;
                     }
                     /* if we start with strength, we'll reset to top */
                     if(startOfRules == TRUE) {
                         src->parsedToken.indirectIndex = 5;
                         top = ucol_tok_doSetTop(src, status);
                         newStrength = UCOL_TOK_RESET;
                         goto EndOfLoop;
                     }
                     newStrength = UCOL_SECONDARY;
                     break;
                 case 0x003C/*'<'*/:
                     if (newStrength != UCOL_TOK_UNSET) {
                         goto EndOfLoop;
                     }
                     /* if we start with strength, we'll reset to top */
                     if(startOfRules == TRUE) {
                         src->parsedToken.indirectIndex = 5;
                         top = ucol_tok_doSetTop(src, status);
                         newStrength = UCOL_TOK_RESET;
                         goto EndOfLoop;
                     }
                     /* before this, do a scan to verify whether this is */
                     /* another strength */
                     if(*(src->current+1) == 0x003C) {
                         src->current++;
                         if(*(src->current+1) == 0x003C) {
                             src->current++; /* three in a row! */
                             newStrength = UCOL_TERTIARY;
                         } else { /* two in a row */
                             newStrength = UCOL_SECONDARY;
                         }
                     } else { /* just one */
                         newStrength = UCOL_PRIMARY;
                     }
                     if(*(src->current+1) == 0x002A) {/*'*'*/
                         src->current++;
                         src->isStarred = TRUE;
                     }
                     break;
                 case 0x0026/*'&'*/:
                     if (newStrength != UCOL_TOK_UNSET) {
                         /**/
                         goto EndOfLoop;
                     }
                     newStrength = UCOL_TOK_RESET; /* PatternEntry::RESET = 0 */
                     break;
                 case 0x005b/*'['*/:
                     /* options - read an option, analyze it */
                     if(u_strchr(src->current, 0x005d /*']'*/) != NULL) {
                         uint8_t result = ucol_uprv_tok_readAndSetOption(src, status);
                         if(U_SUCCESS(*status)) {
                             if(result & UCOL_TOK_TOP) {
                                 if(newStrength == UCOL_TOK_RESET) {
                                     top = ucol_tok_doSetTop(src, status);
                                     if(before) { // This is a combination of before and indirection like '&[before 2][first regular]<b'
                                         src->parsedToken.charsLen+=2;
                                         buff[0] = 0x002d;
                                         buff[1] = before;
                                         ucol_tok_addToExtraCurrent(src, buff, 2, status);
                                     }
                                     src->current++;
                                     goto EndOfLoop;
                                 } else {
                                     *status = U_INVALID_FORMAT_ERROR;
                                     syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                                     DBG_FORMAT_ERROR
                                 }
                             } else if(result & UCOL_TOK_VARIABLE_TOP) {
                                 if(newStrength != UCOL_TOK_RESET && newStrength != UCOL_TOK_UNSET) {
                                     variableTop = TRUE;
                                     src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
                                     src->parsedToken.charsLen = 1;
                                     buff[0] = 0xFFFF;
                                     ucol_tok_addToExtraCurrent(src, buff, 1, status);
                                     src->current++;
                                     goto EndOfLoop;
                                 } else {
                                     *status = U_INVALID_FORMAT_ERROR;
                                     syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                                     DBG_FORMAT_ERROR
                                 }
                             } else if (result & UCOL_TOK_BEFORE){
                                 if(newStrength == UCOL_TOK_RESET) {
                                     before = result & UCOL_TOK_BEFORE;
                                 } else {
                                     *status = U_INVALID_FORMAT_ERROR;
                                     syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                                     DBG_FORMAT_ERROR
                                 }
                             }
                         } else {
                             *status = U_INVALID_FORMAT_ERROR;
                             syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                             DBG_FORMAT_ERROR
                             return NULL;
                         }
                     }
                     break;
                 case 0x0021/*! skip java thai modifier reordering*/:
                     break;
                 case 0x002F/*'/'*/:
                     wasInQuote = FALSE; /* if we were copying source characters, we want to stop now */
                     inChars = FALSE; /* we're now processing expansion */
                     break;
                 case 0x005C /* back slash for escaped chars */:
                     isEscaped = TRUE;
                     break;
                     /* found a quote, we're gonna start copying */
                 case 0x0027/*'\''*/:
                     if (newStrength == UCOL_TOK_UNSET) { /* quote is illegal until we have a strength */
                       *status = U_INVALID_FORMAT_ERROR;
                       syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                       DBG_FORMAT_ERROR
                       return NULL;
                       // enabling rules to start with a non-token character a < b
                       // newStrength = UCOL_TOK_RESET;
                     }
                     inQuote = TRUE;
                     if(inChars) { /* we're doing characters */
                         if(wasInQuote == FALSE) {
                             src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
                         }
                         if (src->parsedToken.charsLen != 0) {
                             ucol_tok_addToExtraCurrent(src, src->current - src->parsedToken.charsLen, src->parsedToken.charsLen, status);
                         }
                         src->parsedToken.charsLen++;
                     } else { /* we're doing an expansion */
                         if(wasInQuote == FALSE) {
                             extensionOffset = (uint32_t)(src->extraCurrent - src->source);
                         }
                         if (newExtensionLen != 0) {
                             ucol_tok_addToExtraCurrent(src, src->current - newExtensionLen, newExtensionLen, status);
                         }
                         newExtensionLen++;
                     }
                     wasInQuote = TRUE;
                     ch = *(++(src->current));
                     if(ch == 0x0027) { /* copy the double quote */
                         ucol_tok_addToExtraCurrent(src, &ch, 1, status);
                         inQuote = FALSE;
                     }
                     break;
                     /* '@' is french only if the strength is not currently set */
                     /* if it is, it's just a regular character in collation rules */
                 case 0x0040/*'@'*/:
                     if (newStrength == UCOL_TOK_UNSET) {
                         src->opts->frenchCollation = UCOL_ON;
                         break;
                     }
                 case 0x007C /*|*/: /* this means we have actually been reading prefix part */
                     // we want to store read characters to the prefix part and continue reading
                     // the characters (proper way would be to restart reading the chars, but in
                     // that case we would have to complicate the token hasher, which I do not
                     // intend to play with. Instead, we will do prefixes when prefixes are due
                     // (before adding the elements).
                     src->parsedToken.prefixOffset = src->parsedToken.charsOffset;
                     src->parsedToken.prefixLen = src->parsedToken.charsLen;
                     if(inChars) { /* we're doing characters */
                         if(wasInQuote == FALSE) {
                             src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
                         }
                         if (src->parsedToken.charsLen != 0) {
                             ucol_tok_addToExtraCurrent(src, src->current - src->parsedToken.charsLen, src->parsedToken.charsLen, status);
                         }
                         src->parsedToken.charsLen++;
                     }
                     wasInQuote = TRUE;
                     do {
                         ch = *(++(src->current));
                         // skip whitespace between '|' and the character
                     } while (PatternProps::isWhiteSpace(ch));
                     break;
                     //charsOffset = 0;
                     //newCharsLen = 0;
                     //break; // We want to store the whole prefix/character sequence. If we break
                     // the '|' is going to get lost.
                 case 0x002D /*-*/: /* A range. */
                     if (newStrength != UCOL_TOK_UNSET) {
                       // While processing the pending token, the isStarred field
                       // is reset, so it needs to be saved for the next
                       // invocation.
                       src->savedIsStarred = src->isStarred;
                       goto EndOfLoop;
                    }
                    src->isStarred = src->savedIsStarred;
                    // Ranges are valid only in starred tokens.
                    if (!src->isStarred) {
                      *status = U_INVALID_FORMAT_ERROR;
                      syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                      DBG_FORMAT_ERROR
                      return NULL;
                    }
                    newStrength = src->parsedToken.strength;
                    src->inRange = TRUE;
                    break;
                 case 0x0023 /*#*/: /* this is a comment, skip everything through the end of line */
                     do {
                         ch = *(++(src->current));
                     } while (!isCharNewLine(ch));
                     break;
                 default:
                     if (newStrength == UCOL_TOK_UNSET) {
                       *status = U_INVALID_FORMAT_ERROR;
                       syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                       DBG_FORMAT_ERROR
                       return NULL;
                     }
                     if (ucol_tok_isSpecialChar(ch) && (inQuote == FALSE)) {
                         *status = U_INVALID_FORMAT_ERROR;
                         syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
                         DBG_FORMAT_ERROR
                         return NULL;
                     }
                     if(ch == 0x0000 && src->current+1 == src->end) {
                         break;
                     }
                     if (inChars) {
                         if(src->parsedToken.charsLen == 0) {
                             src->parsedToken.charsOffset = (uint32_t)(src->current - src->source);
                         }
                         src->parsedToken.charsLen++;
                     } else {
                         if(newExtensionLen == 0) {
                             extensionOffset = (uint32_t)(src->current - src->source);
                         }
                         newExtensionLen++;
                     }
                     break;
                 }
             }
         }
         if(wasInQuote) {
             if(ch != 0x27) {
                 if(inQuote || !PatternProps::isWhiteSpace(ch)) {
                     ucol_tok_addToExtraCurrent(src, &ch, 1, status);
                 }
             }
         }
         src->current++;
     }
 EndOfLoop:
     wasInQuote = FALSE;
     if (newStrength == UCOL_TOK_UNSET) {
         return NULL;
     }
     if (src->parsedToken.charsLen == 0 && top == FALSE) {
         syntaxError(src->source,(int32_t)(src->current-src->source),(int32_t)(src->end-src->source),parseError);
         *status = U_INVALID_FORMAT_ERROR;
         DBG_FORMAT_ERROR
         return NULL;
     }
     src->parsedToken.strength = newStrength;
     src->parsedToken.extensionOffset = extensionOffset;
     src->parsedToken.extensionLen = newExtensionLen;
     src->parsedToken.flags = (UCOL_TOK_VARIABLE_TOP * (variableTop?1:0)) | (UCOL_TOK_TOP * (top?1:0)) | before;
     return src->current;
 }
 /*
  * Parses the next token, keeps the indices in src->parsedToken, and updates the counters.
  * @see ucol_tok_parseNextTokenInternal() for the description of what operators are supported.
  *
  * In addition to what ucol_tok_parseNextTokenInternal() does, this function does the following:
  *  1) ucol_tok_parseNextTokenInternal() returns a range as a single token.  This function separates
  *     it to separate tokens and returns one by one.  In order to do that, the necessary states are
  *     cached as member variables of the token parser.
  *  2) When encountering a range, ucol_tok_parseNextTokenInternal() processes characters up to the
  *     starting character as a single list token (which is separated into individual characters here)
  *     and as another list token starting with the last character in the range.  Before expanding it
  *     as a list of tokens, this function expands the range by filling the intermediate characters and
  *     returns them one by one as separate tokens.
  * Necessary checks are done for invalid combinations.
  */
 U_CAPI const UChar* U_EXPORT2
 ucol_tok_parseNextToken(UColTokenParser *src,
                         UBool startOfRules,
                         UParseError *parseError,
                         UErrorCode *status)
 {
   const UChar *nextToken;
   if (src->inRange) {
     // We are not done processing a range.  Continue it.
     return ucol_tok_processNextCodePointInRange(src, status);
   } else if (src->isStarred) {
     // We are not done processing a starred token.  Continue it.
     return ucol_tok_processNextTokenInStarredList(src);
   }
   // Get the next token.
   nextToken = ucol_tok_parseNextTokenInternal(src, startOfRules, parseError, status);
   if (nextToken == NULL) {
     return NULL;
   }
   if (src->inRange) {
     // A new range has started.
     // Check whether it is a chain of ranges with more than one hyphen.
     if (src->lastRangeCp > 0 && src->lastRangeCp == src->previousCp) {
         *status = U_INVALID_FORMAT_ERROR;
         syntaxError(src->source,src->parsedToken.charsOffset-1,
                     src->parsedToken.charsOffset+src->parsedToken.charsLen, parseError);
         DBG_FORMAT_ERROR
         return NULL;
     }
     // The current token indicates the second code point of the range.
     // Process just that, and then proceed with the star.
     src->currentStarredCharIndex = src->parsedToken.charsOffset;
     U16_NEXT(src->source, src->currentStarredCharIndex,
              (uint32_t)(src->end - src->source), src->lastRangeCp);
     if (src->lastRangeCp <= src->previousCp) {
         *status = U_INVALID_FORMAT_ERROR;
         syntaxError(src->source,src->parsedToken.charsOffset-1,
                     src->parsedToken.charsOffset+src->parsedToken.charsLen,parseError);
         DBG_FORMAT_ERROR
         return NULL;
     }
     // Set current range code point to process the range loop
     src->currentRangeCp = src->previousCp + 1;
     src->lastStarredCharIndex = src->parsedToken.charsOffset + src->parsedToken.charsLen - 1;
     return ucol_tok_processNextCodePointInRange(src, status);
  } else if (src->isStarred) {
     // We define two indices m_currentStarredCharIndex_ and m_lastStarredCharIndex_ so that
     // [m_currentStarredCharIndex_ .. m_lastStarredCharIndex_], both inclusive, need to be
     // separated into several tokens and returned.
     src->currentStarredCharIndex = src->parsedToken.charsOffset;
     src->lastStarredCharIndex =  src->parsedToken.charsOffset + src->parsedToken.charsLen - 1;
     return ucol_tok_processNextTokenInStarredList(src);
   } else {
     // Set previous codepoint
     U16_GET(src->source, 0, src->parsedToken.charsOffset, (uint32_t)(src->end - src->source), src->previousCp);
   }
   return nextToken;
 }
 /*
 Processing Description
 Build a ListList. Each list has a header, which contains two lists (positive
 and negative), a reset token, a baseCE, nextCE, and previousCE. The lists and
 reset may be null.
 As you process, you keep a LAST pointer that points to the last token you
 handled.
 */
 static UColToken *ucol_tok_initAReset(UColTokenParser *src, const UChar *expand, uint32_t *expandNext,
                                       UParseError *parseError, UErrorCode *status)
 {
     if(src->resultLen == src->listCapacity) {
         // Unfortunately, this won't work, as we store addresses of lhs in token
         src->listCapacity *= 2;
         src->lh = (UColTokListHeader *)uprv_realloc(src->lh, src->listCapacity*sizeof(UColTokListHeader));
         if(src->lh == NULL) {
             *status = U_MEMORY_ALLOCATION_ERROR;
             return NULL;
         }
     }
     /* do the reset thing */
     UColToken *sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
     /* test for NULL */
     if (sourceToken == NULL) {
         *status = U_MEMORY_ALLOCATION_ERROR;
         return NULL;
     }
     sourceToken->rulesToParseHdl = &(src->source);
     sourceToken->source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
     sourceToken->expansion = src->parsedToken.extensionLen << 24 | src->parsedToken.extensionOffset;
     sourceToken->debugSource = *(src->source + src->parsedToken.charsOffset);
     sourceToken->debugExpansion = *(src->source + src->parsedToken.extensionOffset);
     // keep the flags around so that we know about before
     sourceToken->flags = src->parsedToken.flags;
     if(src->parsedToken.prefixOffset != 0) {
         // this is a syntax error
         *status = U_INVALID_FORMAT_ERROR;
         syntaxError(src->source,src->parsedToken.charsOffset-1,src->parsedToken.charsOffset+src->parsedToken.charsLen,parseError);
         DBG_FORMAT_ERROR
         uprv_free(sourceToken);
         return 0;
     } else {
         sourceToken->prefix = 0;
     }
     sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
     sourceToken->strength = UCOL_TOK_RESET;
     sourceToken->next = NULL;
     sourceToken->previous = NULL;
     sourceToken->noOfCEs = 0;
     sourceToken->noOfExpCEs = 0;
     sourceToken->listHeader = &src->lh[src->resultLen];
     src->lh[src->resultLen].first = NULL;
     src->lh[src->resultLen].last = NULL;
     src->lh[src->resultLen].first = NULL;
     src->lh[src->resultLen].last = NULL;
     src->lh[src->resultLen].reset = sourceToken;
     /*
 Consider each item: relation, source, and expansion: e.g. ...< x / y ...
     First convert all expansions into normal form. Examples:
     If "xy" doesn't occur earlier in the list or in the UCA, convert &xy * c *
     d * ... into &x * c/y * d * ...
     Note: reset values can never have expansions, although they can cause the
     very next item to have one. They may be contractions, if they are found
     earlier in the list.
     */
     *expandNext = 0;
     if(expand != NULL) {
         /* check to see if there is an expansion */
         if(src->parsedToken.charsLen > 1) {
             uint32_t resetCharsOffset;
             resetCharsOffset = (uint32_t)(expand - src->source);
             sourceToken->source = ((resetCharsOffset - src->parsedToken.charsOffset ) << 24) | src->parsedToken.charsOffset;
             *expandNext = ((src->parsedToken.charsLen + src->parsedToken.charsOffset - resetCharsOffset)<<24) | (resetCharsOffset);
         }
     }
     src->resultLen++;
     uhash_put(src->tailored, sourceToken, sourceToken, status);
     return sourceToken;
 }
 static
 inline UColToken *getVirginBefore(UColTokenParser *src, UColToken *sourceToken, uint8_t strength, UParseError *parseError, UErrorCode *status) {
     if(U_FAILURE(*status)) {
         return NULL;
     }
     /* this is a virgin before - we need to fish the anchor from the UCA */
     collIterate s;
     uint32_t baseCE = UCOL_NOT_FOUND, baseContCE = UCOL_NOT_FOUND;
     uint32_t CE, SecondCE;
     // uint32_t invPos;
     if(sourceToken != NULL) {
         uprv_init_collIterate(src->UCA, src->source+((sourceToken->source)&0xFFFFFF), 1, &s, status);
     } else {
         uprv_init_collIterate(src->UCA, src->source+src->parsedToken.charsOffset /**charsOffset*/, 1, &s, status);
     }
     if(U_FAILURE(*status)) {
         return NULL;
     }
     baseCE = ucol_getNextCE(src->UCA, &s, status) & 0xFFFFFF3F;
     baseContCE = ucol_getNextCE(src->UCA, &s, status);
     if(baseContCE == UCOL_NO_MORE_CES) {
         baseContCE = 0;
     }
     UCAConstants *consts = (UCAConstants *)((uint8_t *)src->UCA->image + src->UCA->image->UCAConsts);
     uint32_t ch = 0;
     uint32_t expandNext = 0;
     UColToken key;
     if((baseCE & 0xFF000000) >= (consts->UCA_PRIMARY_IMPLICIT_MIN<<24) && (baseCE & 0xFF000000) <= (consts->UCA_PRIMARY_IMPLICIT_MAX<<24) ) { /* implicits - */
         uint32_t primary = (baseCE & UCOL_PRIMARYMASK) | ((baseContCE & UCOL_PRIMARYMASK) >> 16);
         uint32_t raw = uprv_uca_getRawFromImplicit(primary);
         ch = uprv_uca_getCodePointFromRaw(raw-1);
         uint32_t primaryCE = uprv_uca_getImplicitFromRaw(raw-1);
         CE = (primaryCE & UCOL_PRIMARYMASK) | 0x0505;
         SecondCE = ((primaryCE << 16) & UCOL_PRIMARYMASK) | UCOL_CONTINUATION_MARKER;
         src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source);
         *src->extraCurrent++ = 0xFFFE;
         *src->extraCurrent++ = (UChar)ch;
         src->parsedToken.charsLen++;
         key.source = (src->parsedToken.charsLen/**newCharsLen*/ << 24) | src->parsedToken.charsOffset/**charsOffset*/;
         key.rulesToParseHdl = &(src->source);
         //sourceToken = (UColToken *)uhash_iget(src->tailored, (int32_t)key);
         sourceToken = (UColToken *)uhash_get(src->tailored, &key);
         if(sourceToken == NULL) {
             src->lh[src->resultLen].baseCE = CE & 0xFFFFFF3F;
             if(isContinuation(SecondCE)) {
                 src->lh[src->resultLen].baseContCE = SecondCE;
             } else {
                 src->lh[src->resultLen].baseContCE = 0;
             }
             src->lh[src->resultLen].nextCE = 0;
             src->lh[src->resultLen].nextContCE = 0;
             src->lh[src->resultLen].previousCE = 0;
             src->lh[src->resultLen].previousContCE = 0;
             src->lh[src->resultLen].indirect = FALSE;
             sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
         }
     } else {
         /* invPos = */ ucol_inv_getPrevCE(src, baseCE, baseContCE, &CE, &SecondCE, strength);
         // we got the previous CE. Now we need to see if the difference between
         // the two CEs is really of the requested strength.
         // if it's a bigger difference (we asked for secondary and got primary), we
         // need to modify the CE.
         if(ucol_getCEStrengthDifference(baseCE, baseContCE, CE, SecondCE) < strength) {
             // adjust the strength
             // now we are in the situation where our baseCE should actually be modified in
             // order to get the CE in the right position.
             if(strength == UCOL_SECONDARY) {
                 CE = baseCE - 0x0200;
             } else { // strength == UCOL_TERTIARY
                 CE = baseCE - 0x02;
             }
             if(baseContCE) {
                 if(strength == UCOL_SECONDARY) {
                     SecondCE = baseContCE - 0x0200;
                 } else { // strength == UCOL_TERTIARY
                     SecondCE = baseContCE - 0x02;
                 }
             }
         }
 #if 0
         // the code below relies on getting a code point from the inverse table, in order to be
         // able to merge the situations like &x < 9 &[before 1]a < d. This won't work:
         // 1. There are many code points that have the same CE
         // 2. The CE to codepoint table (things pointed to by CETable[3*invPos+2] are broken.
         // Also, in case when there is no equivalent strength before an element, we have to actually
         // construct one. For example, &[before 2]a << x won't result in x << a, because the element
         // before a is a primary difference.
         //uint32_t *CETable = (uint32_t *)((uint8_t *)src->invUCA+src->invUCA->table);
         ch = CETable[3*invPos+2];
         if((ch &  UCOL_INV_SIZEMASK) != 0) {
             uint16_t *conts = (uint16_t *)((uint8_t *)src->invUCA+src->invUCA->conts);
             uint32_t offset = (ch & UCOL_INV_OFFSETMASK);
             ch = conts[offset];
         }
         *src->extraCurrent++ = (UChar)ch;
         src->parsedToken.charsOffset = (uint32_t)(src->extraCurrent - src->source - 1);
         src->parsedToken.charsLen = 1;
         // We got an UCA before. However, this might have been tailored.
         // example:
         // &\u30ca = \u306a
         // &[before 3]\u306a<<<\u306a|\u309d
         // uint32_t key = (*newCharsLen << 24) | *charsOffset;
         key.source = (src->parsedToken.charsLen/**newCharsLen*/ << 24) | src->parsedToken.charsOffset/**charsOffset*/;
         key.rulesToParseHdl = &(src->source);
         //sourceToken = (UColToken *)uhash_iget(src->tailored, (int32_t)key);
         sourceToken = (UColToken *)uhash_get(src->tailored, &key);
 #endif
         // here is how it should be. The situation such as &[before 1]a < x, should be
         // resolved exactly as if we wrote &a > x.
         // therefore, I don't really care if the UCA value before a has been changed.
         // However, I do care if the strength between my element and the previous element
         // is bigger then I wanted. So, if CE < baseCE and I wanted &[before 2], then i'll
         // have to construct the base CE.
         // if we found a tailored thing, we have to use the UCA value and construct
         // a new reset token with constructed name
         //if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
         // character to which we want to anchor is already tailored.
         // We need to construct a new token which will be the anchor
         // point
         //*(src->extraCurrent-1) = 0xFFFE;
         //*src->extraCurrent++ = (UChar)ch;
         // grab before
         src->parsedToken.charsOffset -= 10;
         src->parsedToken.charsLen += 10;
         src->lh[src->resultLen].baseCE = CE & 0xFFFFFF3F;
         if(isContinuation(SecondCE)) {
             src->lh[src->resultLen].baseContCE = SecondCE;
         } else {
             src->lh[src->resultLen].baseContCE = 0;
         }
         src->lh[src->resultLen].nextCE = 0;
         src->lh[src->resultLen].nextContCE = 0;
         src->lh[src->resultLen].previousCE = 0;
         src->lh[src->resultLen].previousContCE = 0;
         src->lh[src->resultLen].indirect = FALSE;
         sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
         //}
     }
     return sourceToken;
 }
 uint32_t ucol_tok_assembleTokenList(UColTokenParser *src, UParseError *parseError, UErrorCode *status) {
     UColToken *lastToken = NULL;
     const UChar *parseEnd = NULL;
     uint32_t expandNext = 0;
     UBool variableTop = FALSE;
     UBool top = FALSE;
     uint16_t specs = 0;
     UColTokListHeader *ListList = NULL;
     src->parsedToken.strength = UCOL_TOK_UNSET;
     ListList = src->lh;
     if(U_FAILURE(*status)) {
         return 0;
     }
 #ifdef DEBUG_FOR_CODE_POINTS
     char filename[35];
     sprintf(filename, "/tmp/debug_for_cp_%09d.txt", getpid());
     dfcp_fp = fopen(filename, "a");
     fprintf(stdout, "Output is in the file %s.\n", filename);
 #endif
 #ifdef DEBUG_FOR_COLL_RULES
     std::string s3;
     UnicodeString(src->source).toUTF8String(s3);
     std::cout << "src->source = " << s3 << std::endl;
 #endif
     while(src->current < src->end || src->isStarred) {
         src->parsedToken.prefixOffset = 0;
         parseEnd = ucol_tok_parseNextToken(src,
             (UBool)(lastToken == NULL),
             parseError,
             status);
         specs = src->parsedToken.flags;
         variableTop = ((specs & UCOL_TOK_VARIABLE_TOP) != 0);
         top = ((specs & UCOL_TOK_TOP) != 0);
         if(U_SUCCESS(*status) && parseEnd != NULL) {
             UColToken *sourceToken = NULL;
             //uint32_t key = 0;
             uint32_t lastStrength = UCOL_TOK_UNSET;
             if(lastToken != NULL ) {
                 lastStrength = lastToken->strength;
             }
 #ifdef DEBUG_FOR_CODE_POINTS
             UChar32 cp;
             U16_GET(src->source, 0, src->parsedToken.charsOffset, (uint32_t)(src->extraEnd - src->source), cp);
             fprintf(dfcp_fp, "Code point = %x, Strength = %x\n", cp, src->parsedToken.strength);
 #endif
             //key = newCharsLen << 24 | charsOffset;
             UColToken key;
             key.source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
             key.rulesToParseHdl = &(src->source);
             /*  4 Lookup each source in the CharsToToken map, and find a sourceToken */
             sourceToken = (UColToken *)uhash_get(src->tailored, &key);
             if(src->parsedToken.strength != UCOL_TOK_RESET) {
                 if(lastToken == NULL) { /* this means that rules haven't started properly */
                     *status = U_INVALID_FORMAT_ERROR;
                     syntaxError(src->source,0,(int32_t)(src->end-src->source),parseError);
                     DBG_FORMAT_ERROR
                     return 0;
                 }
                 /*  6 Otherwise (when relation != reset) */
                 if(sourceToken == NULL) {
                     /* If sourceToken is null, create new one, */
                     sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
                     /* test for NULL */
                     if (sourceToken == NULL) {
                         *status = U_MEMORY_ALLOCATION_ERROR;
                         return 0;
                     }
                     sourceToken->rulesToParseHdl = &(src->source);
                     sourceToken->source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
                     sourceToken->debugSource = *(src->source + src->parsedToken.charsOffset);
                     sourceToken->prefix = src->parsedToken.prefixLen << 24 | src->parsedToken.prefixOffset;
                     sourceToken->debugPrefix = *(src->source + src->parsedToken.prefixOffset);
                     sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
                     sourceToken->next = NULL;
                     sourceToken->previous = NULL;
                     sourceToken->noOfCEs = 0;
                     sourceToken->noOfExpCEs = 0;
                     // keep the flags around so that we know about before
                     sourceToken->flags = src->parsedToken.flags;
                     uhash_put(src->tailored, sourceToken, sourceToken, status);
                     if(U_FAILURE(*status)) {
                         return 0;
                     }
                 } else {
                     /* we could have fished out a reset here */
                     if(sourceToken->strength != UCOL_TOK_RESET && lastToken != sourceToken) {
                         /* otherwise remove sourceToken from where it was. */
                         if(sourceToken->next != NULL) {
                             if(sourceToken->next->strength > sourceToken->strength) {
                                 sourceToken->next->strength = sourceToken->strength;
                             }
                             sourceToken->next->previous = sourceToken->previous;
                         } else {
                             sourceToken->listHeader->last = sourceToken->previous;
                         }
                         if(sourceToken->previous != NULL) {
                             sourceToken->previous->next = sourceToken->next;
                         } else {
                             sourceToken->listHeader->first = sourceToken->next;
                         }
                         sourceToken->next = NULL;
                         sourceToken->previous = NULL;
                     }
                 }
                 sourceToken->strength = src->parsedToken.strength;
                 sourceToken->listHeader = lastToken->listHeader;
                 /*
 .  Find the strongest strength in each list, and set strongestP and strongestN
                 accordingly in the headers.
                 */
                 if(lastStrength == UCOL_TOK_RESET
                     || sourceToken->listHeader->first == 0) {
                         /* If LAST is a reset
                         insert sourceToken in the list. */
                         if(sourceToken->listHeader->first == 0) {
                             sourceToken->listHeader->first = sourceToken;
                             sourceToken->listHeader->last = sourceToken;
                         } else { /* we need to find a place for us */
                             /* and we'll get in front of the same strength */
                             if(sourceToken->listHeader->first->strength <= sourceToken->strength) {
                                 sourceToken->next = sourceToken->listHeader->first;
                                 sourceToken->next->previous = sourceToken;
                                 sourceToken->listHeader->first = sourceToken;
                                 sourceToken->previous = NULL;
                             } else {
                                 lastToken = sourceToken->listHeader->first;
                                 while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
                                     lastToken = lastToken->next;
                                 }
                                 if(lastToken->next != NULL) {
                                     lastToken->next->previous = sourceToken;
                                 } else {
                                     sourceToken->listHeader->last = sourceToken;
                                 }
                                 sourceToken->previous = lastToken;
                                 sourceToken->next = lastToken->next;
                                 lastToken->next = sourceToken;
                             }
                         }
                     } else {
                         /* Otherwise (when LAST is not a reset)
                         if polarity (LAST) == polarity(relation), insert sourceToken after LAST,
                         otherwise insert before.
                         when inserting after or before, search to the next position with the same
                         strength in that direction. (This is called postpone insertion).         */
                         if(sourceToken != lastToken) {
                             if(lastToken->polarity == sourceToken->polarity) {
                                 while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
                                     lastToken = lastToken->next;
                                 }
                                 sourceToken->previous = lastToken;
                                 if(lastToken->next != NULL) {
                                     lastToken->next->previous = sourceToken;
                                 } else {
                                     sourceToken->listHeader->last = sourceToken;
                                 }
                                 sourceToken->next = lastToken->next;
                                 lastToken->next = sourceToken;
                             } else {
                                 while(lastToken->previous != NULL && lastToken->previous->strength > sourceToken->strength) {
                                     lastToken = lastToken->previous;
                                 }
                                 sourceToken->next = lastToken;
                                 if(lastToken->previous != NULL) {
                                     lastToken->previous->next = sourceToken;
                                 } else {
                                     sourceToken->listHeader->first = sourceToken;
                                 }
                                 sourceToken->previous = lastToken->previous;
                                 lastToken->previous = sourceToken;
                             }
                         } else { /* repeated one thing twice in rules, stay with the stronger strength */
                             if(lastStrength < sourceToken->strength) {
                                 sourceToken->strength = lastStrength;
                             }
                         }
                     }
                     /* if the token was a variable top, we're gonna put it in */
                     if(variableTop == TRUE && src->varTop == NULL) {
                         variableTop = FALSE;
                         src->varTop = sourceToken;
                     }
                     // Treat the expansions.
                     // There are two types of expansions: explicit (x / y) and reset based propagating expansions
                     // (&abc * d * e <=> &ab * d / c * e / c)
                     // if both of them are in effect for a token, they are combined.
                     sourceToken->expansion = src->parsedToken.extensionLen << 24 | src->parsedToken.extensionOffset;
                     if(expandNext != 0) {
                         if(sourceToken->strength == UCOL_PRIMARY) { /* primary strength kills off the implicit expansion */
                             expandNext = 0;
                         } else if(sourceToken->expansion == 0) { /* if there is no expansion, implicit is just added to the token */
                             sourceToken->expansion = expandNext;
                         } else { /* there is both explicit and implicit expansion. We need to make a combination */
                             uprv_memcpy(src->extraCurrent, src->source + (expandNext & 0xFFFFFF), (expandNext >> 24)*sizeof(UChar));
                             uprv_memcpy(src->extraCurrent+(expandNext >> 24), src->source + src->parsedToken.extensionOffset, src->parsedToken.extensionLen*sizeof(UChar));
                             sourceToken->expansion = (uint32_t)(((expandNext >> 24) + src->parsedToken.extensionLen)<<24 | (uint32_t)(src->extraCurrent - src->source));
                             src->extraCurrent += (expandNext >> 24) + src->parsedToken.extensionLen;
                         }
                     }
                     // This is just for debugging purposes
                     if(sourceToken->expansion != 0) {
                         sourceToken->debugExpansion = *(src->source + src->parsedToken.extensionOffset);
                     } else {
                         sourceToken->debugExpansion = 0;
                     }
                     // if the previous token was a reset before, the strength of this
                     // token must match the strength of before. Otherwise we have an
                     // undefined situation.
                     // In other words, we currently have a cludge which we use to
                     // represent &a >> x. This is written as &[before 2]a << x.
                     if((lastToken->flags & UCOL_TOK_BEFORE) != 0) {
                         uint8_t beforeStrength = (lastToken->flags & UCOL_TOK_BEFORE) - 1;
                         if(beforeStrength != sourceToken->strength) {
                             *status = U_INVALID_FORMAT_ERROR;
                             syntaxError(src->source,0,(int32_t)(src->end-src->source),parseError);
                             DBG_FORMAT_ERROR
                             return 0;
                         }
                     }
             } else {
                 if(lastToken != NULL && lastStrength == UCOL_TOK_RESET) {
                     /* if the previous token was also a reset, */
                     /*this means that we have two consecutive resets */
                     /* and we want to remove the previous one if empty*/
                     if(src->resultLen > 0 && ListList[src->resultLen-1].first == NULL) {
                         src->resultLen--;
                     }
                 }
                 if(sourceToken == NULL) { /* this is a reset, but it might still be somewhere in the tailoring, in shorter form */
                     uint32_t searchCharsLen = src->parsedToken.charsLen;
                     while(searchCharsLen > 1 && sourceToken == NULL) {
                         searchCharsLen--;
                         //key = searchCharsLen << 24 | charsOffset;
                         UColToken key;
                         key.source = searchCharsLen << 24 | src->parsedToken.charsOffset;
                         key.rulesToParseHdl = &(src->source);
                         sourceToken = (UColToken *)uhash_get(src->tailored, &key);
                     }
                     if(sourceToken != NULL) {
                         expandNext = (src->parsedToken.charsLen - searchCharsLen) << 24 | (src->parsedToken.charsOffset + searchCharsLen);
                     }
                 }
                 if((specs & UCOL_TOK_BEFORE) != 0) { /* we're doing before */
                     if(top == FALSE) { /* there is no indirection */
                         uint8_t strength = (specs & UCOL_TOK_BEFORE) - 1;
                         if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
                             /* this is a before that is already ordered in the UCA - so we need to get the previous with good strength */
                             while(sourceToken->strength > strength && sourceToken->previous != NULL) {
                                 sourceToken = sourceToken->previous;
                             }
                             /* here, either we hit the strength or NULL */
                             if(sourceToken->strength == strength) {
                                 if(sourceToken->previous != NULL) {
                                     sourceToken = sourceToken->previous;
                                 } else { /* start of list */
                                     sourceToken = sourceToken->listHeader->reset;
                                 }
                             } else { /* we hit NULL */
                                 /* we should be doing the else part */
                                 sourceToken = sourceToken->listHeader->reset;
                                 sourceToken = getVirginBefore(src, sourceToken, strength, parseError, status);
                             }
                         } else {
                             sourceToken = getVirginBefore(src, sourceToken, strength, parseError, status);
                         }
                     } else { /* this is both before and indirection */
                         top = FALSE;
                         ListList[src->resultLen].previousCE = 0;
                         ListList[src->resultLen].previousContCE = 0;
                         ListList[src->resultLen].indirect = TRUE;
                         /* we need to do slightly more work. we need to get the baseCE using the */
                         /* inverse UCA & getPrevious. The next bound is not set, and will be decided */
                         /* in ucol_bld */
                         uint8_t strength = (specs & UCOL_TOK_BEFORE) - 1;
                         uint32_t baseCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE;
                         uint32_t baseContCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE;//&0xFFFFFF3F;
                         uint32_t CE = UCOL_NOT_FOUND, SecondCE = UCOL_NOT_FOUND;
                         UCAConstants *consts = (UCAConstants *)((uint8_t *)src->UCA->image + src->UCA->image->UCAConsts);
                         if((baseCE & 0xFF000000) >= (consts->UCA_PRIMARY_IMPLICIT_MIN<<24) &&
                            (baseCE & 0xFF000000) <= (consts->UCA_PRIMARY_IMPLICIT_MAX<<24) ) { /* implicits - */
                             uint32_t primary = (baseCE & UCOL_PRIMARYMASK) | ((baseContCE & UCOL_PRIMARYMASK) >> 16);
                             uint32_t raw = uprv_uca_getRawFromImplicit(primary);
                             uint32_t primaryCE = uprv_uca_getImplicitFromRaw(raw-1);
                             CE = (primaryCE & UCOL_PRIMARYMASK) | 0x0505;
                             SecondCE = ((primaryCE << 16) & UCOL_PRIMARYMASK) | UCOL_CONTINUATION_MARKER;
                         } else {
                             /*int32_t invPos = ucol_inv_getPrevCE(baseCE, baseContCE, &CE, &SecondCE, strength);*/
                             ucol_inv_getPrevCE(src, baseCE, baseContCE, &CE, &SecondCE, strength);
                         }
                         ListList[src->resultLen].baseCE = CE;
                         ListList[src->resultLen].baseContCE = SecondCE;
                         ListList[src->resultLen].nextCE = 0;
                         ListList[src->resultLen].nextContCE = 0;
                         sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
                     }
                 }
                 /*  5 If the relation is a reset:
                 If sourceToken is null
                 Create new list, create new sourceToken, make the baseCE from source, put
                 the sourceToken in ListHeader of the new list */
                 if(sourceToken == NULL) {
                     /*
 Consider each item: relation, source, and expansion: e.g. ...< x / y ...
                     First convert all expansions into normal form. Examples:
                     If "xy" doesn't occur earlier in the list or in the UCA, convert &xy * c *
                     d * ... into &x * c/y * d * ...
                     Note: reset values can never have expansions, although they can cause the
                     very next item to have one. They may be contractions, if they are found
                     earlier in the list.
                     */
                     if(top == FALSE) {
                         collIterate s;
                         uint32_t CE = UCOL_NOT_FOUND, SecondCE = UCOL_NOT_FOUND;
                         uprv_init_collIterate(src->UCA, src->source+src->parsedToken.charsOffset, src->parsedToken.charsLen, &s, status);
                         CE = ucol_getNextCE(src->UCA, &s, status);
                         const UChar *expand = s.pos;
                         SecondCE = ucol_getNextCE(src->UCA, &s, status);
                         ListList[src->resultLen].baseCE = CE & 0xFFFFFF3F;
                         if(isContinuation(SecondCE)) {
                             ListList[src->resultLen].baseContCE = SecondCE;
                         } else {
                             ListList[src->resultLen].baseContCE = 0;
                         }
                         ListList[src->resultLen].nextCE = 0;
                         ListList[src->resultLen].nextContCE = 0;
                         ListList[src->resultLen].previousCE = 0;
                         ListList[src->resultLen].previousContCE = 0;
                         ListList[src->resultLen].indirect = FALSE;
                         sourceToken = ucol_tok_initAReset(src, expand, &expandNext, parseError, status);
                     } else { /* top == TRUE */
                         /* just use the supplied values */
                         top = FALSE;
                         ListList[src->resultLen].previousCE = 0;
                         ListList[src->resultLen].previousContCE = 0;
                         ListList[src->resultLen].indirect = TRUE;
                         ListList[src->resultLen].baseCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startCE;
                         ListList[src->resultLen].baseContCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].startContCE;
                         ListList[src->resultLen].nextCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].limitCE;
                         ListList[src->resultLen].nextContCE = ucolIndirectBoundaries[src->parsedToken.indirectIndex].limitContCE;
                         sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
                     }
                 } else { /* reset to something already in rules */
                     top = FALSE;
                 }
             }
             /*  7 After all this, set LAST to point to sourceToken, and goto step 3. */
             lastToken = sourceToken;
         } else {
             if(U_FAILURE(*status)) {
                 return 0;
             }
         }
     }
 #ifdef DEBUG_FOR_CODE_POINTS
     fclose(dfcp_fp);
 #endif
     if(src->resultLen > 0 && ListList[src->resultLen-1].first == NULL) {
         src->resultLen--;
     }
     return src->resultLen;
 }
 const UChar* ucol_tok_getRulesFromBundle(
     void* /*context*/,
     const char* locale,
     const char* type,
     int32_t* pLength,
     UErrorCode* status)
 {
     const UChar* rules = NULL;
     UResourceBundle* bundle;
     UResourceBundle* collations;
     UResourceBundle* collation;
     *pLength = 0;
     bundle = ures_open(U_ICUDATA_COLL, locale, status);
     if(U_SUCCESS(*status)){
         collations = ures_getByKey(bundle, "collations", NULL, status);
         if(U_SUCCESS(*status)){
             collation = ures_getByKey(collations, type, NULL, status);
             if(U_SUCCESS(*status)){
                 rules = ures_getStringByKey(collation, "Sequence", pLength, status);
                 if(U_FAILURE(*status)){
                     *pLength = 0;
                     rules = NULL;
                 }
                 ures_close(collation);
             }
             ures_close(collations);
         }
     }
     ures_close(bundle);
     return rules;
 }
 void ucol_tok_initTokenList(
     UColTokenParser *src,
     const UChar *rules,
     uint32_t rulesLength,
     const UCollator *UCA,
     GetCollationRulesFunction importFunc,
     void* context,
     UErrorCode *status) {
     U_NAMESPACE_USE
     uint32_t nSize = 0;
     uint32_t estimatedSize = (2*rulesLength+UCOL_TOK_EXTRA_RULE_SPACE_SIZE);
     bool needToDeallocRules = false;
     if(U_FAILURE(*status)) {
         return;
     }
     // set everything to zero, so that we can clean up gracefully
     uprv_memset(src, 0, sizeof(UColTokenParser));
     // first we need to find options that don't like to be normalized,
     // like copy and remove...
     //const UChar *openBrace = rules;
     int32_t optionNumber = -1;
     const UChar *setStart = NULL;
     uint32_t i = 0;
     while(i < rulesLength) {
         if(rules[i] == 0x005B) {    // '[': start of an option
             /* Gets the following:
                optionNumber: The index of the option.
                setStart: The pointer at which the option arguments start.
              */
             optionNumber = ucol_uprv_tok_readOption(rules+i+1, rules+rulesLength, &setStart);
             if(optionNumber == OPTION_OPTIMIZE) { /* copy - parts of UCA to tailoring */
                 // [optimize]
                 USet *newSet = ucol_uprv_tok_readAndSetUnicodeSet(setStart, rules+rulesLength, status);
                 if(U_SUCCESS(*status)) {
                     if(src->copySet == NULL) {
                         src->copySet = newSet;
                     } else {
                         uset_addAll(src->copySet, newSet);
                         uset_close(newSet);
                     }
                 } else {
                     return;
                 }
             } else if(optionNumber == OPTION_SUPPRESS_CONTRACTIONS) {
                 USet *newSet = ucol_uprv_tok_readAndSetUnicodeSet(setStart, rules+rulesLength, status);
                 if(U_SUCCESS(*status)) {
                     if(src->removeSet == NULL) {
                         src->removeSet = newSet;
                     } else {
                         uset_addAll(src->removeSet, newSet);
                         uset_close(newSet);
                     }
                 } else {
                     return;
                 }
             } else if(optionNumber == OPTION_IMPORT){
                 // [import <collation-name>]
                 // Find the address of the closing ].
                 UChar* import_end = u_strchr(setStart, 0x005D);
                 int32_t optionEndOffset = (int32_t)(import_end + 1 - rules);
                 // Ignore trailing whitespace.
                 while(PatternProps::isWhiteSpace(*(import_end-1))) {
                     --import_end;
                 }
                 int32_t optionLength = (int32_t)(import_end - setStart);
                 char option[50];
                 if(optionLength >= (int32_t)sizeof(option)) {
                     *status = U_ILLEGAL_ARGUMENT_ERROR;
                     return;
                 }
                 u_UCharsToChars(setStart, option, optionLength);
                 option[optionLength] = 0;
                 *status = U_ZERO_ERROR;
                 char locale[50];
                 int32_t templ;
                 uloc_forLanguageTag(option, locale, (int32_t)sizeof(locale), &templ, status);
                 if(U_FAILURE(*status)) {
                     *status = U_ILLEGAL_ARGUMENT_ERROR;
                     return;
                 }
                 char type[50];
                 if (uloc_getKeywordValue(locale, "collation", type, (int32_t)sizeof(type), status) <= 0 ||
                     U_FAILURE(*status)
                 ) {
                     *status = U_ZERO_ERROR;
                     uprv_strcpy(type, "standard");
                 }
                 // TODO: Use public functions when available, see ticket #8134.
                 char *keywords = (char *)locale_getKeywordsStart(locale);
                 if(keywords != NULL) {
                     *keywords = 0;
                 }
                 int32_t importRulesLength = 0;
                 const UChar* importRules = importFunc(context, locale, type, &importRulesLength, status);
 #ifdef DEBUG_FOR_COLL_RULES
                 std::string s;
                 UnicodeString(importRules).toUTF8String(s);
                 std::cout << "Import rules = " << s << std::endl;
 #endif
                 // Add the length of the imported rules to length of the original rules,
                 // and subtract the length of the import option.
                 uint32_t newRulesLength = rulesLength + importRulesLength - (optionEndOffset - i);
                 UChar* newRules = (UChar*)uprv_malloc(newRulesLength*sizeof(UChar));
 #ifdef DEBUG_FOR_COLL_RULES
                 std::string s1;
                 UnicodeString(rules).toUTF8String(s1);
                 std::cout << "Original rules = " << s1 << std::endl;
 #endif
                 // Copy the section of the original rules leading up to the import
                 uprv_memcpy(newRules, rules, i*sizeof(UChar));
                 // Copy the imported rules
                 uprv_memcpy(newRules+i, importRules, importRulesLength*sizeof(UChar));
                 // Copy the rest of the original rules (minus the import option itself)
                 uprv_memcpy(newRules+i+importRulesLength,
                             rules+optionEndOffset,
                             (rulesLength-optionEndOffset)*sizeof(UChar));
 #ifdef DEBUG_FOR_COLL_RULES
                 std::string s2;
                 UnicodeString(newRules).toUTF8String(s2);
                 std::cout << "Resulting rules = " << s2 << std::endl;
 #endif
                 if(needToDeallocRules){
                     // if needToDeallocRules is set, then we allocated rules, so it's safe to cast and free
                     uprv_free((void*)rules);
                 }
                 needToDeallocRules = true;
                 rules = newRules;
                 rulesLength = newRulesLength;
                 estimatedSize += importRulesLength*2;
                 // First character of the new rules needs to be processed
                 i--;
             }
         }
         //openBrace++;
         i++;
     }
     src->source = (UChar *)uprv_malloc(estimatedSize*sizeof(UChar));
     /* test for NULL */
     if (src->source == NULL) {
         *status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
     uprv_memset(src->source, 0, estimatedSize*sizeof(UChar));
     nSize = unorm_normalize(rules, rulesLength, UNORM_NFD, 0, src->source, estimatedSize, status);
     if(nSize > estimatedSize || *status == U_BUFFER_OVERFLOW_ERROR) {
         *status = U_ZERO_ERROR;
         src->source = (UChar *)uprv_realloc(src->source, (nSize+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar));
         /* test for NULL */
         if (src->source == NULL) {
             *status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
         nSize = unorm_normalize(rules, rulesLength, UNORM_NFD, 0, src->source, nSize+UCOL_TOK_EXTRA_RULE_SPACE_SIZE, status);
     }
     if(needToDeallocRules){
         // if needToDeallocRules is set, then we allocated rules, so it's safe to cast and free
         uprv_free((void*)rules);
     }
     src->current = src->source;
     src->end = src->source+nSize;
     src->sourceCurrent = src->source;
     src->extraCurrent = src->end+1; // Preserve terminating zero in the rule string so that option scanning works correctly
     src->extraEnd = src->source+estimatedSize; //src->end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE;
     src->varTop = NULL;
     src->UCA = UCA;
     src->invUCA = ucol_initInverseUCA(status);
     src->parsedToken.charsLen = 0;
     src->parsedToken.charsOffset = 0;
     src->parsedToken.extensionLen = 0;
     src->parsedToken.extensionOffset = 0;
     src->parsedToken.prefixLen = 0;
     src->parsedToken.prefixOffset = 0;
     src->parsedToken.flags = 0;
     src->parsedToken.strength = UCOL_TOK_UNSET;
     src->buildCCTabFlag = FALSE;
     src->isStarred = FALSE;
     src->inRange = FALSE;
     src->lastRangeCp = 0;
     src->previousCp = 0;
     if(U_FAILURE(*status)) {
         return;
     }
     src->tailored = uhash_open(uhash_hashTokens, uhash_compareTokens, NULL, status);
     if(U_FAILURE(*status)) {
         return;
     }
     uhash_setValueDeleter(src->tailored, uprv_free);
     src->opts = (UColOptionSet *)uprv_malloc(sizeof(UColOptionSet));
     /* test for NULL */
     if (src->opts == NULL) {
         *status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
     uprv_memcpy(src->opts, UCA->options, sizeof(UColOptionSet));
     src->lh = 0;
     src->listCapacity = 1024;
     src->lh = (UColTokListHeader *)uprv_malloc(src->listCapacity*sizeof(UColTokListHeader));
     //Test for NULL
     if (src->lh == NULL) {
         *status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
     uprv_memset(src->lh, 0, src->listCapacity*sizeof(UColTokListHeader));
     src->resultLen = 0;
     UCAConstants *consts = (UCAConstants *)((uint8_t *)src->UCA->image + src->UCA->image->UCAConsts);
     // UCOL_RESET_TOP_VALUE
     setIndirectBoundaries(0, consts->UCA_LAST_NON_VARIABLE, consts->UCA_FIRST_IMPLICIT);
     // UCOL_FIRST_PRIMARY_IGNORABLE
     setIndirectBoundaries(1, consts->UCA_FIRST_PRIMARY_IGNORABLE, 0);
     // UCOL_LAST_PRIMARY_IGNORABLE
     setIndirectBoundaries(2, consts->UCA_LAST_PRIMARY_IGNORABLE, 0);
     // UCOL_FIRST_SECONDARY_IGNORABLE
     setIndirectBoundaries(3, consts->UCA_FIRST_SECONDARY_IGNORABLE, 0);
     // UCOL_LAST_SECONDARY_IGNORABLE
     setIndirectBoundaries(4, consts->UCA_LAST_SECONDARY_IGNORABLE, 0);
     // UCOL_FIRST_TERTIARY_IGNORABLE
     setIndirectBoundaries(5, consts->UCA_FIRST_TERTIARY_IGNORABLE, 0);
     // UCOL_LAST_TERTIARY_IGNORABLE
     setIndirectBoundaries(6, consts->UCA_LAST_TERTIARY_IGNORABLE, 0);
     // UCOL_FIRST_VARIABLE
     setIndirectBoundaries(7, consts->UCA_FIRST_VARIABLE, 0);
     // UCOL_LAST_VARIABLE
     setIndirectBoundaries(8, consts->UCA_LAST_VARIABLE, 0);
     // UCOL_FIRST_NON_VARIABLE
     setIndirectBoundaries(9, consts->UCA_FIRST_NON_VARIABLE, 0);
     // UCOL_LAST_NON_VARIABLE
     setIndirectBoundaries(10, consts->UCA_LAST_NON_VARIABLE, consts->UCA_FIRST_IMPLICIT);
     // UCOL_FIRST_IMPLICIT
     setIndirectBoundaries(11, consts->UCA_FIRST_IMPLICIT, 0);
     // UCOL_LAST_IMPLICIT
     setIndirectBoundaries(12, consts->UCA_LAST_IMPLICIT, consts->UCA_FIRST_TRAILING);
     // UCOL_FIRST_TRAILING
     setIndirectBoundaries(13, consts->UCA_FIRST_TRAILING, 0);
     // UCOL_LAST_TRAILING
     setIndirectBoundaries(14, consts->UCA_LAST_TRAILING, 0);
     ucolIndirectBoundaries[14].limitCE = (consts->UCA_PRIMARY_SPECIAL_MIN<<24);
 }
 void ucol_tok_closeTokenList(UColTokenParser *src) {
     if(src->copySet != NULL) {
         uset_close(src->copySet);
     }
     if(src->removeSet != NULL) {
         uset_close(src->removeSet);
     }
     if(src->tailored != NULL) {
         uhash_close(src->tailored);
     }
     if(src->lh != NULL) {
         uprv_free(src->lh);
     }
     if(src->source != NULL) {
         uprv_free(src->source);
     }
     if(src->opts != NULL) {
         uprv_free(src->opts);
     }
     if (src->reorderCodes != NULL) {
         uprv_free(src->reorderCodes);
     }
 }
 #endif /* #if !UCONFIG_NO_COLLATION */

The Tor Browser / annotate

intl/icu/source/i18n/ucol_tok.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/ucol_tok.cpp