The Tor Browser: comparison intl/icu/source/i18n/identifier

--1:000000000000
+:25147ac22174
+/*
+**********************************************************************
+*   Copyright (C) 2012-2013, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+**********************************************************************
+*/
+#include "unicode/utypes.h"
+#include "unicode/uchar.h"
+#include "unicode/utf16.h"
+#include "identifier_info.h"
+#include "mutex.h"
+#include "scriptset.h"
+#include "ucln_in.h"
+#include "uvector.h"
+U_NAMESPACE_BEGIN
+#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
+static UMutex gInitMutex = U_MUTEX_INITIALIZER;
+static UBool gStaticsAreInitialized = FALSE;
+UnicodeSet *IdentifierInfo::ASCII;
+ScriptSet *IdentifierInfo::JAPANESE;
+ScriptSet *IdentifierInfo::CHINESE;
+ScriptSet *IdentifierInfo::KOREAN;
+ScriptSet *IdentifierInfo::CONFUSABLE_WITH_LATIN;
+UBool IdentifierInfo::cleanup() {
+delete ASCII;
+ASCII = NULL;
+delete JAPANESE;
+JAPANESE = NULL;
+delete CHINESE;
+CHINESE = NULL;
+delete KOREAN;
+KOREAN = NULL;
+delete CONFUSABLE_WITH_LATIN;
+CONFUSABLE_WITH_LATIN = NULL;
+gStaticsAreInitialized = FALSE;
+return TRUE;
+}
+U_CDECL_BEGIN
+static UBool U_CALLCONV
+IdentifierInfo_cleanup(void) {
+return IdentifierInfo::cleanup();
+}
+U_CDECL_END
+IdentifierInfo::IdentifierInfo(UErrorCode &status):
+fIdentifier(NULL), fRequiredScripts(NULL), fScriptSetSet(NULL),
+fCommonAmongAlternates(NULL), fNumerics(NULL), fIdentifierProfile(NULL) {
+if (U_FAILURE(status)) {
+return;
+}
+{
+Mutex lock(&gInitMutex);
+if (!gStaticsAreInitialized) {
+ASCII    = new UnicodeSet(0, 0x7f);
+JAPANESE = new ScriptSet();
+CHINESE  = new ScriptSet();
+KOREAN   = new ScriptSet();
+CONFUSABLE_WITH_LATIN = new ScriptSet();
+if (ASCII == NULL || JAPANESE == NULL || CHINESE == NULL || KOREAN == NULL
+|| CONFUSABLE_WITH_LATIN == NULL) {
+status = U_MEMORY_ALLOCATION_ERROR;
+return;
+}
+ASCII->freeze();
+JAPANESE->set(USCRIPT_LATIN, status).set(USCRIPT_HAN, status).set(USCRIPT_HIRAGANA, status)
+.set(USCRIPT_KATAKANA, status);
+CHINESE->set(USCRIPT_LATIN, status).set(USCRIPT_HAN, status).set(USCRIPT_BOPOMOFO, status);
+KOREAN->set(USCRIPT_LATIN, status).set(USCRIPT_HAN, status).set(USCRIPT_HANGUL, status);
+CONFUSABLE_WITH_LATIN->set(USCRIPT_CYRILLIC, status).set(USCRIPT_GREEK, status)
+.set(USCRIPT_CHEROKEE, status);
+ucln_i18n_registerCleanup(UCLN_I18N_IDENTIFIER_INFO, IdentifierInfo_cleanup);
+gStaticsAreInitialized = TRUE;
+}
+}
+fIdentifier = new UnicodeString();
+fRequiredScripts = new ScriptSet();
+fScriptSetSet = uhash_open(uhash_hashScriptSet, uhash_compareScriptSet, NULL, &status);
+uhash_setKeyDeleter(fScriptSetSet, uhash_deleteScriptSet);
+fCommonAmongAlternates = new ScriptSet();
+fNumerics = new UnicodeSet();
+fIdentifierProfile = new UnicodeSet(0, 0x10FFFF);
+if (U_SUCCESS(status) && (fIdentifier == NULL || fRequiredScripts == NULL || fScriptSetSet == NULL ||
+fCommonAmongAlternates == NULL || fNumerics == NULL || fIdentifierProfile == NULL)) {
+status = U_MEMORY_ALLOCATION_ERROR;
+}
+}
+IdentifierInfo::~IdentifierInfo() {
+delete fIdentifier;
+delete fRequiredScripts;
+uhash_close(fScriptSetSet);
+delete fCommonAmongAlternates;
+delete fNumerics;
+delete fIdentifierProfile;
+}
+IdentifierInfo &IdentifierInfo::clear() {
+fRequiredScripts->resetAll();
+uhash_removeAll(fScriptSetSet);
+fNumerics->clear();
+fCommonAmongAlternates->resetAll();
+return *this;
+}
+IdentifierInfo &IdentifierInfo::setIdentifierProfile(const UnicodeSet &identifierProfile) {
+*fIdentifierProfile = identifierProfile;
+return *this;
+}
+const UnicodeSet &IdentifierInfo::getIdentifierProfile() const {
+return *fIdentifierProfile;
+}
+IdentifierInfo &IdentifierInfo::setIdentifier(const UnicodeString &identifier, UErrorCode &status) {
+if (U_FAILURE(status)) {
+return *this;
+}
+*fIdentifier = identifier;
+clear();
+ScriptSet scriptsForCP;
+UChar32 cp;
+for (int32_t i = 0; i < identifier.length(); i += U16_LENGTH(cp)) {
+cp = identifier.char32At(i);
+// Store a representative character for each kind of decimal digit
+if (u_charType(cp) == U_DECIMAL_DIGIT_NUMBER) {
+// Just store the zero character as a representative for comparison. Unicode guarantees it is cp - value
+fNumerics->add(cp - (UChar32)u_getNumericValue(cp));
+}
+UScriptCode extensions[500];
+int32_t extensionsCount = uscript_getScriptExtensions(cp, extensions, LENGTHOF(extensions), &status);
+if (U_FAILURE(status)) {
+return *this;
+}
+scriptsForCP.resetAll();
+for (int32_t j=0; j<extensionsCount; j++) {
+scriptsForCP.set(extensions[j], status);
+}
+scriptsForCP.reset(USCRIPT_COMMON, status);
+scriptsForCP.reset(USCRIPT_INHERITED, status);
+switch (scriptsForCP.countMembers()) {
+case 0: break;
+case 1:
+// Single script, record it.
+fRequiredScripts->Union(scriptsForCP);
+break;
+default:
+if (!fRequiredScripts->intersects(scriptsForCP)
+&& !uhash_geti(fScriptSetSet, &scriptsForCP)) {
+// If the set hasn't been added already, add it
+//    (Add a copy, fScriptSetSet takes ownership of the copy.)
+uhash_puti(fScriptSetSet, new ScriptSet(scriptsForCP), 1, &status);
+}
+break;
+}
+}
+// Now make a final pass through ScriptSetSet to remove alternates that came before singles.
+// [Kana], [Kana Hira] => [Kana]
+// This is relatively infrequent, so doesn't have to be optimized.
+// We also compute any commonalities among the alternates.
+if (uhash_count(fScriptSetSet) > 0) {
+fCommonAmongAlternates->setAll();
+for (int32_t it = -1;;) {
+const UHashElement *nextHashEl = uhash_nextElement(fScriptSetSet, &it);
+if (nextHashEl == NULL) {
+break;
+}
+ScriptSet *next = static_cast<ScriptSet *>(nextHashEl->key.pointer);
+// [Kana], [Kana Hira] => [Kana]
+if (fRequiredScripts->intersects(*next)) {
+uhash_removeElement(fScriptSetSet, nextHashEl);
+} else {
+fCommonAmongAlternates->intersect(*next);
+// [[Arab Syrc Thaa]; [Arab Syrc]] => [[Arab Syrc]]
+for (int32_t otherIt = -1;;) {
+const UHashElement *otherHashEl = uhash_nextElement(fScriptSetSet, &otherIt);
+if (otherHashEl == NULL) {
+break;
+}
+ScriptSet *other = static_cast<ScriptSet *>(otherHashEl->key.pointer);
+if (next != other && next->contains(*other)) {
+uhash_removeElement(fScriptSetSet, nextHashEl);
+break;
+}
+}
+}
+}
+}
+if (uhash_count(fScriptSetSet) == 0) {
+fCommonAmongAlternates->resetAll();
+}
+return *this;
+}
+const UnicodeString *IdentifierInfo::getIdentifier() const {
+return fIdentifier;
+}
+const ScriptSet *IdentifierInfo::getScripts() const {
+return fRequiredScripts;
+}
+const UHashtable *IdentifierInfo::getAlternates() const {
+return fScriptSetSet;
+}
+const UnicodeSet *IdentifierInfo::getNumerics() const {
+return fNumerics;
+}
+const ScriptSet *IdentifierInfo::getCommonAmongAlternates() const {
+return fCommonAmongAlternates;
+}
+#if !UCONFIG_NO_NORMALIZATION
+URestrictionLevel IdentifierInfo::getRestrictionLevel(UErrorCode &status) const {
+if (!fIdentifierProfile->containsAll(*fIdentifier) || getNumerics()->size() > 1) {
+return USPOOF_UNRESTRICTIVE;
+}
+if (ASCII->containsAll(*fIdentifier)) {
+return USPOOF_ASCII;
+}
+// This is a bit tricky. We look at a number of factors.
+// The number of scripts in the text.
+// Plus 1 if there is some commonality among the alternates (eg [Arab Thaa]; [Arab Syrc])
+// Plus number of alternates otherwise (this only works because we only test cardinality up to 2.)
+// Note: the requiredScripts set omits COMMON and INHERITED; they are taken out at the
+//       time it is created, in setIdentifier().
+int32_t cardinalityPlus = fRequiredScripts->countMembers() +
+(fCommonAmongAlternates->countMembers() == 0 ? uhash_count(fScriptSetSet) : 1);
+if (cardinalityPlus < 2) {
+return USPOOF_HIGHLY_RESTRICTIVE;
+}
+if (containsWithAlternates(*JAPANESE, *fRequiredScripts) || containsWithAlternates(*CHINESE, *fRequiredScripts)
+|| containsWithAlternates(*KOREAN, *fRequiredScripts)) {
+return USPOOF_HIGHLY_RESTRICTIVE;
+}
+if (cardinalityPlus == 2 &&
+fRequiredScripts->test(USCRIPT_LATIN, status) &&
+!fRequiredScripts->intersects(*CONFUSABLE_WITH_LATIN)) {
+return USPOOF_MODERATELY_RESTRICTIVE;
+}
+return USPOOF_MINIMALLY_RESTRICTIVE;
+}
+#endif /* !UCONFIG_NO_NORMALIZATION */
+int32_t IdentifierInfo::getScriptCount() const {
+// Note: Common and Inherited scripts were removed by setIdentifier(), and do not appear in fRequiredScripts.
+int32_t count = fRequiredScripts->countMembers() +
+(fCommonAmongAlternates->countMembers() == 0 ? uhash_count(fScriptSetSet) : 1);
+return count;
+}
+UBool IdentifierInfo::containsWithAlternates(const ScriptSet &container, const ScriptSet &containee) const {
+if (!container.contains(containee)) {
+return FALSE;
+}
+for (int32_t iter = -1; ;) {
+const UHashElement *hashEl = uhash_nextElement(fScriptSetSet, &iter);
+if (hashEl == NULL) {
+break;
+}
+ScriptSet *alternatives = static_cast<ScriptSet *>(hashEl->key.pointer);
+if (!container.intersects(*alternatives)) {
+return false;
+}
+}
+return true;
+}
+UnicodeString &IdentifierInfo::displayAlternates(UnicodeString &dest, const UHashtable *alternates, UErrorCode &status) {
+UVector sorted(status);
+if (U_FAILURE(status)) {
+return dest;
+}
+for (int32_t pos = -1; ;) {
+const UHashElement *el = uhash_nextElement(alternates, &pos);
+if (el == NULL) {
+break;
+}
+ScriptSet *ss = static_cast<ScriptSet *>(el->key.pointer);
+sorted.addElement(ss, status);
+}
+sorted.sort(uhash_compareScriptSet, status);
+UnicodeString separator = UNICODE_STRING_SIMPLE("; ");
+for (int32_t i=0; i<sorted.size(); i++) {
+if (i>0) {
+dest.append(separator);
+}
+ScriptSet *ss = static_cast<ScriptSet *>(sorted.elementAt(i));
+ss->displayScripts(dest);
+}
+return dest;
+}
+U_NAMESPACE_END

The Tor Browser / file comparison

comparison: intl/icu/source/i18n/identifier_info.cpp

intl/icu/source/i18n/identifier_info.cpp