The Tor Browser: intl/icu/source/i18n/csdetect.cpp@b8a032363ba2 (annotated)

intl/icu/source/i18n/csdetect.cpp@b8a032363ba2 (annotated)

intl/icu/source/i18n/csdetect.cpp

Thu, 22 Jan 2015 13:21:57 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Thu, 22 Jan 2015 13:21:57 +0100
branch: TOR_BUG_9701
changeset 15: b8a032363ba2
permissions: -rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

 /*
  **********************************************************************
  *   Copyright (C) 2005-2013, International Business Machines
  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  */
 #include "unicode/utypes.h"
 #if !UCONFIG_NO_CONVERSION
 #include "unicode/ucsdet.h"
 #include "csdetect.h"
 #include "csmatch.h"
 #include "uenumimp.h"
 #include "cmemory.h"
 #include "cstring.h"
 #include "umutex.h"
 #include "ucln_in.h"
 #include "uarrsort.h"
 #include "inputext.h"
 #include "csrsbcs.h"
 #include "csrmbcs.h"
 #include "csrutf8.h"
 #include "csrucode.h"
 #include "csr2022.h"
 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
 #define DELETE_ARRAY(array) uprv_free((void *) (array))
 U_NAMESPACE_BEGIN
 struct CSRecognizerInfo : public UMemory {
     CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled)
         : recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {};
     ~CSRecognizerInfo() {delete recognizer;};
     CharsetRecognizer *recognizer;
     UBool isDefaultEnabled;
 };
 U_NAMESPACE_END
 static icu::CSRecognizerInfo **fCSRecognizers = NULL;
 static icu::UInitOnce gCSRecognizersInitOnce;
 static int32_t fCSRecognizers_size = 0;
 U_CDECL_BEGIN
 static UBool U_CALLCONV csdet_cleanup(void)
 {
     U_NAMESPACE_USE
     if (fCSRecognizers != NULL) {
         for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
             delete fCSRecognizers[r];
             fCSRecognizers[r] = NULL;
         }
         DELETE_ARRAY(fCSRecognizers);
         fCSRecognizers = NULL;
         fCSRecognizers_size = 0;
     }
     gCSRecognizersInitOnce.reset();
     return TRUE;
 }
 static int32_t U_CALLCONV
 charsetMatchComparator(const void * /*context*/, const void *left, const void *right)
 {
     U_NAMESPACE_USE
     const CharsetMatch **csm_l = (const CharsetMatch **) left;
     const CharsetMatch **csm_r = (const CharsetMatch **) right;
     // NOTE: compare is backwards to sort from highest to lowest.
     return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
 }
 static void U_CALLCONV initRecognizers(UErrorCode &status) {
     U_NAMESPACE_USE
     ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
     CSRecognizerInfo *tempArray[] = {
         new CSRecognizerInfo(new CharsetRecog_UTF8(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_1(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_2(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_7_el(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_8_he(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_windows_1251(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_windows_1256(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_KOI8_R(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_sjis(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_gb_18030(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_euc_jp(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_euc_kr(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_big5(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_2022JP(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_2022KR(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_2022CN(), TRUE),
         new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), FALSE),
         new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), FALSE),
         new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), FALSE),
         new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), FALSE)
     };
     int32_t rCount = ARRAY_SIZE(tempArray);
     fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount);
     if (fCSRecognizers == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
     }
     else {
         fCSRecognizers_size = rCount;
         for (int32_t r = 0; r < rCount; r += 1) {
             fCSRecognizers[r] = tempArray[r];
             if (fCSRecognizers[r] == NULL) {
                 status = U_MEMORY_ALLOCATION_ERROR;
             }
         }
     }
 }
 U_CDECL_END
 U_NAMESPACE_BEGIN
 void CharsetDetector::setRecognizers(UErrorCode &status)
 {
     umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status);
 }
 CharsetDetector::CharsetDetector(UErrorCode &status)
   : textIn(new InputText(status)), resultArray(NULL),
     resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE),
     fEnabledRecognizers(NULL)
 {
     if (U_FAILURE(status)) {
         return;
     }
     setRecognizers(status);
     if (U_FAILURE(status)) {
         return;
     }
     resultArray = (CharsetMatch **)uprv_malloc(sizeof(CharsetMatch *)*fCSRecognizers_size);
     if (resultArray == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return;
     }
     for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
         resultArray[i] = new CharsetMatch();
         if (resultArray[i] == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
             break;
         }
     }
 }
 CharsetDetector::~CharsetDetector()
 {
     delete textIn;
     for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
         delete resultArray[i];
     }
     uprv_free(resultArray);
     if (fEnabledRecognizers) {
         uprv_free(fEnabledRecognizers);
     }
 }
 void CharsetDetector::setText(const char *in, int32_t len)
 {
     textIn->setText(in, len);
     fFreshTextSet = TRUE;
 }
 UBool CharsetDetector::setStripTagsFlag(UBool flag)
 {
     UBool temp = fStripTags;
     fStripTags = flag;
     fFreshTextSet = TRUE;
     return temp;
 }
 UBool CharsetDetector::getStripTagsFlag() const
 {
     return fStripTags;
 }
 void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const
 {
     textIn->setDeclaredEncoding(encoding,len);
 }
 int32_t CharsetDetector::getDetectableCount()
 {
     UErrorCode status = U_ZERO_ERROR;
     setRecognizers(status);
     return fCSRecognizers_size;
 }
 const CharsetMatch *CharsetDetector::detect(UErrorCode &status)
 {
     int32_t maxMatchesFound = 0;
     detectAll(maxMatchesFound, status);
     if(maxMatchesFound > 0) {
         return resultArray[0];
     } else {
         return NULL;
     }
 }
 const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status)
 {
     if(!textIn->isSet()) {
         status = U_MISSING_RESOURCE_ERROR;// TODO:  Need to set proper status code for input text not set
         return NULL;
     } else if (fFreshTextSet) {
         CharsetRecognizer *csr;
         int32_t            i;
         textIn->MungeInput(fStripTags);
         // Iterate over all possible charsets, remember all that
         // give a match quality > 0.
         resultCount = 0;
         for (i = 0; i < fCSRecognizers_size; i += 1) {
             csr = fCSRecognizers[i]->recognizer;
             if (csr->match(textIn, resultArray[resultCount])) {
                 resultCount++;
             }
         }
         if (resultCount > 1) {
             uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status);
         }
         fFreshTextSet = FALSE;
     }
     maxMatchesFound = resultCount;
     return resultArray;
 }
 void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status)
 {
     if (U_FAILURE(status)) {
         return;
     }
     int32_t modIdx = -1;
     UBool isDefaultVal = FALSE;
     for (int32_t i = 0; i < fCSRecognizers_size; i++) {
         CSRecognizerInfo *csrinfo = fCSRecognizers[i];
         if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) {
             modIdx = i;
             isDefaultVal = (csrinfo->isDefaultEnabled == enabled);
             break;
         }
     }
     if (modIdx < 0) {
         // No matching encoding found
         status = U_ILLEGAL_ARGUMENT_ERROR;
         return;
     }
     if (fEnabledRecognizers == NULL && !isDefaultVal) {
         // Create an array storing the non default setting
         fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size);
         if (fEnabledRecognizers == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
             return;
         }
         // Initialize the array with default info
         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
             fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled;
         }
     }
     if (fEnabledRecognizers != NULL) {
         fEnabledRecognizers[modIdx] = enabled;
     }
 }
 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
 {
     if( index > fCSRecognizers_size-1 || index < 0) {
         status = U_INDEX_OUTOFBOUNDS_ERROR;
         return 0;
     } else {
         return fCSRecognizers[index]->getName();
     }
 }*/
 U_NAMESPACE_END
 U_CDECL_BEGIN
 typedef struct {
     int32_t currIndex;
     UBool all;
     UBool *enabledRecognizers;
 } Context;
 static void U_CALLCONV
 enumClose(UEnumeration *en) {
     if(en->context != NULL) {
         DELETE_ARRAY(en->context);
     }
     DELETE_ARRAY(en);
 }
 static int32_t U_CALLCONV
 enumCount(UEnumeration *en, UErrorCode *) {
     if (((Context *)en->context)->all) {
         // ucsdet_getAllDetectableCharsets, all charset detector names
         return fCSRecognizers_size;
     }
     // Otherwise, ucsdet_getDetectableCharsets - only enabled ones
     int32_t count = 0;
     UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
     if (enabledArray != NULL) {
         // custom set
         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
             if (enabledArray[i]) {
                 count++;
             }
         }
     } else {
         // default set
         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
             if (fCSRecognizers[i]->isDefaultEnabled) {
                 count++;
             }
         }
     }
     return count;
 }
 static const char* U_CALLCONV
 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {
     const char *currName = NULL;
     if (((Context *)en->context)->currIndex < fCSRecognizers_size) {
         if (((Context *)en->context)->all) {
             // ucsdet_getAllDetectableCharsets, all charset detector names
             currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
             ((Context *)en->context)->currIndex++;
         } else {
             // ucsdet_getDetectableCharsets
             UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
             if (enabledArray != NULL) {
                 // custome set
                 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
                     if (enabledArray[((Context *)en->context)->currIndex]) {
                         currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
                     }
                     ((Context *)en->context)->currIndex++;
                 }
             } else {
                 // default set
                 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
                     if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) {
                         currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
                     }
                     ((Context *)en->context)->currIndex++;
                 }
             }
         }
     }
     if(resultLength != NULL) {
         *resultLength = currName == NULL ? 0 : (int32_t)uprv_strlen(currName);
     }
     return currName;
 }
 static void U_CALLCONV
 enumReset(UEnumeration *en, UErrorCode *) {
     ((Context *)en->context)->currIndex = 0;
 }
 static const UEnumeration gCSDetEnumeration = {
     NULL,
     NULL,
     enumClose,
     enumCount,
     uenum_unextDefault,
     enumNext,
     enumReset
 };
 U_CDECL_END
 U_NAMESPACE_BEGIN
 UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status)
 {
     /* Initialize recognized charsets. */
     setRecognizers(status);
     if(U_FAILURE(status)) {
         return 0;
     }
     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
     if (en == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return 0;
     }
     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
     en->context = (void*)NEW_ARRAY(Context, 1);
     if (en->context == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         DELETE_ARRAY(en);
         return 0;
     }
     uprv_memset(en->context, 0, sizeof(Context));
     ((Context*)en->context)->all = TRUE;
     return en;
 }
 UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const
 {
     if(U_FAILURE(status)) {
         return 0;
     }
     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
     if (en == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         return 0;
     }
     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
     en->context = (void*)NEW_ARRAY(Context, 1);
     if (en->context == NULL) {
         status = U_MEMORY_ALLOCATION_ERROR;
         DELETE_ARRAY(en);
         return 0;
     }
     uprv_memset(en->context, 0, sizeof(Context));
     ((Context*)en->context)->all = FALSE;
     ((Context*)en->context)->enabledRecognizers = fEnabledRecognizers;
     return en;
 }
 U_NAMESPACE_END
 #endif

The Tor Browser / annotate

intl/icu/source/i18n/csdetect.cpp@b8a032363ba2 (annotated)

intl/icu/source/i18n/csdetect.cpp