michael@0: /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "nsLatin1Prober.h"
michael@0: #include "prmem.h"
michael@0: #include <stdio.h>
michael@0: 
michael@0: #define UDF    0        // undefined
michael@0: #define OTH    1        //other
michael@0: #define ASC    2        // ascii capital letter
michael@0: #define ASS    3        // ascii small letter
michael@0: #define ACV    4        // accent capital vowel
michael@0: #define ACO    5        // accent capital other
michael@0: #define ASV    6        // accent small vowel
michael@0: #define ASO    7        // accent small other
michael@0: #define CLASS_NUM   8    // total classes
michael@0: 
michael@0: static const unsigned char Latin1_CharToClass[] = 
michael@0: {
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 00 - 07
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 08 - 0F
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 10 - 17
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 18 - 1F
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 20 - 27
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 28 - 2F
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 30 - 37
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 38 - 3F
michael@0:   OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   // 40 - 47
michael@0:   ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   // 48 - 4F
michael@0:   ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   // 50 - 57
michael@0:   ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   // 58 - 5F
michael@0:   OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   // 60 - 67
michael@0:   ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   // 68 - 6F
michael@0:   ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   // 70 - 77
michael@0:   ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   // 78 - 7F
michael@0:   OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   // 80 - 87
michael@0:   OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   // 88 - 8F
michael@0:   UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 90 - 97
michael@0:   OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   // 98 - 9F
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // A0 - A7
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // A8 - AF
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // B0 - B7
michael@0:   OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // B8 - BF
michael@0:   ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   // C0 - C7
michael@0:   ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   // C8 - CF
michael@0:   ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   // D0 - D7
michael@0:   ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   // D8 - DF
michael@0:   ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   // E0 - E7
michael@0:   ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   // E8 - EF
michael@0:   ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   // F0 - F7
michael@0:   ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO,   // F8 - FF
michael@0: };
michael@0: 
michael@0: 
michael@0: /* 0 : illegal 
michael@0:    1 : very unlikely 
michael@0:    2 : normal 
michael@0:    3 : very likely
michael@0: */
michael@0: static const unsigned char Latin1ClassModel[] = 
michael@0: {
michael@0: /*      UDF OTH ASC ASS ACV ACO ASV ASO  */
michael@0: /*UDF*/  0,  0,  0,  0,  0,  0,  0,  0,
michael@0: /*OTH*/  0,  3,  3,  3,  3,  3,  3,  3,
michael@0: /*ASC*/  0,  3,  3,  3,  3,  3,  3,  3, 
michael@0: /*ASS*/  0,  3,  3,  3,  1,  1,  3,  3,
michael@0: /*ACV*/  0,  3,  3,  3,  1,  2,  1,  2,
michael@0: /*ACO*/  0,  3,  3,  3,  3,  3,  3,  3, 
michael@0: /*ASV*/  0,  3,  1,  3,  1,  1,  1,  3, 
michael@0: /*ASO*/  0,  3,  1,  3,  1,  1,  3,  3,
michael@0: };
michael@0: 
michael@0: void  nsLatin1Prober::Reset(void)
michael@0: {
michael@0:   mState = eDetecting;
michael@0:   mLastCharClass = OTH;
michael@0:   for (int i = 0; i < FREQ_CAT_NUM; i++)
michael@0:     mFreqCounter[i] = 0;
michael@0: }
michael@0: 
michael@0: 
michael@0: nsProbingState nsLatin1Prober::HandleData(const char* aBuf, uint32_t aLen)
michael@0: {
michael@0:   char *newBuf1 = 0;
michael@0:   uint32_t newLen1 = 0;
michael@0: 
michael@0:   if (!FilterWithEnglishLetters(aBuf, aLen, &newBuf1, newLen1)) {
michael@0:     newBuf1 = (char*)aBuf;
michael@0:     newLen1 = aLen;
michael@0:   }
michael@0:   
michael@0:   unsigned char charClass;
michael@0:   unsigned char freq;
michael@0:   for (uint32_t i = 0; i < newLen1; i++)
michael@0:   {
michael@0:     charClass = Latin1_CharToClass[(unsigned char)newBuf1[i]];
michael@0:     freq = Latin1ClassModel[mLastCharClass*CLASS_NUM + charClass];
michael@0:     if (freq == 0) {
michael@0:       mState = eNotMe;
michael@0:       break;
michael@0:     }
michael@0:     mFreqCounter[freq]++;
michael@0:     mLastCharClass = charClass;
michael@0:   }
michael@0: 
michael@0:   if (newBuf1 != aBuf)
michael@0:     PR_FREEIF(newBuf1);
michael@0: 
michael@0:   return mState;
michael@0: }
michael@0: 
michael@0: float nsLatin1Prober::GetConfidence(void)
michael@0: {
michael@0:   if (mState == eNotMe)
michael@0:     return 0.01f;
michael@0:   
michael@0:   float confidence;
michael@0:   uint32_t total = 0;
michael@0:   for (int32_t i = 0; i < FREQ_CAT_NUM; i++)
michael@0:     total += mFreqCounter[i];
michael@0: 
michael@0:   if(!total)
michael@0:     confidence = 0.0f;
michael@0:   else
michael@0:   {
michael@0:     confidence = mFreqCounter[3]*1.0f / total;
michael@0:     confidence -= mFreqCounter[1]*20.0f/total;
michael@0:   }
michael@0: 
michael@0:   if (confidence < 0.0f)
michael@0:     confidence = 0.0f;
michael@0:   
michael@0:   // lower the confidence of latin1 so that other more accurate detector 
michael@0:   // can take priority.
michael@0:   confidence *= 0.50f;
michael@0: 
michael@0:   return confidence;
michael@0: }
michael@0: 
michael@0: #ifdef DEBUG_chardet
michael@0: void  nsLatin1Prober::DumpStatus()
michael@0: {
michael@0:   printf(" Latin1Prober: %1.3f [%s]\r\n", GetConfidence(), GetCharSetName());
michael@0: }
michael@0: #endif
michael@0: 
michael@0: