The Tor Browser / file revision

 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */

 /* This Source Code Form is subject to the terms of the Mozilla Public

  * License, v. 2.0. If a copy of the MPL was not distributed with this

  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

 #include <stdio.h>

 #include "prmem.h"

 #include "nsSBCharSetProber.h"

 #include "nsSBCSGroupProber.h"

 #include "nsHebrewProber.h"

 nsSBCSGroupProber::nsSBCSGroupProber()

 {

   mProbers[0] = new nsSingleByteCharSetProber(&Win1251Model);

   mProbers[1] = new nsSingleByteCharSetProber(&Koi8rModel);

   mProbers[2] = new nsSingleByteCharSetProber(&Latin5Model);

   mProbers[3] = new nsSingleByteCharSetProber(&MacCyrillicModel);

   mProbers[4] = new nsSingleByteCharSetProber(&Ibm866Model);

   mProbers[5] = new nsSingleByteCharSetProber(&Ibm855Model);

   mProbers[6] = new nsSingleByteCharSetProber(&Latin7Model);

   mProbers[7] = new nsSingleByteCharSetProber(&Win1253Model);

   mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel);

   mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel);

   mProbers[10] = new nsSingleByteCharSetProber(&TIS620ThaiModel);

   nsHebrewProber *hebprober = new nsHebrewProber();

   // Notice: Any change in these indexes - 10,11,12 must be reflected

   // in the code below as well.

   mProbers[11] = hebprober;

   mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, false, hebprober); // Logical Hebrew

   mProbers[13] = new nsSingleByteCharSetProber(&Win1255Model, true, hebprober); // Visual Hebrew

   // Tell the Hebrew prober about the logical and visual probers

   if (mProbers[11] && mProbers[12] && mProbers[13]) // all are not null

   {

     hebprober->SetModelProbers(mProbers[12], mProbers[13]);

   }

   else // One or more is null. avoid any Hebrew probing, null them all

   {

     for (uint32_t i = 11; i <= 13; ++i)

     { 

       delete mProbers[i]; 

       mProbers[i] = 0; 

     }

   }

   // disable latin2 before latin1 is available, otherwise all latin1 

   // will be detected as latin2 because of their similarity.

   //mProbers[10] = new nsSingleByteCharSetProber(&Latin2HungarianModel);

   //mProbers[11] = new nsSingleByteCharSetProber(&Win1250HungarianModel);

   Reset();

 }

 nsSBCSGroupProber::~nsSBCSGroupProber()

 {

   for (uint32_t i = 0; i < NUM_OF_SBCS_PROBERS; i++)

   {

     delete mProbers[i];

   }

 }

 const char* nsSBCSGroupProber::GetCharSetName()

 {

   //if we have no answer yet

   if (mBestGuess == -1)

   {

     GetConfidence();

     //no charset seems positive

     if (mBestGuess == -1)

       //we will use default.

       mBestGuess = 0;

   }

   return mProbers[mBestGuess]->GetCharSetName();

 }

 void  nsSBCSGroupProber::Reset(void)

 {

   mActiveNum = 0;

   for (uint32_t i = 0; i < NUM_OF_SBCS_PROBERS; i++)

   {

     if (mProbers[i]) // not null

     {

       mProbers[i]->Reset();

       mIsActive[i] = true;

       ++mActiveNum;

     }

     else

       mIsActive[i] = false;

   }

   mBestGuess = -1;

   mState = eDetecting;

 }

 nsProbingState nsSBCSGroupProber::HandleData(const char* aBuf, uint32_t aLen)

 {

   nsProbingState st;

   uint32_t i;

   char *newBuf1 = 0;

   uint32_t newLen1 = 0;

   //apply filter to original buffer, and we got new buffer back

   //depend on what script it is, we will feed them the new buffer 

   //we got after applying proper filter

   //this is done without any consideration to KeepEnglishLetters

   //of each prober since as of now, there are no probers here which

   //recognize languages with English characters.

   if (!FilterWithoutEnglishLetters(aBuf, aLen, &newBuf1, newLen1))

     goto done;

   if (newLen1 == 0)

     goto done; // Nothing to see here, move on.

   for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)

   {

      if (!mIsActive[i])

        continue;

      st = mProbers[i]->HandleData(newBuf1, newLen1);

      if (st == eFoundIt)

      {

        mBestGuess = i;

        mState = eFoundIt;

        break;

      }

      else if (st == eNotMe)

      {

        mIsActive[i] = false;

        mActiveNum--;

        if (mActiveNum <= 0)

        {

          mState = eNotMe;

          break;

        }

      }

   }

 done:

   PR_FREEIF(newBuf1);

   return mState;

 }

 float nsSBCSGroupProber::GetConfidence(void)

 {

   uint32_t i;

   float bestConf = 0.0, cf;

   switch (mState)

   {

   case eFoundIt:

     return (float)0.99; //sure yes

   case eNotMe:

     return (float)0.01;  //sure no

   default:

     for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)

     {

       if (!mIsActive[i])

         continue;

       cf = mProbers[i]->GetConfidence();

       if (bestConf < cf)

       {

         bestConf = cf;

         mBestGuess = i;

       }

     }

   }

   return bestConf;

 }

 #ifdef DEBUG_chardet

 void nsSBCSGroupProber::DumpStatus()

 {

   uint32_t i;

   float cf;

   cf = GetConfidence();

   printf(" SBCS Group Prober --------begin status \r\n");

   for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)

   {

     if (!mIsActive[i])

       printf("  inactive: [%s] (i.e. confidence is too low).\r\n", mProbers[i]->GetCharSetName());

     else

       mProbers[i]->DumpStatus();

   }

   printf(" SBCS Group found best match [%s] confidence %f.\r\n",  

          mProbers[mBestGuess]->GetCharSetName(), cf);

 }

 #endif