1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/extensions/universalchardet/src/base/nsHebrewProber.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,162 @@
1.4 +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
1.5 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.6 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.8 +
1.9 +#include "nsHebrewProber.h"
1.10 +#include <stdio.h>
1.11 +
1.12 +// windows-1255 / ISO-8859-8 code points of interest
1.13 +#define FINAL_KAF ('\xea')
1.14 +#define NORMAL_KAF ('\xeb')
1.15 +#define FINAL_MEM ('\xed')
1.16 +#define NORMAL_MEM ('\xee')
1.17 +#define FINAL_NUN ('\xef')
1.18 +#define NORMAL_NUN ('\xf0')
1.19 +#define FINAL_PE ('\xf3')
1.20 +#define NORMAL_PE ('\xf4')
1.21 +#define FINAL_TSADI ('\xf5')
1.22 +#define NORMAL_TSADI ('\xf6')
1.23 +
1.24 +// Minimum Visual vs Logical final letter score difference.
1.25 +// If the difference is below this, don't rely solely on the final letter score distance.
1.26 +#define MIN_FINAL_CHAR_DISTANCE (5)
1.27 +
1.28 +// Minimum Visual vs Logical model score difference.
1.29 +// If the difference is below this, don't rely at all on the model score distance.
1.30 +#define MIN_MODEL_DISTANCE (0.01)
1.31 +
1.32 +#define VISUAL_HEBREW_NAME ("ISO-8859-8")
1.33 +#define LOGICAL_HEBREW_NAME ("windows-1255")
1.34 +
1.35 +bool nsHebrewProber::isFinal(char c)
1.36 +{
1.37 + return ((c == FINAL_KAF) || (c == FINAL_MEM) || (c == FINAL_NUN) || (c == FINAL_PE) || (c == FINAL_TSADI));
1.38 +}
1.39 +
1.40 +bool nsHebrewProber::isNonFinal(char c)
1.41 +{
1.42 + return ((c == NORMAL_KAF) || (c == NORMAL_MEM) || (c == NORMAL_NUN) || (c == NORMAL_PE));
1.43 + // The normal Tsadi is not a good Non-Final letter due to words like
1.44 + // 'lechotet' (to chat) containing an apostrophe after the tsadi. This
1.45 + // apostrophe is converted to a space in FilterWithoutEnglishLetters causing
1.46 + // the Non-Final tsadi to appear at an end of a word even though this is not
1.47 + // the case in the original text.
1.48 + // The letters Pe and Kaf rarely display a related behavior of not being a
1.49 + // good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for
1.50 + // example legally end with a Non-Final Pe or Kaf. However, the benefit of
1.51 + // these letters as Non-Final letters outweighs the damage since these words
1.52 + // are quite rare.
1.53 +}
1.54 +
1.55 +/** HandleData
1.56 + * Final letter analysis for logical-visual decision.
1.57 + * Look for evidence that the received buffer is either logical Hebrew or
1.58 + * visual Hebrew.
1.59 + * The following cases are checked:
1.60 + * 1) A word longer than 1 letter, ending with a final letter. This is an
1.61 + * indication that the text is laid out "naturally" since the final letter
1.62 + * really appears at the end. +1 for logical score.
1.63 + * 2) A word longer than 1 letter, ending with a Non-Final letter. In normal
1.64 + * Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, should not end with
1.65 + * the Non-Final form of that letter. Exceptions to this rule are mentioned
1.66 + * above in isNonFinal(). This is an indication that the text is laid out
1.67 + * backwards. +1 for visual score
1.68 + * 3) A word longer than 1 letter, starting with a final letter. Final letters
1.69 + * should not appear at the beginning of a word. This is an indication that
1.70 + * the text is laid out backwards. +1 for visual score.
1.71 + *
1.72 + * The visual score and logical score are accumulated throughout the text and
1.73 + * are finally checked against each other in GetCharSetName().
1.74 + * No checking for final letters in the middle of words is done since that case
1.75 + * is not an indication for either Logical or Visual text.
1.76 + *
1.77 + * The input buffer should not contain any white spaces that are not (' ')
1.78 + * or any low-ascii punctuation marks.
1.79 + */
1.80 +nsProbingState nsHebrewProber::HandleData(const char* aBuf, uint32_t aLen)
1.81 +{
1.82 + // Both model probers say it's not them. No reason to continue.
1.83 + if (GetState() == eNotMe)
1.84 + return eNotMe;
1.85 +
1.86 + const char *curPtr, *endPtr = aBuf+aLen;
1.87 + char cur;
1.88 +
1.89 + for (curPtr = (char*)aBuf; curPtr < endPtr; ++curPtr)
1.90 + {
1.91 + cur = *curPtr;
1.92 + if (cur == ' ') // We stand on a space - a word just ended
1.93 + {
1.94 + if (mBeforePrev != ' ') // *(curPtr-2) was not a space so prev is not a 1 letter word
1.95 + {
1.96 + if (isFinal(mPrev)) // case (1) [-2:not space][-1:final letter][cur:space]
1.97 + ++mFinalCharLogicalScore;
1.98 + else if (isNonFinal(mPrev)) // case (2) [-2:not space][-1:Non-Final letter][cur:space]
1.99 + ++mFinalCharVisualScore;
1.100 + }
1.101 + }
1.102 + else // Not standing on a space
1.103 + {
1.104 + if ((mBeforePrev == ' ') && (isFinal(mPrev)) && (cur != ' ')) // case (3) [-2:space][-1:final letter][cur:not space]
1.105 + ++mFinalCharVisualScore;
1.106 + }
1.107 + mBeforePrev = mPrev;
1.108 + mPrev = cur;
1.109 + }
1.110 +
1.111 + // Forever detecting, till the end or until both model probers return eNotMe (handled above).
1.112 + return eDetecting;
1.113 +}
1.114 +
1.115 +// Make the decision: is it Logical or Visual?
1.116 +const char* nsHebrewProber::GetCharSetName()
1.117 +{
1.118 + // If the final letter score distance is dominant enough, rely on it.
1.119 + int32_t finalsub = mFinalCharLogicalScore - mFinalCharVisualScore;
1.120 + if (finalsub >= MIN_FINAL_CHAR_DISTANCE)
1.121 + return LOGICAL_HEBREW_NAME;
1.122 + if (finalsub <= -(MIN_FINAL_CHAR_DISTANCE))
1.123 + return VISUAL_HEBREW_NAME;
1.124 +
1.125 + // It's not dominant enough, try to rely on the model scores instead.
1.126 + float modelsub = mLogicalProb->GetConfidence() - mVisualProb->GetConfidence();
1.127 + if (modelsub > MIN_MODEL_DISTANCE)
1.128 + return LOGICAL_HEBREW_NAME;
1.129 + if (modelsub < -(MIN_MODEL_DISTANCE))
1.130 + return VISUAL_HEBREW_NAME;
1.131 +
1.132 + // Still no good, back to final letter distance, maybe it'll save the day.
1.133 + if (finalsub < 0)
1.134 + return VISUAL_HEBREW_NAME;
1.135 +
1.136 + // (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
1.137 + return LOGICAL_HEBREW_NAME;
1.138 +}
1.139 +
1.140 +
1.141 +void nsHebrewProber::Reset(void)
1.142 +{
1.143 + mFinalCharLogicalScore = 0;
1.144 + mFinalCharVisualScore = 0;
1.145 +
1.146 + // mPrev and mBeforePrev are initialized to space in order to simulate a word
1.147 + // delimiter at the beginning of the data
1.148 + mPrev = ' ';
1.149 + mBeforePrev = ' ';
1.150 +}
1.151 +
1.152 +nsProbingState nsHebrewProber::GetState(void)
1.153 +{
1.154 + // Remain active as long as any of the model probers are active.
1.155 + if ((mLogicalProb->GetState() == eNotMe) && (mVisualProb->GetState() == eNotMe))
1.156 + return eNotMe;
1.157 + return eDetecting;
1.158 +}
1.159 +
1.160 +#ifdef DEBUG_chardet
1.161 +void nsHebrewProber::DumpStatus()
1.162 +{
1.163 + printf(" HEB: %d - %d [Logical-Visual score]\r\n", mFinalCharLogicalScore, mFinalCharVisualScore);
1.164 +}
1.165 +#endif
