The Tor Browser: comparison extensions/universalchardet/src/base/nsHebrewProber.cpp

--1:000000000000
+:36bb89cb0cb2
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#include "nsHebrewProber.h"
+#include <stdio.h>
+// windows-1255 / ISO-8859-8 code points of interest
+#define FINAL_KAF ('\xea')
+#define NORMAL_KAF ('\xeb')
+#define FINAL_MEM ('\xed')
+#define NORMAL_MEM ('\xee')
+#define FINAL_NUN ('\xef')
+#define NORMAL_NUN ('\xf0')
+#define FINAL_PE ('\xf3')
+#define NORMAL_PE ('\xf4')
+#define FINAL_TSADI ('\xf5')
+#define NORMAL_TSADI ('\xf6')
+// Minimum Visual vs Logical final letter score difference.
+// If the difference is below this, don't rely solely on the final letter score distance.
+#define MIN_FINAL_CHAR_DISTANCE (5)
+// Minimum Visual vs Logical model score difference.
+// If the difference is below this, don't rely at all on the model score distance.
+#define MIN_MODEL_DISTANCE (0.01)
+#define VISUAL_HEBREW_NAME ("ISO-8859-8")
+#define LOGICAL_HEBREW_NAME ("windows-1255")
+bool nsHebrewProber::isFinal(char c)
+{
+return ((c == FINAL_KAF) || (c == FINAL_MEM) || (c == FINAL_NUN) || (c == FINAL_PE) || (c == FINAL_TSADI));
+}
+bool nsHebrewProber::isNonFinal(char c)
+{
+return ((c == NORMAL_KAF) || (c == NORMAL_MEM) || (c == NORMAL_NUN) || (c == NORMAL_PE));
+// The normal Tsadi is not a good Non-Final letter due to words like
+// 'lechotet' (to chat) containing an apostrophe after the tsadi. This
+// apostrophe is converted to a space in FilterWithoutEnglishLetters causing
+// the Non-Final tsadi to appear at an end of a word even though this is not
+// the case in the original text.
+// The letters Pe and Kaf rarely display a related behavior of not being a
+// good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for
+// example legally end with a Non-Final Pe or Kaf. However, the benefit of
+// these letters as Non-Final letters outweighs the damage since these words
+// are quite rare.
+}
+/** HandleData
+* Final letter analysis for logical-visual decision.
+* Look for evidence that the received buffer is either logical Hebrew or
+* visual Hebrew.
+* The following cases are checked:
+* 1) A word longer than 1 letter, ending with a final letter. This is an
+*    indication that the text is laid out "naturally" since the final letter
+*    really appears at the end. +1 for logical score.
+* 2) A word longer than 1 letter, ending with a Non-Final letter. In normal
+*    Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, should not end with
+*    the Non-Final form of that letter. Exceptions to this rule are mentioned
+*    above in isNonFinal(). This is an indication that the text is laid out
+*    backwards. +1 for visual score
+* 3) A word longer than 1 letter, starting with a final letter. Final letters
+*    should not appear at the beginning of a word. This is an indication that
+*    the text is laid out backwards. +1 for visual score.
+*
+* The visual score and logical score are accumulated throughout the text and
+* are finally checked against each other in GetCharSetName().
+* No checking for final letters in the middle of words is done since that case
+* is not an indication for either Logical or Visual text.
+*
+* The input buffer should not contain any white spaces that are not (' ')
+* or any low-ascii punctuation marks.
+*/
+nsProbingState nsHebrewProber::HandleData(const char* aBuf, uint32_t aLen)
+{
+// Both model probers say it's not them. No reason to continue.
+if (GetState() == eNotMe)
+return eNotMe;
+const char *curPtr, *endPtr = aBuf+aLen;
+char cur;
+for (curPtr = (char*)aBuf; curPtr < endPtr; ++curPtr)
+{
+cur = *curPtr;
+if (cur == ' ') // We stand on a space - a word just ended
+{
+if (mBeforePrev != ' ') // *(curPtr-2) was not a space so prev is not a 1 letter word
+{
+if (isFinal(mPrev)) // case (1) [-2:not space][-1:final letter][cur:space]
+++mFinalCharLogicalScore;
+else if (isNonFinal(mPrev)) // case (2) [-2:not space][-1:Non-Final letter][cur:space]
+++mFinalCharVisualScore;
+}
+}
+else  // Not standing on a space
+{
+if ((mBeforePrev == ' ') && (isFinal(mPrev)) && (cur != ' ')) // case (3) [-2:space][-1:final letter][cur:not space]
+++mFinalCharVisualScore;
+}
+mBeforePrev = mPrev;
+mPrev = cur;
+}
+// Forever detecting, till the end or until both model probers return eNotMe (handled above).
+return eDetecting;
+}
+// Make the decision: is it Logical or Visual?
+const char* nsHebrewProber::GetCharSetName()
+{
+// If the final letter score distance is dominant enough, rely on it.
+int32_t finalsub = mFinalCharLogicalScore - mFinalCharVisualScore;
+if (finalsub >= MIN_FINAL_CHAR_DISTANCE)
+return LOGICAL_HEBREW_NAME;
+if (finalsub <= -(MIN_FINAL_CHAR_DISTANCE))
+return VISUAL_HEBREW_NAME;
+// It's not dominant enough, try to rely on the model scores instead.
+float modelsub = mLogicalProb->GetConfidence() - mVisualProb->GetConfidence();
+if (modelsub > MIN_MODEL_DISTANCE)
+return LOGICAL_HEBREW_NAME;
+if (modelsub < -(MIN_MODEL_DISTANCE))
+return VISUAL_HEBREW_NAME;
+// Still no good, back to final letter distance, maybe it'll save the day.
+if (finalsub < 0)
+return VISUAL_HEBREW_NAME;
+// (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
+return LOGICAL_HEBREW_NAME;
+}
+void nsHebrewProber::Reset(void)
+{
+mFinalCharLogicalScore = 0;
+mFinalCharVisualScore = 0;
+// mPrev and mBeforePrev are initialized to space in order to simulate a word
+// delimiter at the beginning of the data
+mPrev = ' ';
+mBeforePrev = ' ';
+}
+nsProbingState nsHebrewProber::GetState(void)
+{
+// Remain active as long as any of the model probers are active.
+if ((mLogicalProb->GetState() == eNotMe) && (mVisualProb->GetState() == eNotMe))
+return eNotMe;
+return eDetecting;
+}
+#ifdef DEBUG_chardet
+void  nsHebrewProber::DumpStatus()
+{
+printf("  HEB: %d - %d [Logical-Visual score]\r\n", mFinalCharLogicalScore, mFinalCharVisualScore);
+}
+#endif

The Tor Browser / file comparison

comparison: extensions/universalchardet/src/base/nsHebrewProber.cpp

extensions/universalchardet/src/base/nsHebrewProber.cpp