extensions/universalchardet/src/base/nsHebrewProber.h@6474c204b198 (annotated)
extensions/universalchardet/src/base/nsHebrewProber.h
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rw-r--r--
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
| michael@0 | 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ |
| michael@0 | 2 | /* This Source Code Form is subject to the terms of the Mozilla Public |
| michael@0 | 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this |
| michael@0 | 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
| michael@0 | 5 | |
| michael@0 | 6 | #ifndef nsHebrewProber_h__ |
| michael@0 | 7 | #define nsHebrewProber_h__ |
| michael@0 | 8 | |
| michael@0 | 9 | #include "nsSBCharSetProber.h" |
| michael@0 | 10 | |
| michael@0 | 11 | // This prober doesn't actually recognize a language or a charset. |
| michael@0 | 12 | // It is a helper prober for the use of the Hebrew model probers |
| michael@0 | 13 | class nsHebrewProber: public nsCharSetProber |
| michael@0 | 14 | { |
| michael@0 | 15 | public: |
| michael@0 | 16 | nsHebrewProber(void) :mLogicalProb(0), mVisualProb(0) { Reset(); } |
| michael@0 | 17 | |
| michael@0 | 18 | virtual ~nsHebrewProber(void) {} |
| michael@0 | 19 | virtual nsProbingState HandleData(const char* aBuf, uint32_t aLen); |
| michael@0 | 20 | virtual const char* GetCharSetName(); |
| michael@0 | 21 | virtual void Reset(void); |
| michael@0 | 22 | |
| michael@0 | 23 | virtual nsProbingState GetState(void); |
| michael@0 | 24 | |
| michael@0 | 25 | virtual float GetConfidence(void) { return (float)0.0; } |
| michael@0 | 26 | |
| michael@0 | 27 | void SetModelProbers(nsCharSetProber *logicalPrb, nsCharSetProber *visualPrb) |
| michael@0 | 28 | { mLogicalProb = logicalPrb; mVisualProb = visualPrb; } |
| michael@0 | 29 | |
| michael@0 | 30 | #ifdef DEBUG_chardet |
| michael@0 | 31 | virtual void DumpStatus(); |
| michael@0 | 32 | #endif |
| michael@0 | 33 | |
| michael@0 | 34 | protected: |
| michael@0 | 35 | static bool isFinal(char c); |
| michael@0 | 36 | static bool isNonFinal(char c); |
| michael@0 | 37 | |
| michael@0 | 38 | int32_t mFinalCharLogicalScore, mFinalCharVisualScore; |
| michael@0 | 39 | |
| michael@0 | 40 | // The two last characters seen in the previous buffer. |
| michael@0 | 41 | char mPrev, mBeforePrev; |
| michael@0 | 42 | |
| michael@0 | 43 | // These probers are owned by the group prober. |
| michael@0 | 44 | nsCharSetProber *mLogicalProb, *mVisualProb; |
| michael@0 | 45 | }; |
| michael@0 | 46 | |
| michael@0 | 47 | /** |
| michael@0 | 48 | * ** General ideas of the Hebrew charset recognition ** |
| michael@0 | 49 | * |
| michael@0 | 50 | * Four main charsets exist in Hebrew: |
| michael@0 | 51 | * "ISO-8859-8" - Visual Hebrew |
| michael@0 | 52 | * "windows-1255" - Logical Hebrew |
| michael@0 | 53 | * "ISO-8859-8-I" - Logical Hebrew |
| michael@0 | 54 | * "x-mac-hebrew" - ?? Logical Hebrew ?? |
| michael@0 | 55 | * |
| michael@0 | 56 | * Both "ISO" charsets use a completely identical set of code points, whereas |
| michael@0 | 57 | * "windows-1255" and "x-mac-hebrew" are two different proper supersets of |
| michael@0 | 58 | * these code points. windows-1255 defines additional characters in the range |
| michael@0 | 59 | * 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific |
| michael@0 | 60 | * diacritics and additional 'Yiddish' ligature letters in the range 0xc0-0xd6. |
| michael@0 | 61 | * x-mac-hebrew defines similar additional code points but with a different |
| michael@0 | 62 | * mapping. |
| michael@0 | 63 | * |
| michael@0 | 64 | * As far as an average Hebrew text with no diacritics is concerned, all four |
| michael@0 | 65 | * charsets are identical with respect to code points. Meaning that for the |
| michael@0 | 66 | * main Hebrew alphabet, all four map the same values to all 27 Hebrew letters |
| michael@0 | 67 | * (including final letters). |
| michael@0 | 68 | * |
| michael@0 | 69 | * The dominant difference between these charsets is their directionality. |
| michael@0 | 70 | * "Visual" directionality means that the text is ordered as if the renderer is |
| michael@0 | 71 | * not aware of a BIDI rendering algorithm. The renderer sees the text and |
| michael@0 | 72 | * draws it from left to right. The text itself when ordered naturally is read |
| michael@0 | 73 | * backwards. A buffer of Visual Hebrew generally looks like so: |
| michael@0 | 74 | * "[last word of first line spelled backwards] [whole line ordered backwards |
| michael@0 | 75 | * and spelled backwards] [first word of first line spelled backwards] |
| michael@0 | 76 | * [end of line] [last word of second line] ... etc' " |
| michael@0 | 77 | * adding punctuation marks, numbers and English text to visual text is |
| michael@0 | 78 | * naturally also "visual" and from left to right. |
| michael@0 | 79 | * |
| michael@0 | 80 | * "Logical" directionality means the text is ordered "naturally" according to |
| michael@0 | 81 | * the order it is read. It is the responsibility of the renderer to display |
| michael@0 | 82 | * the text from right to left. A BIDI algorithm is used to place general |
| michael@0 | 83 | * punctuation marks, numbers and English text in the text. |
| michael@0 | 84 | * |
| michael@0 | 85 | * Texts in x-mac-hebrew are almost impossible to find on the Internet. From |
| michael@0 | 86 | * what little evidence I could find, it seems that its general directionality |
| michael@0 | 87 | * is Logical. |
| michael@0 | 88 | * |
| michael@0 | 89 | * To sum up all of the above, the Hebrew probing mechanism knows about two |
| michael@0 | 90 | * charsets: |
| michael@0 | 91 | * Visual Hebrew - "ISO-8859-8" - backwards text - Words and sentences are |
| michael@0 | 92 | * backwards while line order is natural. For charset recognition purposes |
| michael@0 | 93 | * the line order is unimportant (In fact, for this implementation, even |
| michael@0 | 94 | * word order is unimportant). |
| michael@0 | 95 | * Logical Hebrew - "windows-1255" - normal, naturally ordered text. |
| michael@0 | 96 | * |
| michael@0 | 97 | * "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be |
| michael@0 | 98 | * specifically identified. |
| michael@0 | 99 | * "x-mac-hebrew" is also identified as windows-1255. A text in x-mac-hebrew |
| michael@0 | 100 | * that contain special punctuation marks or diacritics is displayed with |
| michael@0 | 101 | * some unconverted characters showing as question marks. This problem might |
| michael@0 | 102 | * be corrected using another model prober for x-mac-hebrew. Due to the fact |
| michael@0 | 103 | * that x-mac-hebrew texts are so rare, writing another model prober isn't |
| michael@0 | 104 | * worth the effort and performance hit. |
| michael@0 | 105 | * |
| michael@0 | 106 | * *** The Prober *** |
| michael@0 | 107 | * |
| michael@0 | 108 | * The prober is divided between two nsSBCharSetProbers and an nsHebrewProber, |
| michael@0 | 109 | * all of which are managed, created, fed data, inquired and deleted by the |
| michael@0 | 110 | * nsSBCSGroupProber. The two nsSBCharSetProbers identify that the text is in |
| michael@0 | 111 | * fact some kind of Hebrew, Logical or Visual. The final decision about which |
| michael@0 | 112 | * one is it is made by the nsHebrewProber by combining final-letter scores |
| michael@0 | 113 | * with the scores of the two nsSBCharSetProbers to produce a final answer. |
| michael@0 | 114 | * |
| michael@0 | 115 | * The nsSBCSGroupProber is responsible for stripping the original text of HTML |
| michael@0 | 116 | * tags, English characters, numbers, low-ASCII punctuation characters, spaces |
| michael@0 | 117 | * and new lines. It reduces any sequence of such characters to a single space. |
| michael@0 | 118 | * The buffer fed to each prober in the SBCS group prober is pure text in |
| michael@0 | 119 | * high-ASCII. |
| michael@0 | 120 | * The two nsSBCharSetProbers (model probers) share the same language model: |
| michael@0 | 121 | * Win1255Model. |
| michael@0 | 122 | * The first nsSBCharSetProber uses the model normally as any other |
| michael@0 | 123 | * nsSBCharSetProber does, to recognize windows-1255, upon which this model was |
| michael@0 | 124 | * built. The second nsSBCharSetProber is told to make the pair-of-letter |
| michael@0 | 125 | * lookup in the language model backwards. This in practice exactly simulates |
| michael@0 | 126 | * a visual Hebrew model using the windows-1255 logical Hebrew model. |
| michael@0 | 127 | * |
| michael@0 | 128 | * The nsHebrewProber is not using any language model. All it does is look for |
| michael@0 | 129 | * final-letter evidence suggesting the text is either logical Hebrew or visual |
| michael@0 | 130 | * Hebrew. Disjointed from the model probers, the results of the nsHebrewProber |
| michael@0 | 131 | * alone are meaningless. nsHebrewProber always returns 0.00 as confidence |
| michael@0 | 132 | * since it never identifies a charset by itself. Instead, the pointer to the |
| michael@0 | 133 | * nsHebrewProber is passed to the model probers as a helper "Name Prober". |
| michael@0 | 134 | * When the Group prober receives a positive identification from any prober, |
| michael@0 | 135 | * it asks for the name of the charset identified. If the prober queried is a |
| michael@0 | 136 | * Hebrew model prober, the model prober forwards the call to the |
| michael@0 | 137 | * nsHebrewProber to make the final decision. In the nsHebrewProber, the |
| michael@0 | 138 | * decision is made according to the final-letters scores maintained and Both |
| michael@0 | 139 | * model probers scores. The answer is returned in the form of the name of the |
| michael@0 | 140 | * charset identified, either "windows-1255" or "ISO-8859-8". |
| michael@0 | 141 | * |
| michael@0 | 142 | */ |
| michael@0 | 143 | #endif /* nsHebrewProber_h__ */ |
