michael@0: // Copyright 2013 Google Inc. All Rights Reserved.
michael@0: //
michael@0: // Licensed under the Apache License, Version 2.0 (the "License");
michael@0: // you may not use this file except in compliance with the License.
michael@0: // You may obtain a copy of the License at
michael@0: //
michael@0: //     http://www.apache.org/licenses/LICENSE-2.0
michael@0: //
michael@0: // Unless required by applicable law or agreed to in writing, software
michael@0: // distributed under the License is distributed on an "AS IS" BASIS,
michael@0: // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
michael@0: // See the License for the specific language governing permissions and
michael@0: // limitations under the License.
michael@0: 
michael@0: //
michael@0: // Author: dsites@google.com (Dick Sites)
michael@0: //
michael@0: 
michael@0: #ifndef I18N_ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_IMPL_H_
michael@0: #define I18N_ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_IMPL_H_
michael@0: 
michael@0: #include <vector>
michael@0: 
michael@0: #include "../public/compact_lang_det.h"   // For CLDHints, ResultChunkVector
michael@0: #include "integral_types.h"
michael@0: #include "lang_script.h"
michael@0: 
michael@0: namespace CLD2 {
michael@0: 
michael@0: // Internal use flags
michael@0: static const int kCLDFlagFinish = 1;
michael@0: static const int kCLDFlagSqueeze = 2;
michael@0: static const int kCLDFlagRepeats = 4;
michael@0: static const int kCLDFlagTop40 = 8;
michael@0: static const int kCLDFlagShort = 16;
michael@0: static const int kCLDFlagHint = 32;
michael@0: static const int kCLDFlagUseWords = 64;
michael@0: static const int kCLDFlagUNUSED = 128;
michael@0: 
michael@0: // Public use flags, debug output controls, defined in compact_lang_det.h
michael@0: // 0x0100 and above
michael@0: 
michael@0: /***
michael@0: 
michael@0: Flag meanings:
michael@0: 
michael@0: Flags are used in the context of a recursive call from Detect to itself,
michael@0: trying to deal in a more restrictive way with input that was not reliably
michael@0: identified in the top-level call.
michael@0: 
michael@0: Finish -- Do not further recurse; return whatever result ensues, even if it is
michael@0:           unreliable. Typically set in any recursive call to take a second try
michael@0:           on unreliable text.
michael@0: 
michael@0: Squeeze -- For each text run, do an inplace cheapsqueeze to remove chunks of
michael@0:           highly repetitive text and chunks of text with too many 1- and
michael@0:           2-letter words. This avoids scoring repetitive or useless non-text
michael@0:           crap in large files such bogus JPEGs within an HTML file.
michael@0: 
michael@0: Repeats -- When scoring a text run, do a cheap prediction of each character
michael@0:           and do not score a unigram/quadgram if the last character of same is
michael@0:           correctly predicted. This is a slower, finer-grained form of
michael@0:           cheapsqueeze, typically used when the first pass got unreliable
michael@0:           results.
michael@0: 
michael@0: Top40 -- Restrict the set of scored languages to the Google "Top 40", which is
michael@0:           actually 38 languages. This gets rid of about 110 languages that
michael@0:           represent about 0.7% of the web. Typically used when the first pass
michael@0:           got unreliable results.
michael@0: 
michael@0: Short -- DEPRICATED, unused
michael@0: 
michael@0: Hint -- EXPERIMENTAL flag for compact_lang_det_test.cc to indicate a language
michael@0:           hint supplied in parameter plus_one.
michael@0: 
michael@0: UseWords -- In additon to scoring quad/uni/nil-grams, score complete words
michael@0: 
michael@0: 
michael@0: 
michael@0: Tentative decision logic:
michael@0: 
michael@0: In the middle of first pass -- After 4KB of text, look at the front 256 bytes
michael@0:           of every full 4KB buffer. If it compresses very well (say 3:1) or has
michael@0:           lots of spaces (say 1 of every 4 bytes), assume that the input is
michael@0:           large and contains lots of bogus non-text. Recurse, passing the
michael@0:           Squeeze flag to strip out chunks of this non-text.
michael@0: 
michael@0: At the end of the first pass --
michael@0:           If the top language is reliable and >= 70% of the document, return.
michael@0:           Else if the top language is reliable and top+2nd >= say 94%, return.
michael@0:           Else, either the top language is not reliable or there is a lot of
michael@0:           other crap.
michael@0: ***/
michael@0: 
michael@0: 
michael@0:   // Scan interchange-valid UTF-8 bytes and detect most likely language,
michael@0:   // or set of languages.
michael@0:   //
michael@0:   // Design goals:
michael@0:   //   Skip over big stretches of HTML tags
michael@0:   //   Able to return ranges of different languages
michael@0:   //   Relatively small tables and relatively fast processing
michael@0:   //   Thread safe
michael@0:   //
michael@0: 
michael@0:   typedef struct {
michael@0:     int perscript_count;
michael@0:     const Language* perscript_lang;
michael@0:   } PerScriptPair;
michael@0: 
michael@0:   typedef struct {
michael@0:     // Constants for hashing 4-7 byte quadgram to 32 bits
michael@0:     const int kQuadHashB4Shift;
michael@0:     const int kQuadHashB4bShift;
michael@0:     const int kQuadHashB5Shift;
michael@0:     const int kQuadHashB5bShift;
michael@0:     // Constants for hashing 32 bits to kQuadKeyTable subscript/key
michael@0:     const int kHashvalToSubShift;
michael@0:     const uint32 kHashvalToSubMask;
michael@0:     const int kHashvalToKeyShift;
michael@0:     const uint32 kHashvalToKeyMask;
michael@0:     const int kHashvalAssociativity;
michael@0:     // Pointers to the actual tables
michael@0:     const PerScriptPair* kPerScriptPair;
michael@0:     const uint16* kQuadKeyTable;
michael@0:     const uint32* kQuadValueTable;
michael@0:   } LangDetObj;
michael@0: 
michael@0:   // For HTML documents, tags are skipped, along with <script> ... </script>
michael@0:   // and <style> ... </style> sequences, and entities are expanded.
michael@0:   //
michael@0:   // We distinguish between bytes of the raw input buffer and bytes of non-tag
michael@0:   // text letters. Since tags can be over 50% of the bytes of an HTML Page,
michael@0:   // and are nearly all seven-bit ASCII English, we prefer to distinguish
michael@0:   // language mixture fractions based on just the non-tag text.
michael@0:   //
michael@0:   // Inputs: text and text_length
michael@0:   //  is_plain_text if true says to NOT parse/skip HTML tags nor entities
michael@0:   // Outputs:
michael@0:   //  language3 is an array of the top 3 languages or UNKNOWN_LANGUAGE
michael@0:   //  percent3 is an array of the text percentages 0..100 of the top 3 languages
michael@0:   //  normalized_score3 is an array of internal scores, normalized to the
michael@0:   //    average score for each language over a body of training text. A
michael@0:   //    normalized score significantly away from 1.0 indicates very skewed text
michael@0:   //    or gibberish.
michael@0:   //
michael@0:   //  text_bytes is the amount of non-tag/letters-only text found
michael@0:   //  is_reliable set true if the returned Language is at least 2**30 times more
michael@0:   //  probable then the second-best Language
michael@0:   //
michael@0:   // Return value: the most likely Language for the majority of the input text
michael@0:   //  Length 0 input and text with no reliable letter sequences returns
michael@0:   //  UNKNOWN_LANGUAGE
michael@0:   //
michael@0:   // Subsetting: For fast detection over large documents, these routines will
michael@0:   // only scan up to a fixed limit (currently 160KB of non-tag letters).
michael@0:   //
michael@0: 
michael@0:   Language DetectLanguageSummaryV2(
michael@0:                         const char* buffer,
michael@0:                         int buffer_length,
michael@0:                         bool is_plain_text,
michael@0:                         const CLDHints* cld_hints,
michael@0:                         bool allow_extended_lang,
michael@0:                         int flags,
michael@0:                         Language plus_one,
michael@0:                         Language* language3,
michael@0:                         int* percent3,
michael@0:                         double* normalized_score3,
michael@0:                         ResultChunkVector* resultchunkvector,
michael@0:                         int* text_bytes,
michael@0:                         bool* is_reliable);
michael@0: 
michael@0:   // For unit testing:
michael@0:   // Remove portions of text that have a high density of spaces, or that are
michael@0:   // overly repetitive, squeezing the remaining text in-place to the front
michael@0:   // of the input buffer.
michael@0:   // Return the new, possibly-shorter length
michael@0:   int CheapSqueezeInplace(char* isrc, int srclen, int ichunksize);
michael@0: 
michael@0: }       // End namespace CLD2
michael@0: 
michael@0: #endif  // I18N_ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_IMPL_H_