1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/intl/icu/source/i18n/csrsbcs.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1259 @@
1.4 +/*
1.5 + **********************************************************************
1.6 + * Copyright (C) 2005-2013, International Business Machines
1.7 + * Corporation and others. All Rights Reserved.
1.8 + **********************************************************************
1.9 + */
1.10 +
1.11 +#include "unicode/utypes.h"
1.12 +
1.13 +#include "cmemory.h"
1.14 +
1.15 +#if !UCONFIG_NO_CONVERSION
1.16 +#include "csrsbcs.h"
1.17 +#include "csmatch.h"
1.18 +
1.19 +#define N_GRAM_SIZE 3
1.20 +#define N_GRAM_MASK 0xFFFFFF
1.21 +#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
1.22 +
1.23 +U_NAMESPACE_BEGIN
1.24 +
1.25 +NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap)
1.26 + : ngram(0), byteIndex(0)
1.27 +{
1.28 + ngramList = theNgramList;
1.29 + charMap = theCharMap;
1.30 +
1.31 + ngramCount = hitCount = 0;
1.32 +}
1.33 +
1.34 +/*
1.35 + * Binary search for value in table, which must have exactly 64 entries.
1.36 + */
1.37 +
1.38 +int32_t NGramParser::search(const int32_t *table, int32_t value)
1.39 +{
1.40 + int32_t index = 0;
1.41 +
1.42 + if (table[index + 32] <= value) {
1.43 + index += 32;
1.44 + }
1.45 +
1.46 + if (table[index + 16] <= value) {
1.47 + index += 16;
1.48 + }
1.49 +
1.50 + if (table[index + 8] <= value) {
1.51 + index += 8;
1.52 + }
1.53 +
1.54 + if (table[index + 4] <= value) {
1.55 + index += 4;
1.56 + }
1.57 +
1.58 + if (table[index + 2] <= value) {
1.59 + index += 2;
1.60 + }
1.61 +
1.62 + if (table[index + 1] <= value) {
1.63 + index += 1;
1.64 + }
1.65 +
1.66 + if (table[index] > value) {
1.67 + index -= 1;
1.68 + }
1.69 +
1.70 + if (index < 0 || table[index] != value) {
1.71 + return -1;
1.72 + }
1.73 +
1.74 + return index;
1.75 +}
1.76 +
1.77 +void NGramParser::lookup(int32_t thisNgram)
1.78 +{
1.79 + ngramCount += 1;
1.80 +
1.81 + if (search(ngramList, thisNgram) >= 0) {
1.82 + hitCount += 1;
1.83 + }
1.84 +
1.85 +}
1.86 +
1.87 +void NGramParser::addByte(int32_t b)
1.88 +{
1.89 + ngram = ((ngram << 8) + b) & N_GRAM_MASK;
1.90 + lookup(ngram);
1.91 +}
1.92 +
1.93 +int32_t NGramParser::nextByte(InputText *det)
1.94 +{
1.95 + if (byteIndex >= det->fInputLen) {
1.96 + return -1;
1.97 + }
1.98 +
1.99 + return det->fInputBytes[byteIndex++];
1.100 +}
1.101 +
1.102 +void NGramParser::parseCharacters(InputText *det)
1.103 +{
1.104 + int32_t b;
1.105 + bool ignoreSpace = FALSE;
1.106 +
1.107 + while ((b = nextByte(det)) >= 0) {
1.108 + uint8_t mb = charMap[b];
1.109 +
1.110 + // TODO: 0x20 might not be a space in all character sets...
1.111 + if (mb != 0) {
1.112 + if (!(mb == 0x20 && ignoreSpace)) {
1.113 + addByte(mb);
1.114 + }
1.115 +
1.116 + ignoreSpace = (mb == 0x20);
1.117 + }
1.118 + }
1.119 +}
1.120 +
1.121 +int32_t NGramParser::parse(InputText *det)
1.122 +{
1.123 + parseCharacters(det);
1.124 +
1.125 + // TODO: Is this OK? The buffer could have ended in the middle of a word...
1.126 + addByte(0x20);
1.127 +
1.128 + double rawPercent = (double) hitCount / (double) ngramCount;
1.129 +
1.130 + // if (rawPercent <= 2.0) {
1.131 + // return 0;
1.132 + // }
1.133 +
1.134 + // TODO - This is a bit of a hack to take care of a case
1.135 + // were we were getting a confidence of 135...
1.136 + if (rawPercent > 0.33) {
1.137 + return 98;
1.138 + }
1.139 +
1.140 + return (int32_t) (rawPercent * 300.0);
1.141 +}
1.142 +
1.143 +static const uint8_t unshapeMap_IBM420[] = {
1.144 +/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */
1.145 +/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.146 +/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.147 +/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.148 +/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.149 +/* 4- */ 0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,
1.150 +/* 5- */ 0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
1.151 +/* 6- */ 0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.152 +/* 7- */ 0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,
1.153 +/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F,
1.154 +/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E,
1.155 +/* A- */ 0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF,
1.156 +/* B- */ 0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF,
1.157 +/* C- */ 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF,
1.158 +/* D- */ 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF,
1.159 +/* E- */ 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.160 +/* F- */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
1.161 +};
1.162 +
1.163 +NGramParser_IBM420::NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap):NGramParser(theNgramList, theCharMap)
1.164 +{
1.165 + alef = 0x00;
1.166 +}
1.167 +
1.168 +
1.169 +int32_t NGramParser_IBM420::isLamAlef(int32_t b)
1.170 +{
1.171 + if(b == 0xB2 || b == 0xB3){
1.172 + return 0x47;
1.173 + }else if(b == 0xB4 || b == 0xB5){
1.174 + return 0x49;
1.175 + }else if(b == 0xB8 || b == 0xB9){
1.176 + return 0x56;
1.177 + }else
1.178 + return 0x00;
1.179 +}
1.180 +
1.181 +/*
1.182 +* Arabic shaping needs to be done manually. Cannot call ArabicShaping class
1.183 +* because CharsetDetector is dealing with bytes not Unicode code points. We could
1.184 +* convert the bytes to Unicode code points but that would leave us dependent
1.185 +* on CharsetICU which we try to avoid. IBM420 converter amongst different versions
1.186 +* of JDK can produce different results and therefore is also avoided.
1.187 +*/
1.188 +int32_t NGramParser_IBM420::nextByte(InputText *det)
1.189 +{
1.190 +
1.191 + if (byteIndex >= det->fInputLen || det->fInputBytes[byteIndex] == 0) {
1.192 + return -1;
1.193 + }
1.194 + int next;
1.195 +
1.196 + alef = isLamAlef(det->fInputBytes[byteIndex]);
1.197 + if(alef != 0x00)
1.198 + next = 0xB1 & 0xFF;
1.199 + else
1.200 + next = unshapeMap_IBM420[det->fInputBytes[byteIndex]& 0xFF] & 0xFF;
1.201 +
1.202 + byteIndex++;
1.203 +
1.204 + return next;
1.205 +}
1.206 +
1.207 +void NGramParser_IBM420::parseCharacters(InputText *det)
1.208 +{
1.209 + int32_t b;
1.210 + bool ignoreSpace = FALSE;
1.211 +
1.212 + while ((b = nextByte(det)) >= 0) {
1.213 + uint8_t mb = charMap[b];
1.214 +
1.215 + // TODO: 0x20 might not be a space in all character sets...
1.216 + if (mb != 0) {
1.217 + if (!(mb == 0x20 && ignoreSpace)) {
1.218 + addByte(mb);
1.219 + }
1.220 + ignoreSpace = (mb == 0x20);
1.221 + }
1.222 +
1.223 + if(alef != 0x00){
1.224 + mb = charMap[alef & 0xFF];
1.225 +
1.226 + // TODO: 0x20 might not be a space in all character sets...
1.227 + if (mb != 0) {
1.228 + if (!(mb == 0x20 && ignoreSpace)) {
1.229 + addByte(mb);
1.230 + }
1.231 +
1.232 + ignoreSpace = (mb == 0x20);
1.233 + }
1.234 +
1.235 + }
1.236 + }
1.237 +}
1.238 +
1.239 +CharsetRecog_sbcs::CharsetRecog_sbcs()
1.240 +{
1.241 + // nothing else to do
1.242 +}
1.243 +
1.244 +CharsetRecog_sbcs::~CharsetRecog_sbcs()
1.245 +{
1.246 + // nothing to do
1.247 +}
1.248 +
1.249 +int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const
1.250 +{
1.251 + NGramParser parser(ngrams, byteMap);
1.252 + int32_t result;
1.253 +
1.254 + result = parser.parse(det);
1.255 +
1.256 + return result;
1.257 +}
1.258 +
1.259 +static const uint8_t charMap_8859_1[] = {
1.260 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.261 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.262 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.263 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.264 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.265 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.266 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.267 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.268 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.269 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.270 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.271 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.272 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.273 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.274 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.275 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.276 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.277 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.278 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.279 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.280 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.281 + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.282 + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
1.283 + 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.284 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.285 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.286 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
1.287 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
1.288 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.289 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.290 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
1.291 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
1.292 +};
1.293 +
1.294 +static const uint8_t charMap_8859_2[] = {
1.295 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.296 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.297 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.298 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.299 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.300 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.301 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.302 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.303 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.304 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.305 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.306 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.307 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.308 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.309 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.310 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.311 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.312 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.313 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.314 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.315 + 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20,
1.316 + 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
1.317 + 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7,
1.318 + 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
1.319 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.320 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.321 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
1.322 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
1.323 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.324 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.325 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
1.326 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
1.327 +};
1.328 +
1.329 +static const uint8_t charMap_8859_5[] = {
1.330 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.331 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.332 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.333 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.334 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.335 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.336 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.337 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.338 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.339 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.340 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.341 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.342 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.343 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.344 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.345 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.346 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.347 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.348 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.349 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.350 + 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.351 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
1.352 + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
1.353 + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
1.354 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.355 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.356 + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
1.357 + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
1.358 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.359 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.360 + 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.361 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
1.362 +};
1.363 +
1.364 +static const uint8_t charMap_8859_6[] = {
1.365 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.366 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.367 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.368 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.369 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.370 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.371 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.372 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.373 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.374 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.375 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.376 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.377 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.378 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.379 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.380 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.381 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.382 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.383 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.384 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.385 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.386 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.387 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.388 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.389 + 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
1.390 + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
1.391 + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
1.392 + 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.393 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.394 + 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.395 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.396 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.397 +};
1.398 +
1.399 +static const uint8_t charMap_8859_7[] = {
1.400 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.401 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.402 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.403 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.404 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.405 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.406 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.407 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.408 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.409 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.410 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.411 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.412 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.413 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.414 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.415 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.416 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.417 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.418 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.419 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.420 + 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20,
1.421 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.422 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20,
1.423 + 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE,
1.424 + 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.425 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.426 + 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.427 + 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF,
1.428 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.429 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.430 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.431 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
1.432 +};
1.433 +
1.434 +static const uint8_t charMap_8859_8[] = {
1.435 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.436 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.437 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.438 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.439 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.440 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.441 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.442 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.443 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.444 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.445 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.446 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.447 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.448 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.449 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.450 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.451 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.452 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.453 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.454 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.455 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.456 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.457 + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
1.458 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.459 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.460 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.461 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.462 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.463 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.464 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.465 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.466 + 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.467 +};
1.468 +
1.469 +static const uint8_t charMap_8859_9[] = {
1.470 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.471 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.472 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.473 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.474 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.475 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.476 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.477 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.478 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.479 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.480 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.481 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.482 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.483 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.484 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.485 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.486 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.487 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.488 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.489 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.490 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.491 + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.492 + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
1.493 + 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.494 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.495 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.496 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
1.497 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF,
1.498 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.499 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.500 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
1.501 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
1.502 +};
1.503 +
1.504 +static const int32_t ngrams_windows_1251[] = {
1.505 + 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE,
1.506 + 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED,
1.507 + 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2,
1.508 + 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520,
1.509 +};
1.510 +
1.511 +static const uint8_t charMap_windows_1251[] = {
1.512 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.513 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.514 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.515 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.516 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.517 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.518 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.519 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.520 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.521 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.522 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.523 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.524 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.525 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.526 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.527 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.528 + 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
1.529 + 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
1.530 + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.531 + 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
1.532 + 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20,
1.533 + 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF,
1.534 + 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20,
1.535 + 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF,
1.536 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.537 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.538 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.539 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
1.540 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.541 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.542 + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
1.543 + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
1.544 +};
1.545 +
1.546 +static const int32_t ngrams_windows_1256[] = {
1.547 + 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8,
1.548 + 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD,
1.549 + 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20,
1.550 + 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420,
1.551 +};
1.552 +
1.553 +static const uint8_t charMap_windows_1256[] = {
1.554 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.555 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.556 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.557 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.558 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.559 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.560 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.561 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.562 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.563 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.564 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.565 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.566 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.567 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.568 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.569 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.570 + 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
1.571 + 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F,
1.572 + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.573 + 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F,
1.574 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.575 + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
1.576 + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
1.577 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.578 + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
1.579 + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
1.580 + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20,
1.581 + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
1.582 + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
1.583 + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1.584 + 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20,
1.585 + 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF,
1.586 +};
1.587 +
1.588 +static const int32_t ngrams_KOI8_R[] = {
1.589 + 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1,
1.590 + 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE,
1.591 + 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1,
1.592 + 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF,
1.593 +};
1.594 +
1.595 +static const uint8_t charMap_KOI8_R[] = {
1.596 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.597 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.598 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.599 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.600 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
1.601 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.602 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.603 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.604 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.605 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.606 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.607 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.608 + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
1.609 + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1.610 + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
1.611 + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
1.612 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.613 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.614 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.615 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.616 + 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
1.617 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.618 + 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
1.619 + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
1.620 + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
1.621 + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
1.622 + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
1.623 + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
1.624 + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
1.625 + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
1.626 + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
1.627 + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
1.628 +};
1.629 +
1.630 +static const int32_t ngrams_IBM424_he_rtl[] = {
1.631 + 0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641,
1.632 + 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045,
1.633 + 0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056,
1.634 + 0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069,
1.635 +};
1.636 +
1.637 +static const int32_t ngrams_IBM424_he_ltr[] = {
1.638 + 0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141,
1.639 + 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054,
1.640 + 0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940,
1.641 + 0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651,
1.642 +};
1.643 +
1.644 +static const uint8_t charMap_IBM424_he[] = {
1.645 +/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */
1.646 +/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.647 +/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.648 +/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.649 +/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.650 +/* 4- */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.651 +/* 5- */ 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.652 +/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.653 +/* 7- */ 0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40,
1.654 +/* 8- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.655 +/* 9- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.656 +/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.657 +/* B- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.658 +/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.659 +/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.660 +/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.661 +/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.662 +};
1.663 +
1.664 +static const int32_t ngrams_IBM420_ar_rtl[] = {
1.665 + 0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158,
1.666 + 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB,
1.667 + 0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40,
1.668 + 0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,
1.669 +};
1.670 +
1.671 +static const int32_t ngrams_IBM420_ar_ltr[] = {
1.672 + 0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF,
1.673 + 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD,
1.674 + 0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156,
1.675 + 0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156
1.676 +};
1.677 +
1.678 +static const uint8_t charMap_IBM420_ar[]= {
1.679 +/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */
1.680 +/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.681 +/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.682 +/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.683 +/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.684 +/* 4- */ 0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.685 +/* 5- */ 0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.686 +/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.687 +/* 7- */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1.688 +/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
1.689 +/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F,
1.690 +/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF,
1.691 +/* B- */ 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
1.692 +/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF,
1.693 +/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
1.694 +/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF,
1.695 +/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40,
1.696 +};
1.697 +
1.698 +//ISO-8859-1,2,5,6,7,8,9 Ngrams
1.699 +
1.700 +struct NGramsPlusLang {
1.701 + const int32_t ngrams[64];
1.702 + const char * lang;
1.703 +};
1.704 +
1.705 +static const NGramsPlusLang ngrams_8859_1[] = {
1.706 + {
1.707 + {
1.708 + 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F,
1.709 + 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74,
1.710 + 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420,
1.711 + 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320,
1.712 + },
1.713 + "en"
1.714 + },
1.715 + {
1.716 + {
1.717 + 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620,
1.718 + 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320,
1.719 + 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520,
1.720 + 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572,
1.721 + },
1.722 + "da"
1.723 + },
1.724 + {
1.725 + {
1.726 + 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F,
1.727 + 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220,
1.728 + 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465,
1.729 + 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572,
1.730 + },
1.731 + "de"
1.732 + },
1.733 + {
1.734 + {
1.735 + 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
1.736 + 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C,
1.737 + 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064,
1.738 + 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20,
1.739 + },
1.740 + "es"
1.741 + },
1.742 + {
1.743 + {
1.744 + 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E,
1.745 + 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20,
1.746 + 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420,
1.747 + 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220,
1.748 + },
1.749 + "fr"
1.750 + },
1.751 + {
1.752 + {
1.753 + 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073,
1.754 + 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220,
1.755 + 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20,
1.756 + 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F,
1.757 + },
1.758 + "it"
1.759 + },
1.760 + {
1.761 + {
1.762 + 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665,
1.763 + 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E,
1.764 + 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F,
1.765 + 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F,
1.766 + },
1.767 + "nl"
1.768 + },
1.769 + {
1.770 + {
1.771 + 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469,
1.772 + 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474,
1.773 + 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65,
1.774 + 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572,
1.775 + },
1.776 + "no"
1.777 + },
1.778 + {
1.779 + {
1.780 + 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
1.781 + 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20,
1.782 + 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065,
1.783 + 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F,
1.784 + },
1.785 + "pt"
1.786 + },
1.787 + {
1.788 + {
1.789 + 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469,
1.790 + 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220,
1.791 + 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20,
1.792 + 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220,
1.793 + },
1.794 + "sv"
1.795 + }
1.796 +};
1.797 +
1.798 +
1.799 +static const NGramsPlusLang ngrams_8859_2[] = {
1.800 + {
1.801 + {
1.802 + 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F,
1.803 + 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465,
1.804 + 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865,
1.805 + 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564,
1.806 + },
1.807 + "cs"
1.808 + },
1.809 + {
1.810 + {
1.811 + 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69,
1.812 + 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20,
1.813 + 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061,
1.814 + 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320,
1.815 + },
1.816 + "hu"
1.817 + },
1.818 + {
1.819 + {
1.820 + 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779,
1.821 + 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20,
1.822 + 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769,
1.823 + 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720,
1.824 + },
1.825 + "pl"
1.826 + },
1.827 + {
1.828 + {
1.829 + 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69,
1.830 + 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070,
1.831 + 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72,
1.832 + 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20,
1.833 + },
1.834 + "ro"
1.835 + }
1.836 +};
1.837 +
1.838 +static const int32_t ngrams_8859_5_ru[] = {
1.839 + 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE,
1.840 + 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD,
1.841 + 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2,
1.842 + 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520,
1.843 +};
1.844 +
1.845 +static const int32_t ngrams_8859_6_ar[] = {
1.846 + 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8,
1.847 + 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1,
1.848 + 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20,
1.849 + 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620,
1.850 +};
1.851 +
1.852 +static const int32_t ngrams_8859_7_el[] = {
1.853 + 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7,
1.854 + 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120,
1.855 + 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5,
1.856 + 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20,
1.857 +};
1.858 +
1.859 +static const int32_t ngrams_8859_8_I_he[] = {
1.860 + 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0,
1.861 + 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4,
1.862 + 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE,
1.863 + 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,
1.864 +};
1.865 +
1.866 +static const int32_t ngrams_8859_8_he[] = {
1.867 + 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0,
1.868 + 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC,
1.869 + 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920,
1.870 + 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,
1.871 +};
1.872 +
1.873 +static const int32_t ngrams_8859_9_tr[] = {
1.874 + 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961,
1.875 + 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062,
1.876 + 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062,
1.877 + 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD,
1.878 +};
1.879 +
1.880 +CharsetRecog_8859_1::~CharsetRecog_8859_1()
1.881 +{
1.882 + // nothing to do
1.883 +}
1.884 +
1.885 +UBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const {
1.886 + const char *name = textIn->fC1Bytes? "windows-1252" : "ISO-8859-1";
1.887 + uint32_t i;
1.888 + int32_t bestConfidenceSoFar = -1;
1.889 + for (i=0; i < ARRAY_SIZE(ngrams_8859_1) ; i++) {
1.890 + const int32_t *ngrams = ngrams_8859_1[i].ngrams;
1.891 + const char *lang = ngrams_8859_1[i].lang;
1.892 + int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_1);
1.893 + if (confidence > bestConfidenceSoFar) {
1.894 + results->set(textIn, this, confidence, name, lang);
1.895 + bestConfidenceSoFar = confidence;
1.896 + }
1.897 + }
1.898 + return (bestConfidenceSoFar > 0);
1.899 +}
1.900 +
1.901 +const char *CharsetRecog_8859_1::getName() const
1.902 +{
1.903 + return "ISO-8859-1";
1.904 +}
1.905 +
1.906 +
1.907 +CharsetRecog_8859_2::~CharsetRecog_8859_2()
1.908 +{
1.909 + // nothing to do
1.910 +}
1.911 +
1.912 +UBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const {
1.913 + const char *name = textIn->fC1Bytes? "windows-1250" : "ISO-8859-2";
1.914 + uint32_t i;
1.915 + int32_t bestConfidenceSoFar = -1;
1.916 + for (i=0; i < ARRAY_SIZE(ngrams_8859_2) ; i++) {
1.917 + const int32_t *ngrams = ngrams_8859_2[i].ngrams;
1.918 + const char *lang = ngrams_8859_2[i].lang;
1.919 + int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_2);
1.920 + if (confidence > bestConfidenceSoFar) {
1.921 + results->set(textIn, this, confidence, name, lang);
1.922 + bestConfidenceSoFar = confidence;
1.923 + }
1.924 + }
1.925 + return (bestConfidenceSoFar > 0);
1.926 +}
1.927 +
1.928 +const char *CharsetRecog_8859_2::getName() const
1.929 +{
1.930 + return "ISO-8859-2";
1.931 +}
1.932 +
1.933 +
1.934 +CharsetRecog_8859_5::~CharsetRecog_8859_5()
1.935 +{
1.936 + // nothing to do
1.937 +}
1.938 +
1.939 +const char *CharsetRecog_8859_5::getName() const
1.940 +{
1.941 + return "ISO-8859-5";
1.942 +}
1.943 +
1.944 +CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru()
1.945 +{
1.946 + // nothing to do
1.947 +}
1.948 +
1.949 +const char *CharsetRecog_8859_5_ru::getLanguage() const
1.950 +{
1.951 + return "ru";
1.952 +}
1.953 +
1.954 +UBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const
1.955 +{
1.956 + int32_t confidence = match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5);
1.957 + results->set(textIn, this, confidence);
1.958 + return (confidence > 0);
1.959 +}
1.960 +
1.961 +CharsetRecog_8859_6::~CharsetRecog_8859_6()
1.962 +{
1.963 + // nothing to do
1.964 +}
1.965 +
1.966 +const char *CharsetRecog_8859_6::getName() const
1.967 +{
1.968 + return "ISO-8859-6";
1.969 +}
1.970 +
1.971 +CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar()
1.972 +{
1.973 + // nothing to do
1.974 +}
1.975 +
1.976 +const char *CharsetRecog_8859_6_ar::getLanguage() const
1.977 +{
1.978 + return "ar";
1.979 +}
1.980 +
1.981 +UBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const
1.982 +{
1.983 + int32_t confidence = match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6);
1.984 + results->set(textIn, this, confidence);
1.985 + return (confidence > 0);
1.986 +}
1.987 +
1.988 +CharsetRecog_8859_7::~CharsetRecog_8859_7()
1.989 +{
1.990 + // nothing to do
1.991 +}
1.992 +
1.993 +const char *CharsetRecog_8859_7::getName() const
1.994 +{
1.995 + return "ISO-8859-7";
1.996 +}
1.997 +
1.998 +CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el()
1.999 +{
1.1000 + // nothing to do
1.1001 +}
1.1002 +
1.1003 +const char *CharsetRecog_8859_7_el::getLanguage() const
1.1004 +{
1.1005 + return "el";
1.1006 +}
1.1007 +
1.1008 +UBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const
1.1009 +{
1.1010 + const char *name = textIn->fC1Bytes? "windows-1253" : "ISO-8859-7";
1.1011 + int32_t confidence = match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7);
1.1012 + results->set(textIn, this, confidence, name, "el");
1.1013 + return (confidence > 0);
1.1014 +}
1.1015 +
1.1016 +CharsetRecog_8859_8::~CharsetRecog_8859_8()
1.1017 +{
1.1018 + // nothing to do
1.1019 +}
1.1020 +
1.1021 +const char *CharsetRecog_8859_8::getName() const
1.1022 +{
1.1023 + return "ISO-8859-8";
1.1024 +}
1.1025 +
1.1026 +CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he ()
1.1027 +{
1.1028 + // nothing to do
1.1029 +}
1.1030 +
1.1031 +const char *CharsetRecog_8859_8_I_he::getName() const
1.1032 +{
1.1033 + return "ISO-8859-8-I";
1.1034 +}
1.1035 +
1.1036 +const char *CharsetRecog_8859_8_I_he::getLanguage() const
1.1037 +{
1.1038 + return "he";
1.1039 +}
1.1040 +
1.1041 +UBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const
1.1042 +{
1.1043 + const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8-I";
1.1044 + int32_t confidence = match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8);
1.1045 + results->set(textIn, this, confidence, name, "he");
1.1046 + return (confidence > 0);
1.1047 +}
1.1048 +
1.1049 +CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he()
1.1050 +{
1.1051 + // od ot gnihton
1.1052 +}
1.1053 +
1.1054 +const char *CharsetRecog_8859_8_he::getLanguage() const
1.1055 +{
1.1056 + return "he";
1.1057 +}
1.1058 +
1.1059 +UBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const
1.1060 +{
1.1061 + const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8";
1.1062 + int32_t confidence = match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8);
1.1063 + results->set(textIn, this, confidence, name, "he");
1.1064 + return (confidence > 0);
1.1065 +}
1.1066 +
1.1067 +CharsetRecog_8859_9::~CharsetRecog_8859_9()
1.1068 +{
1.1069 + // nothing to do
1.1070 +}
1.1071 +
1.1072 +const char *CharsetRecog_8859_9::getName() const
1.1073 +{
1.1074 + return "ISO-8859-9";
1.1075 +}
1.1076 +
1.1077 +CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr ()
1.1078 +{
1.1079 + // nothing to do
1.1080 +}
1.1081 +
1.1082 +const char *CharsetRecog_8859_9_tr::getLanguage() const
1.1083 +{
1.1084 + return "tr";
1.1085 +}
1.1086 +
1.1087 +UBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const
1.1088 +{
1.1089 + const char *name = textIn->fC1Bytes? "windows-1254" : "ISO-8859-9";
1.1090 + int32_t confidence = match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9);
1.1091 + results->set(textIn, this, confidence, name, "tr");
1.1092 + return (confidence > 0);
1.1093 +}
1.1094 +
1.1095 +CharsetRecog_windows_1256::~CharsetRecog_windows_1256()
1.1096 +{
1.1097 + // nothing to do
1.1098 +}
1.1099 +
1.1100 +const char *CharsetRecog_windows_1256::getName() const
1.1101 +{
1.1102 + return "windows-1256";
1.1103 +}
1.1104 +
1.1105 +const char *CharsetRecog_windows_1256::getLanguage() const
1.1106 +{
1.1107 + return "ar";
1.1108 +}
1.1109 +
1.1110 +UBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const
1.1111 +{
1.1112 + int32_t confidence = match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256);
1.1113 + results->set(textIn, this, confidence);
1.1114 + return (confidence > 0);
1.1115 +}
1.1116 +
1.1117 +CharsetRecog_windows_1251::~CharsetRecog_windows_1251()
1.1118 +{
1.1119 + // nothing to do
1.1120 +}
1.1121 +
1.1122 +const char *CharsetRecog_windows_1251::getName() const
1.1123 +{
1.1124 + return "windows-1251";
1.1125 +}
1.1126 +
1.1127 +const char *CharsetRecog_windows_1251::getLanguage() const
1.1128 +{
1.1129 + return "ru";
1.1130 +}
1.1131 +
1.1132 +UBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const
1.1133 +{
1.1134 + int32_t confidence = match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251);
1.1135 + results->set(textIn, this, confidence);
1.1136 + return (confidence > 0);
1.1137 +}
1.1138 +
1.1139 +CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R()
1.1140 +{
1.1141 + // nothing to do
1.1142 +}
1.1143 +
1.1144 +const char *CharsetRecog_KOI8_R::getName() const
1.1145 +{
1.1146 + return "KOI8-R";
1.1147 +}
1.1148 +
1.1149 +const char *CharsetRecog_KOI8_R::getLanguage() const
1.1150 +{
1.1151 + return "ru";
1.1152 +}
1.1153 +
1.1154 +UBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const
1.1155 +{
1.1156 + int32_t confidence = match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R);
1.1157 + results->set(textIn, this, confidence);
1.1158 + return (confidence > 0);
1.1159 +}
1.1160 +
1.1161 +CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he()
1.1162 +{
1.1163 + // nothing to do
1.1164 +}
1.1165 +
1.1166 +const char *CharsetRecog_IBM424_he::getLanguage() const
1.1167 +{
1.1168 + return "he";
1.1169 +}
1.1170 +
1.1171 +CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl()
1.1172 +{
1.1173 + // nothing to do
1.1174 +}
1.1175 +
1.1176 +const char *CharsetRecog_IBM424_he_rtl::getName() const
1.1177 +{
1.1178 + return "IBM424_rtl";
1.1179 +}
1.1180 +
1.1181 +UBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const
1.1182 +{
1.1183 + int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he);
1.1184 + results->set(textIn, this, confidence);
1.1185 + return (confidence > 0);
1.1186 +}
1.1187 +
1.1188 +CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr()
1.1189 +{
1.1190 + // nothing to do
1.1191 +}
1.1192 +
1.1193 +const char *CharsetRecog_IBM424_he_ltr::getName() const
1.1194 +{
1.1195 + return "IBM424_ltr";
1.1196 +}
1.1197 +
1.1198 +UBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const
1.1199 +{
1.1200 + int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he);
1.1201 + results->set(textIn, this, confidence);
1.1202 + return (confidence > 0);
1.1203 +}
1.1204 +
1.1205 +CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar()
1.1206 +{
1.1207 + // nothing to do
1.1208 +}
1.1209 +
1.1210 +const char *CharsetRecog_IBM420_ar::getLanguage() const
1.1211 +{
1.1212 + return "ar";
1.1213 +}
1.1214 +
1.1215 +
1.1216 +int32_t CharsetRecog_IBM420_ar::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const
1.1217 +{
1.1218 + NGramParser_IBM420 parser(ngrams, byteMap);
1.1219 + int32_t result;
1.1220 +
1.1221 + result = parser.parse(det);
1.1222 +
1.1223 + return result;
1.1224 +}
1.1225 +
1.1226 +CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl()
1.1227 +{
1.1228 + // nothing to do
1.1229 +}
1.1230 +
1.1231 +const char *CharsetRecog_IBM420_ar_rtl::getName() const
1.1232 +{
1.1233 + return "IBM420_rtl";
1.1234 +}
1.1235 +
1.1236 +UBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const
1.1237 +{
1.1238 + int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar);
1.1239 + results->set(textIn, this, confidence);
1.1240 + return (confidence > 0);
1.1241 +}
1.1242 +
1.1243 +CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr()
1.1244 +{
1.1245 + // nothing to do
1.1246 +}
1.1247 +
1.1248 +const char *CharsetRecog_IBM420_ar_ltr::getName() const
1.1249 +{
1.1250 + return "IBM420_ltr";
1.1251 +}
1.1252 +
1.1253 +UBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const
1.1254 +{
1.1255 + int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar);
1.1256 + results->set(textIn, this, confidence);
1.1257 + return (confidence > 0);
1.1258 +}
1.1259 +
1.1260 +U_NAMESPACE_END
1.1261 +#endif
1.1262 +
