The Tor Browser / file comparison

comparison: intl/icu/source/i18n/csdetect.cpp

intl/icu/source/i18n/csdetect.cpp

branch
TOR_BUG_3246
changeset 4
fc2d59ddac77
equal deleted inserted replaced
-1:000000000000 0:a83bb1f24089
1 /*
2 **********************************************************************
3 * Copyright (C) 2005-2013, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 */
7
8 #include "unicode/utypes.h"
9
10 #if !UCONFIG_NO_CONVERSION
11
12 #include "unicode/ucsdet.h"
13
14 #include "csdetect.h"
15 #include "csmatch.h"
16 #include "uenumimp.h"
17
18 #include "cmemory.h"
19 #include "cstring.h"
20 #include "umutex.h"
21 #include "ucln_in.h"
22 #include "uarrsort.h"
23 #include "inputext.h"
24 #include "csrsbcs.h"
25 #include "csrmbcs.h"
26 #include "csrutf8.h"
27 #include "csrucode.h"
28 #include "csr2022.h"
29
30 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
31
32 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
33 #define DELETE_ARRAY(array) uprv_free((void *) (array))
34
35 U_NAMESPACE_BEGIN
36
37 struct CSRecognizerInfo : public UMemory {
38 CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled)
39 : recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {};
40
41 ~CSRecognizerInfo() {delete recognizer;};
42
43 CharsetRecognizer *recognizer;
44 UBool isDefaultEnabled;
45 };
46
47 U_NAMESPACE_END
48
49 static icu::CSRecognizerInfo **fCSRecognizers = NULL;
50 static icu::UInitOnce gCSRecognizersInitOnce;
51 static int32_t fCSRecognizers_size = 0;
52
53 U_CDECL_BEGIN
54 static UBool U_CALLCONV csdet_cleanup(void)
55 {
56 U_NAMESPACE_USE
57 if (fCSRecognizers != NULL) {
58 for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
59 delete fCSRecognizers[r];
60 fCSRecognizers[r] = NULL;
61 }
62
63 DELETE_ARRAY(fCSRecognizers);
64 fCSRecognizers = NULL;
65 fCSRecognizers_size = 0;
66 }
67 gCSRecognizersInitOnce.reset();
68
69 return TRUE;
70 }
71
72 static int32_t U_CALLCONV
73 charsetMatchComparator(const void * /*context*/, const void *left, const void *right)
74 {
75 U_NAMESPACE_USE
76
77 const CharsetMatch **csm_l = (const CharsetMatch **) left;
78 const CharsetMatch **csm_r = (const CharsetMatch **) right;
79
80 // NOTE: compare is backwards to sort from highest to lowest.
81 return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
82 }
83
84 static void U_CALLCONV initRecognizers(UErrorCode &status) {
85 U_NAMESPACE_USE
86 ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
87 CSRecognizerInfo *tempArray[] = {
88 new CSRecognizerInfo(new CharsetRecog_UTF8(), TRUE),
89
90 new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), TRUE),
91 new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), TRUE),
92 new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), TRUE),
93 new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), TRUE),
94
95 new CSRecognizerInfo(new CharsetRecog_8859_1(), TRUE),
96 new CSRecognizerInfo(new CharsetRecog_8859_2(), TRUE),
97 new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), TRUE),
98 new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), TRUE),
99 new CSRecognizerInfo(new CharsetRecog_8859_7_el(), TRUE),
100 new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), TRUE),
101 new CSRecognizerInfo(new CharsetRecog_8859_8_he(), TRUE),
102 new CSRecognizerInfo(new CharsetRecog_windows_1251(), TRUE),
103 new CSRecognizerInfo(new CharsetRecog_windows_1256(), TRUE),
104 new CSRecognizerInfo(new CharsetRecog_KOI8_R(), TRUE),
105 new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), TRUE),
106 new CSRecognizerInfo(new CharsetRecog_sjis(), TRUE),
107 new CSRecognizerInfo(new CharsetRecog_gb_18030(), TRUE),
108 new CSRecognizerInfo(new CharsetRecog_euc_jp(), TRUE),
109 new CSRecognizerInfo(new CharsetRecog_euc_kr(), TRUE),
110 new CSRecognizerInfo(new CharsetRecog_big5(), TRUE),
111
112 new CSRecognizerInfo(new CharsetRecog_2022JP(), TRUE),
113 new CSRecognizerInfo(new CharsetRecog_2022KR(), TRUE),
114 new CSRecognizerInfo(new CharsetRecog_2022CN(), TRUE),
115
116 new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), FALSE),
117 new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), FALSE),
118 new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), FALSE),
119 new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), FALSE)
120 };
121 int32_t rCount = ARRAY_SIZE(tempArray);
122
123 fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount);
124
125 if (fCSRecognizers == NULL) {
126 status = U_MEMORY_ALLOCATION_ERROR;
127 }
128 else {
129 fCSRecognizers_size = rCount;
130 for (int32_t r = 0; r < rCount; r += 1) {
131 fCSRecognizers[r] = tempArray[r];
132 if (fCSRecognizers[r] == NULL) {
133 status = U_MEMORY_ALLOCATION_ERROR;
134 }
135 }
136 }
137 }
138
139 U_CDECL_END
140
141 U_NAMESPACE_BEGIN
142
143 void CharsetDetector::setRecognizers(UErrorCode &status)
144 {
145 umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status);
146 }
147
148 CharsetDetector::CharsetDetector(UErrorCode &status)
149 : textIn(new InputText(status)), resultArray(NULL),
150 resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE),
151 fEnabledRecognizers(NULL)
152 {
153 if (U_FAILURE(status)) {
154 return;
155 }
156
157 setRecognizers(status);
158
159 if (U_FAILURE(status)) {
160 return;
161 }
162
163 resultArray = (CharsetMatch **)uprv_malloc(sizeof(CharsetMatch *)*fCSRecognizers_size);
164
165 if (resultArray == NULL) {
166 status = U_MEMORY_ALLOCATION_ERROR;
167 return;
168 }
169
170 for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
171 resultArray[i] = new CharsetMatch();
172
173 if (resultArray[i] == NULL) {
174 status = U_MEMORY_ALLOCATION_ERROR;
175 break;
176 }
177 }
178 }
179
180 CharsetDetector::~CharsetDetector()
181 {
182 delete textIn;
183
184 for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
185 delete resultArray[i];
186 }
187
188 uprv_free(resultArray);
189
190 if (fEnabledRecognizers) {
191 uprv_free(fEnabledRecognizers);
192 }
193 }
194
195 void CharsetDetector::setText(const char *in, int32_t len)
196 {
197 textIn->setText(in, len);
198 fFreshTextSet = TRUE;
199 }
200
201 UBool CharsetDetector::setStripTagsFlag(UBool flag)
202 {
203 UBool temp = fStripTags;
204 fStripTags = flag;
205 fFreshTextSet = TRUE;
206 return temp;
207 }
208
209 UBool CharsetDetector::getStripTagsFlag() const
210 {
211 return fStripTags;
212 }
213
214 void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const
215 {
216 textIn->setDeclaredEncoding(encoding,len);
217 }
218
219 int32_t CharsetDetector::getDetectableCount()
220 {
221 UErrorCode status = U_ZERO_ERROR;
222
223 setRecognizers(status);
224
225 return fCSRecognizers_size;
226 }
227
228 const CharsetMatch *CharsetDetector::detect(UErrorCode &status)
229 {
230 int32_t maxMatchesFound = 0;
231
232 detectAll(maxMatchesFound, status);
233
234 if(maxMatchesFound > 0) {
235 return resultArray[0];
236 } else {
237 return NULL;
238 }
239 }
240
241 const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status)
242 {
243 if(!textIn->isSet()) {
244 status = U_MISSING_RESOURCE_ERROR;// TODO: Need to set proper status code for input text not set
245
246 return NULL;
247 } else if (fFreshTextSet) {
248 CharsetRecognizer *csr;
249 int32_t i;
250
251 textIn->MungeInput(fStripTags);
252
253 // Iterate over all possible charsets, remember all that
254 // give a match quality > 0.
255 resultCount = 0;
256 for (i = 0; i < fCSRecognizers_size; i += 1) {
257 csr = fCSRecognizers[i]->recognizer;
258 if (csr->match(textIn, resultArray[resultCount])) {
259 resultCount++;
260 }
261 }
262
263 if (resultCount > 1) {
264 uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status);
265 }
266 fFreshTextSet = FALSE;
267 }
268
269 maxMatchesFound = resultCount;
270
271 return resultArray;
272 }
273
274 void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status)
275 {
276 if (U_FAILURE(status)) {
277 return;
278 }
279
280 int32_t modIdx = -1;
281 UBool isDefaultVal = FALSE;
282 for (int32_t i = 0; i < fCSRecognizers_size; i++) {
283 CSRecognizerInfo *csrinfo = fCSRecognizers[i];
284 if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) {
285 modIdx = i;
286 isDefaultVal = (csrinfo->isDefaultEnabled == enabled);
287 break;
288 }
289 }
290 if (modIdx < 0) {
291 // No matching encoding found
292 status = U_ILLEGAL_ARGUMENT_ERROR;
293 return;
294 }
295
296 if (fEnabledRecognizers == NULL && !isDefaultVal) {
297 // Create an array storing the non default setting
298 fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size);
299 if (fEnabledRecognizers == NULL) {
300 status = U_MEMORY_ALLOCATION_ERROR;
301 return;
302 }
303 // Initialize the array with default info
304 for (int32_t i = 0; i < fCSRecognizers_size; i++) {
305 fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled;
306 }
307 }
308
309 if (fEnabledRecognizers != NULL) {
310 fEnabledRecognizers[modIdx] = enabled;
311 }
312 }
313
314 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
315 {
316 if( index > fCSRecognizers_size-1 || index < 0) {
317 status = U_INDEX_OUTOFBOUNDS_ERROR;
318
319 return 0;
320 } else {
321 return fCSRecognizers[index]->getName();
322 }
323 }*/
324
325 U_NAMESPACE_END
326
327 U_CDECL_BEGIN
328 typedef struct {
329 int32_t currIndex;
330 UBool all;
331 UBool *enabledRecognizers;
332 } Context;
333
334
335
336 static void U_CALLCONV
337 enumClose(UEnumeration *en) {
338 if(en->context != NULL) {
339 DELETE_ARRAY(en->context);
340 }
341
342 DELETE_ARRAY(en);
343 }
344
345 static int32_t U_CALLCONV
346 enumCount(UEnumeration *en, UErrorCode *) {
347 if (((Context *)en->context)->all) {
348 // ucsdet_getAllDetectableCharsets, all charset detector names
349 return fCSRecognizers_size;
350 }
351
352 // Otherwise, ucsdet_getDetectableCharsets - only enabled ones
353 int32_t count = 0;
354 UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
355 if (enabledArray != NULL) {
356 // custom set
357 for (int32_t i = 0; i < fCSRecognizers_size; i++) {
358 if (enabledArray[i]) {
359 count++;
360 }
361 }
362 } else {
363 // default set
364 for (int32_t i = 0; i < fCSRecognizers_size; i++) {
365 if (fCSRecognizers[i]->isDefaultEnabled) {
366 count++;
367 }
368 }
369 }
370 return count;
371 }
372
373 static const char* U_CALLCONV
374 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {
375 const char *currName = NULL;
376
377 if (((Context *)en->context)->currIndex < fCSRecognizers_size) {
378 if (((Context *)en->context)->all) {
379 // ucsdet_getAllDetectableCharsets, all charset detector names
380 currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
381 ((Context *)en->context)->currIndex++;
382 } else {
383 // ucsdet_getDetectableCharsets
384 UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
385 if (enabledArray != NULL) {
386 // custome set
387 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
388 if (enabledArray[((Context *)en->context)->currIndex]) {
389 currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
390 }
391 ((Context *)en->context)->currIndex++;
392 }
393 } else {
394 // default set
395 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
396 if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) {
397 currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
398 }
399 ((Context *)en->context)->currIndex++;
400 }
401 }
402 }
403 }
404
405 if(resultLength != NULL) {
406 *resultLength = currName == NULL ? 0 : (int32_t)uprv_strlen(currName);
407 }
408
409 return currName;
410 }
411
412
413 static void U_CALLCONV
414 enumReset(UEnumeration *en, UErrorCode *) {
415 ((Context *)en->context)->currIndex = 0;
416 }
417
418 static const UEnumeration gCSDetEnumeration = {
419 NULL,
420 NULL,
421 enumClose,
422 enumCount,
423 uenum_unextDefault,
424 enumNext,
425 enumReset
426 };
427
428 U_CDECL_END
429
430 U_NAMESPACE_BEGIN
431
432 UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status)
433 {
434
435 /* Initialize recognized charsets. */
436 setRecognizers(status);
437
438 if(U_FAILURE(status)) {
439 return 0;
440 }
441
442 UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
443 if (en == NULL) {
444 status = U_MEMORY_ALLOCATION_ERROR;
445 return 0;
446 }
447 memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
448 en->context = (void*)NEW_ARRAY(Context, 1);
449 if (en->context == NULL) {
450 status = U_MEMORY_ALLOCATION_ERROR;
451 DELETE_ARRAY(en);
452 return 0;
453 }
454 uprv_memset(en->context, 0, sizeof(Context));
455 ((Context*)en->context)->all = TRUE;
456 return en;
457 }
458
459 UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const
460 {
461 if(U_FAILURE(status)) {
462 return 0;
463 }
464
465 UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
466 if (en == NULL) {
467 status = U_MEMORY_ALLOCATION_ERROR;
468 return 0;
469 }
470 memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
471 en->context = (void*)NEW_ARRAY(Context, 1);
472 if (en->context == NULL) {
473 status = U_MEMORY_ALLOCATION_ERROR;
474 DELETE_ARRAY(en);
475 return 0;
476 }
477 uprv_memset(en->context, 0, sizeof(Context));
478 ((Context*)en->context)->all = FALSE;
479 ((Context*)en->context)->enabledRecognizers = fEnabledRecognizers;
480 return en;
481 }
482
483 U_NAMESPACE_END
484
485 #endif

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial