The Tor Browser / file comparison
comparison: intl/icu/source/i18n/csrucode.cpp
intl/icu/source/i18n/csrucode.cpp
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
equal
deleted
inserted
replaced
| -1:000000000000 | 0:b4ee803cbd41 |
|---|---|
| 1 /* | |
| 2 ********************************************************************** | |
| 3 * Copyright (C) 2005-2013, International Business Machines | |
| 4 * Corporation and others. All Rights Reserved. | |
| 5 ********************************************************************** | |
| 6 */ | |
| 7 | |
| 8 #include "unicode/utypes.h" | |
| 9 | |
| 10 #if !UCONFIG_NO_CONVERSION | |
| 11 | |
| 12 #include "csrucode.h" | |
| 13 #include "csmatch.h" | |
| 14 | |
| 15 U_NAMESPACE_BEGIN | |
| 16 | |
| 17 CharsetRecog_Unicode::~CharsetRecog_Unicode() | |
| 18 { | |
| 19 // nothing to do | |
| 20 } | |
| 21 | |
| 22 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE() | |
| 23 { | |
| 24 // nothing to do | |
| 25 } | |
| 26 | |
| 27 const char *CharsetRecog_UTF_16_BE::getName() const | |
| 28 { | |
| 29 return "UTF-16BE"; | |
| 30 } | |
| 31 | |
| 32 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const | |
| 33 { | |
| 34 const uint8_t *input = textIn->fRawInput; | |
| 35 int32_t confidence = 0; | |
| 36 int32_t length = textIn->fRawLength; | |
| 37 | |
| 38 if (length >=2 && input[0] == 0xFE && input[1] == 0xFF) { | |
| 39 confidence = 100; | |
| 40 } | |
| 41 | |
| 42 // TODO: Do some statastics to check for unsigned UTF-16BE | |
| 43 results->set(textIn, this, confidence); | |
| 44 return (confidence > 0); | |
| 45 } | |
| 46 | |
| 47 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE() | |
| 48 { | |
| 49 // nothing to do | |
| 50 } | |
| 51 | |
| 52 const char *CharsetRecog_UTF_16_LE::getName() const | |
| 53 { | |
| 54 return "UTF-16LE"; | |
| 55 } | |
| 56 | |
| 57 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const | |
| 58 { | |
| 59 const uint8_t *input = textIn->fRawInput; | |
| 60 int32_t confidence = 0; | |
| 61 int32_t length = textIn->fRawLength; | |
| 62 | |
| 63 if (length >= 4 && input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) { | |
| 64 confidence = 100; | |
| 65 } | |
| 66 | |
| 67 // TODO: Do some statastics to check for unsigned UTF-16LE | |
| 68 results->set(textIn, this, confidence); | |
| 69 return (confidence > 0); | |
| 70 } | |
| 71 | |
| 72 CharsetRecog_UTF_32::~CharsetRecog_UTF_32() | |
| 73 { | |
| 74 // nothing to do | |
| 75 } | |
| 76 | |
| 77 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const | |
| 78 { | |
| 79 const uint8_t *input = textIn->fRawInput; | |
| 80 int32_t limit = (textIn->fRawLength / 4) * 4; | |
| 81 int32_t numValid = 0; | |
| 82 int32_t numInvalid = 0; | |
| 83 bool hasBOM = FALSE; | |
| 84 int32_t confidence = 0; | |
| 85 | |
| 86 if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) { | |
| 87 hasBOM = TRUE; | |
| 88 } | |
| 89 | |
| 90 for(int32_t i = 0; i < limit; i += 4) { | |
| 91 int32_t ch = getChar(input, i); | |
| 92 | |
| 93 if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) { | |
| 94 numInvalid += 1; | |
| 95 } else { | |
| 96 numValid += 1; | |
| 97 } | |
| 98 } | |
| 99 | |
| 100 | |
| 101 // Cook up some sort of confidence score, based on presense of a BOM | |
| 102 // and the existence of valid and/or invalid multi-byte sequences. | |
| 103 if (hasBOM && numInvalid==0) { | |
| 104 confidence = 100; | |
| 105 } else if (hasBOM && numValid > numInvalid*10) { | |
| 106 confidence = 80; | |
| 107 } else if (numValid > 3 && numInvalid == 0) { | |
| 108 confidence = 100; | |
| 109 } else if (numValid > 0 && numInvalid == 0) { | |
| 110 confidence = 80; | |
| 111 } else if (numValid > numInvalid*10) { | |
| 112 // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance. | |
| 113 confidence = 25; | |
| 114 } | |
| 115 | |
| 116 results->set(textIn, this, confidence); | |
| 117 return (confidence > 0); | |
| 118 } | |
| 119 | |
| 120 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE() | |
| 121 { | |
| 122 // nothing to do | |
| 123 } | |
| 124 | |
| 125 const char *CharsetRecog_UTF_32_BE::getName() const | |
| 126 { | |
| 127 return "UTF-32BE"; | |
| 128 } | |
| 129 | |
| 130 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const | |
| 131 { | |
| 132 return input[index + 0] << 24 | input[index + 1] << 16 | | |
| 133 input[index + 2] << 8 | input[index + 3]; | |
| 134 } | |
| 135 | |
| 136 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE() | |
| 137 { | |
| 138 // nothing to do | |
| 139 } | |
| 140 | |
| 141 const char *CharsetRecog_UTF_32_LE::getName() const | |
| 142 { | |
| 143 return "UTF-32LE"; | |
| 144 } | |
| 145 | |
| 146 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const | |
| 147 { | |
| 148 return input[index + 3] << 24 | input[index + 2] << 16 | | |
| 149 input[index + 1] << 8 | input[index + 0]; | |
| 150 } | |
| 151 | |
| 152 U_NAMESPACE_END | |
| 153 #endif | |
| 154 |
