intl/icu/source/i18n/csrutf8.cpp@6474c204b198 (annotated)
intl/icu/source/i18n/csrutf8.cpp
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rw-r--r--
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
| michael@0 | 1 | /* |
| michael@0 | 2 | ********************************************************************** |
| michael@0 | 3 | * Copyright (C) 2005-2012, International Business Machines |
| michael@0 | 4 | * Corporation and others. All Rights Reserved. |
| michael@0 | 5 | ********************************************************************** |
| michael@0 | 6 | */ |
| michael@0 | 7 | |
| michael@0 | 8 | #include "unicode/utypes.h" |
| michael@0 | 9 | |
| michael@0 | 10 | #if !UCONFIG_NO_CONVERSION |
| michael@0 | 11 | |
| michael@0 | 12 | #include "csrutf8.h" |
| michael@0 | 13 | #include "csmatch.h" |
| michael@0 | 14 | |
| michael@0 | 15 | U_NAMESPACE_BEGIN |
| michael@0 | 16 | |
| michael@0 | 17 | CharsetRecog_UTF8::~CharsetRecog_UTF8() |
| michael@0 | 18 | { |
| michael@0 | 19 | // nothing to do |
| michael@0 | 20 | } |
| michael@0 | 21 | |
| michael@0 | 22 | const char *CharsetRecog_UTF8::getName() const |
| michael@0 | 23 | { |
| michael@0 | 24 | return "UTF-8"; |
| michael@0 | 25 | } |
| michael@0 | 26 | |
| michael@0 | 27 | UBool CharsetRecog_UTF8::match(InputText* input, CharsetMatch *results) const { |
| michael@0 | 28 | bool hasBOM = FALSE; |
| michael@0 | 29 | int32_t numValid = 0; |
| michael@0 | 30 | int32_t numInvalid = 0; |
| michael@0 | 31 | const uint8_t *inputBytes = input->fRawInput; |
| michael@0 | 32 | int32_t i; |
| michael@0 | 33 | int32_t trailBytes = 0; |
| michael@0 | 34 | int32_t confidence; |
| michael@0 | 35 | |
| michael@0 | 36 | if (input->fRawLength >= 3 && |
| michael@0 | 37 | inputBytes[0] == 0xEF && inputBytes[1] == 0xBB && inputBytes[2] == 0xBF) { |
| michael@0 | 38 | hasBOM = TRUE; |
| michael@0 | 39 | } |
| michael@0 | 40 | |
| michael@0 | 41 | // Scan for multi-byte sequences |
| michael@0 | 42 | for (i=0; i < input->fRawLength; i += 1) { |
| michael@0 | 43 | int32_t b = inputBytes[i]; |
| michael@0 | 44 | |
| michael@0 | 45 | if ((b & 0x80) == 0) { |
| michael@0 | 46 | continue; // ASCII |
| michael@0 | 47 | } |
| michael@0 | 48 | |
| michael@0 | 49 | // Hi bit on char found. Figure out how long the sequence should be |
| michael@0 | 50 | if ((b & 0x0E0) == 0x0C0) { |
| michael@0 | 51 | trailBytes = 1; |
| michael@0 | 52 | } else if ((b & 0x0F0) == 0x0E0) { |
| michael@0 | 53 | trailBytes = 2; |
| michael@0 | 54 | } else if ((b & 0x0F8) == 0xF0) { |
| michael@0 | 55 | trailBytes = 3; |
| michael@0 | 56 | } else { |
| michael@0 | 57 | numInvalid += 1; |
| michael@0 | 58 | |
| michael@0 | 59 | if (numInvalid > 5) { |
| michael@0 | 60 | break; |
| michael@0 | 61 | } |
| michael@0 | 62 | |
| michael@0 | 63 | trailBytes = 0; |
| michael@0 | 64 | } |
| michael@0 | 65 | |
| michael@0 | 66 | // Verify that we've got the right number of trail bytes in the sequence |
| michael@0 | 67 | for (;;) { |
| michael@0 | 68 | i += 1; |
| michael@0 | 69 | |
| michael@0 | 70 | if (i >= input->fRawLength) { |
| michael@0 | 71 | break; |
| michael@0 | 72 | } |
| michael@0 | 73 | |
| michael@0 | 74 | b = inputBytes[i]; |
| michael@0 | 75 | |
| michael@0 | 76 | if ((b & 0xC0) != 0x080) { |
| michael@0 | 77 | numInvalid += 1; |
| michael@0 | 78 | break; |
| michael@0 | 79 | } |
| michael@0 | 80 | |
| michael@0 | 81 | if (--trailBytes == 0) { |
| michael@0 | 82 | numValid += 1; |
| michael@0 | 83 | break; |
| michael@0 | 84 | } |
| michael@0 | 85 | } |
| michael@0 | 86 | |
| michael@0 | 87 | } |
| michael@0 | 88 | |
| michael@0 | 89 | // Cook up some sort of confidence score, based on presense of a BOM |
| michael@0 | 90 | // and the existence of valid and/or invalid multi-byte sequences. |
| michael@0 | 91 | confidence = 0; |
| michael@0 | 92 | if (hasBOM && numInvalid == 0) { |
| michael@0 | 93 | confidence = 100; |
| michael@0 | 94 | } else if (hasBOM && numValid > numInvalid*10) { |
| michael@0 | 95 | confidence = 80; |
| michael@0 | 96 | } else if (numValid > 3 && numInvalid == 0) { |
| michael@0 | 97 | confidence = 100; |
| michael@0 | 98 | } else if (numValid > 0 && numInvalid == 0) { |
| michael@0 | 99 | confidence = 80; |
| michael@0 | 100 | } else if (numValid == 0 && numInvalid == 0) { |
| michael@0 | 101 | // Plain ASCII. |
| michael@0 | 102 | confidence = 10; |
| michael@0 | 103 | } else if (numValid > numInvalid*10) { |
| michael@0 | 104 | // Probably corruput utf-8 data. Valid sequences aren't likely by chance. |
| michael@0 | 105 | confidence = 25; |
| michael@0 | 106 | } |
| michael@0 | 107 | |
| michael@0 | 108 | results->set(input, this, confidence); |
| michael@0 | 109 | return (confidence > 0); |
| michael@0 | 110 | } |
| michael@0 | 111 | |
| michael@0 | 112 | U_NAMESPACE_END |
| michael@0 | 113 | #endif |
