The Tor Browser: intl/icu/source/i18n/csrucode.cpp@6474c204b198

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

     1 /*

     2  **********************************************************************

     3  *   Copyright (C) 2005-2013, International Business Machines

     4  *   Corporation and others.  All Rights Reserved.

     5  **********************************************************************

     6  */

     8 #include "unicode/utypes.h"

    10 #if !UCONFIG_NO_CONVERSION

    12 #include "csrucode.h"

    13 #include "csmatch.h"

    15 U_NAMESPACE_BEGIN

    17 CharsetRecog_Unicode::~CharsetRecog_Unicode()

    18 {

    19     // nothing to do

    20 }

    22 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()

    23 {

    24     // nothing to do

    25 }

    27 const char *CharsetRecog_UTF_16_BE::getName() const

    28 {

    29     return "UTF-16BE";

    30 }

    32 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const

    33 {

    34     const uint8_t *input = textIn->fRawInput;

    35     int32_t confidence = 0;

    36     int32_t length = textIn->fRawLength;

    38     if (length >=2 && input[0] == 0xFE && input[1] == 0xFF) {

    39         confidence = 100;

    40     }

    42     // TODO: Do some statastics to check for unsigned UTF-16BE

    43     results->set(textIn, this, confidence);

    44     return (confidence > 0);

    45 }

    47 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()

    48 {

    49     // nothing to do

    50 }

    52 const char *CharsetRecog_UTF_16_LE::getName() const

    53 {

    54     return "UTF-16LE";

    55 }

    57 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const

    58 {

    59     const uint8_t *input = textIn->fRawInput;

    60     int32_t confidence = 0;

    61     int32_t length = textIn->fRawLength;

    63     if (length >= 4 && input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) {

    64         confidence = 100;

    65     }

    67     // TODO: Do some statastics to check for unsigned UTF-16LE

    68     results->set(textIn, this, confidence);

    69     return (confidence > 0);

    70 }

    72 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()

    73 {

    74     // nothing to do

    75 }

    77 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const

    78 {

    79     const uint8_t *input = textIn->fRawInput;

    80     int32_t limit = (textIn->fRawLength / 4) * 4;

    81     int32_t numValid = 0;

    82     int32_t numInvalid = 0;

    83     bool hasBOM = FALSE;

    84     int32_t confidence = 0;

    86     if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {

    87         hasBOM = TRUE;

    88     }

    90     for(int32_t i = 0; i < limit; i += 4) {

    91         int32_t ch = getChar(input, i);

    93         if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {

    94             numInvalid += 1;

    95         } else {

    96             numValid += 1;

    97         }

    98     }

   101     // Cook up some sort of confidence score, based on presense of a BOM

   102     //    and the existence of valid and/or invalid multi-byte sequences.

   103     if (hasBOM && numInvalid==0) {

   104         confidence = 100;

   105     } else if (hasBOM && numValid > numInvalid*10) {

   106         confidence = 80;

   107     } else if (numValid > 3 && numInvalid == 0) {

   108         confidence = 100;

   109     } else if (numValid > 0 && numInvalid == 0) {

   110         confidence = 80;

   111     } else if (numValid > numInvalid*10) {

   112         // Probably corruput UTF-32BE data.  Valid sequences aren't likely by chance.

   113         confidence = 25;

   114     }

   116     results->set(textIn, this, confidence);

   117     return (confidence > 0);

   118 }

   120 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()

   121 {

   122     // nothing to do

   123 }

   125 const char *CharsetRecog_UTF_32_BE::getName() const

   126 {

   127     return "UTF-32BE";

   128 }

   130 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const

   131 {

   132     return input[index + 0] << 24 | input[index + 1] << 16 |

   133            input[index + 2] <<  8 | input[index + 3];

   134 }

   136 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()

   137 {

   138     // nothing to do

   139 }

   141 const char *CharsetRecog_UTF_32_LE::getName() const

   142 {

   143     return "UTF-32LE";

   144 }

   146 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const

   147 {

   148     return input[index + 3] << 24 | input[index + 2] << 16 |

   149            input[index + 1] <<  8 | input[index + 0];

   150 }

   152 U_NAMESPACE_END

   153 #endif

The Tor Browser / file revision