The Tor Browser: browser/components/translation/cld2/internal/getonescriptspan.cc@6474c204b198 (annotated)

browser/components/translation/cld2/internal/getonescriptspan.cc@6474c204b198 (annotated)

browser/components/translation/cld2/internal/getonescriptspan.cc

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 // Copyright 2013 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
 // Author: dsites@google.com (Dick Sites)
 //
 #include "getonescriptspan.h"
 #include <string.h>
 #include "fixunicodevalue.h"
 #include "lang_script.h"
 #include "port.h"
 #include "utf8statetable.h"
 #include "utf8prop_lettermarkscriptnum.h"
 #include "utf8repl_lettermarklower.h"
 #include "utf8scannot_lettermarkspecial.h"
 namespace CLD2 {
 // Alphabetical order for binary search, from
 // generated_entities.cc
 extern const int kNameToEntitySize;
 extern const CharIntPair kNameToEntity[];
 static const int kMaxUpToWordBoundary = 50;       // span < this make longer,
                                                   // else make shorter
 static const int kMaxAdvanceToWordBoundary = 10;  // +/- this many bytes
                                                   // to round to word boundary,
                                                   // direction above
 static const char kSpecialSymbol[256] = {       // true for < > &
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 ,0,0,0,0,0,1,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,1,0,1,0,
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 ,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
 };
 #define LT 0      // <
 #define GT 1      // >
 #define EX 2      // !
 #define HY 3      // -
 #define QU 4      // "
 #define AP 5      // '
 #define SL 6      // /
 #define S_ 7
 #define C_ 8
 #define R_ 9
 #define I_ 10
 #define P_ 11
 #define T_ 12
 #define Y_ 13
 #define L_ 14
 #define E_ 15
 #define CR 16     // <cr> or <lf>
 #define NL 17     // non-letter: ASCII whitespace, digit, punctuation
 #define PL 18     // possible letter, incl. &
 #define xx 19     // <unused>
 // Map byte to one of ~20 interesting categories for cheap tag parsing
 static const uint8 kCharToSub[256] = {
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,CR,NL, NL,CR,NL,NL,
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
   NL,EX,QU,NL, NL,NL,PL,AP, NL,NL,NL,NL, NL,HY,NL,SL,
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, LT,NL,GT,NL,
   PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
   P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
   PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
   P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
 };
 #undef LT
 #undef GT
 #undef EX
 #undef HY
 #undef QU
 #undef AP
 #undef SL
 #undef S_
 #undef C_
 #undef R_
 #undef I_
 #undef P_
 #undef T_
 #undef Y_
 #undef L_
 #undef E_
 #undef CR
 #undef NL
 #undef PL
 #undef xx
 #define OK 0
 #define X_ 1
 static const int kMaxExitStateLettersMarksOnly = 1;
 static const int kMaxExitStateAllText = 2;
 // State machine to do cheap parse of non-letter strings incl. tags
 // advances <tag>
 //          |    |
 // advances <tag> ... </tag>  for <script> <style>
 //          |               |
 // advances <!-- ... <tag> ... -->
 //          |                     |
 // advances <tag
 //          ||  (0)
 // advances <tag <tag2>
 //          ||  (0)
 //
 // We start in state [0] at a non-letter and make at least one transition
 // When scanning for just letters, arriving back at state [0] or [1] exits
 //   the state machine.
 // When scanning for any non-tag text, arriving at state [2] also exits
 static const uint8 kTagParseTbl_0[] = {
 // <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
 , 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [0] OK    exit state
   X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, // [1] error exit state
 , 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [2] NL*   [exit state]
   X_, 2, 4, 9, 10,11, 9,13,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [3] <
   X_, 2, 9, 5, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [4] <!
   X_, 2, 9, 6, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [5] <!-
 , 6, 6, 7,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [6] <!--.*
 , 6, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [7] <!--.*-
 , 2, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [8] <!--.*--
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [9] <.*
 ,10,10,10,  9,10,10,10, 10,10,10,10, 10,10,10,10, 12,10,10,X_, // [10] <.*"
 ,11,11,11, 11, 9,11,11, 11,11,11,11, 11,11,11,11, 12,11,11,X_, // [11] <.*'
   X_, 2,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,X_, // [12] <.* no " '
 // <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
   X_, 2, 9, 9, 10,11, 9, 9, 14, 9, 9, 9, 28, 9, 9, 9,  9, 9, 9,X_, // [13] <S
   X_, 2, 9, 9, 10,11, 9, 9,  9,15, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [14] <SC
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9,16, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [15] <SCR
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9,17,  9, 9, 9, 9,  9, 9, 9,X_, // [16] <SCRI
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9, 18, 9, 9, 9,  9, 9, 9,X_, // [17] <SCRIP
   X_,19, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 19,19, 9,X_, // [18] <SCRIPT
 ,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [19] <SCRIPT .*
 ,19,19,19, 19,19,21,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [20] <SCRIPT .*<
 ,19,19,19, 19,19,19,22, 19,19,19,19, 19,19,19,19, 21,21,19,X_, // [21] <SCRIPT .*</ allow SP CR LF
 ,19,19,19, 19,19,19,19, 23,19,19,19, 19,19,19,19, 19,19,19,X_, // [22] <SCRIPT .*</S
 ,19,19,19, 19,19,19,19, 19,24,19,19, 19,19,19,19, 19,19,19,X_, // [23] <SCRIPT .*</SC
 ,19,19,19, 19,19,19,19, 19,19,25,19, 19,19,19,19, 19,19,19,X_, // [24] <SCRIPT .*</SCR
 ,19,19,19, 19,19,19,19, 19,19,19,26, 19,19,19,19, 19,19,19,X_, // [25] <SCRIPT .*</SCRI
 ,19,19,19, 19,19,19,19, 19,19,19,19, 27,19,19,19, 19,19,19,X_, // [26] <SCRIPT .*</SCRIP
 , 2,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [27] <SCRIPT .*</SCRIPT
 // <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9,29, 9, 9,  9, 9, 9,X_, // [28] <ST
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9,30, 9,  9, 9, 9,X_, // [29] <STY
   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9,31,  9, 9, 9,X_, // [30] <STYL
   X_,32, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 32,32, 9,X_, // [31] <STYLE
 ,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [32] <STYLE .*
 ,32,32,32, 32,32,34,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [33] <STYLE .*<
 ,32,32,32, 32,32,32,35, 32,32,32,32, 32,32,32,32, 34,34,32,X_, // [34] <STYLE .*</ allow SP CR LF
 ,32,32,32, 32,32,32,32, 32,32,32,32, 36,32,32,32, 32,32,32,X_, // [35] <STYLE .*</S
 ,32,32,32, 32,32,32,32, 32,32,32,32, 32,37,32,32, 32,32,32,X_, // [36] <STYLE .*</ST
 ,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,38,32, 32,32,32,X_, // [37] <STYLE .*</STY
 ,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,39, 32,32,32,X_, // [38] <STYLE .*</STYL
 , 2,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [39] <STYLE .*</STYLE
 };
 #undef OK
 #undef X_
 enum
 {
   UTFmax        = 4,            // maximum bytes per rune
   Runesync      = 0x80,         // cannot represent part of a UTF sequence (<)
   Runeself      = 0x80,         // rune and UTF sequences are the same (<)
   Runeerror     = 0xFFFD,       // decoding error in UTF
   Runemax       = 0x10FFFF,     // maximum rune value
 };
 // Debugging. Not thread safe.
 static char gDisplayPiece[32];
 const uint8 gCharlen[16] = {1,1,1,1, 1,1,1,1, 1,1,1,1, 2,2,3,4};
 char* DisplayPiece(const char* next_byte_, int byte_length_) {
   // Copy up to 8 UTF-8 chars to buffer
   int k = 0;    // byte count
   int n = 0;    // character count
   for (int i = 0; i < byte_length_; ++i) {
     char c = next_byte_[i];
     if ((c & 0xc0) != 0x80) {
       // Beginning of a UTF-8 character
       int charlen = gCharlen[static_cast<uint8>(c) >> 4];
       if (i + charlen > byte_length_) {break;} // Not enough room for full char
       if (k >= (32 - 7)) {break;}   // Not necessarily enough room
       if (n >= 8) {break;}          // Enough characters already
       ++n;
     }
     if (c == '<') {
       memcpy(&gDisplayPiece[k], "&lt;", 4); k += 4;
     } else if (c == '>') {
       memcpy(&gDisplayPiece[k], "&gt;", 4); k += 4;
     } else if (c == '&') {
       memcpy(&gDisplayPiece[k], "&amp;", 5); k += 5;
     } else if (c == '\'') {
       memcpy(&gDisplayPiece[k], "&apos;", 6); k += 6;
     } else if (c == '"') {
       memcpy(&gDisplayPiece[k], "&quot;", 6); k += 6;
     } else {
       gDisplayPiece[k++] = c;
     }
   }
   gDisplayPiece[k++] = '\0';
   return gDisplayPiece;
 }
 // runetochar copies (encodes) one rune, pointed to by r, to at most
 // UTFmax bytes starting at s and returns the number of bytes generated.
 int runetochar(char *str, const char32 *rune) {
   // Convert to unsigned for range check.
   unsigned long c;
   // 1 char 00-7F
   c = *rune;
   if(c <= 0x7F) {
     str[0] = c;
     return 1;
   }
   // 2 char 0080-07FF
   if(c <= 0x07FF) {
     str[0] = 0xC0 | (c >> 1*6);
     str[1] = 0x80 | (c & 0x3F);
     return 2;
   }
   // Range check
   if (c > Runemax) {
     c = Runeerror;
   }
   // 3 char 0800-FFFF
   if (c <= 0xFFFF) {
     str[0] = 0xE0 |  (c >> 2*6);
     str[1] = 0x80 | ((c >> 1*6) & 0x3F);
     str[2] = 0x80 |  (c & 0x3F);
     return 3;
   }
   // 4 char 10000-1FFFFF
   str[0] = 0xF0 | (c >> 3*6);
   str[1] = 0x80 | ((c >> 2*6) & 0x3F);
   str[2] = 0x80 | ((c >> 1*6) & 0x3F);
   str[3] = 0x80 | (c & 0x3F);
   return 4;
 }
 // Useful for converting an entity to an ascii value.
 // RETURNS unicode value, or -1 if entity isn't valid.  Don't include & or ;
 int LookupEntity(const char* entity_name, int entity_len) {
   // Make a C string
   if (entity_len >= 16) {return -1;}    // All real entities are shorter
   char temp[16];
   memcpy(temp, entity_name, entity_len);
   temp[entity_len] = '\0';
   int match = BinarySearch(temp, 0, kNameToEntitySize, kNameToEntity);
   if (match >= 0) {return kNameToEntity[match].i;}
   return -1;
 }
 bool ascii_isdigit(char c) {
   return ('0' <= c) && (c <= '9');
 }
 bool ascii_isxdigit(char c) {
   if (('0' <= c) && (c <= '9')) {return true;}
   if (('a' <= c) && (c <= 'f')) {return true;}
   if (('A' <= c) && (c <= 'F')) {return true;}
   return false;
 }
 bool ascii_isalnum(char c) {
   if (('0' <= c) && (c <= '9')) {return true;}
   if (('a' <= c) && (c <= 'z')) {return true;}
   if (('A' <= c) && (c <= 'Z')) {return true;}
   return false;
 }
 int hex_digit_to_int(char c) {
   if (('0' <= c) && (c <= '9')) {return c - '0';}
   if (('a' <= c) && (c <= 'f')) {return c - 'a' + 10;}
   if (('A' <= c) && (c <= 'F')) {return c - 'A' + 10;}
   return 0;
 }
 static int32 strto32_base10(const char* nptr, const char* limit,
                             const char **endptr) {
   *endptr = nptr;
   while (nptr < limit && *nptr == '0') {
     ++nptr;
   }
   if (nptr == limit || !ascii_isdigit(*nptr))
     return -1;
   const char* end_digits_run = nptr;
   while (end_digits_run < limit && ascii_isdigit(*end_digits_run)) {
     ++end_digits_run;
   }
   *endptr = end_digits_run;
   const int num_digits = end_digits_run - nptr;
   // kint32max == 2147483647.
   if (num_digits < 9 ||
       (num_digits == 10 && memcmp(nptr, "2147483647", 10) <= 0)) {
     int value = 0;
     for (; nptr < end_digits_run; ++nptr) {
       value *= 10;
       value += *nptr - '0';
     }
     // Overflow past the last valid unicode codepoint
     // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
     return FixUnicodeValue(value);
   } else {
     // Overflow: can't fit in an int32;
     // returns the replacement character 0xFFFD.
     return 0xFFFD;
   }
 }
 static int32 strto32_base16(const char* nptr, const char* limit,
                             const char **endptr) {
   *endptr = nptr;
   while (nptr < limit && *nptr == '0') {
     ++nptr;
   }
   if (nptr == limit || !ascii_isxdigit(*nptr)) {
     return -1;
   }
   const char* end_xdigits_run = nptr;
   while (end_xdigits_run < limit && ascii_isxdigit(*end_xdigits_run)) {
     ++end_xdigits_run;
   }
   *endptr = end_xdigits_run;
   const int num_xdigits = end_xdigits_run - nptr;
   // kint32max == 0x7FFFFFFF.
   if (num_xdigits < 8 || (num_xdigits == 8 && nptr[0] < '8')) {
     int value = 0;
     for (; nptr < end_xdigits_run; ++nptr) {
       value <<= 4;
       value += hex_digit_to_int(*nptr);
     }
     // Overflow past the last valid unicode codepoint
     // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
     return FixUnicodeValue(value);
   } else {
     // Overflow: can't fit in an int32;
     // returns the replacement character 0xFFFD.
     return 0xFFFD;
   }
 }
 // Unescape the current character pointed to by src.  SETS the number
 // of chars read for the conversion (in UTF8).  If src isn't a valid entity,
 // just consume the & and RETURN -1.  If src doesn't point to & -- which it
 // should -- set src_consumed to 0 and RETURN -1.
 int ReadEntity(const char* src, int srcn, int* src_consumed) {
   const char* const srcend = src + srcn;
   if (srcn == 0 || *src != '&') {      // input should start with an ampersand
     *src_consumed = 0;
     return -1;
   }
   *src_consumed = 1;                   // we'll get the & at least
   // The standards are a bit unclear on when an entity ends.  Certainly a ";"
   // ends one, but spaces probably do too.  We follow the lead of both IE and
   // Netscape, which as far as we can tell end numeric entities (1st case below)
   // at any non-digit, and end character entities (2nd case) at any non-alnum.
   const char* entstart, *entend;  // where the entity starts and ends
   entstart = src + 1;             // read past the &
   int entval;                     // UCS2 value of the entity
   if ( *entstart == '#' ) {       // -- 1st case: numeric entity
     if ( entstart + 2 >= srcend ) {
       return -1;                  // no way a legitimate number could fit
     } else if ( entstart[1] == 'x' || entstart[1] == 'X' ) {   // hex numeric
       entval = strto32_base16(entstart + 2, srcend, &entend);
     } else {                                  // decimal numeric entity
       entval = strto32_base10(entstart+1, srcend, &entend);
     }
     if (entval == -1 || entend > srcend) {
       return -1;                 // not entirely correct, but close enough
     }
   } else {                       // -- 2nd case: character entity
     for (entend = entstart;
          entend < srcend && ascii_isalnum(*entend);
          ++entend ) {
       // entity consists of alphanumeric chars
     }
     entval = LookupEntity(entstart, entend - entstart);
     if (entval < 0) {
       return -1;  // not a legal entity name
     }
     // Now we do a strange-seeming IE6-compatibility check: if entval is
     // >= 256, it *must* be followed by a semicolon or it's not considered
     // an entity.  The problem is lots of the newfangled entity names, like
     // "lang", also occur in URL CGI arguments: "/search?q=test&lang=en".
     // When these links are written in HTML, it would be really bad if the
     // "&lang" were treated as an entity, which is what the spec says
     // *should* happen (even when the HTML is inside an "A HREF" tag!)
     // IE ignores the spec for these new, high-value entities, so we do too.
     if ( entval >= 256 && !(entend < srcend && *entend == ';') ) {
       return -1;                 // make non-;-terminated entity illegal
     }
   }
   // Finally, figure out how much src was consumed
   if ( entend < srcend && *entend == ';' ) {
     entend++;                    // standard says ; terminator is special
   }
   *src_consumed = entend - src;
   return entval;
 }
 // Src points to '&'
 // Writes entity value to dst. Returns take(src), put(dst) byte counts
 void EntityToBuffer(const char* src, int len, char* dst,
                     int* tlen, int* plen) {
   char32 entval = ReadEntity(src, len, tlen);
   // ReadEntity does this already: entval = FixUnicodeValue(entval);
   // Convert UTF-32 to UTF-8
   if (entval > 0) {
     *plen = runetochar(dst, &entval);
   } else {
     // Illegal entity; ignore the '&'
     *tlen = 1;
     *plen = 0;
   }
 }
 // Returns true if character is < > or &, none of which are letters
 bool inline IsSpecial(char c) {
   if ((c & 0xe0) == 0x20) {
     return kSpecialSymbol[static_cast<uint8>(c)];
   }
   return false;
 }
 // Quick Skip to next letter or < > & or to end of string (eos)
 // Always return is_letter for eos
 int ScanToLetterOrSpecial(const char* src, int len) {
   int bytes_consumed;
   StringPiece str(src, len);
   UTF8GenericScan(&utf8scannot_lettermarkspecial_obj, str, &bytes_consumed);
   return bytes_consumed;
 }
 // src points to non-letter, such as tag-opening '<'
 // Return length from here to next possible letter
 // On another < before >, return 1
 // advances <tag>
 //          |    |
 // advances <tag> ... </tag>  for <script> <style>
 //          |               |
 // advances <!-- ... <tag> ... -->
 //          |                     |
 // advances <tag
 //          |    | end of string
 // advances <tag <tag2>
 //          ||
 int ScanToPossibleLetter(const char* isrc, int len, int max_exit_state) {
   const uint8* src = reinterpret_cast<const uint8*>(isrc);
   const uint8* srclimit = src + len;
   const uint8* tagParseTbl = kTagParseTbl_0;
   int e = 0;
   while (src < srclimit) {
     e = tagParseTbl[kCharToSub[*src++]];
     if (e <= max_exit_state) {
       // We overshot by one byte
       --src;
       break;
     }
     tagParseTbl = &kTagParseTbl_0[e * 20];
   }
   if (src >= srclimit) {
     // We fell off the end of the text.
     // It looks like the most common case for this is a truncated file, not
     // mismatched angle brackets. So we pretend that the last char was '>'
     return len;
   }
   // OK to be in state 0 or state 2 at exit
   if ((e != 0) && (e != 2)) {
     // Error, '<' followed by '<'
     // We want to back up to first <, then advance by one byte past it
     int offset = src - reinterpret_cast<const uint8*>(isrc);
     // Backscan to first '<' and return enough length to just get past it
     --offset;   // back up over the second '<', which caused us to stop
     while ((0 < offset) && (isrc[offset] != '<')) {
       // Find the first '<', which is unmatched
       --offset;
     }
     // skip to just beyond first '<'
     return offset + 1;
   }
   return src - reinterpret_cast<const uint8*>(isrc);
 }
 ScriptScanner::ScriptScanner(const char* buffer,
                              int buffer_length,
                              bool is_plain_text)
   : start_byte_(buffer),
   next_byte_(buffer),
   next_byte_limit_(buffer + buffer_length),
   byte_length_(buffer_length),
   is_plain_text_(is_plain_text),
   letters_marks_only_(true),
   one_script_only_(true),
   exit_state_(kMaxExitStateLettersMarksOnly) {
     script_buffer_ = new char[kMaxScriptBuffer];
     script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
     map2original_.Clear();    // map from script_buffer_ to buffer
     map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
 }
 // Extended version to allow spans of any non-tag text and spans of mixed script
 ScriptScanner::ScriptScanner(const char* buffer,
                              int buffer_length,
                              bool is_plain_text,
                              bool any_text,
                              bool any_script)
   : start_byte_(buffer),
   next_byte_(buffer),
   next_byte_limit_(buffer + buffer_length),
   byte_length_(buffer_length),
   is_plain_text_(is_plain_text),
   letters_marks_only_(!any_text),
   one_script_only_(!any_script),
   exit_state_(any_text ? kMaxExitStateAllText : kMaxExitStateLettersMarksOnly) {
     script_buffer_ = new char[kMaxScriptBuffer];
     script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
     map2original_.Clear();    // map from script_buffer_ to buffer
     map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
 }
 ScriptScanner::~ScriptScanner() {
   delete[] script_buffer_;
   delete[] script_buffer_lower_;
 }
 // Get to the first real non-tag letter or entity that is a letter
 // Sets script of that letter
 // Return len if no more letters
 int ScriptScanner::SkipToFrontOfSpan(const char* src, int len, int* script) {
   int sc = UNKNOWN_ULSCRIPT;
   int skip = 0;
   int tlen, plen;
   // Do run of non-letters (tag | &NL | NL)*
   tlen = 0;
   while (skip < len) {
     // Do fast scan to next interesting byte
     // int oldskip = skip;
     skip += ScanToLetterOrSpecial(src + skip, len - skip);
     // Check for no more letters/specials
     if (skip >= len) {
       // All done
       *script = sc;
       return len;
     }
     // We are at a letter, nonletter, tag, or entity
     if (IsSpecial(src[skip]) && !is_plain_text_) {
       if (src[skip] == '<') {
         // Begining of tag; skip to end and go around again
         tlen = ScanToPossibleLetter(src + skip, len - skip,
                                     exit_state_);
         sc = 0;
       } else if (src[skip] == '>') {
         // Unexpected end of tag; skip it and go around again
         tlen = 1;         // Over the >
         sc = 0;
       } else if (src[skip] == '&') {
         // Expand entity, no advance
         char temp[4];
         EntityToBuffer(src + skip, len - skip,
                        temp, &tlen, &plen);
         sc = GetUTF8LetterScriptNum(temp);
       }
     } else {
       // Update 1..4 bytes
       tlen = UTF8OneCharLen(src + skip);
       sc = GetUTF8LetterScriptNum(src + skip);
     }
     if (sc != 0) {break;}           // Letter found
     skip += tlen;                   // Else advance
   }
   *script = sc;
   return skip;
 }
 // These are for ASCII-only tag names
 // Compare one letter uplow to c, ignoring case of uplowp
 inline bool EqCase(char uplow, char c) {
   return (uplow | 0x20) == c;
 }
 // These are for ASCII-only tag names
 // Return true for space / < > etc. all less than 0x40
 inline bool NeqLetter(char c) {
   return c < 0x40;
 }
 // These are for ASCII-only tag names
 // Return true for space \n false for \r
 inline bool WS(char c) {
   return (c == ' ') || (c == '\n');
 }
 // Canonical CR or LF
 static const char LF = '\n';
 // The naive loop scans from next_byte_ to script_buffer_ until full.
 // But this can leave an awkward hard-to-identify short fragment at the
 // end of the input. We would prefer to make the next-to-last fragment
 // shorter and the last fragment longer.
 // Copy next run of non-tag characters to buffer [NUL terminated]
 // This just replaces tags with space or \n and removes entities.
 // Tags <br> <p> and <tr> are replaced with \n. Non-letter sequences
 // including \r or \n are replaced by \n. All other tags and skipped text
 // are replaced with ASCII space.
 //
 // Buffer ALWAYS has leading space and trailing space space space NUL
 bool ScriptScanner::GetOneTextSpan(LangSpan* span) {
   span->text = script_buffer_;
   span->text_bytes = 0;
   span->offset = next_byte_ - start_byte_;
   span->ulscript = UNKNOWN_ULSCRIPT;
   span->lang = UNKNOWN_LANGUAGE;
   span->truncated = false;
   int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
   if ((kMaxScriptBytes <= byte_length_) &&
       (byte_length_ < (2 * kMaxScriptBytes))) {
     // Try to split the last two fragments in half
     put_soft_limit = byte_length_ / 2;
   }
   script_buffer_[0] = ' ';  // Always a space at front of output
   script_buffer_[1] = '\0';
   int take = 0;
   int put = 1;              // Start after the initial space
   int tlen, plen;
   if (byte_length_ <= 0) {
     return false;          // No more text to be found
   }
   // Go over alternating spans of text and tags,
   // copying letters to buffer with single spaces for each run of non-letters
   bool last_byte_was_space = false;
   while (take < byte_length_) {
     char c = next_byte_[take];
     if (c == '\r') {c = LF;}      // Canonical CR or LF
     if (c == '\n') {c = LF;}      // Canonical CR or LF
     if (IsSpecial(c) && !is_plain_text_) {
       if (c == '<') {
         // Replace tag with space
         c = ' ';                      // for almost-full test below
         // or if <p> <br> <tr>, replace with \n
         if (take < (byte_length_ - 3)) {
           if (EqCase(next_byte_[take + 1], 'p') &&
               NeqLetter(next_byte_[take + 2])) {
             c = LF;
           }
           if (EqCase(next_byte_[take + 1], 'b') &&
               EqCase(next_byte_[take + 2], 'r') &&
               NeqLetter(next_byte_[take + 3])) {
             c = LF;
           }
           if (EqCase(next_byte_[take + 1], 't') &&
               EqCase(next_byte_[take + 2], 'r') &&
               NeqLetter(next_byte_[take + 3])) {
             c = LF;
           }
         }
         // Begining of tag; skip to end and go around again
         tlen = 1 + ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
                                     exit_state_);
         // Copy one byte, compressing spaces
         if (!last_byte_was_space || !WS(c)) {
           script_buffer_[put++] = c;      // Advance dest
           last_byte_was_space = WS(c);
         }
       } else if (c == '>') {
         // Unexpected end of tag; copy it and go around again
         tlen = 1;         // Over the >
         script_buffer_[put++] = c;    // Advance dest
       } else if (c == '&') {
         // Expand entity, no advance
         EntityToBuffer(next_byte_ + take, byte_length_ - take,
                        script_buffer_ + put, &tlen, &plen);
         put += plen;                  // Advance dest
       }
       take += tlen;                   // Advance source
     } else {
       // Copy one byte, compressing spaces
       if (!last_byte_was_space || !WS(c)) {
         script_buffer_[put++] = c;      // Advance dest
         last_byte_was_space = WS(c);
       }
       ++take;                         // Advance source
     }
     if (WS(c) &&
         (put >= put_soft_limit)) {
       // Buffer is almost full
       span->truncated = true;
       break;
     }
     if (put >= kMaxScriptBytes) {
       // Buffer is completely full
       span->truncated = true;
       break;
     }
   }
   // Almost done. Back up to a character boundary if needed
   while ((0 < take) && ((next_byte_[take] & 0xc0) == 0x80)) {
     // Back up over continuation byte
     --take;
     --put;
   }
   // Update input position
   next_byte_ += take;
   byte_length_ -= take;
   // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
   //                          kMaxScriptBytes |   | put
   script_buffer_[put + 0] = ' ';
   script_buffer_[put + 1] = ' ';
   script_buffer_[put + 2] = ' ';
   script_buffer_[put + 3] = '\0';
   span->text_bytes = put;       // Does not include the last four chars above
   return true;
 }
 // Copy next run of same-script non-tag letters to buffer [NUL terminated]
 // Buffer ALWAYS has leading space and trailing space space space NUL
 bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
   if (!letters_marks_only_) {
     // Return non-tag text, including punctuation and digits
     return GetOneTextSpan(span);
   }
   span->text = script_buffer_;
   span->text_bytes = 0;
   span->offset = next_byte_ - start_byte_;
   span->ulscript = UNKNOWN_ULSCRIPT;
   span->lang = UNKNOWN_LANGUAGE;
   span->truncated = false;
   // struct timeval script_start, script_mid, script_end;
   int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
   if ((kMaxScriptBytes <= byte_length_) &&
       (byte_length_ < (2 * kMaxScriptBytes))) {
     // Try to split the last two fragments in half
     put_soft_limit = byte_length_ / 2;
   }
   int spanscript;           // The script of this span
   int sc = UNKNOWN_ULSCRIPT;  // The script of next character
   int tlen = 0;
   int plen = 0;
   script_buffer_[0] = ' ';  // Always a space at front of output
   script_buffer_[1] = '\0';
   int take = 0;
   int put = 1;              // Start after the initial space
   // Build offsets from span->text back to start_byte_ + span->offset
   // This mapping reflects deletion of non-letters, expansion of
   // entities, etc.
   map2original_.Clear();
   map2original_.Delete(span->offset);   // So that MapBack(0) gives offset
   // Get to the first real non-tag letter or entity that is a letter
   int skip = SkipToFrontOfSpan(next_byte_, byte_length_, &spanscript);
   next_byte_ += skip;
   byte_length_ -= skip;
   if (skip != 1) {
     map2original_.Delete(skip);
     map2original_.Insert(1);
   } else {
     map2original_.Copy(1);
   }
   if (byte_length_ <= 0) {
     map2original_.Reset();
     return false;               // No more letters to be found
   }
   // There is at least one letter, so we know the script for this span
   span->ulscript = (ULScript)spanscript;
   // Go over alternating spans of same-script letters and non-letters,
   // copying letters to buffer with single spaces for each run of non-letters
   while (take < byte_length_) {
     // Copy run of letters in same script (&LS | LS)*
     int letter_count = 0;              // Keep track of word length
     bool need_break = false;
     while (take < byte_length_) {
       // We are at a letter, nonletter, tag, or entity
       if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
         if (next_byte_[take] == '<') {
           // Begining of tag
           sc = 0;
           break;
         } else if (next_byte_[take] == '>') {
           // Unexpected end of tag
           sc = 0;
           break;
         } else if (next_byte_[take] == '&') {
           // Copy entity, no advance
           EntityToBuffer(next_byte_ + take, byte_length_ - take,
                          script_buffer_ + put, &tlen, &plen);
           sc = GetUTF8LetterScriptNum(script_buffer_ + put);
         }
       } else {
         // Real letter, safely copy up to 4 bytes, increment by 1..4
         // Will update by 1..4 bytes at Advance, below
         tlen = plen = UTF8OneCharLen(next_byte_ + take);
         if (take < (byte_length_ - 3)) {
           // X86 fast case, does unaligned load/store
           UNALIGNED_STORE32(script_buffer_ + put,
                             UNALIGNED_LOAD32(next_byte_ + take));
         } else {
           // Slow case, happens 1-3 times per input document
           memcpy(script_buffer_ + put, next_byte_ + take, plen);
         }
         sc = GetUTF8LetterScriptNum(next_byte_ + take);
       }
       // Allow continue across a single letter in a different script:
       // A B D = three scripts, c = common script, i = inherited script,
       // - = don't care, ( = take position before the += below
       //  AAA(A-    continue
       //
       //  AAA(BA    continue
       //  AAA(BB    break
       //  AAA(Bc    continue (breaks after B)
       //  AAA(BD    break
       //  AAA(Bi    break
       //
       //  AAA(c-    break
       //
       //  AAA(i-    continue
       //
       if ((sc != spanscript) && (sc != ULScript_Inherited)) {
         // Might need to break this script span
         if (sc == ULScript_Common) {
           need_break = true;
         } else {
           // Look at next following character, ignoring entity as Common
           int sc2 = GetUTF8LetterScriptNum(next_byte_ + take + tlen);
           if ((sc2 != ULScript_Common) && (sc2 != spanscript)) {
             // We found a non-trivial change of script
             if (one_script_only_) {
               need_break = true;
             }
           }
         }
       }
       if (need_break) {break;}  // Non-letter or letter in wrong script
       take += tlen;                   // Advance
       put += plen;                    // Advance
       // Update the offset map to reflect take/put lengths
       if (tlen == plen) {
         map2original_.Copy(tlen);
       } else if (tlen < plen) {
         map2original_.Copy(tlen);
         map2original_.Insert(plen - tlen);
       } else {    // plen < tlen
         map2original_.Copy(plen);
         map2original_.Delete(tlen - plen);
       }
       ++letter_count;
       if (put >= kMaxScriptBytes) {
         // Buffer is full
         span->truncated = true;
         break;
       }
     }     // End while letters
     // Do run of non-letters (tag | &NL | NL)*
     while (take < byte_length_) {
       // Do fast scan to next interesting byte
       tlen = ScanToLetterOrSpecial(next_byte_ + take, byte_length_ - take);
       take += tlen;
       map2original_.Delete(tlen);
       if (take >= byte_length_) {break;}    // Might have scanned to end
       // We are at a letter, nonletter, tag, or entity
       if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
         if (next_byte_[take] == '<') {
           // Begining of tag; skip to end and go around again
           tlen = ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
                                       exit_state_);
           sc = 0;
         } else if (next_byte_[take] == '>') {
           // Unexpected end of tag; skip it and go around again
           tlen = 1;         // Over the >
           sc = 0;
         } else if (next_byte_[take] == '&') {
           // Expand entity, no advance
           EntityToBuffer(next_byte_ + take, byte_length_ - take,
                          script_buffer_ + put, &tlen, &plen);
           sc = GetUTF8LetterScriptNum(script_buffer_ + put);
         }
       } else {
         // Update 1..4
         tlen = UTF8OneCharLen(next_byte_ + take);
         sc = GetUTF8LetterScriptNum(next_byte_ + take);
       }
       if (sc != 0) {break;}           // Letter found
       take += tlen;                   // Else advance
       map2original_.Delete(tlen);
     }     // End while not-letters
     script_buffer_[put++] = ' ';
     map2original_.Insert(1);
     // Letter in wrong script ?
     if ((sc != spanscript) && (sc != ULScript_Inherited)) {break;}
     if (put >= put_soft_limit) {
       // Buffer is almost full
       span->truncated = true;
       break;
     }
   }
   // Almost done. Back up to a character boundary if needed
   while ((0 < take) && (take < byte_length_) &&
          ((next_byte_[take] & 0xc0) == 0x80)) {
     // Back up over continuation byte
     --take;
     --put;
   }
   // Update input position
   next_byte_ += take;
   byte_length_ -= take;
   // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
   //                          kMaxScriptBytes |   | put
   script_buffer_[put + 0] = ' ';
   script_buffer_[put + 1] = ' ';
   script_buffer_[put + 2] = ' ';
   script_buffer_[put + 3] = '\0';
   map2original_.Insert(4);
   map2original_.Reset();
   span->text_bytes = put;       // Does not include the last four chars above
   return true;
 }
 // Force Latin, Cyrillic, Armenian, Greek scripts to be lowercase
 // List changes with each version of Unicode, so just always lowercase
 // Unicode 6.2.0:
 //   ARMENIAN COPTIC CYRILLIC DESERET GEORGIAN GLAGOLITIC GREEK LATIN
 void ScriptScanner::LowerScriptSpan(LangSpan* span) {
   // If needed, lowercase all the text. If we do it sooner, might miss
   // lowercasing an entity such as &Aacute;
   // We only need to do this for Latn and Cyrl scripts
   map2uplow_.Clear();
   // Full Unicode lowercase of the entire buffer, including
   // four pad bytes off the end.
   // Ahhh. But the last byte 0x00 is not interchange-valid, so we do 3 pad
   // bytes and put the 0x00 in explicitly.
   // Build an offset map from script_buffer_lower_ back to script_buffer_
   int consumed, filled, changed;
   StringPiece istr(span->text, span->text_bytes + 3);
   StringPiece ostr(script_buffer_lower_, kMaxScriptLowerBuffer);
   UTF8GenericReplace(&utf8repl_lettermarklower_obj,
                             istr, ostr, is_plain_text_,
                             &consumed, &filled, &changed, &map2uplow_);
   script_buffer_lower_[filled] = '\0';
   span->text = script_buffer_lower_;
   span->text_bytes = filled - 3;
   map2uplow_.Reset();
 }
 // Copy next run of same-script non-tag letters to buffer [NUL terminated]
 // Force Latin, Cyrillic, Greek scripts to be lowercase
 // Buffer ALWAYS has leading space and trailing space space space NUL
 bool ScriptScanner::GetOneScriptSpanLower(LangSpan* span) {
   bool ok = GetOneScriptSpan(span);
   LowerScriptSpan(span);
   return ok;
 }
 // Maps byte offset in most recent GetOneScriptSpan/Lower
 // span->text [0..text_bytes] into an additional byte offset from
 // span->offset, to get back to corresponding text in the original
 // input buffer.
 // text_offset must be the first byte
 // of a UTF-8 character, or just beyond the last character. Normally this
 // routine is called with the first byte of an interesting range and
 // again with the first byte of the following range.
 int ScriptScanner::MapBack(int text_offset) {
   return map2original_.MapBack(map2uplow_.MapBack(text_offset));
 }
 // Gets lscript number for letters; always returns
 //   0 (common script) for non-letters
 int GetUTF8LetterScriptNum(const char* src) {
   int srclen = UTF8OneCharLen(src);
   const uint8* usrc = reinterpret_cast<const uint8*>(src);
   return UTF8GenericPropertyTwoByte(&utf8prop_lettermarkscriptnum_obj,
                                     &usrc, &srclen);
 }
 }  // namespace CLD2

The Tor Browser / annotate

browser/components/translation/cld2/internal/getonescriptspan.cc@6474c204b198 (annotated)

browser/components/translation/cld2/internal/getonescriptspan.cc