michael@0: // Copyright 2013 Google Inc. All Rights Reserved.
michael@0: //
michael@0: // Licensed under the Apache License, Version 2.0 (the "License");
michael@0: // you may not use this file except in compliance with the License.
michael@0: // You may obtain a copy of the License at
michael@0: //
michael@0: //     http://www.apache.org/licenses/LICENSE-2.0
michael@0: //
michael@0: // Unless required by applicable law or agreed to in writing, software
michael@0: // distributed under the License is distributed on an "AS IS" BASIS,
michael@0: // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
michael@0: // See the License for the specific language governing permissions and
michael@0: // limitations under the License.
michael@0: 
michael@0: //
michael@0: // Author: dsites@google.com (Dick Sites)
michael@0: //
michael@0: 
michael@0: 
michael@0: #include "getonescriptspan.h"
michael@0: #include <string.h>
michael@0: 
michael@0: #include "fixunicodevalue.h"
michael@0: #include "lang_script.h"
michael@0: #include "port.h"
michael@0: #include "utf8statetable.h"
michael@0: 
michael@0: #include "utf8prop_lettermarkscriptnum.h"
michael@0: #include "utf8repl_lettermarklower.h"
michael@0: #include "utf8scannot_lettermarkspecial.h"
michael@0: 
michael@0: 
michael@0: namespace CLD2 {
michael@0: 
michael@0: // Alphabetical order for binary search, from
michael@0: // generated_entities.cc
michael@0: extern const int kNameToEntitySize;
michael@0: extern const CharIntPair kNameToEntity[];
michael@0: 
michael@0: static const int kMaxUpToWordBoundary = 50;       // span < this make longer,
michael@0:                                                   // else make shorter
michael@0: static const int kMaxAdvanceToWordBoundary = 10;  // +/- this many bytes
michael@0:                                                   // to round to word boundary,
michael@0:                                                   // direction above
michael@0: 
michael@0: static const char kSpecialSymbol[256] = {       // true for < > &
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0:   0,0,0,0,0,0,1,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,1,0,1,0,
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0: 
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0:   0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
michael@0: };
michael@0: 
michael@0: 
michael@0: 
michael@0: #define LT 0      // <
michael@0: #define GT 1      // >
michael@0: #define EX 2      // !
michael@0: #define HY 3      // -
michael@0: #define QU 4      // "
michael@0: #define AP 5      // '
michael@0: #define SL 6      // /
michael@0: #define S_ 7
michael@0: #define C_ 8
michael@0: #define R_ 9
michael@0: #define I_ 10
michael@0: #define P_ 11
michael@0: #define T_ 12
michael@0: #define Y_ 13
michael@0: #define L_ 14
michael@0: #define E_ 15
michael@0: #define CR 16     // <cr> or <lf>
michael@0: #define NL 17     // non-letter: ASCII whitespace, digit, punctuation
michael@0: #define PL 18     // possible letter, incl. &
michael@0: #define xx 19     // <unused>
michael@0: 
michael@0: // Map byte to one of ~20 interesting categories for cheap tag parsing
michael@0: static const uint8 kCharToSub[256] = {
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,CR,NL, NL,CR,NL,NL,
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
michael@0:   NL,EX,QU,NL, NL,NL,PL,AP, NL,NL,NL,NL, NL,HY,NL,SL,
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, LT,NL,GT,NL,
michael@0: 
michael@0:   PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
michael@0:   P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
michael@0:   PL,PL,PL,C_, PL,E_,PL,PL, PL,I_,PL,PL, L_,PL,PL,PL,
michael@0:   P_,PL,R_,S_, T_,PL,PL,PL, PL,Y_,PL,NL, NL,NL,NL,NL,
michael@0: 
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
michael@0:   NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL, NL,NL,NL,NL,
michael@0: 
michael@0:   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
michael@0:   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
michael@0:   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
michael@0:   PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL, PL,PL,PL,PL,
michael@0: };
michael@0: 
michael@0: #undef LT
michael@0: #undef GT
michael@0: #undef EX
michael@0: #undef HY
michael@0: #undef QU
michael@0: #undef AP
michael@0: #undef SL
michael@0: #undef S_
michael@0: #undef C_
michael@0: #undef R_
michael@0: #undef I_
michael@0: #undef P_
michael@0: #undef T_
michael@0: #undef Y_
michael@0: #undef L_
michael@0: #undef E_
michael@0: #undef CR
michael@0: #undef NL
michael@0: #undef PL
michael@0: #undef xx
michael@0: 
michael@0: 
michael@0: #define OK 0
michael@0: #define X_ 1
michael@0: 
michael@0: 
michael@0: static const int kMaxExitStateLettersMarksOnly = 1;
michael@0: static const int kMaxExitStateAllText = 2;
michael@0: 
michael@0: 
michael@0: // State machine to do cheap parse of non-letter strings incl. tags
michael@0: // advances <tag>
michael@0: //          |    |
michael@0: // advances <tag> ... </tag>  for <script> <style>
michael@0: //          |               |
michael@0: // advances <!-- ... <tag> ... -->
michael@0: //          |                     |
michael@0: // advances <tag
michael@0: //          ||  (0)
michael@0: // advances <tag <tag2>
michael@0: //          ||  (0)
michael@0: //
michael@0: // We start in state [0] at a non-letter and make at least one transition
michael@0: // When scanning for just letters, arriving back at state [0] or [1] exits
michael@0: //   the state machine.
michael@0: // When scanning for any non-tag text, arriving at state [2] also exits
michael@0: static const uint8 kTagParseTbl_0[] = {
michael@0: // <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
michael@0:    3, 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [0] OK    exit state
michael@0:   X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, X_,X_,X_,X_, // [1] error exit state
michael@0:    3, 2, 2, 2,  2, 2, 2,OK, OK,OK,OK,OK, OK,OK,OK,OK,  2, 2,OK,X_, // [2] NL*   [exit state]
michael@0:   X_, 2, 4, 9, 10,11, 9,13,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [3] <
michael@0:   X_, 2, 9, 5, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [4] <!
michael@0:   X_, 2, 9, 6, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [5] <!-
michael@0:    6, 6, 6, 7,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [6] <!--.*
michael@0:    6, 6, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [7] <!--.*-
michael@0:    6, 2, 6, 8,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6, 6,  6, 6, 6,X_, // [8] <!--.*--
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [9] <.*
michael@0:   10,10,10,10,  9,10,10,10, 10,10,10,10, 10,10,10,10, 12,10,10,X_, // [10] <.*"
michael@0:   11,11,11,11, 11, 9,11,11, 11,11,11,11, 11,11,11,11, 12,11,11,X_, // [11] <.*'
michael@0:   X_, 2,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,12, 12,12,12,X_, // [12] <.* no " '
michael@0: 
michael@0: // <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
michael@0:   X_, 2, 9, 9, 10,11, 9, 9, 14, 9, 9, 9, 28, 9, 9, 9,  9, 9, 9,X_, // [13] <S
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9,15, 9, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [14] <SC
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9,16, 9,  9, 9, 9, 9,  9, 9, 9,X_, // [15] <SCR
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9,17,  9, 9, 9, 9,  9, 9, 9,X_, // [16] <SCRI
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9, 18, 9, 9, 9,  9, 9, 9,X_, // [17] <SCRIP
michael@0:   X_,19, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 19,19, 9,X_, // [18] <SCRIPT
michael@0:   20,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [19] <SCRIPT .*
michael@0:   19,19,19,19, 19,19,21,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [20] <SCRIPT .*<
michael@0:   19,19,19,19, 19,19,19,22, 19,19,19,19, 19,19,19,19, 21,21,19,X_, // [21] <SCRIPT .*</ allow SP CR LF
michael@0:   19,19,19,19, 19,19,19,19, 23,19,19,19, 19,19,19,19, 19,19,19,X_, // [22] <SCRIPT .*</S
michael@0:   19,19,19,19, 19,19,19,19, 19,24,19,19, 19,19,19,19, 19,19,19,X_, // [23] <SCRIPT .*</SC
michael@0:   19,19,19,19, 19,19,19,19, 19,19,25,19, 19,19,19,19, 19,19,19,X_, // [24] <SCRIPT .*</SCR
michael@0:   19,19,19,19, 19,19,19,19, 19,19,19,26, 19,19,19,19, 19,19,19,X_, // [25] <SCRIPT .*</SCRI
michael@0:   19,19,19,19, 19,19,19,19, 19,19,19,19, 27,19,19,19, 19,19,19,X_, // [26] <SCRIPT .*</SCRIP
michael@0:   19, 2,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,19, 19,19,19,X_, // [27] <SCRIPT .*</SCRIPT
michael@0: 
michael@0: // <  >  !  -   "  '  /  S   C  R  I  P   T  Y  L  E  CR NL PL xx
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9,29, 9, 9,  9, 9, 9,X_, // [28] <ST
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9,30, 9,  9, 9, 9,X_, // [29] <STY
michael@0:   X_, 2, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9,31,  9, 9, 9,X_, // [30] <STYL
michael@0:   X_,32, 9, 9, 10,11, 9, 9,  9, 9, 9, 9,  9, 9, 9, 9, 32,32, 9,X_, // [31] <STYLE
michael@0:   33,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [32] <STYLE .*
michael@0:   32,32,32,32, 32,32,34,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [33] <STYLE .*<
michael@0:   32,32,32,32, 32,32,32,35, 32,32,32,32, 32,32,32,32, 34,34,32,X_, // [34] <STYLE .*</ allow SP CR LF
michael@0:   32,32,32,32, 32,32,32,32, 32,32,32,32, 36,32,32,32, 32,32,32,X_, // [35] <STYLE .*</S
michael@0:   32,32,32,32, 32,32,32,32, 32,32,32,32, 32,37,32,32, 32,32,32,X_, // [36] <STYLE .*</ST
michael@0:   32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,38,32, 32,32,32,X_, // [37] <STYLE .*</STY
michael@0:   32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,39, 32,32,32,X_, // [38] <STYLE .*</STYL
michael@0:   32, 2,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,32, 32,32,32,X_, // [39] <STYLE .*</STYLE
michael@0: };
michael@0: 
michael@0: #undef OK
michael@0: #undef X_
michael@0: 
michael@0: enum
michael@0: {
michael@0:   UTFmax        = 4,            // maximum bytes per rune
michael@0:   Runesync      = 0x80,         // cannot represent part of a UTF sequence (<)
michael@0:   Runeself      = 0x80,         // rune and UTF sequences are the same (<)
michael@0:   Runeerror     = 0xFFFD,       // decoding error in UTF
michael@0:   Runemax       = 0x10FFFF,     // maximum rune value
michael@0: };
michael@0: 
michael@0: // Debugging. Not thread safe.
michael@0: static char gDisplayPiece[32];
michael@0: const uint8 gCharlen[16] = {1,1,1,1, 1,1,1,1, 1,1,1,1, 2,2,3,4};
michael@0: char* DisplayPiece(const char* next_byte_, int byte_length_) {
michael@0:   // Copy up to 8 UTF-8 chars to buffer
michael@0:   int k = 0;    // byte count
michael@0:   int n = 0;    // character count
michael@0:   for (int i = 0; i < byte_length_; ++i) {
michael@0:     char c = next_byte_[i];
michael@0:     if ((c & 0xc0) != 0x80) {
michael@0:       // Beginning of a UTF-8 character
michael@0:       int charlen = gCharlen[static_cast<uint8>(c) >> 4];
michael@0:       if (i + charlen > byte_length_) {break;} // Not enough room for full char
michael@0:       if (k >= (32 - 7)) {break;}   // Not necessarily enough room
michael@0:       if (n >= 8) {break;}          // Enough characters already
michael@0:       ++n;
michael@0:     }
michael@0:     if (c == '<') {
michael@0:       memcpy(&gDisplayPiece[k], "&lt;", 4); k += 4;
michael@0:     } else if (c == '>') {
michael@0:       memcpy(&gDisplayPiece[k], "&gt;", 4); k += 4;
michael@0:     } else if (c == '&') {
michael@0:       memcpy(&gDisplayPiece[k], "&amp;", 5); k += 5;
michael@0:     } else if (c == '\'') {
michael@0:       memcpy(&gDisplayPiece[k], "&apos;", 6); k += 6;
michael@0:     } else if (c == '"') {
michael@0:       memcpy(&gDisplayPiece[k], "&quot;", 6); k += 6;
michael@0:     } else {
michael@0:       gDisplayPiece[k++] = c;
michael@0:     }
michael@0:   }
michael@0:   gDisplayPiece[k++] = '\0';
michael@0:   return gDisplayPiece;
michael@0: }
michael@0: 
michael@0: 
michael@0: 
michael@0: // runetochar copies (encodes) one rune, pointed to by r, to at most
michael@0: // UTFmax bytes starting at s and returns the number of bytes generated.
michael@0: int runetochar(char *str, const char32 *rune) {
michael@0:   // Convert to unsigned for range check.
michael@0:   unsigned long c;
michael@0: 
michael@0:   // 1 char 00-7F
michael@0:   c = *rune;
michael@0:   if(c <= 0x7F) {
michael@0:     str[0] = c;
michael@0:     return 1;
michael@0:   }
michael@0: 
michael@0:   // 2 char 0080-07FF
michael@0:   if(c <= 0x07FF) {
michael@0:     str[0] = 0xC0 | (c >> 1*6);
michael@0:     str[1] = 0x80 | (c & 0x3F);
michael@0:     return 2;
michael@0:   }
michael@0: 
michael@0:   // Range check
michael@0:   if (c > Runemax) {
michael@0:     c = Runeerror;
michael@0:   }
michael@0: 
michael@0:   // 3 char 0800-FFFF
michael@0:   if (c <= 0xFFFF) {
michael@0:     str[0] = 0xE0 |  (c >> 2*6);
michael@0:     str[1] = 0x80 | ((c >> 1*6) & 0x3F);
michael@0:     str[2] = 0x80 |  (c & 0x3F);
michael@0:     return 3;
michael@0:   }
michael@0: 
michael@0:   // 4 char 10000-1FFFFF
michael@0:   str[0] = 0xF0 | (c >> 3*6);
michael@0:   str[1] = 0x80 | ((c >> 2*6) & 0x3F);
michael@0:   str[2] = 0x80 | ((c >> 1*6) & 0x3F);
michael@0:   str[3] = 0x80 | (c & 0x3F);
michael@0:   return 4;
michael@0: }
michael@0: 
michael@0: 
michael@0: 
michael@0: // Useful for converting an entity to an ascii value.
michael@0: // RETURNS unicode value, or -1 if entity isn't valid.  Don't include & or ;
michael@0: int LookupEntity(const char* entity_name, int entity_len) {
michael@0:   // Make a C string
michael@0:   if (entity_len >= 16) {return -1;}    // All real entities are shorter
michael@0:   char temp[16];
michael@0:   memcpy(temp, entity_name, entity_len);
michael@0:   temp[entity_len] = '\0';
michael@0:   int match = BinarySearch(temp, 0, kNameToEntitySize, kNameToEntity);
michael@0:   if (match >= 0) {return kNameToEntity[match].i;}
michael@0:   return -1;
michael@0: }
michael@0: 
michael@0: bool ascii_isdigit(char c) {
michael@0:   return ('0' <= c) && (c <= '9');
michael@0: }
michael@0: bool ascii_isxdigit(char c) {
michael@0:   if (('0' <= c) && (c <= '9')) {return true;}
michael@0:   if (('a' <= c) && (c <= 'f')) {return true;}
michael@0:   if (('A' <= c) && (c <= 'F')) {return true;}
michael@0:   return false;
michael@0: }
michael@0: bool ascii_isalnum(char c) {
michael@0:   if (('0' <= c) && (c <= '9')) {return true;}
michael@0:   if (('a' <= c) && (c <= 'z')) {return true;}
michael@0:   if (('A' <= c) && (c <= 'Z')) {return true;}
michael@0:   return false;
michael@0: }
michael@0: int hex_digit_to_int(char c) {
michael@0:   if (('0' <= c) && (c <= '9')) {return c - '0';}
michael@0:   if (('a' <= c) && (c <= 'f')) {return c - 'a' + 10;}
michael@0:   if (('A' <= c) && (c <= 'F')) {return c - 'A' + 10;}
michael@0:   return 0;
michael@0: }
michael@0: 
michael@0: static int32 strto32_base10(const char* nptr, const char* limit,
michael@0:                             const char **endptr) {
michael@0:   *endptr = nptr;
michael@0:   while (nptr < limit && *nptr == '0') {
michael@0:     ++nptr;
michael@0:   }
michael@0:   if (nptr == limit || !ascii_isdigit(*nptr))
michael@0:     return -1;
michael@0:   const char* end_digits_run = nptr;
michael@0:   while (end_digits_run < limit && ascii_isdigit(*end_digits_run)) {
michael@0:     ++end_digits_run;
michael@0:   }
michael@0:   *endptr = end_digits_run;
michael@0:   const int num_digits = end_digits_run - nptr;
michael@0:   // kint32max == 2147483647.
michael@0:   if (num_digits < 9 ||
michael@0:       (num_digits == 10 && memcmp(nptr, "2147483647", 10) <= 0)) {
michael@0:     int value = 0;
michael@0:     for (; nptr < end_digits_run; ++nptr) {
michael@0:       value *= 10;
michael@0:       value += *nptr - '0';
michael@0:     }
michael@0:     // Overflow past the last valid unicode codepoint
michael@0:     // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
michael@0:     return FixUnicodeValue(value);
michael@0:   } else {
michael@0:     // Overflow: can't fit in an int32;
michael@0:     // returns the replacement character 0xFFFD.
michael@0:     return 0xFFFD;
michael@0:   }
michael@0: }
michael@0: 
michael@0: static int32 strto32_base16(const char* nptr, const char* limit,
michael@0:                             const char **endptr) {
michael@0:   *endptr = nptr;
michael@0:   while (nptr < limit && *nptr == '0') {
michael@0:     ++nptr;
michael@0:   }
michael@0:   if (nptr == limit || !ascii_isxdigit(*nptr)) {
michael@0:     return -1;
michael@0:   }
michael@0:   const char* end_xdigits_run = nptr;
michael@0:   while (end_xdigits_run < limit && ascii_isxdigit(*end_xdigits_run)) {
michael@0:     ++end_xdigits_run;
michael@0:   }
michael@0:   *endptr = end_xdigits_run;
michael@0:   const int num_xdigits = end_xdigits_run - nptr;
michael@0:   // kint32max == 0x7FFFFFFF.
michael@0:   if (num_xdigits < 8 || (num_xdigits == 8 && nptr[0] < '8')) {
michael@0:     int value = 0;
michael@0:     for (; nptr < end_xdigits_run; ++nptr) {
michael@0:       value <<= 4;
michael@0:       value += hex_digit_to_int(*nptr);
michael@0:     }
michael@0:     // Overflow past the last valid unicode codepoint
michael@0:     // (0x10ffff) is converted to U+FFFD by FixUnicodeValue().
michael@0:     return FixUnicodeValue(value);
michael@0:   } else {
michael@0:     // Overflow: can't fit in an int32;
michael@0:     // returns the replacement character 0xFFFD.
michael@0:     return 0xFFFD;
michael@0:   }
michael@0: }
michael@0: 
michael@0: // Unescape the current character pointed to by src.  SETS the number
michael@0: // of chars read for the conversion (in UTF8).  If src isn't a valid entity,
michael@0: // just consume the & and RETURN -1.  If src doesn't point to & -- which it
michael@0: // should -- set src_consumed to 0 and RETURN -1.
michael@0: int ReadEntity(const char* src, int srcn, int* src_consumed) {
michael@0:   const char* const srcend = src + srcn;
michael@0: 
michael@0:   if (srcn == 0 || *src != '&') {      // input should start with an ampersand
michael@0:     *src_consumed = 0;
michael@0:     return -1;
michael@0:   }
michael@0:   *src_consumed = 1;                   // we'll get the & at least
michael@0: 
michael@0:   // The standards are a bit unclear on when an entity ends.  Certainly a ";"
michael@0:   // ends one, but spaces probably do too.  We follow the lead of both IE and
michael@0:   // Netscape, which as far as we can tell end numeric entities (1st case below)
michael@0:   // at any non-digit, and end character entities (2nd case) at any non-alnum.
michael@0:   const char* entstart, *entend;  // where the entity starts and ends
michael@0:   entstart = src + 1;             // read past the &
michael@0:   int entval;                     // UCS2 value of the entity
michael@0:   if ( *entstart == '#' ) {       // -- 1st case: numeric entity
michael@0:     if ( entstart + 2 >= srcend ) {
michael@0:       return -1;                  // no way a legitimate number could fit
michael@0:     } else if ( entstart[1] == 'x' || entstart[1] == 'X' ) {   // hex numeric
michael@0:       entval = strto32_base16(entstart + 2, srcend, &entend);
michael@0:     } else {                                  // decimal numeric entity
michael@0:       entval = strto32_base10(entstart+1, srcend, &entend);
michael@0:     }
michael@0:     if (entval == -1 || entend > srcend) {
michael@0:       return -1;                 // not entirely correct, but close enough
michael@0:     }
michael@0:   } else {                       // -- 2nd case: character entity
michael@0:     for (entend = entstart;
michael@0:          entend < srcend && ascii_isalnum(*entend);
michael@0:          ++entend ) {
michael@0:       // entity consists of alphanumeric chars
michael@0:     }
michael@0:     entval = LookupEntity(entstart, entend - entstart);
michael@0:     if (entval < 0) {
michael@0:       return -1;  // not a legal entity name
michael@0:     }
michael@0:     // Now we do a strange-seeming IE6-compatibility check: if entval is
michael@0:     // >= 256, it *must* be followed by a semicolon or it's not considered
michael@0:     // an entity.  The problem is lots of the newfangled entity names, like
michael@0:     // "lang", also occur in URL CGI arguments: "/search?q=test&lang=en".
michael@0:     // When these links are written in HTML, it would be really bad if the
michael@0:     // "&lang" were treated as an entity, which is what the spec says
michael@0:     // *should* happen (even when the HTML is inside an "A HREF" tag!)
michael@0:     // IE ignores the spec for these new, high-value entities, so we do too.
michael@0:     if ( entval >= 256 && !(entend < srcend && *entend == ';') ) {
michael@0:       return -1;                 // make non-;-terminated entity illegal
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Finally, figure out how much src was consumed
michael@0:   if ( entend < srcend && *entend == ';' ) {
michael@0:     entend++;                    // standard says ; terminator is special
michael@0:   }
michael@0:   *src_consumed = entend - src;
michael@0:   return entval;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Src points to '&'
michael@0: // Writes entity value to dst. Returns take(src), put(dst) byte counts
michael@0: void EntityToBuffer(const char* src, int len, char* dst,
michael@0:                     int* tlen, int* plen) {
michael@0:   char32 entval = ReadEntity(src, len, tlen);
michael@0: 
michael@0:   // ReadEntity does this already: entval = FixUnicodeValue(entval);
michael@0: 
michael@0:   // Convert UTF-32 to UTF-8
michael@0:   if (entval > 0) {
michael@0:     *plen = runetochar(dst, &entval);
michael@0:   } else {
michael@0:     // Illegal entity; ignore the '&'
michael@0:     *tlen = 1;
michael@0:     *plen = 0;
michael@0:   }
michael@0: }
michael@0: 
michael@0: // Returns true if character is < > or &, none of which are letters
michael@0: bool inline IsSpecial(char c) {
michael@0:   if ((c & 0xe0) == 0x20) {
michael@0:     return kSpecialSymbol[static_cast<uint8>(c)];
michael@0:   }
michael@0:   return false;
michael@0: }
michael@0: 
michael@0: // Quick Skip to next letter or < > & or to end of string (eos)
michael@0: // Always return is_letter for eos
michael@0: int ScanToLetterOrSpecial(const char* src, int len) {
michael@0:   int bytes_consumed;
michael@0:   StringPiece str(src, len);
michael@0:   UTF8GenericScan(&utf8scannot_lettermarkspecial_obj, str, &bytes_consumed);
michael@0:   return bytes_consumed;
michael@0: }
michael@0: 
michael@0: 
michael@0: 
michael@0: 
michael@0: // src points to non-letter, such as tag-opening '<'
michael@0: // Return length from here to next possible letter
michael@0: // On another < before >, return 1
michael@0: // advances <tag>
michael@0: //          |    |
michael@0: // advances <tag> ... </tag>  for <script> <style>
michael@0: //          |               |
michael@0: // advances <!-- ... <tag> ... -->
michael@0: //          |                     |
michael@0: // advances <tag
michael@0: //          |    | end of string
michael@0: // advances <tag <tag2>
michael@0: //          ||
michael@0: int ScanToPossibleLetter(const char* isrc, int len, int max_exit_state) {
michael@0:   const uint8* src = reinterpret_cast<const uint8*>(isrc);
michael@0:   const uint8* srclimit = src + len;
michael@0:   const uint8* tagParseTbl = kTagParseTbl_0;
michael@0:   int e = 0;
michael@0:   while (src < srclimit) {
michael@0:     e = tagParseTbl[kCharToSub[*src++]];
michael@0:     if (e <= max_exit_state) {
michael@0:       // We overshot by one byte
michael@0:       --src;
michael@0:       break;
michael@0:     }
michael@0:     tagParseTbl = &kTagParseTbl_0[e * 20];
michael@0:   }
michael@0: 
michael@0:   if (src >= srclimit) {
michael@0:     // We fell off the end of the text.
michael@0:     // It looks like the most common case for this is a truncated file, not
michael@0:     // mismatched angle brackets. So we pretend that the last char was '>'
michael@0:     return len;
michael@0:   }
michael@0: 
michael@0:   // OK to be in state 0 or state 2 at exit
michael@0:   if ((e != 0) && (e != 2)) {
michael@0:     // Error, '<' followed by '<'
michael@0:     // We want to back up to first <, then advance by one byte past it
michael@0:     int offset = src - reinterpret_cast<const uint8*>(isrc);
michael@0: 
michael@0:     // Backscan to first '<' and return enough length to just get past it
michael@0:     --offset;   // back up over the second '<', which caused us to stop
michael@0:     while ((0 < offset) && (isrc[offset] != '<')) {
michael@0:       // Find the first '<', which is unmatched
michael@0:       --offset;
michael@0:     }
michael@0:     // skip to just beyond first '<'
michael@0:     return offset + 1;
michael@0:   }
michael@0: 
michael@0:   return src - reinterpret_cast<const uint8*>(isrc);
michael@0: }
michael@0: 
michael@0: 
michael@0: ScriptScanner::ScriptScanner(const char* buffer,
michael@0:                              int buffer_length,
michael@0:                              bool is_plain_text)
michael@0:   : start_byte_(buffer),
michael@0:   next_byte_(buffer),
michael@0:   next_byte_limit_(buffer + buffer_length),
michael@0:   byte_length_(buffer_length),
michael@0:   is_plain_text_(is_plain_text),
michael@0:   letters_marks_only_(true),
michael@0:   one_script_only_(true),
michael@0:   exit_state_(kMaxExitStateLettersMarksOnly) {
michael@0:     script_buffer_ = new char[kMaxScriptBuffer];
michael@0:     script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
michael@0:     map2original_.Clear();    // map from script_buffer_ to buffer
michael@0:     map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
michael@0: }
michael@0: 
michael@0: // Extended version to allow spans of any non-tag text and spans of mixed script
michael@0: ScriptScanner::ScriptScanner(const char* buffer,
michael@0:                              int buffer_length,
michael@0:                              bool is_plain_text,
michael@0:                              bool any_text,
michael@0:                              bool any_script)
michael@0:   : start_byte_(buffer),
michael@0:   next_byte_(buffer),
michael@0:   next_byte_limit_(buffer + buffer_length),
michael@0:   byte_length_(buffer_length),
michael@0:   is_plain_text_(is_plain_text),
michael@0:   letters_marks_only_(!any_text),
michael@0:   one_script_only_(!any_script),
michael@0:   exit_state_(any_text ? kMaxExitStateAllText : kMaxExitStateLettersMarksOnly) {
michael@0:     script_buffer_ = new char[kMaxScriptBuffer];
michael@0:     script_buffer_lower_ = new char[kMaxScriptLowerBuffer];
michael@0:     map2original_.Clear();    // map from script_buffer_ to buffer
michael@0:     map2uplow_.Clear();       // map from script_buffer_lower_ to script_buffer_
michael@0: }
michael@0: 
michael@0: 
michael@0: ScriptScanner::~ScriptScanner() {
michael@0:   delete[] script_buffer_;
michael@0:   delete[] script_buffer_lower_;
michael@0: }
michael@0: 
michael@0: 
michael@0: 
michael@0: 
michael@0: // Get to the first real non-tag letter or entity that is a letter
michael@0: // Sets script of that letter
michael@0: // Return len if no more letters
michael@0: int ScriptScanner::SkipToFrontOfSpan(const char* src, int len, int* script) {
michael@0:   int sc = UNKNOWN_ULSCRIPT;
michael@0:   int skip = 0;
michael@0:   int tlen, plen;
michael@0: 
michael@0:   // Do run of non-letters (tag | &NL | NL)*
michael@0:   tlen = 0;
michael@0:   while (skip < len) {
michael@0:     // Do fast scan to next interesting byte
michael@0:     // int oldskip = skip;
michael@0:     skip += ScanToLetterOrSpecial(src + skip, len - skip);
michael@0: 
michael@0:     // Check for no more letters/specials
michael@0:     if (skip >= len) {
michael@0:       // All done
michael@0:       *script = sc;
michael@0:       return len;
michael@0:     }
michael@0: 
michael@0:     // We are at a letter, nonletter, tag, or entity
michael@0:     if (IsSpecial(src[skip]) && !is_plain_text_) {
michael@0:       if (src[skip] == '<') {
michael@0:         // Begining of tag; skip to end and go around again
michael@0:         tlen = ScanToPossibleLetter(src + skip, len - skip,
michael@0:                                     exit_state_);
michael@0:         sc = 0;
michael@0:       } else if (src[skip] == '>') {
michael@0:         // Unexpected end of tag; skip it and go around again
michael@0:         tlen = 1;         // Over the >
michael@0:         sc = 0;
michael@0:       } else if (src[skip] == '&') {
michael@0:         // Expand entity, no advance
michael@0:         char temp[4];
michael@0:         EntityToBuffer(src + skip, len - skip,
michael@0:                        temp, &tlen, &plen);
michael@0:         sc = GetUTF8LetterScriptNum(temp);
michael@0:       }
michael@0:     } else {
michael@0:       // Update 1..4 bytes
michael@0:       tlen = UTF8OneCharLen(src + skip);
michael@0:       sc = GetUTF8LetterScriptNum(src + skip);
michael@0:     }
michael@0:     if (sc != 0) {break;}           // Letter found
michael@0:     skip += tlen;                   // Else advance
michael@0:   }
michael@0: 
michael@0:   *script = sc;
michael@0:   return skip;
michael@0: }
michael@0: 
michael@0: 
michael@0: // These are for ASCII-only tag names
michael@0: // Compare one letter uplow to c, ignoring case of uplowp
michael@0: inline bool EqCase(char uplow, char c) {
michael@0:   return (uplow | 0x20) == c;
michael@0: }
michael@0: 
michael@0: // These are for ASCII-only tag names
michael@0: // Return true for space / < > etc. all less than 0x40
michael@0: inline bool NeqLetter(char c) {
michael@0:   return c < 0x40;
michael@0: }
michael@0: 
michael@0: // These are for ASCII-only tag names
michael@0: // Return true for space \n false for \r
michael@0: inline bool WS(char c) {
michael@0:   return (c == ' ') || (c == '\n');
michael@0: }
michael@0: 
michael@0: // Canonical CR or LF
michael@0: static const char LF = '\n';
michael@0: 
michael@0: 
michael@0: // The naive loop scans from next_byte_ to script_buffer_ until full.
michael@0: // But this can leave an awkward hard-to-identify short fragment at the
michael@0: // end of the input. We would prefer to make the next-to-last fragment
michael@0: // shorter and the last fragment longer.
michael@0: 
michael@0: // Copy next run of non-tag characters to buffer [NUL terminated]
michael@0: // This just replaces tags with space or \n and removes entities.
michael@0: // Tags <br> <p> and <tr> are replaced with \n. Non-letter sequences
michael@0: // including \r or \n are replaced by \n. All other tags and skipped text
michael@0: // are replaced with ASCII space.
michael@0: //
michael@0: // Buffer ALWAYS has leading space and trailing space space space NUL
michael@0: bool ScriptScanner::GetOneTextSpan(LangSpan* span) {
michael@0:   span->text = script_buffer_;
michael@0:   span->text_bytes = 0;
michael@0:   span->offset = next_byte_ - start_byte_;
michael@0:   span->ulscript = UNKNOWN_ULSCRIPT;
michael@0:   span->lang = UNKNOWN_LANGUAGE;
michael@0:   span->truncated = false;
michael@0: 
michael@0:   int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
michael@0:   if ((kMaxScriptBytes <= byte_length_) &&
michael@0:       (byte_length_ < (2 * kMaxScriptBytes))) {
michael@0:     // Try to split the last two fragments in half
michael@0:     put_soft_limit = byte_length_ / 2;
michael@0:   }
michael@0: 
michael@0:   script_buffer_[0] = ' ';  // Always a space at front of output
michael@0:   script_buffer_[1] = '\0';
michael@0:   int take = 0;
michael@0:   int put = 1;              // Start after the initial space
michael@0:   int tlen, plen;
michael@0: 
michael@0:   if (byte_length_ <= 0) {
michael@0:     return false;          // No more text to be found
michael@0:   }
michael@0: 
michael@0:   // Go over alternating spans of text and tags,
michael@0:   // copying letters to buffer with single spaces for each run of non-letters
michael@0:   bool last_byte_was_space = false;
michael@0:   while (take < byte_length_) {
michael@0:     char c = next_byte_[take];
michael@0:     if (c == '\r') {c = LF;}      // Canonical CR or LF
michael@0:     if (c == '\n') {c = LF;}      // Canonical CR or LF
michael@0: 
michael@0:     if (IsSpecial(c) && !is_plain_text_) {
michael@0:       if (c == '<') {
michael@0:         // Replace tag with space
michael@0:         c = ' ';                      // for almost-full test below
michael@0:         // or if <p> <br> <tr>, replace with \n
michael@0:         if (take < (byte_length_ - 3)) {
michael@0:           if (EqCase(next_byte_[take + 1], 'p') &&
michael@0:               NeqLetter(next_byte_[take + 2])) {
michael@0:             c = LF;
michael@0:           }
michael@0:           if (EqCase(next_byte_[take + 1], 'b') &&
michael@0:               EqCase(next_byte_[take + 2], 'r') &&
michael@0:               NeqLetter(next_byte_[take + 3])) {
michael@0:             c = LF;
michael@0:           }
michael@0:           if (EqCase(next_byte_[take + 1], 't') &&
michael@0:               EqCase(next_byte_[take + 2], 'r') &&
michael@0:               NeqLetter(next_byte_[take + 3])) {
michael@0:             c = LF;
michael@0:           }
michael@0:         }
michael@0:         // Begining of tag; skip to end and go around again
michael@0:         tlen = 1 + ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
michael@0:                                     exit_state_);
michael@0:         // Copy one byte, compressing spaces
michael@0:         if (!last_byte_was_space || !WS(c)) {
michael@0:           script_buffer_[put++] = c;      // Advance dest
michael@0:           last_byte_was_space = WS(c);
michael@0:         }
michael@0:       } else if (c == '>') {
michael@0:         // Unexpected end of tag; copy it and go around again
michael@0:         tlen = 1;         // Over the >
michael@0:         script_buffer_[put++] = c;    // Advance dest
michael@0:       } else if (c == '&') {
michael@0:         // Expand entity, no advance
michael@0:         EntityToBuffer(next_byte_ + take, byte_length_ - take,
michael@0:                        script_buffer_ + put, &tlen, &plen);
michael@0:         put += plen;                  // Advance dest
michael@0:       }
michael@0:       take += tlen;                   // Advance source
michael@0:     } else {
michael@0:       // Copy one byte, compressing spaces
michael@0:       if (!last_byte_was_space || !WS(c)) {
michael@0:         script_buffer_[put++] = c;      // Advance dest
michael@0:         last_byte_was_space = WS(c);
michael@0:       }
michael@0:       ++take;                         // Advance source
michael@0:     }
michael@0: 
michael@0:     if (WS(c) &&
michael@0:         (put >= put_soft_limit)) {
michael@0:       // Buffer is almost full
michael@0:       span->truncated = true;
michael@0:       break;
michael@0:     }
michael@0:     if (put >= kMaxScriptBytes) {
michael@0:       // Buffer is completely full
michael@0:       span->truncated = true;
michael@0:       break;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Almost done. Back up to a character boundary if needed
michael@0:   while ((0 < take) && ((next_byte_[take] & 0xc0) == 0x80)) {
michael@0:     // Back up over continuation byte
michael@0:     --take;
michael@0:     --put;
michael@0:   }
michael@0: 
michael@0:   // Update input position
michael@0:   next_byte_ += take;
michael@0:   byte_length_ -= take;
michael@0: 
michael@0:   // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
michael@0:   //                          kMaxScriptBytes |   | put
michael@0:   script_buffer_[put + 0] = ' ';
michael@0:   script_buffer_[put + 1] = ' ';
michael@0:   script_buffer_[put + 2] = ' ';
michael@0:   script_buffer_[put + 3] = '\0';
michael@0: 
michael@0:   span->text_bytes = put;       // Does not include the last four chars above
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Copy next run of same-script non-tag letters to buffer [NUL terminated]
michael@0: // Buffer ALWAYS has leading space and trailing space space space NUL
michael@0: bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
michael@0:   if (!letters_marks_only_) {
michael@0:     // Return non-tag text, including punctuation and digits
michael@0:     return GetOneTextSpan(span);
michael@0:   }
michael@0: 
michael@0:   span->text = script_buffer_;
michael@0:   span->text_bytes = 0;
michael@0:   span->offset = next_byte_ - start_byte_;
michael@0:   span->ulscript = UNKNOWN_ULSCRIPT;
michael@0:   span->lang = UNKNOWN_LANGUAGE;
michael@0:   span->truncated = false;
michael@0: 
michael@0:   // struct timeval script_start, script_mid, script_end;
michael@0: 
michael@0:   int put_soft_limit = kMaxScriptBytes - kWithinScriptTail;
michael@0:   if ((kMaxScriptBytes <= byte_length_) &&
michael@0:       (byte_length_ < (2 * kMaxScriptBytes))) {
michael@0:     // Try to split the last two fragments in half
michael@0:     put_soft_limit = byte_length_ / 2;
michael@0:   }
michael@0: 
michael@0: 
michael@0:   int spanscript;           // The script of this span
michael@0:   int sc = UNKNOWN_ULSCRIPT;  // The script of next character
michael@0:   int tlen = 0;
michael@0:   int plen = 0;
michael@0: 
michael@0:   script_buffer_[0] = ' ';  // Always a space at front of output
michael@0:   script_buffer_[1] = '\0';
michael@0:   int take = 0;
michael@0:   int put = 1;              // Start after the initial space
michael@0: 
michael@0:   // Build offsets from span->text back to start_byte_ + span->offset
michael@0:   // This mapping reflects deletion of non-letters, expansion of
michael@0:   // entities, etc.
michael@0:   map2original_.Clear();
michael@0:   map2original_.Delete(span->offset);   // So that MapBack(0) gives offset
michael@0: 
michael@0:   // Get to the first real non-tag letter or entity that is a letter
michael@0:   int skip = SkipToFrontOfSpan(next_byte_, byte_length_, &spanscript);
michael@0:   next_byte_ += skip;
michael@0:   byte_length_ -= skip;
michael@0: 
michael@0:   if (skip != 1) {
michael@0:     map2original_.Delete(skip);
michael@0:     map2original_.Insert(1);
michael@0:   } else {
michael@0:     map2original_.Copy(1);
michael@0:   }
michael@0:   if (byte_length_ <= 0) {
michael@0:     map2original_.Reset();
michael@0:     return false;               // No more letters to be found
michael@0:   }
michael@0: 
michael@0:   // There is at least one letter, so we know the script for this span
michael@0:   span->ulscript = (ULScript)spanscript;
michael@0: 
michael@0: 
michael@0:   // Go over alternating spans of same-script letters and non-letters,
michael@0:   // copying letters to buffer with single spaces for each run of non-letters
michael@0:   while (take < byte_length_) {
michael@0:     // Copy run of letters in same script (&LS | LS)*
michael@0:     int letter_count = 0;              // Keep track of word length
michael@0:     bool need_break = false;
michael@0: 
michael@0:     while (take < byte_length_) {
michael@0:       // We are at a letter, nonletter, tag, or entity
michael@0:       if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
michael@0:         if (next_byte_[take] == '<') {
michael@0:           // Begining of tag
michael@0:           sc = 0;
michael@0:           break;
michael@0:         } else if (next_byte_[take] == '>') {
michael@0:           // Unexpected end of tag
michael@0:           sc = 0;
michael@0:           break;
michael@0:         } else if (next_byte_[take] == '&') {
michael@0:           // Copy entity, no advance
michael@0:           EntityToBuffer(next_byte_ + take, byte_length_ - take,
michael@0:                          script_buffer_ + put, &tlen, &plen);
michael@0:           sc = GetUTF8LetterScriptNum(script_buffer_ + put);
michael@0:         }
michael@0:       } else {
michael@0:         // Real letter, safely copy up to 4 bytes, increment by 1..4
michael@0:         // Will update by 1..4 bytes at Advance, below
michael@0:         tlen = plen = UTF8OneCharLen(next_byte_ + take);
michael@0:         if (take < (byte_length_ - 3)) {
michael@0:           // X86 fast case, does unaligned load/store
michael@0:           UNALIGNED_STORE32(script_buffer_ + put,
michael@0:                             UNALIGNED_LOAD32(next_byte_ + take));
michael@0: 
michael@0:         } else {
michael@0:           // Slow case, happens 1-3 times per input document
michael@0:           memcpy(script_buffer_ + put, next_byte_ + take, plen);
michael@0:         }
michael@0:         sc = GetUTF8LetterScriptNum(next_byte_ + take);
michael@0:       }
michael@0: 
michael@0:       // Allow continue across a single letter in a different script:
michael@0:       // A B D = three scripts, c = common script, i = inherited script,
michael@0:       // - = don't care, ( = take position before the += below
michael@0:       //  AAA(A-    continue
michael@0:       //
michael@0:       //  AAA(BA    continue
michael@0:       //  AAA(BB    break
michael@0:       //  AAA(Bc    continue (breaks after B)
michael@0:       //  AAA(BD    break
michael@0:       //  AAA(Bi    break
michael@0:       //
michael@0:       //  AAA(c-    break
michael@0:       //
michael@0:       //  AAA(i-    continue
michael@0:       //
michael@0: 
michael@0:       if ((sc != spanscript) && (sc != ULScript_Inherited)) {
michael@0:         // Might need to break this script span
michael@0:         if (sc == ULScript_Common) {
michael@0:           need_break = true;
michael@0:         } else {
michael@0:           // Look at next following character, ignoring entity as Common
michael@0:           int sc2 = GetUTF8LetterScriptNum(next_byte_ + take + tlen);
michael@0:           if ((sc2 != ULScript_Common) && (sc2 != spanscript)) {
michael@0:             // We found a non-trivial change of script
michael@0:             if (one_script_only_) {
michael@0:               need_break = true;
michael@0:             }
michael@0:           }
michael@0:         }
michael@0:       }
michael@0:       if (need_break) {break;}  // Non-letter or letter in wrong script
michael@0: 
michael@0:       take += tlen;                   // Advance
michael@0:       put += plen;                    // Advance
michael@0: 
michael@0:       // Update the offset map to reflect take/put lengths
michael@0:       if (tlen == plen) {
michael@0:         map2original_.Copy(tlen);
michael@0:       } else if (tlen < plen) {
michael@0:         map2original_.Copy(tlen);
michael@0:         map2original_.Insert(plen - tlen);
michael@0:       } else {    // plen < tlen
michael@0:         map2original_.Copy(plen);
michael@0:         map2original_.Delete(tlen - plen);
michael@0:       }
michael@0: 
michael@0:       ++letter_count;
michael@0:       if (put >= kMaxScriptBytes) {
michael@0:         // Buffer is full
michael@0:         span->truncated = true;
michael@0:         break;
michael@0:       }
michael@0:     }     // End while letters
michael@0: 
michael@0:     // Do run of non-letters (tag | &NL | NL)*
michael@0:     while (take < byte_length_) {
michael@0:       // Do fast scan to next interesting byte
michael@0:       tlen = ScanToLetterOrSpecial(next_byte_ + take, byte_length_ - take);
michael@0:       take += tlen;
michael@0:       map2original_.Delete(tlen);
michael@0:       if (take >= byte_length_) {break;}    // Might have scanned to end
michael@0: 
michael@0:       // We are at a letter, nonletter, tag, or entity
michael@0:       if (IsSpecial(next_byte_[take]) && !is_plain_text_) {
michael@0:         if (next_byte_[take] == '<') {
michael@0:           // Begining of tag; skip to end and go around again
michael@0:           tlen = ScanToPossibleLetter(next_byte_ + take, byte_length_ - take,
michael@0:                                       exit_state_);
michael@0:           sc = 0;
michael@0:         } else if (next_byte_[take] == '>') {
michael@0:           // Unexpected end of tag; skip it and go around again
michael@0:           tlen = 1;         // Over the >
michael@0:           sc = 0;
michael@0:         } else if (next_byte_[take] == '&') {
michael@0:           // Expand entity, no advance
michael@0:           EntityToBuffer(next_byte_ + take, byte_length_ - take,
michael@0:                          script_buffer_ + put, &tlen, &plen);
michael@0:           sc = GetUTF8LetterScriptNum(script_buffer_ + put);
michael@0:         }
michael@0:       } else {
michael@0:         // Update 1..4
michael@0:         tlen = UTF8OneCharLen(next_byte_ + take);
michael@0:         sc = GetUTF8LetterScriptNum(next_byte_ + take);
michael@0:       }
michael@0:       if (sc != 0) {break;}           // Letter found
michael@0:       take += tlen;                   // Else advance
michael@0:       map2original_.Delete(tlen);
michael@0:     }     // End while not-letters
michael@0: 
michael@0:     script_buffer_[put++] = ' ';
michael@0:     map2original_.Insert(1);
michael@0: 
michael@0:     // Letter in wrong script ?
michael@0:     if ((sc != spanscript) && (sc != ULScript_Inherited)) {break;}
michael@0:     if (put >= put_soft_limit) {
michael@0:       // Buffer is almost full
michael@0:       span->truncated = true;
michael@0:       break;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Almost done. Back up to a character boundary if needed
michael@0:   while ((0 < take) && (take < byte_length_) &&
michael@0:          ((next_byte_[take] & 0xc0) == 0x80)) {
michael@0:     // Back up over continuation byte
michael@0:     --take;
michael@0:     --put;
michael@0:   }
michael@0: 
michael@0:   // Update input position
michael@0:   next_byte_ += take;
michael@0:   byte_length_ -= take;
michael@0: 
michael@0:   // Put four more spaces/NUL. Worst case is abcd _ _ _ \0
michael@0:   //                          kMaxScriptBytes |   | put
michael@0:   script_buffer_[put + 0] = ' ';
michael@0:   script_buffer_[put + 1] = ' ';
michael@0:   script_buffer_[put + 2] = ' ';
michael@0:   script_buffer_[put + 3] = '\0';
michael@0:   map2original_.Insert(4);
michael@0:   map2original_.Reset();
michael@0: 
michael@0:   span->text_bytes = put;       // Does not include the last four chars above
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: // Force Latin, Cyrillic, Armenian, Greek scripts to be lowercase
michael@0: // List changes with each version of Unicode, so just always lowercase
michael@0: // Unicode 6.2.0:
michael@0: //   ARMENIAN COPTIC CYRILLIC DESERET GEORGIAN GLAGOLITIC GREEK LATIN
michael@0: void ScriptScanner::LowerScriptSpan(LangSpan* span) {
michael@0:   // If needed, lowercase all the text. If we do it sooner, might miss
michael@0:   // lowercasing an entity such as &Aacute;
michael@0:   // We only need to do this for Latn and Cyrl scripts
michael@0:   map2uplow_.Clear();
michael@0:   // Full Unicode lowercase of the entire buffer, including
michael@0:   // four pad bytes off the end.
michael@0:   // Ahhh. But the last byte 0x00 is not interchange-valid, so we do 3 pad
michael@0:   // bytes and put the 0x00 in explicitly.
michael@0:   // Build an offset map from script_buffer_lower_ back to script_buffer_
michael@0:   int consumed, filled, changed;
michael@0:   StringPiece istr(span->text, span->text_bytes + 3);
michael@0:   StringPiece ostr(script_buffer_lower_, kMaxScriptLowerBuffer);
michael@0: 
michael@0:   UTF8GenericReplace(&utf8repl_lettermarklower_obj,
michael@0:                             istr, ostr, is_plain_text_,
michael@0:                             &consumed, &filled, &changed, &map2uplow_);
michael@0:   script_buffer_lower_[filled] = '\0';
michael@0:   span->text = script_buffer_lower_;
michael@0:   span->text_bytes = filled - 3;
michael@0:   map2uplow_.Reset();
michael@0: }
michael@0: 
michael@0: // Copy next run of same-script non-tag letters to buffer [NUL terminated]
michael@0: // Force Latin, Cyrillic, Greek scripts to be lowercase
michael@0: // Buffer ALWAYS has leading space and trailing space space space NUL
michael@0: bool ScriptScanner::GetOneScriptSpanLower(LangSpan* span) {
michael@0:   bool ok = GetOneScriptSpan(span);
michael@0:   LowerScriptSpan(span);
michael@0:   return ok;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Maps byte offset in most recent GetOneScriptSpan/Lower
michael@0: // span->text [0..text_bytes] into an additional byte offset from
michael@0: // span->offset, to get back to corresponding text in the original
michael@0: // input buffer.
michael@0: // text_offset must be the first byte
michael@0: // of a UTF-8 character, or just beyond the last character. Normally this
michael@0: // routine is called with the first byte of an interesting range and
michael@0: // again with the first byte of the following range.
michael@0: int ScriptScanner::MapBack(int text_offset) {
michael@0:   return map2original_.MapBack(map2uplow_.MapBack(text_offset));
michael@0: }
michael@0: 
michael@0: 
michael@0: // Gets lscript number for letters; always returns
michael@0: //   0 (common script) for non-letters
michael@0: int GetUTF8LetterScriptNum(const char* src) {
michael@0:   int srclen = UTF8OneCharLen(src);
michael@0:   const uint8* usrc = reinterpret_cast<const uint8*>(src);
michael@0:   return UTF8GenericPropertyTwoByte(&utf8prop_lettermarkscriptnum_obj,
michael@0:                                     &usrc, &srclen);
michael@0: }
michael@0: 
michael@0: }  // namespace CLD2
michael@0: 
michael@0: