The Tor Browser: gfx/angle/src/compiler/preprocessor/Tokenizer.l@141e0f1194b1 (annotated)

gfx/angle/src/compiler/preprocessor/Tokenizer.l@141e0f1194b1 (annotated)

gfx/angle/src/compiler/preprocessor/Tokenizer.l

Wed, 31 Dec 2014 07:16:47 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 07:16:47 +0100
branch: TOR_BUG_9701
changeset 3: 141e0f1194b1
permissions: -rw-r--r--

Revert simplistic fix pending revisit of Mozilla integration attempt.

 /*
 //
 // Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 This file contains the Lex specification for GLSL ES preprocessor.
 Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
 http://msdn.microsoft.com/en-us/library/2scxys89.aspx
 IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 */
 %top{
 //
 // Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // This file is auto-generated by generate_parser.sh. DO NOT EDIT!
 }
 %{
 #include "Tokenizer.h"
 #include "DiagnosticsBase.h"
 #include "Token.h"
 #if defined(__GNUC__)
 // Triggered by the auto-generated yy_fatal_error function.
 #pragma GCC diagnostic ignored "-Wmissing-noreturn"
 #endif
 typedef std::string YYSTYPE;
 typedef pp::SourceLocation YYLTYPE;
 // Use the unused yycolumn variable to track file (string) number.
 #define yyfileno yycolumn
 #define YY_USER_INIT                   \
     do {                               \
         yyfileno = 0;                  \
         yylineno = 1;                  \
         yyextra->leadingSpace = false; \
         yyextra->lineStart = true;     \
     } while(0);
 #define YY_USER_ACTION                                              \
     do                                                              \
     {                                                               \
         pp::Input* input = &yyextra->input;                         \
         pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
         while ((scanLoc->sIndex < input->count()) &&                \
                (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
         {                                                           \
             scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
             ++yyfileno; yylineno = 1;                               \
         }                                                           \
         yylloc->file = yyfileno;                                    \
         yylloc->line = yylineno;                                    \
         scanLoc->cIndex += yyleng;                                  \
     } while(0);
 #define YY_INPUT(buf, result, maxSize) \
     result = yyextra->input.read(buf, maxSize);
 %}
 %option noyywrap nounput never-interactive
 %option reentrant bison-bridge bison-locations
 %option prefix="pp"
 %option extra-type="pp::Tokenizer::Context*"
 %x COMMENT
 NEWLINE     \n|\r|\r\n
 IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
 PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]
 DECIMAL_CONSTANT      [1-9][0-9]*
 OCTAL_CONSTANT        0[0-7]*
 HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+
 DIGIT                [0-9]
 EXPONENT_PART        [eE][+-]?{DIGIT}+
 FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 %%
     /* Line comment */
 "//"[^\r\n]*
     /* Block comment */
     /* Line breaks are just counted - not returned. */
     /* The comment is replaced by a single space. */
 "/*" { BEGIN(COMMENT); }
 <COMMENT>[^*\r\n]+
 <COMMENT>"*"
 <COMMENT>{NEWLINE} { ++yylineno; }
 <COMMENT>"*/" {
     yyextra->leadingSpace = true;
     BEGIN(INITIAL);
 }
 # {
     // # is only valid at start of line for preprocessor directives.
     yylval->assign(1, yytext[0]);
     return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
 }
 {IDENTIFIER} {
     yylval->assign(yytext, yyleng);
     return pp::Token::IDENTIFIER;
 }
 {DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
     yylval->assign(yytext, yyleng);
     return pp::Token::CONST_INT;
 }
 ({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
     yylval->assign(yytext, yyleng);
     return pp::Token::CONST_FLOAT;
 }
     /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
     /* Rule to catch all invalid integers and floats. */
 ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
     yylval->assign(yytext, yyleng);
     return pp::Token::PP_NUMBER;
 }
 "++" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_INC;
 }
 "--" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_DEC;
 }
 "<<" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_LEFT;
 }
 ">>" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_RIGHT;
 }
 "<=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_LE;
 }
 ">=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_GE;
 }
 "==" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_EQ;
 }
 "!=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_NE;
 }
 "&&" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_AND;
 }
 "^^" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_XOR;
 }
 "||" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_OR;
 }
 "+=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_ADD_ASSIGN;
 }
 "-=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_SUB_ASSIGN;
 }
 "*=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_MUL_ASSIGN;
 }
 "/=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_DIV_ASSIGN;
 }
 "%=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_MOD_ASSIGN;
 }
 "<<=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_LEFT_ASSIGN;
 }
 ">>=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_RIGHT_ASSIGN;
 }
 "&=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_AND_ASSIGN;
 }
 "^=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_XOR_ASSIGN;
 }
 "|=" {
     yylval->assign(yytext, yyleng);
     return pp::Token::OP_OR_ASSIGN;
 }
 {PUNCTUATOR} {
     yylval->assign(1, yytext[0]);
     return yytext[0];
 }
 [ \t\v\f]+   { yyextra->leadingSpace = true; }
 {NEWLINE} {
     ++yylineno;
     yylval->assign(1, '\n');
     return '\n';
 }
 \\{NEWLINE} { ++yylineno; }
 . {
     yylval->assign(1, yytext[0]);
     return pp::Token::PP_OTHER;
 }
 <*><<EOF>> {
     // YY_USER_ACTION is not invoked for handling EOF.
     // Set the location for EOF token manually.
     pp::Input* input = &yyextra->input;
     pp::Input::Location* scanLoc = &yyextra->scanLoc;
     yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
     if (scanLoc->sIndex != sIndexMax)
     {
         // We can only reach here if there are empty strings at the
         // end of the input.
         scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
         // FIXME: this is not 64-bit clean.
         yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
     }
     yylloc->file = yyfileno;
     yylloc->line = yylineno;
     yylval->clear();
     if (YY_START == COMMENT)
     {
         yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
                                      pp::SourceLocation(yyfileno, yylineno),
                                      "");
     }
     yyterminate();
 }
 %%
 namespace pp {
 // TODO(alokp): Maximum token length should ideally be specified by
 // the preprocessor client, i.e., the compiler.
 const size_t Tokenizer::kMaxTokenLength = 256;
 Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
     mContext.diagnostics = diagnostics;
 }
 Tokenizer::~Tokenizer()
 {
     destroyScanner();
 }
 bool Tokenizer::init(size_t count, const char* const string[], const int length[])
 {
     if ((count > 0) && (string == 0)) return false;
     mContext.input = Input(count, string, length);
     return initScanner();
 }
 void Tokenizer::setFileNumber(int file)
 {
     // We use column number as file number.
     // See macro yyfileno.
     yyset_column(file, mHandle);
 }
 void Tokenizer::setLineNumber(int line)
 {
     yyset_lineno(line, mHandle);
 }
 void Tokenizer::lex(Token* token)
 {
     token->type = yylex(&token->text, &token->location, mHandle);
     if (token->text.size() > kMaxTokenLength)
     {
         mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
                                      token->location, token->text);
         token->text.erase(kMaxTokenLength);
     }
     token->flags = 0;
     token->setAtStartOfLine(mContext.lineStart);
     mContext.lineStart = token->type == '\n';
     token->setHasLeadingSpace(mContext.leadingSpace);
     mContext.leadingSpace = false;
 }
 bool Tokenizer::initScanner()
 {
     if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
         return false;
     yyrestart(0, mHandle);
     return true;
 }
 void Tokenizer::destroyScanner()
 {
     if (mHandle == NULL)
         return;
     yylex_destroy(mHandle);
     mHandle = NULL;
 }
 }  // namespace pp

The Tor Browser / annotate

gfx/angle/src/compiler/preprocessor/Tokenizer.l@141e0f1194b1 (annotated)

gfx/angle/src/compiler/preprocessor/Tokenizer.l