The Tor Browser: gfx/angle/src/compiler/preprocessor/Tokenizer.l@141e0f1194b1

     1 /*

     2 //

     3 // Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.

     4 // Use of this source code is governed by a BSD-style license that can be

     5 // found in the LICENSE file.

     6 //

     8 This file contains the Lex specification for GLSL ES preprocessor.

     9 Based on Microsoft Visual Studio 2010 Preprocessor Grammar:

    10 http://msdn.microsoft.com/en-us/library/2scxys89.aspx

    12 IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.

    13 */

    15 %top{

    16 //

    17 // Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.

    18 // Use of this source code is governed by a BSD-style license that can be

    19 // found in the LICENSE file.

    20 //

    22 // This file is auto-generated by generate_parser.sh. DO NOT EDIT!

    23 }

    25 %{

    26 #include "Tokenizer.h"

    28 #include "DiagnosticsBase.h"

    29 #include "Token.h"

    31 #if defined(__GNUC__)

    32 // Triggered by the auto-generated yy_fatal_error function.

    33 #pragma GCC diagnostic ignored "-Wmissing-noreturn"

    34 #endif

    36 typedef std::string YYSTYPE;

    37 typedef pp::SourceLocation YYLTYPE;

    39 // Use the unused yycolumn variable to track file (string) number.

    40 #define yyfileno yycolumn

    42 #define YY_USER_INIT                   \

    43     do {                               \

    44         yyfileno = 0;                  \

    45         yylineno = 1;                  \

    46         yyextra->leadingSpace = false; \

    47         yyextra->lineStart = true;     \

    48     } while(0);

    50 #define YY_USER_ACTION                                              \

    51     do                                                              \

    52     {                                                               \

    53         pp::Input* input = &yyextra->input;                         \

    54         pp::Input::Location* scanLoc = &yyextra->scanLoc;           \

    55         while ((scanLoc->sIndex < input->count()) &&                \

    56                (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \

    57         {                                                           \

    58             scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \

    59             ++yyfileno; yylineno = 1;                               \

    60         }                                                           \

    61         yylloc->file = yyfileno;                                    \

    62         yylloc->line = yylineno;                                    \

    63         scanLoc->cIndex += yyleng;                                  \

    64     } while(0);

    66 #define YY_INPUT(buf, result, maxSize) \

    67     result = yyextra->input.read(buf, maxSize);

    69 %}

    71 %option noyywrap nounput never-interactive

    72 %option reentrant bison-bridge bison-locations

    73 %option prefix="pp"

    74 %option extra-type="pp::Tokenizer::Context*"

    75 %x COMMENT

    77 NEWLINE     \n|\r|\r\n

    78 IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*

    79 PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]

    81 DECIMAL_CONSTANT      [1-9][0-9]*

    82 OCTAL_CONSTANT        0[0-7]*

    83 HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+

    85 DIGIT                [0-9]

    86 EXPONENT_PART        [eE][+-]?{DIGIT}+

    87 FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")

    89 %%

    91     /* Line comment */

    92 "//"[^\r\n]*

    94     /* Block comment */

    95     /* Line breaks are just counted - not returned. */

    96     /* The comment is replaced by a single space. */

    97 "/*" { BEGIN(COMMENT); }

    98 <COMMENT>[^*\r\n]+

    99 <COMMENT>"*"

   100 <COMMENT>{NEWLINE} { ++yylineno; }

   101 <COMMENT>"*/" {

   102     yyextra->leadingSpace = true;

   103     BEGIN(INITIAL);

   104 }

   106 # {

   107     // # is only valid at start of line for preprocessor directives.

   108     yylval->assign(1, yytext[0]);

   109     return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;

   110 }

   112 {IDENTIFIER} {

   113     yylval->assign(yytext, yyleng);

   114     return pp::Token::IDENTIFIER;

   115 }

   117 {DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {

   118     yylval->assign(yytext, yyleng);

   119     return pp::Token::CONST_INT;

   120 }

   122 ({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {

   123     yylval->assign(yytext, yyleng);

   124     return pp::Token::CONST_FLOAT;

   125 }

   127     /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */

   128     /* Rule to catch all invalid integers and floats. */

   129 ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {

   130     yylval->assign(yytext, yyleng);

   131     return pp::Token::PP_NUMBER;

   132 }

   134 "++" {

   135     yylval->assign(yytext, yyleng);

   136     return pp::Token::OP_INC;

   137 }

   138 "--" {

   139     yylval->assign(yytext, yyleng);

   140     return pp::Token::OP_DEC;

   141 }

   142 "<<" {

   143     yylval->assign(yytext, yyleng);

   144     return pp::Token::OP_LEFT;

   145 }

   146 ">>" {

   147     yylval->assign(yytext, yyleng);

   148     return pp::Token::OP_RIGHT;

   149 }

   150 "<=" {

   151     yylval->assign(yytext, yyleng);

   152     return pp::Token::OP_LE;

   153 }

   154 ">=" {

   155     yylval->assign(yytext, yyleng);

   156     return pp::Token::OP_GE;

   157 }

   158 "==" {

   159     yylval->assign(yytext, yyleng);

   160     return pp::Token::OP_EQ;

   161 }

   162 "!=" {

   163     yylval->assign(yytext, yyleng);

   164     return pp::Token::OP_NE;

   165 }

   166 "&&" {

   167     yylval->assign(yytext, yyleng);

   168     return pp::Token::OP_AND;

   169 }

   170 "^^" {

   171     yylval->assign(yytext, yyleng);

   172     return pp::Token::OP_XOR;

   173 }

   174 "||" {

   175     yylval->assign(yytext, yyleng);

   176     return pp::Token::OP_OR;

   177 }

   178 "+=" {

   179     yylval->assign(yytext, yyleng);

   180     return pp::Token::OP_ADD_ASSIGN;

   181 }

   182 "-=" {

   183     yylval->assign(yytext, yyleng);

   184     return pp::Token::OP_SUB_ASSIGN;

   185 }

   186 "*=" {

   187     yylval->assign(yytext, yyleng);

   188     return pp::Token::OP_MUL_ASSIGN;

   189 }

   190 "/=" {

   191     yylval->assign(yytext, yyleng);

   192     return pp::Token::OP_DIV_ASSIGN;

   193 }

   194 "%=" {

   195     yylval->assign(yytext, yyleng);

   196     return pp::Token::OP_MOD_ASSIGN;

   197 }

   198 "<<=" {

   199     yylval->assign(yytext, yyleng);

   200     return pp::Token::OP_LEFT_ASSIGN;

   201 }

   202 ">>=" {

   203     yylval->assign(yytext, yyleng);

   204     return pp::Token::OP_RIGHT_ASSIGN;

   205 }

   206 "&=" {

   207     yylval->assign(yytext, yyleng);

   208     return pp::Token::OP_AND_ASSIGN;

   209 }

   210 "^=" {

   211     yylval->assign(yytext, yyleng);

   212     return pp::Token::OP_XOR_ASSIGN;

   213 }

   214 "|=" {

   215     yylval->assign(yytext, yyleng);

   216     return pp::Token::OP_OR_ASSIGN;

   217 }

   219 {PUNCTUATOR} {

   220     yylval->assign(1, yytext[0]);

   221     return yytext[0];

   222 }

   224 [ \t\v\f]+   { yyextra->leadingSpace = true; }

   226 {NEWLINE} {

   227     ++yylineno;

   228     yylval->assign(1, '\n');

   229     return '\n';

   230 }

   232 \\{NEWLINE} { ++yylineno; }

   234 . {

   235     yylval->assign(1, yytext[0]);

   236     return pp::Token::PP_OTHER;

   237 }

   239 <*><<EOF>> {

   240     // YY_USER_ACTION is not invoked for handling EOF.

   241     // Set the location for EOF token manually.

   242     pp::Input* input = &yyextra->input;

   243     pp::Input::Location* scanLoc = &yyextra->scanLoc;

   244     yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;

   245     if (scanLoc->sIndex != sIndexMax)

   246     {

   247         // We can only reach here if there are empty strings at the

   248         // end of the input.

   249         scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;

   250         // FIXME: this is not 64-bit clean.

   251         yyfileno = static_cast<int>(sIndexMax); yylineno = 1;

   252     }

   253     yylloc->file = yyfileno;

   254     yylloc->line = yylineno;

   255     yylval->clear();

   257     if (YY_START == COMMENT)

   258     {

   259         yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,

   260                                      pp::SourceLocation(yyfileno, yylineno),

   261                                      "");

   262     }

   263     yyterminate();

   264 }

   266 %%

   268 namespace pp {

   270 // TODO(alokp): Maximum token length should ideally be specified by

   271 // the preprocessor client, i.e., the compiler.

   272 const size_t Tokenizer::kMaxTokenLength = 256;

   274 Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)

   275 {

   276     mContext.diagnostics = diagnostics;

   277 }

   279 Tokenizer::~Tokenizer()

   280 {

   281     destroyScanner();

   282 }

   284 bool Tokenizer::init(size_t count, const char* const string[], const int length[])

   285 {

   286     if ((count > 0) && (string == 0)) return false;

   288     mContext.input = Input(count, string, length);

   289     return initScanner();

   290 }

   292 void Tokenizer::setFileNumber(int file)

   293 {

   294     // We use column number as file number.

   295     // See macro yyfileno.

   296     yyset_column(file, mHandle);

   297 }

   299 void Tokenizer::setLineNumber(int line)

   300 {

   301     yyset_lineno(line, mHandle);

   302 }

   304 void Tokenizer::lex(Token* token)

   305 {

   306     token->type = yylex(&token->text, &token->location, mHandle);

   307     if (token->text.size() > kMaxTokenLength)

   308     {

   309         mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,

   310                                      token->location, token->text);

   311         token->text.erase(kMaxTokenLength);

   312     }

   314     token->flags = 0;

   316     token->setAtStartOfLine(mContext.lineStart);

   317     mContext.lineStart = token->type == '\n';

   319     token->setHasLeadingSpace(mContext.leadingSpace);

   320     mContext.leadingSpace = false;

   321 }

   323 bool Tokenizer::initScanner()

   324 {

   325     if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))

   326         return false;

   328     yyrestart(0, mHandle);

   329     return true;

   330 }

   332 void Tokenizer::destroyScanner()

   333 {

   334     if (mHandle == NULL)

   335         return;

   337     yylex_destroy(mHandle);

   338     mHandle = NULL;

   339 }

   341 }  // namespace pp

The Tor Browser / file revision

gfx/angle/src/compiler/preprocessor/Tokenizer.l@141e0f1194b1

gfx/angle/src/compiler/preprocessor/Tokenizer.l