michael@0: /*
michael@0: //
michael@0: // Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
michael@0: // Use of this source code is governed by a BSD-style license that can be
michael@0: // found in the LICENSE file.
michael@0: //
michael@0: 
michael@0: This file contains the Lex specification for GLSL ES preprocessor.
michael@0: Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
michael@0: http://msdn.microsoft.com/en-us/library/2scxys89.aspx
michael@0: 
michael@0: IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
michael@0: */
michael@0: 
michael@0: %top{
michael@0: //
michael@0: // Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
michael@0: // Use of this source code is governed by a BSD-style license that can be
michael@0: // found in the LICENSE file.
michael@0: //
michael@0: 
michael@0: // This file is auto-generated by generate_parser.sh. DO NOT EDIT!
michael@0: }
michael@0: 
michael@0: %{
michael@0: #include "Tokenizer.h"
michael@0: 
michael@0: #include "DiagnosticsBase.h"
michael@0: #include "Token.h"
michael@0: 
michael@0: #if defined(__GNUC__)
michael@0: // Triggered by the auto-generated yy_fatal_error function.
michael@0: #pragma GCC diagnostic ignored "-Wmissing-noreturn"
michael@0: #endif
michael@0: 
michael@0: typedef std::string YYSTYPE;
michael@0: typedef pp::SourceLocation YYLTYPE;
michael@0: 
michael@0: // Use the unused yycolumn variable to track file (string) number.
michael@0: #define yyfileno yycolumn
michael@0: 
michael@0: #define YY_USER_INIT                   \
michael@0:     do {                               \
michael@0:         yyfileno = 0;                  \
michael@0:         yylineno = 1;                  \
michael@0:         yyextra->leadingSpace = false; \
michael@0:         yyextra->lineStart = true;     \
michael@0:     } while(0);
michael@0: 
michael@0: #define YY_USER_ACTION                                              \
michael@0:     do                                                              \
michael@0:     {                                                               \
michael@0:         pp::Input* input = &yyextra->input;                         \
michael@0:         pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
michael@0:         while ((scanLoc->sIndex < input->count()) &&                \
michael@0:                (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
michael@0:         {                                                           \
michael@0:             scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
michael@0:             ++yyfileno; yylineno = 1;                               \
michael@0:         }                                                           \
michael@0:         yylloc->file = yyfileno;                                    \
michael@0:         yylloc->line = yylineno;                                    \
michael@0:         scanLoc->cIndex += yyleng;                                  \
michael@0:     } while(0);
michael@0: 
michael@0: #define YY_INPUT(buf, result, maxSize) \
michael@0:     result = yyextra->input.read(buf, maxSize);
michael@0: 
michael@0: %}
michael@0: 
michael@0: %option noyywrap nounput never-interactive
michael@0: %option reentrant bison-bridge bison-locations
michael@0: %option prefix="pp"
michael@0: %option extra-type="pp::Tokenizer::Context*"
michael@0: %x COMMENT
michael@0: 
michael@0: NEWLINE     \n|\r|\r\n
michael@0: IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
michael@0: PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]
michael@0: 
michael@0: DECIMAL_CONSTANT      [1-9][0-9]*
michael@0: OCTAL_CONSTANT        0[0-7]*
michael@0: HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+
michael@0: 
michael@0: DIGIT                [0-9]
michael@0: EXPONENT_PART        [eE][+-]?{DIGIT}+
michael@0: FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
michael@0: 
michael@0: %%
michael@0: 
michael@0:     /* Line comment */
michael@0: "//"[^\r\n]*
michael@0: 
michael@0:     /* Block comment */
michael@0:     /* Line breaks are just counted - not returned. */
michael@0:     /* The comment is replaced by a single space. */ 
michael@0: "/*" { BEGIN(COMMENT); }
michael@0: <COMMENT>[^*\r\n]+
michael@0: <COMMENT>"*"
michael@0: <COMMENT>{NEWLINE} { ++yylineno; }
michael@0: <COMMENT>"*/" {
michael@0:     yyextra->leadingSpace = true;
michael@0:     BEGIN(INITIAL);
michael@0: }
michael@0: 
michael@0: # {
michael@0:     // # is only valid at start of line for preprocessor directives.
michael@0:     yylval->assign(1, yytext[0]);
michael@0:     return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
michael@0: }
michael@0: 
michael@0: {IDENTIFIER} {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::IDENTIFIER;
michael@0: }
michael@0: 
michael@0: {DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::CONST_INT;
michael@0: }
michael@0: 
michael@0: ({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::CONST_FLOAT;
michael@0: }
michael@0: 
michael@0:     /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
michael@0:     /* Rule to catch all invalid integers and floats. */
michael@0: ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::PP_NUMBER;
michael@0: }
michael@0: 
michael@0: "++" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_INC;
michael@0: }
michael@0: "--" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_DEC;
michael@0: }
michael@0: "<<" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_LEFT;
michael@0: }
michael@0: ">>" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_RIGHT;
michael@0: }
michael@0: "<=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_LE;
michael@0: }
michael@0: ">=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_GE;
michael@0: }
michael@0: "==" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_EQ;
michael@0: }
michael@0: "!=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_NE;
michael@0: }
michael@0: "&&" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_AND;
michael@0: }
michael@0: "^^" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_XOR;
michael@0: }
michael@0: "||" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_OR;
michael@0: }
michael@0: "+=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_ADD_ASSIGN;
michael@0: }
michael@0: "-=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_SUB_ASSIGN;
michael@0: }
michael@0: "*=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_MUL_ASSIGN;
michael@0: }
michael@0: "/=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_DIV_ASSIGN;
michael@0: }
michael@0: "%=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_MOD_ASSIGN;
michael@0: }
michael@0: "<<=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_LEFT_ASSIGN;
michael@0: }
michael@0: ">>=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_RIGHT_ASSIGN;
michael@0: }
michael@0: "&=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_AND_ASSIGN;
michael@0: }
michael@0: "^=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_XOR_ASSIGN;
michael@0: }
michael@0: "|=" {
michael@0:     yylval->assign(yytext, yyleng);
michael@0:     return pp::Token::OP_OR_ASSIGN;
michael@0: }
michael@0: 
michael@0: {PUNCTUATOR} {
michael@0:     yylval->assign(1, yytext[0]);
michael@0:     return yytext[0];
michael@0: }
michael@0: 
michael@0: [ \t\v\f]+   { yyextra->leadingSpace = true; }
michael@0: 
michael@0: {NEWLINE} {
michael@0:     ++yylineno;
michael@0:     yylval->assign(1, '\n');
michael@0:     return '\n';
michael@0: }
michael@0: 
michael@0: \\{NEWLINE} { ++yylineno; }
michael@0: 
michael@0: . {
michael@0:     yylval->assign(1, yytext[0]);
michael@0:     return pp::Token::PP_OTHER;
michael@0: }
michael@0: 
michael@0: <*><<EOF>> {
michael@0:     // YY_USER_ACTION is not invoked for handling EOF.
michael@0:     // Set the location for EOF token manually.
michael@0:     pp::Input* input = &yyextra->input;
michael@0:     pp::Input::Location* scanLoc = &yyextra->scanLoc;
michael@0:     yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
michael@0:     if (scanLoc->sIndex != sIndexMax)
michael@0:     {
michael@0:         // We can only reach here if there are empty strings at the
michael@0:         // end of the input.
michael@0:         scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
michael@0:         // FIXME: this is not 64-bit clean.
michael@0:         yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
michael@0:     }
michael@0:     yylloc->file = yyfileno;
michael@0:     yylloc->line = yylineno;
michael@0:     yylval->clear();
michael@0: 
michael@0:     if (YY_START == COMMENT)
michael@0:     {
michael@0:         yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
michael@0:                                      pp::SourceLocation(yyfileno, yylineno),
michael@0:                                      "");
michael@0:     }
michael@0:     yyterminate();
michael@0: }
michael@0: 
michael@0: %%
michael@0: 
michael@0: namespace pp {
michael@0: 
michael@0: // TODO(alokp): Maximum token length should ideally be specified by
michael@0: // the preprocessor client, i.e., the compiler.
michael@0: const size_t Tokenizer::kMaxTokenLength = 256;
michael@0: 
michael@0: Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
michael@0: {
michael@0:     mContext.diagnostics = diagnostics;
michael@0: }
michael@0: 
michael@0: Tokenizer::~Tokenizer()
michael@0: {
michael@0:     destroyScanner();
michael@0: }
michael@0: 
michael@0: bool Tokenizer::init(size_t count, const char* const string[], const int length[])
michael@0: {
michael@0:     if ((count > 0) && (string == 0)) return false;
michael@0: 
michael@0:     mContext.input = Input(count, string, length);
michael@0:     return initScanner();
michael@0: }
michael@0: 
michael@0: void Tokenizer::setFileNumber(int file)
michael@0: {
michael@0:     // We use column number as file number.
michael@0:     // See macro yyfileno.
michael@0:     yyset_column(file, mHandle);
michael@0: }
michael@0: 
michael@0: void Tokenizer::setLineNumber(int line)
michael@0: {
michael@0:     yyset_lineno(line, mHandle);
michael@0: }
michael@0: 
michael@0: void Tokenizer::lex(Token* token)
michael@0: {
michael@0:     token->type = yylex(&token->text, &token->location, mHandle);
michael@0:     if (token->text.size() > kMaxTokenLength)
michael@0:     {
michael@0:         mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
michael@0:                                      token->location, token->text);
michael@0:         token->text.erase(kMaxTokenLength);
michael@0:     }
michael@0: 
michael@0:     token->flags = 0;
michael@0: 
michael@0:     token->setAtStartOfLine(mContext.lineStart);
michael@0:     mContext.lineStart = token->type == '\n';
michael@0: 
michael@0:     token->setHasLeadingSpace(mContext.leadingSpace);
michael@0:     mContext.leadingSpace = false;
michael@0: }
michael@0: 
michael@0: bool Tokenizer::initScanner()
michael@0: {
michael@0:     if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
michael@0:         return false;
michael@0: 
michael@0:     yyrestart(0, mHandle);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: void Tokenizer::destroyScanner()
michael@0: {
michael@0:     if (mHandle == NULL)
michael@0:         return;
michael@0: 
michael@0:     yylex_destroy(mHandle);
michael@0:     mHandle = NULL;
michael@0: }
michael@0: 
michael@0: }  // namespace pp
michael@0: