1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/gfx/angle/src/compiler/preprocessor/Tokenizer.l Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,342 @@
1.4 +/*
1.5 +//
1.6 +// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
1.7 +// Use of this source code is governed by a BSD-style license that can be
1.8 +// found in the LICENSE file.
1.9 +//
1.10 +
1.11 +This file contains the Lex specification for GLSL ES preprocessor.
1.12 +Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
1.13 +http://msdn.microsoft.com/en-us/library/2scxys89.aspx
1.14 +
1.15 +IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
1.16 +*/
1.17 +
1.18 +%top{
1.19 +//
1.20 +// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
1.21 +// Use of this source code is governed by a BSD-style license that can be
1.22 +// found in the LICENSE file.
1.23 +//
1.24 +
1.25 +// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
1.26 +}
1.27 +
1.28 +%{
1.29 +#include "Tokenizer.h"
1.30 +
1.31 +#include "DiagnosticsBase.h"
1.32 +#include "Token.h"
1.33 +
1.34 +#if defined(__GNUC__)
1.35 +// Triggered by the auto-generated yy_fatal_error function.
1.36 +#pragma GCC diagnostic ignored "-Wmissing-noreturn"
1.37 +#endif
1.38 +
1.39 +typedef std::string YYSTYPE;
1.40 +typedef pp::SourceLocation YYLTYPE;
1.41 +
1.42 +// Use the unused yycolumn variable to track file (string) number.
1.43 +#define yyfileno yycolumn
1.44 +
1.45 +#define YY_USER_INIT \
1.46 + do { \
1.47 + yyfileno = 0; \
1.48 + yylineno = 1; \
1.49 + yyextra->leadingSpace = false; \
1.50 + yyextra->lineStart = true; \
1.51 + } while(0);
1.52 +
1.53 +#define YY_USER_ACTION \
1.54 + do \
1.55 + { \
1.56 + pp::Input* input = &yyextra->input; \
1.57 + pp::Input::Location* scanLoc = &yyextra->scanLoc; \
1.58 + while ((scanLoc->sIndex < input->count()) && \
1.59 + (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
1.60 + { \
1.61 + scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
1.62 + ++yyfileno; yylineno = 1; \
1.63 + } \
1.64 + yylloc->file = yyfileno; \
1.65 + yylloc->line = yylineno; \
1.66 + scanLoc->cIndex += yyleng; \
1.67 + } while(0);
1.68 +
1.69 +#define YY_INPUT(buf, result, maxSize) \
1.70 + result = yyextra->input.read(buf, maxSize);
1.71 +
1.72 +%}
1.73 +
1.74 +%option noyywrap nounput never-interactive
1.75 +%option reentrant bison-bridge bison-locations
1.76 +%option prefix="pp"
1.77 +%option extra-type="pp::Tokenizer::Context*"
1.78 +%x COMMENT
1.79 +
1.80 +NEWLINE \n|\r|\r\n
1.81 +IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]*
1.82 +PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?]
1.83 +
1.84 +DECIMAL_CONSTANT [1-9][0-9]*
1.85 +OCTAL_CONSTANT 0[0-7]*
1.86 +HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+
1.87 +
1.88 +DIGIT [0-9]
1.89 +EXPONENT_PART [eE][+-]?{DIGIT}+
1.90 +FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
1.91 +
1.92 +%%
1.93 +
1.94 + /* Line comment */
1.95 +"//"[^\r\n]*
1.96 +
1.97 + /* Block comment */
1.98 + /* Line breaks are just counted - not returned. */
1.99 + /* The comment is replaced by a single space. */
1.100 +"/*" { BEGIN(COMMENT); }
1.101 +<COMMENT>[^*\r\n]+
1.102 +<COMMENT>"*"
1.103 +<COMMENT>{NEWLINE} { ++yylineno; }
1.104 +<COMMENT>"*/" {
1.105 + yyextra->leadingSpace = true;
1.106 + BEGIN(INITIAL);
1.107 +}
1.108 +
1.109 +# {
1.110 + // # is only valid at start of line for preprocessor directives.
1.111 + yylval->assign(1, yytext[0]);
1.112 + return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
1.113 +}
1.114 +
1.115 +{IDENTIFIER} {
1.116 + yylval->assign(yytext, yyleng);
1.117 + return pp::Token::IDENTIFIER;
1.118 +}
1.119 +
1.120 +{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
1.121 + yylval->assign(yytext, yyleng);
1.122 + return pp::Token::CONST_INT;
1.123 +}
1.124 +
1.125 +({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
1.126 + yylval->assign(yytext, yyleng);
1.127 + return pp::Token::CONST_FLOAT;
1.128 +}
1.129 +
1.130 + /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
1.131 + /* Rule to catch all invalid integers and floats. */
1.132 +({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
1.133 + yylval->assign(yytext, yyleng);
1.134 + return pp::Token::PP_NUMBER;
1.135 +}
1.136 +
1.137 +"++" {
1.138 + yylval->assign(yytext, yyleng);
1.139 + return pp::Token::OP_INC;
1.140 +}
1.141 +"--" {
1.142 + yylval->assign(yytext, yyleng);
1.143 + return pp::Token::OP_DEC;
1.144 +}
1.145 +"<<" {
1.146 + yylval->assign(yytext, yyleng);
1.147 + return pp::Token::OP_LEFT;
1.148 +}
1.149 +">>" {
1.150 + yylval->assign(yytext, yyleng);
1.151 + return pp::Token::OP_RIGHT;
1.152 +}
1.153 +"<=" {
1.154 + yylval->assign(yytext, yyleng);
1.155 + return pp::Token::OP_LE;
1.156 +}
1.157 +">=" {
1.158 + yylval->assign(yytext, yyleng);
1.159 + return pp::Token::OP_GE;
1.160 +}
1.161 +"==" {
1.162 + yylval->assign(yytext, yyleng);
1.163 + return pp::Token::OP_EQ;
1.164 +}
1.165 +"!=" {
1.166 + yylval->assign(yytext, yyleng);
1.167 + return pp::Token::OP_NE;
1.168 +}
1.169 +"&&" {
1.170 + yylval->assign(yytext, yyleng);
1.171 + return pp::Token::OP_AND;
1.172 +}
1.173 +"^^" {
1.174 + yylval->assign(yytext, yyleng);
1.175 + return pp::Token::OP_XOR;
1.176 +}
1.177 +"||" {
1.178 + yylval->assign(yytext, yyleng);
1.179 + return pp::Token::OP_OR;
1.180 +}
1.181 +"+=" {
1.182 + yylval->assign(yytext, yyleng);
1.183 + return pp::Token::OP_ADD_ASSIGN;
1.184 +}
1.185 +"-=" {
1.186 + yylval->assign(yytext, yyleng);
1.187 + return pp::Token::OP_SUB_ASSIGN;
1.188 +}
1.189 +"*=" {
1.190 + yylval->assign(yytext, yyleng);
1.191 + return pp::Token::OP_MUL_ASSIGN;
1.192 +}
1.193 +"/=" {
1.194 + yylval->assign(yytext, yyleng);
1.195 + return pp::Token::OP_DIV_ASSIGN;
1.196 +}
1.197 +"%=" {
1.198 + yylval->assign(yytext, yyleng);
1.199 + return pp::Token::OP_MOD_ASSIGN;
1.200 +}
1.201 +"<<=" {
1.202 + yylval->assign(yytext, yyleng);
1.203 + return pp::Token::OP_LEFT_ASSIGN;
1.204 +}
1.205 +">>=" {
1.206 + yylval->assign(yytext, yyleng);
1.207 + return pp::Token::OP_RIGHT_ASSIGN;
1.208 +}
1.209 +"&=" {
1.210 + yylval->assign(yytext, yyleng);
1.211 + return pp::Token::OP_AND_ASSIGN;
1.212 +}
1.213 +"^=" {
1.214 + yylval->assign(yytext, yyleng);
1.215 + return pp::Token::OP_XOR_ASSIGN;
1.216 +}
1.217 +"|=" {
1.218 + yylval->assign(yytext, yyleng);
1.219 + return pp::Token::OP_OR_ASSIGN;
1.220 +}
1.221 +
1.222 +{PUNCTUATOR} {
1.223 + yylval->assign(1, yytext[0]);
1.224 + return yytext[0];
1.225 +}
1.226 +
1.227 +[ \t\v\f]+ { yyextra->leadingSpace = true; }
1.228 +
1.229 +{NEWLINE} {
1.230 + ++yylineno;
1.231 + yylval->assign(1, '\n');
1.232 + return '\n';
1.233 +}
1.234 +
1.235 +\\{NEWLINE} { ++yylineno; }
1.236 +
1.237 +. {
1.238 + yylval->assign(1, yytext[0]);
1.239 + return pp::Token::PP_OTHER;
1.240 +}
1.241 +
1.242 +<*><<EOF>> {
1.243 + // YY_USER_ACTION is not invoked for handling EOF.
1.244 + // Set the location for EOF token manually.
1.245 + pp::Input* input = &yyextra->input;
1.246 + pp::Input::Location* scanLoc = &yyextra->scanLoc;
1.247 + yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
1.248 + if (scanLoc->sIndex != sIndexMax)
1.249 + {
1.250 + // We can only reach here if there are empty strings at the
1.251 + // end of the input.
1.252 + scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
1.253 + // FIXME: this is not 64-bit clean.
1.254 + yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
1.255 + }
1.256 + yylloc->file = yyfileno;
1.257 + yylloc->line = yylineno;
1.258 + yylval->clear();
1.259 +
1.260 + if (YY_START == COMMENT)
1.261 + {
1.262 + yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
1.263 + pp::SourceLocation(yyfileno, yylineno),
1.264 + "");
1.265 + }
1.266 + yyterminate();
1.267 +}
1.268 +
1.269 +%%
1.270 +
1.271 +namespace pp {
1.272 +
1.273 +// TODO(alokp): Maximum token length should ideally be specified by
1.274 +// the preprocessor client, i.e., the compiler.
1.275 +const size_t Tokenizer::kMaxTokenLength = 256;
1.276 +
1.277 +Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
1.278 +{
1.279 + mContext.diagnostics = diagnostics;
1.280 +}
1.281 +
1.282 +Tokenizer::~Tokenizer()
1.283 +{
1.284 + destroyScanner();
1.285 +}
1.286 +
1.287 +bool Tokenizer::init(size_t count, const char* const string[], const int length[])
1.288 +{
1.289 + if ((count > 0) && (string == 0)) return false;
1.290 +
1.291 + mContext.input = Input(count, string, length);
1.292 + return initScanner();
1.293 +}
1.294 +
1.295 +void Tokenizer::setFileNumber(int file)
1.296 +{
1.297 + // We use column number as file number.
1.298 + // See macro yyfileno.
1.299 + yyset_column(file, mHandle);
1.300 +}
1.301 +
1.302 +void Tokenizer::setLineNumber(int line)
1.303 +{
1.304 + yyset_lineno(line, mHandle);
1.305 +}
1.306 +
1.307 +void Tokenizer::lex(Token* token)
1.308 +{
1.309 + token->type = yylex(&token->text, &token->location, mHandle);
1.310 + if (token->text.size() > kMaxTokenLength)
1.311 + {
1.312 + mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
1.313 + token->location, token->text);
1.314 + token->text.erase(kMaxTokenLength);
1.315 + }
1.316 +
1.317 + token->flags = 0;
1.318 +
1.319 + token->setAtStartOfLine(mContext.lineStart);
1.320 + mContext.lineStart = token->type == '\n';
1.321 +
1.322 + token->setHasLeadingSpace(mContext.leadingSpace);
1.323 + mContext.leadingSpace = false;
1.324 +}
1.325 +
1.326 +bool Tokenizer::initScanner()
1.327 +{
1.328 + if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
1.329 + return false;
1.330 +
1.331 + yyrestart(0, mHandle);
1.332 + return true;
1.333 +}
1.334 +
1.335 +void Tokenizer::destroyScanner()
1.336 +{
1.337 + if (mHandle == NULL)
1.338 + return;
1.339 +
1.340 + yylex_destroy(mHandle);
1.341 + mHandle = NULL;
1.342 +}
1.343 +
1.344 +} // namespace pp
1.345 +
