The Tor Browser: comparison gfx/angle/src/compiler/preprocessor/Tokenizer.l

--1:000000000000
+:74909364a6f7
+/*
+//
+// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+This file contains the Lex specification for GLSL ES preprocessor.
+Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
+http://msdn.microsoft.com/en-us/library/2scxys89.aspx
+IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
+*/
+%top{
+//
+// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
+}
+%{
+#include "Tokenizer.h"
+#include "DiagnosticsBase.h"
+#include "Token.h"
+#if defined(__GNUC__)
+// Triggered by the auto-generated yy_fatal_error function.
+#pragma GCC diagnostic ignored "-Wmissing-noreturn"
+#endif
+typedef std::string YYSTYPE;
+typedef pp::SourceLocation YYLTYPE;
+// Use the unused yycolumn variable to track file (string) number.
+#define yyfileno yycolumn
+#define YY_USER_INIT                   \
+do {                               \
+yyfileno = 0;                  \
+yylineno = 1;                  \
+yyextra->leadingSpace = false; \
+yyextra->lineStart = true;     \
+} while(0);
+#define YY_USER_ACTION                                              \
+do                                                              \
+{                                                               \
+pp::Input* input = &yyextra->input;                         \
+pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
+while ((scanLoc->sIndex < input->count()) &&                \
+(scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
+{                                                           \
+scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
+++yyfileno; yylineno = 1;                               \
+}                                                           \
+yylloc->file = yyfileno;                                    \
+yylloc->line = yylineno;                                    \
+scanLoc->cIndex += yyleng;                                  \
+} while(0);
+#define YY_INPUT(buf, result, maxSize) \
+result = yyextra->input.read(buf, maxSize);
+%}
+%option noyywrap nounput never-interactive
+%option reentrant bison-bridge bison-locations
+%option prefix="pp"
+%option extra-type="pp::Tokenizer::Context*"
+%x COMMENT
+NEWLINE     \n|\r|\r\n
+IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
+PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]
+DECIMAL_CONSTANT      [1-9][0-9]*
+OCTAL_CONSTANT        0[0-7]*
+HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+
+DIGIT                [0-9]
+EXPONENT_PART        [eE][+-]?{DIGIT}+
+FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
+%%
+/* Line comment */
+"//"[^\r\n]*
+/* Block comment */
+/* Line breaks are just counted - not returned. */
+/* The comment is replaced by a single space. */
+"/*" { BEGIN(COMMENT); }
+<COMMENT>[^*\r\n]+
+<COMMENT>"*"
+<COMMENT>{NEWLINE} { ++yylineno; }
+<COMMENT>"*/" {
+yyextra->leadingSpace = true;
+BEGIN(INITIAL);
+}
+# {
+// # is only valid at start of line for preprocessor directives.
+yylval->assign(1, yytext[0]);
+return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
+}
+{IDENTIFIER} {
+yylval->assign(yytext, yyleng);
+return pp::Token::IDENTIFIER;
+}
+{DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
+yylval->assign(yytext, yyleng);
+return pp::Token::CONST_INT;
+}
+({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
+yylval->assign(yytext, yyleng);
+return pp::Token::CONST_FLOAT;
+}
+/* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
+/* Rule to catch all invalid integers and floats. */
+({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
+yylval->assign(yytext, yyleng);
+return pp::Token::PP_NUMBER;
+}
+"++" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_INC;
+}
+"--" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_DEC;
+}
+"<<" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_LEFT;
+}
+">>" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_RIGHT;
+}
+"<=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_LE;
+}
+">=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_GE;
+}
+"==" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_EQ;
+}
+"!=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_NE;
+}
+"&&" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_AND;
+}
+"^^" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_XOR;
+}
+"||" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_OR;
+}
+"+=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_ADD_ASSIGN;
+}
+"-=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_SUB_ASSIGN;
+}
+"*=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_MUL_ASSIGN;
+}
+"/=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_DIV_ASSIGN;
+}
+"%=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_MOD_ASSIGN;
+}
+"<<=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_LEFT_ASSIGN;
+}
+">>=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_RIGHT_ASSIGN;
+}
+"&=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_AND_ASSIGN;
+}
+"^=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_XOR_ASSIGN;
+}
+"|=" {
+yylval->assign(yytext, yyleng);
+return pp::Token::OP_OR_ASSIGN;
+}
+{PUNCTUATOR} {
+yylval->assign(1, yytext[0]);
+return yytext[0];
+}
+[ \t\v\f]+   { yyextra->leadingSpace = true; }
+{NEWLINE} {
+++yylineno;
+yylval->assign(1, '\n');
+return '\n';
+}
+\\{NEWLINE} { ++yylineno; }
+. {
+yylval->assign(1, yytext[0]);
+return pp::Token::PP_OTHER;
+}
+<*><<EOF>> {
+// YY_USER_ACTION is not invoked for handling EOF.
+// Set the location for EOF token manually.
+pp::Input* input = &yyextra->input;
+pp::Input::Location* scanLoc = &yyextra->scanLoc;
+yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
+if (scanLoc->sIndex != sIndexMax)
+{
+// We can only reach here if there are empty strings at the
+// end of the input.
+scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
+// FIXME: this is not 64-bit clean.
+yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
+}
+yylloc->file = yyfileno;
+yylloc->line = yylineno;
+yylval->clear();
+if (YY_START == COMMENT)
+{
+yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+pp::SourceLocation(yyfileno, yylineno),
+"");
+}
+yyterminate();
+}
+%%
+namespace pp {
+// TODO(alokp): Maximum token length should ideally be specified by
+// the preprocessor client, i.e., the compiler.
+const size_t Tokenizer::kMaxTokenLength = 256;
+Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
+{
+mContext.diagnostics = diagnostics;
+}
+Tokenizer::~Tokenizer()
+{
+destroyScanner();
+}
+bool Tokenizer::init(size_t count, const char* const string[], const int length[])
+{
+if ((count > 0) && (string == 0)) return false;
+mContext.input = Input(count, string, length);
+return initScanner();
+}
+void Tokenizer::setFileNumber(int file)
+{
+// We use column number as file number.
+// See macro yyfileno.
+yyset_column(file, mHandle);
+}
+void Tokenizer::setLineNumber(int line)
+{
+yyset_lineno(line, mHandle);
+}
+void Tokenizer::lex(Token* token)
+{
+token->type = yylex(&token->text, &token->location, mHandle);
+if (token->text.size() > kMaxTokenLength)
+{
+mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
+token->location, token->text);
+token->text.erase(kMaxTokenLength);
+}
+token->flags = 0;
+token->setAtStartOfLine(mContext.lineStart);
+mContext.lineStart = token->type == '\n';
+token->setHasLeadingSpace(mContext.leadingSpace);
+mContext.leadingSpace = false;
+}
+bool Tokenizer::initScanner()
+{
+if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
+return false;
+yyrestart(0, mHandle);
+return true;
+}
+void Tokenizer::destroyScanner()
+{
+if (mHandle == NULL)
+return;
+yylex_destroy(mHandle);
+mHandle = NULL;
+}
+}  // namespace pp

The Tor Browser / file comparison

comparison: gfx/angle/src/compiler/preprocessor/Tokenizer.l

gfx/angle/src/compiler/preprocessor/Tokenizer.l