michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- michael@0: * vim: set ts=8 sts=4 et sw=4 tw=99: michael@0: * This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: // JS lexical scanner. michael@0: michael@0: #include "frontend/TokenStream.h" michael@0: michael@0: #include "mozilla/PodOperations.h" michael@0: michael@0: #include michael@0: #include michael@0: #include michael@0: #include michael@0: michael@0: #include "jsatom.h" michael@0: #include "jscntxt.h" michael@0: #include "jsexn.h" michael@0: #include "jsnum.h" michael@0: #include "jsworkers.h" michael@0: michael@0: #include "frontend/BytecodeCompiler.h" michael@0: #include "js/CharacterEncoding.h" michael@0: #include "vm/Keywords.h" michael@0: #include "vm/StringBuffer.h" michael@0: michael@0: using namespace js; michael@0: using namespace js::frontend; michael@0: using namespace js::unicode; michael@0: michael@0: using mozilla::Maybe; michael@0: using mozilla::PodAssign; michael@0: using mozilla::PodCopy; michael@0: using mozilla::PodZero; michael@0: michael@0: struct KeywordInfo { michael@0: const char *chars; // C string with keyword text michael@0: TokenKind tokentype; michael@0: JSVersion version; michael@0: }; michael@0: michael@0: static const KeywordInfo keywords[] = { michael@0: #define KEYWORD_INFO(keyword, name, type, version) \ michael@0: {js_##keyword##_str, type, version}, michael@0: FOR_EACH_JAVASCRIPT_KEYWORD(KEYWORD_INFO) michael@0: #undef KEYWORD_INFO michael@0: }; michael@0: michael@0: // Returns a KeywordInfo for the specified characters, or nullptr if the string michael@0: // is not a keyword. michael@0: static const KeywordInfo * michael@0: FindKeyword(const jschar *s, size_t length) michael@0: { michael@0: JS_ASSERT(length != 0); michael@0: michael@0: size_t i; michael@0: const KeywordInfo *kw; michael@0: const char *chars; michael@0: michael@0: #define JSKW_LENGTH() length michael@0: #define JSKW_AT(column) s[column] michael@0: #define JSKW_GOT_MATCH(index) i = (index); goto got_match; michael@0: #define JSKW_TEST_GUESS(index) i = (index); goto test_guess; michael@0: #define JSKW_NO_MATCH() goto no_match; michael@0: #include "jsautokw.h" michael@0: #undef JSKW_NO_MATCH michael@0: #undef JSKW_TEST_GUESS michael@0: #undef JSKW_GOT_MATCH michael@0: #undef JSKW_AT michael@0: #undef JSKW_LENGTH michael@0: michael@0: got_match: michael@0: return &keywords[i]; michael@0: michael@0: test_guess: michael@0: kw = &keywords[i]; michael@0: chars = kw->chars; michael@0: do { michael@0: if (*s++ != (unsigned char)(*chars++)) michael@0: goto no_match; michael@0: } while (--length != 0); michael@0: return kw; michael@0: michael@0: no_match: michael@0: return nullptr; michael@0: } michael@0: michael@0: bool michael@0: frontend::IsIdentifier(JSLinearString *str) michael@0: { michael@0: const jschar *chars = str->chars(); michael@0: size_t length = str->length(); michael@0: michael@0: if (length == 0) michael@0: return false; michael@0: jschar c = *chars; michael@0: if (!IsIdentifierStart(c)) michael@0: return false; michael@0: const jschar *end = chars + length; michael@0: while (++chars != end) { michael@0: c = *chars; michael@0: if (!IsIdentifierPart(c)) michael@0: return false; michael@0: } michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: frontend::IsKeyword(JSLinearString *str) michael@0: { michael@0: return FindKeyword(str->chars(), str->length()) != nullptr; michael@0: } michael@0: michael@0: TokenStream::SourceCoords::SourceCoords(ExclusiveContext *cx, uint32_t ln) michael@0: : lineStartOffsets_(cx), initialLineNum_(ln), lastLineIndex_(0) michael@0: { michael@0: // This is actually necessary! Removing it causes compile errors on michael@0: // GCC and clang. You could try declaring this: michael@0: // michael@0: // const uint32_t TokenStream::SourceCoords::MAX_PTR; michael@0: // michael@0: // which fixes the GCC/clang error, but causes bustage on Windows. Sigh. michael@0: // michael@0: uint32_t maxPtr = MAX_PTR; michael@0: michael@0: // The first line begins at buffer offset 0. MAX_PTR is the sentinel. The michael@0: // appends cannot fail because |lineStartOffsets_| has statically-allocated michael@0: // elements. michael@0: JS_ASSERT(lineStartOffsets_.capacity() >= 2); michael@0: (void)lineStartOffsets_.reserve(2); michael@0: lineStartOffsets_.infallibleAppend(0); michael@0: lineStartOffsets_.infallibleAppend(maxPtr); michael@0: } michael@0: michael@0: MOZ_ALWAYS_INLINE void michael@0: TokenStream::SourceCoords::add(uint32_t lineNum, uint32_t lineStartOffset) michael@0: { michael@0: uint32_t lineIndex = lineNumToIndex(lineNum); michael@0: uint32_t sentinelIndex = lineStartOffsets_.length() - 1; michael@0: michael@0: JS_ASSERT(lineStartOffsets_[0] == 0 && lineStartOffsets_[sentinelIndex] == MAX_PTR); michael@0: michael@0: if (lineIndex == sentinelIndex) { michael@0: // We haven't seen this newline before. Update lineStartOffsets_. michael@0: // We ignore any failures due to OOM -- because we always have a michael@0: // sentinel node, it'll just be like the newline wasn't present. I.e. michael@0: // the line numbers will be wrong, but the code won't crash or anything michael@0: // like that. michael@0: lineStartOffsets_[lineIndex] = lineStartOffset; michael@0: michael@0: uint32_t maxPtr = MAX_PTR; michael@0: (void)lineStartOffsets_.append(maxPtr); michael@0: michael@0: } else { michael@0: // We have seen this newline before (and ungot it). Do nothing (other michael@0: // than checking it hasn't mysteriously changed). michael@0: JS_ASSERT(lineStartOffsets_[lineIndex] == lineStartOffset); michael@0: } michael@0: } michael@0: michael@0: MOZ_ALWAYS_INLINE bool michael@0: TokenStream::SourceCoords::fill(const TokenStream::SourceCoords &other) michael@0: { michael@0: JS_ASSERT(lineStartOffsets_.back() == MAX_PTR); michael@0: JS_ASSERT(other.lineStartOffsets_.back() == MAX_PTR); michael@0: michael@0: if (lineStartOffsets_.length() >= other.lineStartOffsets_.length()) michael@0: return true; michael@0: michael@0: uint32_t sentinelIndex = lineStartOffsets_.length() - 1; michael@0: lineStartOffsets_[sentinelIndex] = other.lineStartOffsets_[sentinelIndex]; michael@0: michael@0: for (size_t i = sentinelIndex + 1; i < other.lineStartOffsets_.length(); i++) { michael@0: if (!lineStartOffsets_.append(other.lineStartOffsets_[i])) michael@0: return false; michael@0: } michael@0: return true; michael@0: } michael@0: michael@0: MOZ_ALWAYS_INLINE uint32_t michael@0: TokenStream::SourceCoords::lineIndexOf(uint32_t offset) const michael@0: { michael@0: uint32_t iMin, iMax, iMid; michael@0: michael@0: if (lineStartOffsets_[lastLineIndex_] <= offset) { michael@0: // If we reach here, offset is on a line the same as or higher than michael@0: // last time. Check first for the +0, +1, +2 cases, because they michael@0: // typically cover 85--98% of cases. michael@0: if (offset < lineStartOffsets_[lastLineIndex_ + 1]) michael@0: return lastLineIndex_; // lineIndex is same as last time michael@0: michael@0: // If we reach here, there must be at least one more entry (plus the michael@0: // sentinel). Try it. michael@0: lastLineIndex_++; michael@0: if (offset < lineStartOffsets_[lastLineIndex_ + 1]) michael@0: return lastLineIndex_; // lineIndex is one higher than last time michael@0: michael@0: // The same logic applies here. michael@0: lastLineIndex_++; michael@0: if (offset < lineStartOffsets_[lastLineIndex_ + 1]) { michael@0: return lastLineIndex_; // lineIndex is two higher than last time michael@0: } michael@0: michael@0: // No luck. Oh well, we have a better-than-default starting point for michael@0: // the binary search. michael@0: iMin = lastLineIndex_ + 1; michael@0: JS_ASSERT(iMin < lineStartOffsets_.length() - 1); // -1 due to the sentinel michael@0: michael@0: } else { michael@0: iMin = 0; michael@0: } michael@0: michael@0: // This is a binary search with deferred detection of equality, which was michael@0: // marginally faster in this case than a standard binary search. michael@0: // The -2 is because |lineStartOffsets_.length() - 1| is the sentinel, and we michael@0: // want one before that. michael@0: iMax = lineStartOffsets_.length() - 2; michael@0: while (iMax > iMin) { michael@0: iMid = iMin + (iMax - iMin) / 2; michael@0: if (offset >= lineStartOffsets_[iMid + 1]) michael@0: iMin = iMid + 1; // offset is above lineStartOffsets_[iMid] michael@0: else michael@0: iMax = iMid; // offset is below or within lineStartOffsets_[iMid] michael@0: } michael@0: JS_ASSERT(iMax == iMin); michael@0: JS_ASSERT(lineStartOffsets_[iMin] <= offset && offset < lineStartOffsets_[iMin + 1]); michael@0: lastLineIndex_ = iMin; michael@0: return iMin; michael@0: } michael@0: michael@0: uint32_t michael@0: TokenStream::SourceCoords::lineNum(uint32_t offset) const michael@0: { michael@0: uint32_t lineIndex = lineIndexOf(offset); michael@0: return lineIndexToNum(lineIndex); michael@0: } michael@0: michael@0: uint32_t michael@0: TokenStream::SourceCoords::columnIndex(uint32_t offset) const michael@0: { michael@0: uint32_t lineIndex = lineIndexOf(offset); michael@0: uint32_t lineStartOffset = lineStartOffsets_[lineIndex]; michael@0: JS_ASSERT(offset >= lineStartOffset); michael@0: return offset - lineStartOffset; michael@0: } michael@0: michael@0: void michael@0: TokenStream::SourceCoords::lineNumAndColumnIndex(uint32_t offset, uint32_t *lineNum, michael@0: uint32_t *columnIndex) const michael@0: { michael@0: uint32_t lineIndex = lineIndexOf(offset); michael@0: *lineNum = lineIndexToNum(lineIndex); michael@0: uint32_t lineStartOffset = lineStartOffsets_[lineIndex]; michael@0: JS_ASSERT(offset >= lineStartOffset); michael@0: *columnIndex = offset - lineStartOffset; michael@0: } michael@0: michael@0: #ifdef _MSC_VER michael@0: #pragma warning(push) michael@0: #pragma warning(disable:4351) michael@0: #endif michael@0: michael@0: // Initialize members that aren't initialized in |init|. michael@0: TokenStream::TokenStream(ExclusiveContext *cx, const ReadOnlyCompileOptions &options, michael@0: const jschar *base, size_t length, StrictModeGetter *smg) michael@0: : srcCoords(cx, options.lineno), michael@0: options_(options), michael@0: tokens(), michael@0: cursor(), michael@0: lookahead(), michael@0: lineno(options.lineno), michael@0: flags(), michael@0: linebase(base - options.column), michael@0: prevLinebase(nullptr), michael@0: userbuf(cx, base - options.column, length + options.column), // See comment below michael@0: filename(options.filename()), michael@0: displayURL_(nullptr), michael@0: sourceMapURL_(nullptr), michael@0: tokenbuf(cx), michael@0: cx(cx), michael@0: originPrincipals(options.originPrincipals(cx)), michael@0: strictModeGetter(smg) michael@0: { michael@0: // The caller must ensure that a reference is held on the supplied principals michael@0: // throughout compilation. michael@0: JS_ASSERT_IF(originPrincipals, originPrincipals->refcount > 0); michael@0: michael@0: // Column numbers are computed as offsets from the current line's base, so the michael@0: // initial line's base must be included in the buffer. linebase and userbuf michael@0: // were adjusted above, and if we are starting tokenization part way through michael@0: // this line then adjust the next character. michael@0: userbuf.setAddressOfNextRawChar(base); michael@0: michael@0: // Nb: the following tables could be static, but initializing them here is michael@0: // much easier. Don't worry, the time to initialize them for each michael@0: // TokenStream is trivial. See bug 639420. michael@0: michael@0: // See getChar() for an explanation of maybeEOL[]. michael@0: memset(maybeEOL, 0, sizeof(maybeEOL)); michael@0: maybeEOL[unsigned('\n')] = true; michael@0: maybeEOL[unsigned('\r')] = true; michael@0: maybeEOL[unsigned(LINE_SEPARATOR & 0xff)] = true; michael@0: maybeEOL[unsigned(PARA_SEPARATOR & 0xff)] = true; michael@0: michael@0: // See getTokenInternal() for an explanation of maybeStrSpecial[]. michael@0: memset(maybeStrSpecial, 0, sizeof(maybeStrSpecial)); michael@0: maybeStrSpecial[unsigned('"')] = true; michael@0: maybeStrSpecial[unsigned('\'')] = true; michael@0: maybeStrSpecial[unsigned('\\')] = true; michael@0: maybeStrSpecial[unsigned('\n')] = true; michael@0: maybeStrSpecial[unsigned('\r')] = true; michael@0: maybeStrSpecial[unsigned(LINE_SEPARATOR & 0xff)] = true; michael@0: maybeStrSpecial[unsigned(PARA_SEPARATOR & 0xff)] = true; michael@0: maybeStrSpecial[unsigned(EOF & 0xff)] = true; michael@0: michael@0: // See Parser::assignExpr() for an explanation of isExprEnding[]. michael@0: memset(isExprEnding, 0, sizeof(isExprEnding)); michael@0: isExprEnding[TOK_COMMA] = 1; michael@0: isExprEnding[TOK_SEMI] = 1; michael@0: isExprEnding[TOK_COLON] = 1; michael@0: isExprEnding[TOK_RP] = 1; michael@0: isExprEnding[TOK_RB] = 1; michael@0: isExprEnding[TOK_RC] = 1; michael@0: } michael@0: michael@0: #ifdef _MSC_VER michael@0: #pragma warning(pop) michael@0: #endif michael@0: michael@0: TokenStream::~TokenStream() michael@0: { michael@0: js_free(displayURL_); michael@0: js_free(sourceMapURL_); michael@0: michael@0: JS_ASSERT_IF(originPrincipals, originPrincipals->refcount); michael@0: } michael@0: michael@0: // Use the fastest available getc. michael@0: #if defined(HAVE_GETC_UNLOCKED) michael@0: # define fast_getc getc_unlocked michael@0: #elif defined(HAVE__GETC_NOLOCK) michael@0: # define fast_getc _getc_nolock michael@0: #else michael@0: # define fast_getc getc michael@0: #endif michael@0: michael@0: MOZ_ALWAYS_INLINE void michael@0: TokenStream::updateLineInfoForEOL() michael@0: { michael@0: prevLinebase = linebase; michael@0: linebase = userbuf.addressOfNextRawChar(); michael@0: lineno++; michael@0: srcCoords.add(lineno, linebase - userbuf.base()); michael@0: } michael@0: michael@0: MOZ_ALWAYS_INLINE void michael@0: TokenStream::updateFlagsForEOL() michael@0: { michael@0: flags.isDirtyLine = false; michael@0: } michael@0: michael@0: // This gets the next char, normalizing all EOL sequences to '\n' as it goes. michael@0: int32_t michael@0: TokenStream::getChar() michael@0: { michael@0: int32_t c; michael@0: if (MOZ_LIKELY(userbuf.hasRawChars())) { michael@0: c = userbuf.getRawChar(); michael@0: michael@0: // Normalize the jschar if it was a newline. We need to detect any of michael@0: // these four characters: '\n' (0x000a), '\r' (0x000d), michael@0: // LINE_SEPARATOR (0x2028), PARA_SEPARATOR (0x2029). Testing for each michael@0: // one in turn is slow, so we use a single probabilistic check, and if michael@0: // that succeeds, test for them individually. michael@0: // michael@0: // We use the bottom 8 bits to index into a lookup table, succeeding michael@0: // when d&0xff is 0xa, 0xd, 0x28 or 0x29. Among ASCII chars (which michael@0: // are by the far the most common) this gives false positives for '(' michael@0: // (0x0028) and ')' (0x0029). We could avoid those by incorporating michael@0: // the 13th bit of d into the lookup, but that requires extra shifting michael@0: // and masking and isn't worthwhile. See TokenStream::TokenStream() michael@0: // for the initialization of the relevant entries in the table. michael@0: if (MOZ_UNLIKELY(maybeEOL[c & 0xff])) { michael@0: if (c == '\n') michael@0: goto eol; michael@0: if (c == '\r') { michael@0: // If it's a \r\n sequence: treat as a single EOL, skip over the \n. michael@0: if (userbuf.hasRawChars()) michael@0: userbuf.matchRawChar('\n'); michael@0: goto eol; michael@0: } michael@0: if (c == LINE_SEPARATOR || c == PARA_SEPARATOR) michael@0: goto eol; michael@0: } michael@0: return c; michael@0: } michael@0: michael@0: flags.isEOF = true; michael@0: return EOF; michael@0: michael@0: eol: michael@0: updateLineInfoForEOL(); michael@0: return '\n'; michael@0: } michael@0: michael@0: // This gets the next char. It does nothing special with EOL sequences, not michael@0: // even updating the line counters. It can be used safely if (a) the michael@0: // resulting char is guaranteed to be ungotten (by ungetCharIgnoreEOL()) if michael@0: // it's an EOL, and (b) the line-related state (lineno, linebase) is not used michael@0: // before it's ungotten. michael@0: int32_t michael@0: TokenStream::getCharIgnoreEOL() michael@0: { michael@0: if (MOZ_LIKELY(userbuf.hasRawChars())) michael@0: return userbuf.getRawChar(); michael@0: michael@0: flags.isEOF = true; michael@0: return EOF; michael@0: } michael@0: michael@0: void michael@0: TokenStream::ungetChar(int32_t c) michael@0: { michael@0: if (c == EOF) michael@0: return; michael@0: JS_ASSERT(!userbuf.atStart()); michael@0: userbuf.ungetRawChar(); michael@0: if (c == '\n') { michael@0: #ifdef DEBUG michael@0: int32_t c2 = userbuf.peekRawChar(); michael@0: JS_ASSERT(TokenBuf::isRawEOLChar(c2)); michael@0: #endif michael@0: michael@0: // If it's a \r\n sequence, also unget the \r. michael@0: if (!userbuf.atStart()) michael@0: userbuf.matchRawCharBackwards('\r'); michael@0: michael@0: JS_ASSERT(prevLinebase); // we should never get more than one EOL char michael@0: linebase = prevLinebase; michael@0: prevLinebase = nullptr; michael@0: lineno--; michael@0: } else { michael@0: JS_ASSERT(userbuf.peekRawChar() == c); michael@0: } michael@0: } michael@0: michael@0: void michael@0: TokenStream::ungetCharIgnoreEOL(int32_t c) michael@0: { michael@0: if (c == EOF) michael@0: return; michael@0: JS_ASSERT(!userbuf.atStart()); michael@0: userbuf.ungetRawChar(); michael@0: } michael@0: michael@0: // Return true iff |n| raw characters can be read from this without reading past michael@0: // EOF or a newline, and copy those characters into |cp| if so. The characters michael@0: // are not consumed: use skipChars(n) to do so after checking that the consumed michael@0: // characters had appropriate values. michael@0: bool michael@0: TokenStream::peekChars(int n, jschar *cp) michael@0: { michael@0: int i, j; michael@0: int32_t c; michael@0: michael@0: for (i = 0; i < n; i++) { michael@0: c = getCharIgnoreEOL(); michael@0: if (c == EOF) michael@0: break; michael@0: if (c == '\n') { michael@0: ungetCharIgnoreEOL(c); michael@0: break; michael@0: } michael@0: cp[i] = jschar(c); michael@0: } michael@0: for (j = i - 1; j >= 0; j--) michael@0: ungetCharIgnoreEOL(cp[j]); michael@0: return i == n; michael@0: } michael@0: michael@0: const jschar * michael@0: TokenStream::TokenBuf::findEOLMax(const jschar *p, size_t max) michael@0: { michael@0: JS_ASSERT(base_ <= p && p <= limit_); michael@0: michael@0: size_t n = 0; michael@0: while (true) { michael@0: if (p >= limit_) michael@0: break; michael@0: if (n >= max) michael@0: break; michael@0: if (TokenBuf::isRawEOLChar(*p++)) michael@0: break; michael@0: n++; michael@0: } michael@0: return p; michael@0: } michael@0: michael@0: void michael@0: TokenStream::advance(size_t position) michael@0: { michael@0: const jschar *end = userbuf.base() + position; michael@0: while (userbuf.addressOfNextRawChar() < end) michael@0: getChar(); michael@0: michael@0: Token *cur = &tokens[cursor]; michael@0: cur->pos.begin = userbuf.addressOfNextRawChar() - userbuf.base(); michael@0: cur->type = TOK_ERROR; michael@0: lookahead = 0; michael@0: } michael@0: michael@0: void michael@0: TokenStream::tell(Position *pos) michael@0: { michael@0: pos->buf = userbuf.addressOfNextRawChar(/* allowPoisoned = */ true); michael@0: pos->flags = flags; michael@0: pos->lineno = lineno; michael@0: pos->linebase = linebase; michael@0: pos->prevLinebase = prevLinebase; michael@0: pos->lookahead = lookahead; michael@0: pos->currentToken = currentToken(); michael@0: for (unsigned i = 0; i < lookahead; i++) michael@0: pos->lookaheadTokens[i] = tokens[(cursor + 1 + i) & ntokensMask]; michael@0: } michael@0: michael@0: void michael@0: TokenStream::seek(const Position &pos) michael@0: { michael@0: userbuf.setAddressOfNextRawChar(pos.buf, /* allowPoisoned = */ true); michael@0: flags = pos.flags; michael@0: lineno = pos.lineno; michael@0: linebase = pos.linebase; michael@0: prevLinebase = pos.prevLinebase; michael@0: lookahead = pos.lookahead; michael@0: michael@0: tokens[cursor] = pos.currentToken; michael@0: for (unsigned i = 0; i < lookahead; i++) michael@0: tokens[(cursor + 1 + i) & ntokensMask] = pos.lookaheadTokens[i]; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::seek(const Position &pos, const TokenStream &other) michael@0: { michael@0: if (!srcCoords.fill(other.srcCoords)) michael@0: return false; michael@0: seek(pos); michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::reportStrictModeErrorNumberVA(uint32_t offset, bool strictMode, unsigned errorNumber, michael@0: va_list args) michael@0: { michael@0: // In strict mode code, this is an error, not merely a warning. michael@0: unsigned flags = JSREPORT_STRICT; michael@0: if (strictMode) michael@0: flags |= JSREPORT_ERROR; michael@0: else if (options().extraWarningsOption) michael@0: flags |= JSREPORT_WARNING; michael@0: else michael@0: return true; michael@0: michael@0: return reportCompileErrorNumberVA(offset, flags, errorNumber, args); michael@0: } michael@0: michael@0: void michael@0: CompileError::throwError(JSContext *cx) michael@0: { michael@0: // If there's a runtime exception type associated with this error michael@0: // number, set that as the pending exception. For errors occuring at michael@0: // compile time, this is very likely to be a JSEXN_SYNTAXERR. michael@0: // michael@0: // If an exception is thrown but not caught, the JSREPORT_EXCEPTION michael@0: // flag will be set in report.flags. Proper behavior for an error michael@0: // reporter is to ignore a report with this flag for all but top-level michael@0: // compilation errors. The exception will remain pending, and so long michael@0: // as the non-top-level "load", "eval", or "compile" native function michael@0: // returns false, the top-level reporter will eventually receive the michael@0: // uncaught exception report. michael@0: if (!js_ErrorToException(cx, message, &report, nullptr, nullptr)) michael@0: CallErrorReporter(cx, message, &report); michael@0: } michael@0: michael@0: CompileError::~CompileError() michael@0: { michael@0: js_free((void*)report.uclinebuf); michael@0: js_free((void*)report.linebuf); michael@0: js_free((void*)report.ucmessage); michael@0: js_free(message); michael@0: message = nullptr; michael@0: michael@0: if (report.messageArgs) { michael@0: if (argumentsType == ArgumentsAreASCII) { michael@0: unsigned i = 0; michael@0: while (report.messageArgs[i]) michael@0: js_free((void*)report.messageArgs[i++]); michael@0: } michael@0: js_free(report.messageArgs); michael@0: } michael@0: michael@0: PodZero(&report); michael@0: } michael@0: michael@0: bool michael@0: TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigned errorNumber, michael@0: va_list args) michael@0: { michael@0: bool warning = JSREPORT_IS_WARNING(flags); michael@0: michael@0: if (warning && options().werrorOption) { michael@0: flags &= ~JSREPORT_WARNING; michael@0: warning = false; michael@0: } michael@0: michael@0: // On the main thread, report the error immediately. When compiling off michael@0: // thread, save the error so that the main thread can report it later. michael@0: CompileError tempErr; michael@0: CompileError &err = cx->isJSContext() ? tempErr : cx->addPendingCompileError(); michael@0: michael@0: err.report.flags = flags; michael@0: err.report.errorNumber = errorNumber; michael@0: err.report.filename = filename; michael@0: err.report.originPrincipals = originPrincipals; michael@0: if (offset == NoOffset) { michael@0: err.report.lineno = 0; michael@0: err.report.column = 0; michael@0: } else { michael@0: err.report.lineno = srcCoords.lineNum(offset); michael@0: err.report.column = srcCoords.columnIndex(offset); michael@0: } michael@0: michael@0: err.argumentsType = (flags & JSREPORT_UC) ? ArgumentsAreUnicode : ArgumentsAreASCII; michael@0: michael@0: if (!js_ExpandErrorArguments(cx, js_GetErrorMessage, nullptr, errorNumber, &err.message, michael@0: &err.report, err.argumentsType, args)) michael@0: { michael@0: return false; michael@0: } michael@0: michael@0: // Given a token, T, that we want to complain about: if T's (starting) michael@0: // lineno doesn't match TokenStream's lineno, that means we've scanned past michael@0: // the line that T starts on, which makes it hard to print some or all of michael@0: // T's (starting) line for context. michael@0: // michael@0: // So we don't even try, leaving report.linebuf and friends zeroed. This michael@0: // means that any error involving a multi-line token (e.g. an unterminated michael@0: // multi-line string literal) won't have a context printed. michael@0: if (offset != NoOffset && err.report.lineno == lineno) { michael@0: const jschar *tokenStart = userbuf.base() + offset; michael@0: michael@0: // We show only a portion (a "window") of the line around the erroneous michael@0: // token -- the first char in the token, plus |windowRadius| chars michael@0: // before it and |windowRadius - 1| chars after it. This is because michael@0: // lines can be very long and printing the whole line is (a) not that michael@0: // helpful, and (b) can waste a lot of memory. See bug 634444. michael@0: static const size_t windowRadius = 60; michael@0: michael@0: // Truncate at the front if necessary. michael@0: const jschar *windowBase = (linebase + windowRadius < tokenStart) michael@0: ? tokenStart - windowRadius michael@0: : linebase; michael@0: uint32_t windowOffset = tokenStart - windowBase; michael@0: michael@0: // Find EOL, or truncate at the back if necessary. michael@0: const jschar *windowLimit = userbuf.findEOLMax(tokenStart, windowRadius); michael@0: size_t windowLength = windowLimit - windowBase; michael@0: JS_ASSERT(windowLength <= windowRadius * 2); michael@0: michael@0: // Create the windowed strings. michael@0: StringBuffer windowBuf(cx); michael@0: if (!windowBuf.append(windowBase, windowLength) || !windowBuf.append((jschar)0)) michael@0: return false; michael@0: michael@0: // Unicode and char versions of the window into the offending source michael@0: // line, without final \n. michael@0: err.report.uclinebuf = windowBuf.extractWellSized(); michael@0: if (!err.report.uclinebuf) michael@0: return false; michael@0: TwoByteChars tbchars(err.report.uclinebuf, windowLength); michael@0: err.report.linebuf = LossyTwoByteCharsToNewLatin1CharsZ(cx, tbchars).c_str(); michael@0: if (!err.report.linebuf) michael@0: return false; michael@0: michael@0: err.report.tokenptr = err.report.linebuf + windowOffset; michael@0: err.report.uctokenptr = err.report.uclinebuf + windowOffset; michael@0: } michael@0: michael@0: if (cx->isJSContext()) michael@0: err.throwError(cx->asJSContext()); michael@0: michael@0: return warning; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::reportStrictModeError(unsigned errorNumber, ...) michael@0: { michael@0: va_list args; michael@0: va_start(args, errorNumber); michael@0: bool result = reportStrictModeErrorNumberVA(currentToken().pos.begin, strictMode(), michael@0: errorNumber, args); michael@0: va_end(args); michael@0: return result; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::reportError(unsigned errorNumber, ...) michael@0: { michael@0: va_list args; michael@0: va_start(args, errorNumber); michael@0: bool result = reportCompileErrorNumberVA(currentToken().pos.begin, JSREPORT_ERROR, errorNumber, michael@0: args); michael@0: va_end(args); michael@0: return result; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::reportWarning(unsigned errorNumber, ...) michael@0: { michael@0: va_list args; michael@0: va_start(args, errorNumber); michael@0: bool result = reportCompileErrorNumberVA(currentToken().pos.begin, JSREPORT_WARNING, michael@0: errorNumber, args); michael@0: va_end(args); michael@0: return result; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::reportStrictWarningErrorNumberVA(uint32_t offset, unsigned errorNumber, va_list args) michael@0: { michael@0: if (!options().extraWarningsOption) michael@0: return true; michael@0: michael@0: return reportCompileErrorNumberVA(offset, JSREPORT_STRICT|JSREPORT_WARNING, errorNumber, args); michael@0: } michael@0: michael@0: void michael@0: TokenStream::reportAsmJSError(uint32_t offset, unsigned errorNumber, ...) michael@0: { michael@0: va_list args; michael@0: va_start(args, errorNumber); michael@0: reportCompileErrorNumberVA(offset, JSREPORT_WARNING, errorNumber, args); michael@0: va_end(args); michael@0: } michael@0: michael@0: // We have encountered a '\': check for a Unicode escape sequence after it. michael@0: // Return 'true' and the character code value (by value) if we found a michael@0: // Unicode escape sequence. Otherwise, return 'false'. In both cases, do not michael@0: // advance along the buffer. michael@0: bool michael@0: TokenStream::peekUnicodeEscape(int *result) michael@0: { michael@0: jschar cp[5]; michael@0: michael@0: if (peekChars(5, cp) && cp[0] == 'u' && michael@0: JS7_ISHEX(cp[1]) && JS7_ISHEX(cp[2]) && michael@0: JS7_ISHEX(cp[3]) && JS7_ISHEX(cp[4])) michael@0: { michael@0: *result = (((((JS7_UNHEX(cp[1]) << 4) michael@0: + JS7_UNHEX(cp[2])) << 4) michael@0: + JS7_UNHEX(cp[3])) << 4) michael@0: + JS7_UNHEX(cp[4]); michael@0: return true; michael@0: } michael@0: return false; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::matchUnicodeEscapeIdStart(int32_t *cp) michael@0: { michael@0: if (peekUnicodeEscape(cp) && IsIdentifierStart(*cp)) { michael@0: skipChars(5); michael@0: return true; michael@0: } michael@0: return false; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::matchUnicodeEscapeIdent(int32_t *cp) michael@0: { michael@0: if (peekUnicodeEscape(cp) && IsIdentifierPart(*cp)) { michael@0: skipChars(5); michael@0: return true; michael@0: } michael@0: return false; michael@0: } michael@0: michael@0: // Helper function which returns true if the first length(q) characters in p are michael@0: // the same as the characters in q. michael@0: static bool michael@0: CharsMatch(const jschar *p, const char *q) { michael@0: while (*q) { michael@0: if (*p++ != *q++) michael@0: return false; michael@0: } michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::getDirectives(bool isMultiline, bool shouldWarnDeprecated) michael@0: { michael@0: // Match directive comments used in debugging, such as "//# sourceURL" and michael@0: // "//# sourceMappingURL". Use of "//@" instead of "//#" is deprecated. michael@0: // michael@0: // To avoid a crashing bug in IE, several JavaScript transpilers wrap single michael@0: // line comments containing a source mapping URL inside a multiline michael@0: // comment. To avoid potentially expensive lookahead and backtracking, we michael@0: // only check for this case if we encounter a '#' character. michael@0: michael@0: if (!getDisplayURL(isMultiline, shouldWarnDeprecated)) michael@0: return false; michael@0: if (!getSourceMappingURL(isMultiline, shouldWarnDeprecated)) michael@0: return false; michael@0: michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::getDirective(bool isMultiline, bool shouldWarnDeprecated, michael@0: const char *directive, int directiveLength, michael@0: const char *errorMsgPragma, jschar **destination) { michael@0: JS_ASSERT(directiveLength <= 18); michael@0: jschar peeked[18]; michael@0: int32_t c; michael@0: michael@0: if (peekChars(directiveLength, peeked) && CharsMatch(peeked, directive)) { michael@0: if (shouldWarnDeprecated && michael@0: !reportWarning(JSMSG_DEPRECATED_PRAGMA, errorMsgPragma)) michael@0: return false; michael@0: michael@0: skipChars(directiveLength); michael@0: tokenbuf.clear(); michael@0: michael@0: while ((c = peekChar()) && c != EOF && !IsSpaceOrBOM2(c)) { michael@0: getChar(); michael@0: // Debugging directives can occur in both single- and multi-line michael@0: // comments. If we're currently inside a multi-line comment, we also michael@0: // need to recognize multi-line comment terminators. michael@0: if (isMultiline && c == '*' && peekChar() == '/') { michael@0: ungetChar('*'); michael@0: break; michael@0: } michael@0: tokenbuf.append(c); michael@0: } michael@0: michael@0: if (tokenbuf.empty()) michael@0: // The directive's URL was missing, but this is not quite an michael@0: // exception that we should stop and drop everything for. michael@0: return true; michael@0: michael@0: size_t length = tokenbuf.length(); michael@0: michael@0: js_free(*destination); michael@0: *destination = cx->pod_malloc(length + 1); michael@0: if (!*destination) michael@0: return false; michael@0: michael@0: PodCopy(*destination, tokenbuf.begin(), length); michael@0: (*destination)[length] = '\0'; michael@0: } michael@0: michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::getDisplayURL(bool isMultiline, bool shouldWarnDeprecated) michael@0: { michael@0: // Match comments of the form "//# sourceURL=" or michael@0: // "/\* //# sourceURL= *\/" michael@0: // michael@0: // Note that while these are labeled "sourceURL" in the source text, michael@0: // internally we refer to it as a "displayURL" to distinguish what the michael@0: // developer would like to refer to the source as from the source's actual michael@0: // URL. michael@0: michael@0: return getDirective(isMultiline, shouldWarnDeprecated, " sourceURL=", 11, michael@0: "sourceURL", &displayURL_); michael@0: } michael@0: michael@0: bool michael@0: TokenStream::getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated) michael@0: { michael@0: // Match comments of the form "//# sourceMappingURL=" or michael@0: // "/\* //# sourceMappingURL= *\/" michael@0: michael@0: return getDirective(isMultiline, shouldWarnDeprecated, " sourceMappingURL=", 18, michael@0: "sourceMappingURL", &sourceMapURL_); michael@0: } michael@0: michael@0: MOZ_ALWAYS_INLINE Token * michael@0: TokenStream::newToken(ptrdiff_t adjust) michael@0: { michael@0: cursor = (cursor + 1) & ntokensMask; michael@0: Token *tp = &tokens[cursor]; michael@0: tp->pos.begin = userbuf.addressOfNextRawChar() + adjust - userbuf.base(); michael@0: michael@0: // NOTE: tp->pos.end is not set until the very end of getTokenInternal(). michael@0: MOZ_MAKE_MEM_UNDEFINED(&tp->pos.end, sizeof(tp->pos.end)); michael@0: michael@0: return tp; michael@0: } michael@0: michael@0: MOZ_ALWAYS_INLINE JSAtom * michael@0: TokenStream::atomize(ExclusiveContext *cx, CharBuffer &cb) michael@0: { michael@0: return AtomizeChars(cx, cb.begin(), cb.length()); michael@0: } michael@0: michael@0: #ifdef DEBUG michael@0: static bool michael@0: IsTokenSane(Token *tp) michael@0: { michael@0: // Nb: TOK_EOL should never be used in an actual Token; it should only be michael@0: // returned as a TokenKind from peekTokenSameLine(). michael@0: if (tp->type < TOK_ERROR || tp->type >= TOK_LIMIT || tp->type == TOK_EOL) michael@0: return false; michael@0: michael@0: if (tp->pos.end < tp->pos.begin) michael@0: return false; michael@0: michael@0: return true; michael@0: } michael@0: #endif michael@0: michael@0: bool michael@0: TokenStream::putIdentInTokenbuf(const jschar *identStart) michael@0: { michael@0: int32_t c, qc; michael@0: const jschar *tmp = userbuf.addressOfNextRawChar(); michael@0: userbuf.setAddressOfNextRawChar(identStart); michael@0: michael@0: tokenbuf.clear(); michael@0: for (;;) { michael@0: c = getCharIgnoreEOL(); michael@0: if (!IsIdentifierPart(c)) { michael@0: if (c != '\\' || !matchUnicodeEscapeIdent(&qc)) michael@0: break; michael@0: c = qc; michael@0: } michael@0: if (!tokenbuf.append(c)) { michael@0: userbuf.setAddressOfNextRawChar(tmp); michael@0: return false; michael@0: } michael@0: } michael@0: userbuf.setAddressOfNextRawChar(tmp); michael@0: return true; michael@0: } michael@0: michael@0: bool michael@0: TokenStream::checkForKeyword(const jschar *s, size_t length, TokenKind *ttp) michael@0: { michael@0: const KeywordInfo *kw = FindKeyword(s, length); michael@0: if (!kw) michael@0: return true; michael@0: michael@0: if (kw->tokentype == TOK_RESERVED) michael@0: return reportError(JSMSG_RESERVED_ID, kw->chars); michael@0: michael@0: if (kw->tokentype != TOK_STRICT_RESERVED) { michael@0: if (kw->version <= versionNumber()) { michael@0: // Working keyword. michael@0: if (ttp) { michael@0: *ttp = kw->tokentype; michael@0: return true; michael@0: } michael@0: return reportError(JSMSG_RESERVED_ID, kw->chars); michael@0: } michael@0: michael@0: // The keyword is not in this version. Treat it as an identifier, unless michael@0: // it is let which we treat as TOK_STRICT_RESERVED by falling through to michael@0: // the code below (ES5 forbids it in strict mode). michael@0: if (kw->tokentype != TOK_LET) michael@0: return true; michael@0: } michael@0: michael@0: // Strict reserved word. michael@0: return reportStrictModeError(JSMSG_RESERVED_ID, kw->chars); michael@0: } michael@0: michael@0: enum FirstCharKind { michael@0: // A jschar has the 'OneChar' kind if it, by itself, constitutes a valid michael@0: // token that cannot also be a prefix of a longer token. E.g. ';' has the michael@0: // OneChar kind, but '+' does not, because '++' and '+=' are valid longer tokens michael@0: // that begin with '+'. michael@0: // michael@0: // The few token kinds satisfying these properties cover roughly 35--45% michael@0: // of the tokens seen in practice. michael@0: // michael@0: // We represent the 'OneChar' kind with any positive value less than michael@0: // TOK_LIMIT. This representation lets us associate each one-char token michael@0: // jschar with a TokenKind and thus avoid a subsequent jschar-to-TokenKind michael@0: // conversion. michael@0: OneChar_Min = 0, michael@0: OneChar_Max = TOK_LIMIT - 1, michael@0: michael@0: Space = TOK_LIMIT, michael@0: Ident, michael@0: Dec, michael@0: String, michael@0: EOL, michael@0: BasePrefix, michael@0: Other, michael@0: michael@0: LastCharKind = Other michael@0: }; michael@0: michael@0: // OneChar: 40, 41, 44, 58, 59, 63, 91, 93, 123, 125, 126: michael@0: // '(', ')', ',', ':', ';', '?', '[', ']', '{', '}', '~' michael@0: // Ident: 36, 65..90, 95, 97..122: '$', 'A'..'Z', '_', 'a'..'z' michael@0: // Dot: 46: '.' michael@0: // Equals: 61: '=' michael@0: // String: 34, 39: '"', '\'' michael@0: // Dec: 49..57: '1'..'9' michael@0: // Plus: 43: '+' michael@0: // BasePrefix: 48: '0' michael@0: // Space: 9, 11, 12, 32: '\t', '\v', '\f', ' ' michael@0: // EOL: 10, 13: '\n', '\r' michael@0: // michael@0: #define T_COMMA TOK_COMMA michael@0: #define T_COLON TOK_COLON michael@0: #define T_BITNOT TOK_BITNOT michael@0: #define _______ Other michael@0: static const uint8_t firstCharKinds[] = { michael@0: /* 0 1 2 3 4 5 6 7 8 9 */ michael@0: /* 0+ */ _______, _______, _______, _______, _______, _______, _______, _______, _______, Space, michael@0: /* 10+ */ EOL, Space, Space, EOL, _______, _______, _______, _______, _______, _______, michael@0: /* 20+ */ _______, _______, _______, _______, _______, _______, _______, _______, _______, _______, michael@0: /* 30+ */ _______, _______, Space, _______, String, _______, Ident, _______, _______, String, michael@0: /* 40+ */ TOK_LP, TOK_RP, _______, _______, T_COMMA,_______, _______, _______,BasePrefix, Dec, michael@0: /* 50+ */ Dec, Dec, Dec, Dec, Dec, Dec, Dec, Dec, T_COLON,TOK_SEMI, michael@0: /* 60+ */ _______, _______, _______,TOK_HOOK, _______, Ident, Ident, Ident, Ident, Ident, michael@0: /* 70+ */ Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, michael@0: /* 80+ */ Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, michael@0: /* 90+ */ Ident, TOK_LB, _______, TOK_RB, _______, Ident, _______, Ident, Ident, Ident, michael@0: /* 100+ */ Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, michael@0: /* 110+ */ Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, Ident, michael@0: /* 120+ */ Ident, Ident, Ident, TOK_LC, _______, TOK_RC,T_BITNOT, _______ michael@0: }; michael@0: #undef T_COMMA michael@0: #undef T_COLON michael@0: #undef T_BITNOT michael@0: #undef _______ michael@0: michael@0: static_assert(LastCharKind < (1 << (sizeof(firstCharKinds[0]) * 8)), michael@0: "Elements of firstCharKinds[] are too small"); michael@0: michael@0: TokenKind michael@0: TokenStream::getTokenInternal(Modifier modifier) michael@0: { michael@0: int c, qc; michael@0: Token *tp; michael@0: FirstCharKind c1kind; michael@0: const jschar *numStart; michael@0: bool hasExp; michael@0: DecimalPoint decimalPoint; michael@0: const jschar *identStart; michael@0: bool hadUnicodeEscape; michael@0: michael@0: retry: michael@0: if (MOZ_UNLIKELY(!userbuf.hasRawChars())) { michael@0: tp = newToken(0); michael@0: tp->type = TOK_EOF; michael@0: flags.isEOF = true; michael@0: goto out; michael@0: } michael@0: michael@0: c = userbuf.getRawChar(); michael@0: JS_ASSERT(c != EOF); michael@0: michael@0: // Chars not in the range 0..127 are rare. Getting them out of the way michael@0: // early allows subsequent checking to be faster. michael@0: if (MOZ_UNLIKELY(c >= 128)) { michael@0: if (IsSpaceOrBOM2(c)) { michael@0: if (c == LINE_SEPARATOR || c == PARA_SEPARATOR) { michael@0: updateLineInfoForEOL(); michael@0: updateFlagsForEOL(); michael@0: } michael@0: michael@0: goto retry; michael@0: } michael@0: michael@0: tp = newToken(-1); michael@0: michael@0: // '$' and '_' don't pass IsLetter, but they're < 128 so never appear here. michael@0: JS_STATIC_ASSERT('$' < 128 && '_' < 128); michael@0: if (IsLetter(c)) { michael@0: identStart = userbuf.addressOfNextRawChar() - 1; michael@0: hadUnicodeEscape = false; michael@0: goto identifier; michael@0: } michael@0: michael@0: goto badchar; michael@0: } michael@0: michael@0: // Get the token kind, based on the first char. The ordering of c1kind michael@0: // comparison is based on the frequency of tokens in real code -- Parsemark michael@0: // (which represents typical JS code on the web) and the Unreal demo (which michael@0: // represents asm.js code). michael@0: // michael@0: // Parsemark Unreal michael@0: // OneChar 32.9% 39.7% michael@0: // Space 25.0% 0.6% michael@0: // Ident 19.2% 36.4% michael@0: // Dec 7.2% 5.1% michael@0: // String 7.9% 0.0% michael@0: // EOL 1.7% 0.0% michael@0: // BasePrefix 0.4% 4.9% michael@0: // Other 5.7% 13.3% michael@0: // michael@0: // The ordering is based mostly only Parsemark frequencies, with Unreal michael@0: // frequencies used to break close categories (e.g. |Dec| and |String|). michael@0: // |Other| is biggish, but no other token kind is common enough for it to michael@0: // be worth adding extra values to FirstCharKind. michael@0: // michael@0: c1kind = FirstCharKind(firstCharKinds[c]); michael@0: michael@0: // Look for an unambiguous single-char token. michael@0: // michael@0: if (c1kind <= OneChar_Max) { michael@0: tp = newToken(-1); michael@0: tp->type = TokenKind(c1kind); michael@0: goto out; michael@0: } michael@0: michael@0: // Skip over non-EOL whitespace chars. michael@0: // michael@0: if (c1kind == Space) michael@0: goto retry; michael@0: michael@0: // Look for an identifier. michael@0: // michael@0: if (c1kind == Ident) { michael@0: tp = newToken(-1); michael@0: identStart = userbuf.addressOfNextRawChar() - 1; michael@0: hadUnicodeEscape = false; michael@0: michael@0: identifier: michael@0: for (;;) { michael@0: c = getCharIgnoreEOL(); michael@0: if (c == EOF) michael@0: break; michael@0: if (!IsIdentifierPart(c)) { michael@0: if (c != '\\' || !matchUnicodeEscapeIdent(&qc)) michael@0: break; michael@0: hadUnicodeEscape = true; michael@0: } michael@0: } michael@0: ungetCharIgnoreEOL(c); michael@0: michael@0: // Identifiers containing no Unicode escapes can be processed directly michael@0: // from userbuf. The rest must use the escapes converted via tokenbuf michael@0: // before atomizing. michael@0: const jschar *chars; michael@0: size_t length; michael@0: if (hadUnicodeEscape) { michael@0: if (!putIdentInTokenbuf(identStart)) michael@0: goto error; michael@0: michael@0: chars = tokenbuf.begin(); michael@0: length = tokenbuf.length(); michael@0: } else { michael@0: chars = identStart; michael@0: length = userbuf.addressOfNextRawChar() - identStart; michael@0: } michael@0: michael@0: // Check for keywords unless the parser told us not to. michael@0: if (modifier != KeywordIsName) { michael@0: tp->type = TOK_NAME; michael@0: if (!checkForKeyword(chars, length, &tp->type)) michael@0: goto error; michael@0: if (tp->type != TOK_NAME) michael@0: goto out; michael@0: } michael@0: michael@0: JSAtom *atom = AtomizeChars(cx, chars, length); michael@0: if (!atom) michael@0: goto error; michael@0: tp->type = TOK_NAME; michael@0: tp->setName(atom->asPropertyName()); michael@0: goto out; michael@0: } michael@0: michael@0: // Look for a decimal number. michael@0: // michael@0: if (c1kind == Dec) { michael@0: tp = newToken(-1); michael@0: numStart = userbuf.addressOfNextRawChar() - 1; michael@0: michael@0: decimal: michael@0: decimalPoint = NoDecimal; michael@0: hasExp = false; michael@0: while (JS7_ISDEC(c)) michael@0: c = getCharIgnoreEOL(); michael@0: michael@0: if (c == '.') { michael@0: decimalPoint = HasDecimal; michael@0: decimal_dot: michael@0: do { michael@0: c = getCharIgnoreEOL(); michael@0: } while (JS7_ISDEC(c)); michael@0: } michael@0: if (c == 'e' || c == 'E') { michael@0: hasExp = true; michael@0: c = getCharIgnoreEOL(); michael@0: if (c == '+' || c == '-') michael@0: c = getCharIgnoreEOL(); michael@0: if (!JS7_ISDEC(c)) { michael@0: ungetCharIgnoreEOL(c); michael@0: reportError(JSMSG_MISSING_EXPONENT); michael@0: goto error; michael@0: } michael@0: do { michael@0: c = getCharIgnoreEOL(); michael@0: } while (JS7_ISDEC(c)); michael@0: } michael@0: ungetCharIgnoreEOL(c); michael@0: michael@0: if (c != EOF && IsIdentifierStart(c)) { michael@0: reportError(JSMSG_IDSTART_AFTER_NUMBER); michael@0: goto error; michael@0: } michael@0: michael@0: // Unlike identifiers and strings, numbers cannot contain escaped michael@0: // chars, so we don't need to use tokenbuf. Instead we can just michael@0: // convert the jschars in userbuf directly to the numeric value. michael@0: double dval; michael@0: if (!((decimalPoint == HasDecimal) || hasExp)) { michael@0: if (!GetDecimalInteger(cx, numStart, userbuf.addressOfNextRawChar(), &dval)) michael@0: goto error; michael@0: } else { michael@0: const jschar *dummy; michael@0: if (!js_strtod(cx, numStart, userbuf.addressOfNextRawChar(), &dummy, &dval)) michael@0: goto error; michael@0: } michael@0: tp->type = TOK_NUMBER; michael@0: tp->setNumber(dval, decimalPoint); michael@0: goto out; michael@0: } michael@0: michael@0: // Look for a string. michael@0: // michael@0: if (c1kind == String) { michael@0: tp = newToken(-1); michael@0: qc = c; michael@0: tokenbuf.clear(); michael@0: while (true) { michael@0: // We need to detect any of these chars: " or ', \n (or its michael@0: // equivalents), \\, EOF. We use maybeStrSpecial[] in a manner michael@0: // similar to maybeEOL[], see above. Because we detect EOL michael@0: // sequences here and put them back immediately, we can use michael@0: // getCharIgnoreEOL(). michael@0: c = getCharIgnoreEOL(); michael@0: if (maybeStrSpecial[c & 0xff]) { michael@0: if (c == qc) michael@0: break; michael@0: if (c == '\\') { michael@0: switch (c = getChar()) { michael@0: case 'b': c = '\b'; break; michael@0: case 'f': c = '\f'; break; michael@0: case 'n': c = '\n'; break; michael@0: case 'r': c = '\r'; break; michael@0: case 't': c = '\t'; break; michael@0: case 'v': c = '\v'; break; michael@0: michael@0: default: michael@0: if ('0' <= c && c < '8') { michael@0: int32_t val = JS7_UNDEC(c); michael@0: michael@0: c = peekChar(); michael@0: // Strict mode code allows only \0, then a non-digit. michael@0: if (val != 0 || JS7_ISDEC(c)) { michael@0: if (!reportStrictModeError(JSMSG_DEPRECATED_OCTAL)) michael@0: goto error; michael@0: flags.sawOctalEscape = true; michael@0: } michael@0: if ('0' <= c && c < '8') { michael@0: val = 8 * val + JS7_UNDEC(c); michael@0: getChar(); michael@0: c = peekChar(); michael@0: if ('0' <= c && c < '8') { michael@0: int32_t save = val; michael@0: val = 8 * val + JS7_UNDEC(c); michael@0: if (val <= 0377) michael@0: getChar(); michael@0: else michael@0: val = save; michael@0: } michael@0: } michael@0: michael@0: c = jschar(val); michael@0: } else if (c == 'u') { michael@0: jschar cp[4]; michael@0: if (peekChars(4, cp) && michael@0: JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1]) && michael@0: JS7_ISHEX(cp[2]) && JS7_ISHEX(cp[3])) { michael@0: c = (((((JS7_UNHEX(cp[0]) << 4) michael@0: + JS7_UNHEX(cp[1])) << 4) michael@0: + JS7_UNHEX(cp[2])) << 4) michael@0: + JS7_UNHEX(cp[3]); michael@0: skipChars(4); michael@0: } else { michael@0: reportError(JSMSG_MALFORMED_ESCAPE, "Unicode"); michael@0: goto error; michael@0: } michael@0: } else if (c == 'x') { michael@0: jschar cp[2]; michael@0: if (peekChars(2, cp) && michael@0: JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1])) { michael@0: c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]); michael@0: skipChars(2); michael@0: } else { michael@0: reportError(JSMSG_MALFORMED_ESCAPE, "hexadecimal"); michael@0: goto error; michael@0: } michael@0: } else if (c == '\n') { michael@0: // ES5 7.8.4: an escaped line terminator represents michael@0: // no character. michael@0: continue; michael@0: } michael@0: break; michael@0: } michael@0: } else if (TokenBuf::isRawEOLChar(c) || c == EOF) { michael@0: ungetCharIgnoreEOL(c); michael@0: reportError(JSMSG_UNTERMINATED_STRING); michael@0: goto error; michael@0: } michael@0: } michael@0: if (!tokenbuf.append(c)) michael@0: goto error; michael@0: } michael@0: JSAtom *atom = atomize(cx, tokenbuf); michael@0: if (!atom) michael@0: goto error; michael@0: tp->type = TOK_STRING; michael@0: tp->setAtom(atom); michael@0: goto out; michael@0: } michael@0: michael@0: // Skip over EOL chars, updating line state along the way. michael@0: // michael@0: if (c1kind == EOL) { michael@0: // If it's a \r\n sequence: treat as a single EOL, skip over the \n. michael@0: if (c == '\r' && userbuf.hasRawChars()) michael@0: userbuf.matchRawChar('\n'); michael@0: updateLineInfoForEOL(); michael@0: updateFlagsForEOL(); michael@0: goto retry; michael@0: } michael@0: michael@0: // Look for a hexadecimal, octal, or binary number. michael@0: // michael@0: if (c1kind == BasePrefix) { michael@0: tp = newToken(-1); michael@0: int radix; michael@0: c = getCharIgnoreEOL(); michael@0: if (c == 'x' || c == 'X') { michael@0: radix = 16; michael@0: c = getCharIgnoreEOL(); michael@0: if (!JS7_ISHEX(c)) { michael@0: ungetCharIgnoreEOL(c); michael@0: reportError(JSMSG_MISSING_HEXDIGITS); michael@0: goto error; michael@0: } michael@0: numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0x' michael@0: while (JS7_ISHEX(c)) michael@0: c = getCharIgnoreEOL(); michael@0: } else if (c == 'b' || c == 'B') { michael@0: radix = 2; michael@0: c = getCharIgnoreEOL(); michael@0: if (c != '0' && c != '1') { michael@0: ungetCharIgnoreEOL(c); michael@0: reportError(JSMSG_MISSING_BINARY_DIGITS); michael@0: goto error; michael@0: } michael@0: numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0b' michael@0: while (c == '0' || c == '1') michael@0: c = getCharIgnoreEOL(); michael@0: } else if (c == 'o' || c == 'O') { michael@0: radix = 8; michael@0: c = getCharIgnoreEOL(); michael@0: if (c < '0' || c > '7') { michael@0: ungetCharIgnoreEOL(c); michael@0: reportError(JSMSG_MISSING_OCTAL_DIGITS); michael@0: goto error; michael@0: } michael@0: numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0o' michael@0: while ('0' <= c && c <= '7') michael@0: c = getCharIgnoreEOL(); michael@0: } else if (JS7_ISDEC(c)) { michael@0: radix = 8; michael@0: numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0' michael@0: while (JS7_ISDEC(c)) { michael@0: // Octal integer literals are not permitted in strict mode code. michael@0: if (!reportStrictModeError(JSMSG_DEPRECATED_OCTAL)) michael@0: goto error; michael@0: michael@0: // Outside strict mode, we permit 08 and 09 as decimal numbers, michael@0: // which makes our behaviour a superset of the ECMA numeric michael@0: // grammar. We might not always be so permissive, so we warn michael@0: // about it. michael@0: if (c >= '8') { michael@0: if (!reportWarning(JSMSG_BAD_OCTAL, c == '8' ? "08" : "09")) { michael@0: goto error; michael@0: } michael@0: goto decimal; // use the decimal scanner for the rest of the number michael@0: } michael@0: c = getCharIgnoreEOL(); michael@0: } michael@0: } else { michael@0: // '0' not followed by 'x', 'X' or a digit; scan as a decimal number. michael@0: numStart = userbuf.addressOfNextRawChar() - 1; michael@0: goto decimal; michael@0: } michael@0: ungetCharIgnoreEOL(c); michael@0: michael@0: if (c != EOF && IsIdentifierStart(c)) { michael@0: reportError(JSMSG_IDSTART_AFTER_NUMBER); michael@0: goto error; michael@0: } michael@0: michael@0: double dval; michael@0: const jschar *dummy; michael@0: if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), radix, &dummy, &dval)) michael@0: goto error; michael@0: tp->type = TOK_NUMBER; michael@0: tp->setNumber(dval, NoDecimal); michael@0: goto out; michael@0: } michael@0: michael@0: // This handles everything else. michael@0: // michael@0: JS_ASSERT(c1kind == Other); michael@0: tp = newToken(-1); michael@0: switch (c) { michael@0: case '.': michael@0: c = getCharIgnoreEOL(); michael@0: if (JS7_ISDEC(c)) { michael@0: numStart = userbuf.addressOfNextRawChar() - 2; michael@0: decimalPoint = HasDecimal; michael@0: hasExp = false; michael@0: goto decimal_dot; michael@0: } michael@0: if (c == '.') { michael@0: if (matchChar('.')) { michael@0: tp->type = TOK_TRIPLEDOT; michael@0: goto out; michael@0: } michael@0: } michael@0: ungetCharIgnoreEOL(c); michael@0: tp->type = TOK_DOT; michael@0: goto out; michael@0: michael@0: case '=': michael@0: if (matchChar('=')) michael@0: tp->type = matchChar('=') ? TOK_STRICTEQ : TOK_EQ; michael@0: else if (matchChar('>')) michael@0: tp->type = TOK_ARROW; michael@0: else michael@0: tp->type = TOK_ASSIGN; michael@0: goto out; michael@0: michael@0: case '+': michael@0: if (matchChar('+')) michael@0: tp->type = TOK_INC; michael@0: else michael@0: tp->type = matchChar('=') ? TOK_ADDASSIGN : TOK_ADD; michael@0: goto out; michael@0: michael@0: case '\\': michael@0: hadUnicodeEscape = matchUnicodeEscapeIdStart(&qc); michael@0: if (hadUnicodeEscape) { michael@0: identStart = userbuf.addressOfNextRawChar() - 6; michael@0: goto identifier; michael@0: } michael@0: goto badchar; michael@0: michael@0: case '|': michael@0: if (matchChar('|')) michael@0: tp->type = TOK_OR; michael@0: else michael@0: tp->type = matchChar('=') ? TOK_BITORASSIGN : TOK_BITOR; michael@0: goto out; michael@0: michael@0: case '^': michael@0: tp->type = matchChar('=') ? TOK_BITXORASSIGN : TOK_BITXOR; michael@0: goto out; michael@0: michael@0: case '&': michael@0: if (matchChar('&')) michael@0: tp->type = TOK_AND; michael@0: else michael@0: tp->type = matchChar('=') ? TOK_BITANDASSIGN : TOK_BITAND; michael@0: goto out; michael@0: michael@0: case '!': michael@0: if (matchChar('=')) michael@0: tp->type = matchChar('=') ? TOK_STRICTNE : TOK_NE; michael@0: else michael@0: tp->type = TOK_NOT; michael@0: goto out; michael@0: michael@0: case '<': michael@0: // NB: treat HTML begin-comment as comment-till-end-of-line. michael@0: if (matchChar('!')) { michael@0: if (matchChar('-')) { michael@0: if (matchChar('-')) michael@0: goto skipline; michael@0: ungetChar('-'); michael@0: } michael@0: ungetChar('!'); michael@0: } michael@0: if (matchChar('<')) { michael@0: tp->type = matchChar('=') ? TOK_LSHASSIGN : TOK_LSH; michael@0: } else { michael@0: tp->type = matchChar('=') ? TOK_LE : TOK_LT; michael@0: } michael@0: goto out; michael@0: michael@0: case '>': michael@0: if (matchChar('>')) { michael@0: if (matchChar('>')) michael@0: tp->type = matchChar('=') ? TOK_URSHASSIGN : TOK_URSH; michael@0: else michael@0: tp->type = matchChar('=') ? TOK_RSHASSIGN : TOK_RSH; michael@0: } else { michael@0: tp->type = matchChar('=') ? TOK_GE : TOK_GT; michael@0: } michael@0: goto out; michael@0: michael@0: case '*': michael@0: tp->type = matchChar('=') ? TOK_MULASSIGN : TOK_MUL; michael@0: goto out; michael@0: michael@0: case '/': michael@0: // Look for a single-line comment. michael@0: if (matchChar('/')) { michael@0: c = peekChar(); michael@0: if (c == '@' || c == '#') { michael@0: bool shouldWarn = getChar() == '@'; michael@0: if (!getDirectives(false, shouldWarn)) michael@0: goto error; michael@0: } michael@0: michael@0: skipline: michael@0: while ((c = getChar()) != EOF && c != '\n') michael@0: continue; michael@0: ungetChar(c); michael@0: cursor = (cursor - 1) & ntokensMask; michael@0: goto retry; michael@0: } michael@0: michael@0: // Look for a multi-line comment. michael@0: if (matchChar('*')) { michael@0: unsigned linenoBefore = lineno; michael@0: while ((c = getChar()) != EOF && michael@0: !(c == '*' && matchChar('/'))) { michael@0: if (c == '@' || c == '#') { michael@0: bool shouldWarn = c == '@'; michael@0: if (!getDirectives(true, shouldWarn)) michael@0: goto error; michael@0: } michael@0: } michael@0: if (c == EOF) { michael@0: reportError(JSMSG_UNTERMINATED_COMMENT); michael@0: goto error; michael@0: } michael@0: if (linenoBefore != lineno) michael@0: updateFlagsForEOL(); michael@0: cursor = (cursor - 1) & ntokensMask; michael@0: goto retry; michael@0: } michael@0: michael@0: // Look for a regexp. michael@0: if (modifier == Operand) { michael@0: tokenbuf.clear(); michael@0: michael@0: bool inCharClass = false; michael@0: for (;;) { michael@0: c = getChar(); michael@0: if (c == '\\') { michael@0: if (!tokenbuf.append(c)) michael@0: goto error; michael@0: c = getChar(); michael@0: } else if (c == '[') { michael@0: inCharClass = true; michael@0: } else if (c == ']') { michael@0: inCharClass = false; michael@0: } else if (c == '/' && !inCharClass) { michael@0: // For compat with IE, allow unescaped / in char classes. michael@0: break; michael@0: } michael@0: if (c == '\n' || c == EOF) { michael@0: ungetChar(c); michael@0: reportError(JSMSG_UNTERMINATED_REGEXP); michael@0: goto error; michael@0: } michael@0: if (!tokenbuf.append(c)) michael@0: goto error; michael@0: } michael@0: michael@0: RegExpFlag reflags = NoFlags; michael@0: unsigned length = tokenbuf.length() + 1; michael@0: while (true) { michael@0: c = peekChar(); michael@0: if (c == 'g' && !(reflags & GlobalFlag)) michael@0: reflags = RegExpFlag(reflags | GlobalFlag); michael@0: else if (c == 'i' && !(reflags & IgnoreCaseFlag)) michael@0: reflags = RegExpFlag(reflags | IgnoreCaseFlag); michael@0: else if (c == 'm' && !(reflags & MultilineFlag)) michael@0: reflags = RegExpFlag(reflags | MultilineFlag); michael@0: else if (c == 'y' && !(reflags & StickyFlag)) michael@0: reflags = RegExpFlag(reflags | StickyFlag); michael@0: else michael@0: break; michael@0: getChar(); michael@0: length++; michael@0: } michael@0: michael@0: c = peekChar(); michael@0: if (JS7_ISLET(c)) { michael@0: char buf[2] = { '\0', '\0' }; michael@0: tp->pos.begin += length + 1; michael@0: buf[0] = char(c); michael@0: reportError(JSMSG_BAD_REGEXP_FLAG, buf); michael@0: (void) getChar(); michael@0: goto error; michael@0: } michael@0: tp->type = TOK_REGEXP; michael@0: tp->setRegExpFlags(reflags); michael@0: goto out; michael@0: } michael@0: michael@0: tp->type = matchChar('=') ? TOK_DIVASSIGN : TOK_DIV; michael@0: goto out; michael@0: michael@0: case '%': michael@0: tp->type = matchChar('=') ? TOK_MODASSIGN : TOK_MOD; michael@0: goto out; michael@0: michael@0: case '-': michael@0: if (matchChar('-')) { michael@0: if (peekChar() == '>' && !flags.isDirtyLine) michael@0: goto skipline; michael@0: tp->type = TOK_DEC; michael@0: } else { michael@0: tp->type = matchChar('=') ? TOK_SUBASSIGN : TOK_SUB; michael@0: } michael@0: goto out; michael@0: michael@0: badchar: michael@0: default: michael@0: reportError(JSMSG_ILLEGAL_CHARACTER); michael@0: goto error; michael@0: } michael@0: michael@0: MOZ_ASSUME_UNREACHABLE("should have jumped to |out| or |error|"); michael@0: michael@0: out: michael@0: flags.isDirtyLine = true; michael@0: tp->pos.end = userbuf.addressOfNextRawChar() - userbuf.base(); michael@0: JS_ASSERT(IsTokenSane(tp)); michael@0: return tp->type; michael@0: michael@0: error: michael@0: flags.isDirtyLine = true; michael@0: tp->pos.end = userbuf.addressOfNextRawChar() - userbuf.base(); michael@0: tp->type = TOK_ERROR; michael@0: JS_ASSERT(IsTokenSane(tp)); michael@0: onError(); michael@0: return TOK_ERROR; michael@0: } michael@0: michael@0: void michael@0: TokenStream::onError() michael@0: { michael@0: flags.hadError = true; michael@0: #ifdef DEBUG michael@0: // Poisoning userbuf on error establishes an invariant: once an erroneous michael@0: // token has been seen, userbuf will not be consulted again. This is true michael@0: // because the parser will either (a) deal with the TOK_ERROR token by michael@0: // aborting parsing immediately; or (b) if the TOK_ERROR token doesn't michael@0: // match what it expected, it will unget the token, and the next getToken() michael@0: // call will immediately return the just-gotten TOK_ERROR token again michael@0: // without consulting userbuf, thanks to the lookahead buffer. michael@0: userbuf.poison(); michael@0: #endif michael@0: } michael@0: michael@0: JS_FRIEND_API(int) michael@0: js_fgets(char *buf, int size, FILE *file) michael@0: { michael@0: int n, i, c; michael@0: bool crflag; michael@0: michael@0: n = size - 1; michael@0: if (n < 0) michael@0: return -1; michael@0: michael@0: crflag = false; michael@0: for (i = 0; i < n && (c = fast_getc(file)) != EOF; i++) { michael@0: buf[i] = c; michael@0: if (c == '\n') { // any \n ends a line michael@0: i++; // keep the \n; we know there is room for \0 michael@0: break; michael@0: } michael@0: if (crflag) { // \r not followed by \n ends line at the \r michael@0: ungetc(c, file); michael@0: break; // and overwrite c in buf with \0 michael@0: } michael@0: crflag = (c == '\r'); michael@0: } michael@0: michael@0: buf[i] = '\0'; michael@0: return i; michael@0: } michael@0: michael@0: #ifdef DEBUG michael@0: const char * michael@0: TokenKindToString(TokenKind tt) michael@0: { michael@0: switch (tt) { michael@0: case TOK_ERROR: return "TOK_ERROR"; michael@0: case TOK_EOF: return "TOK_EOF"; michael@0: case TOK_EOL: return "TOK_EOL"; michael@0: case TOK_SEMI: return "TOK_SEMI"; michael@0: case TOK_COMMA: return "TOK_COMMA"; michael@0: case TOK_HOOK: return "TOK_HOOK"; michael@0: case TOK_COLON: return "TOK_COLON"; michael@0: case TOK_OR: return "TOK_OR"; michael@0: case TOK_AND: return "TOK_AND"; michael@0: case TOK_BITOR: return "TOK_BITOR"; michael@0: case TOK_BITXOR: return "TOK_BITXOR"; michael@0: case TOK_BITAND: return "TOK_BITAND"; michael@0: case TOK_ADD: return "TOK_ADD"; michael@0: case TOK_SUB: return "TOK_SUB"; michael@0: case TOK_MUL: return "TOK_MUL"; michael@0: case TOK_DIV: return "TOK_DIV"; michael@0: case TOK_MOD: return "TOK_MOD"; michael@0: case TOK_INC: return "TOK_INC"; michael@0: case TOK_DEC: return "TOK_DEC"; michael@0: case TOK_DOT: return "TOK_DOT"; michael@0: case TOK_TRIPLEDOT: return "TOK_TRIPLEDOT"; michael@0: case TOK_LB: return "TOK_LB"; michael@0: case TOK_RB: return "TOK_RB"; michael@0: case TOK_LC: return "TOK_LC"; michael@0: case TOK_RC: return "TOK_RC"; michael@0: case TOK_LP: return "TOK_LP"; michael@0: case TOK_RP: return "TOK_RP"; michael@0: case TOK_ARROW: return "TOK_ARROW"; michael@0: case TOK_NAME: return "TOK_NAME"; michael@0: case TOK_NUMBER: return "TOK_NUMBER"; michael@0: case TOK_STRING: return "TOK_STRING"; michael@0: case TOK_REGEXP: return "TOK_REGEXP"; michael@0: case TOK_TRUE: return "TOK_TRUE"; michael@0: case TOK_FALSE: return "TOK_FALSE"; michael@0: case TOK_NULL: return "TOK_NULL"; michael@0: case TOK_THIS: return "TOK_THIS"; michael@0: case TOK_FUNCTION: return "TOK_FUNCTION"; michael@0: case TOK_IF: return "TOK_IF"; michael@0: case TOK_ELSE: return "TOK_ELSE"; michael@0: case TOK_SWITCH: return "TOK_SWITCH"; michael@0: case TOK_CASE: return "TOK_CASE"; michael@0: case TOK_DEFAULT: return "TOK_DEFAULT"; michael@0: case TOK_WHILE: return "TOK_WHILE"; michael@0: case TOK_DO: return "TOK_DO"; michael@0: case TOK_FOR: return "TOK_FOR"; michael@0: case TOK_BREAK: return "TOK_BREAK"; michael@0: case TOK_CONTINUE: return "TOK_CONTINUE"; michael@0: case TOK_IN: return "TOK_IN"; michael@0: case TOK_VAR: return "TOK_VAR"; michael@0: case TOK_CONST: return "TOK_CONST"; michael@0: case TOK_WITH: return "TOK_WITH"; michael@0: case TOK_RETURN: return "TOK_RETURN"; michael@0: case TOK_NEW: return "TOK_NEW"; michael@0: case TOK_DELETE: return "TOK_DELETE"; michael@0: case TOK_TRY: return "TOK_TRY"; michael@0: case TOK_CATCH: return "TOK_CATCH"; michael@0: case TOK_FINALLY: return "TOK_FINALLY"; michael@0: case TOK_THROW: return "TOK_THROW"; michael@0: case TOK_INSTANCEOF: return "TOK_INSTANCEOF"; michael@0: case TOK_DEBUGGER: return "TOK_DEBUGGER"; michael@0: case TOK_YIELD: return "TOK_YIELD"; michael@0: case TOK_LET: return "TOK_LET"; michael@0: case TOK_RESERVED: return "TOK_RESERVED"; michael@0: case TOK_STRICT_RESERVED: return "TOK_STRICT_RESERVED"; michael@0: case TOK_STRICTEQ: return "TOK_STRICTEQ"; michael@0: case TOK_EQ: return "TOK_EQ"; michael@0: case TOK_STRICTNE: return "TOK_STRICTNE"; michael@0: case TOK_NE: return "TOK_NE"; michael@0: case TOK_TYPEOF: return "TOK_TYPEOF"; michael@0: case TOK_VOID: return "TOK_VOID"; michael@0: case TOK_NOT: return "TOK_NOT"; michael@0: case TOK_BITNOT: return "TOK_BITNOT"; michael@0: case TOK_LT: return "TOK_LT"; michael@0: case TOK_LE: return "TOK_LE"; michael@0: case TOK_GT: return "TOK_GT"; michael@0: case TOK_GE: return "TOK_GE"; michael@0: case TOK_LSH: return "TOK_LSH"; michael@0: case TOK_RSH: return "TOK_RSH"; michael@0: case TOK_URSH: return "TOK_URSH"; michael@0: case TOK_ASSIGN: return "TOK_ASSIGN"; michael@0: case TOK_ADDASSIGN: return "TOK_ADDASSIGN"; michael@0: case TOK_SUBASSIGN: return "TOK_SUBASSIGN"; michael@0: case TOK_BITORASSIGN: return "TOK_BITORASSIGN"; michael@0: case TOK_BITXORASSIGN: return "TOK_BITXORASSIGN"; michael@0: case TOK_BITANDASSIGN: return "TOK_BITANDASSIGN"; michael@0: case TOK_LSHASSIGN: return "TOK_LSHASSIGN"; michael@0: case TOK_RSHASSIGN: return "TOK_RSHASSIGN"; michael@0: case TOK_URSHASSIGN: return "TOK_URSHASSIGN"; michael@0: case TOK_MULASSIGN: return "TOK_MULASSIGN"; michael@0: case TOK_DIVASSIGN: return "TOK_DIVASSIGN"; michael@0: case TOK_MODASSIGN: return "TOK_MODASSIGN"; michael@0: case TOK_EXPORT: return "TOK_EXPORT"; michael@0: case TOK_IMPORT: return "TOK_IMPORT"; michael@0: case TOK_LIMIT: break; michael@0: } michael@0: michael@0: return ""; michael@0: } michael@0: #endif