The Tor Browser: layout/style/nsCSSScanner.cpp@141e0f1194b1 (annotated)

layout/style/nsCSSScanner.cpp@141e0f1194b1 (annotated)

layout/style/nsCSSScanner.cpp

Wed, 31 Dec 2014 07:16:47 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 07:16:47 +0100
branch: TOR_BUG_9701
changeset 3: 141e0f1194b1
permissions: -rw-r--r--

Revert simplistic fix pending revisit of Mozilla integration attempt.

 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 /* tokenization of CSS style sheets */
 #include "nsCSSScanner.h"
 #include "nsStyleUtil.h"
 #include "nsISupportsImpl.h"
 #include "mozilla/ArrayUtils.h"
 #include "mozilla/css/ErrorReporter.h"
 #include "mozilla/Likely.h"
 #include <algorithm>
 /* Character class tables and related helper functions. */
 static const uint8_t IS_HEX_DIGIT  = 0x01;
 static const uint8_t IS_IDSTART    = 0x02;
 static const uint8_t IS_IDCHAR     = 0x04;
 static const uint8_t IS_URL_CHAR   = 0x08;
 static const uint8_t IS_HSPACE     = 0x10;
 static const uint8_t IS_VSPACE     = 0x20;
 static const uint8_t IS_SPACE      = IS_HSPACE|IS_VSPACE;
 static const uint8_t IS_STRING     = 0x40;
 #define H    IS_HSPACE
 #define V    IS_VSPACE
 #define I    IS_IDCHAR
 #define J    IS_IDSTART
 #define U    IS_URL_CHAR
 #define S    IS_STRING
 #define X    IS_HEX_DIGIT
 #define SH    S|H
 #define SU    S|U
 #define SUI   S|U|I
 #define SUIJ  S|U|I|J
 #define SUIX  S|U|I|X
 #define SUIJX S|U|I|J|X
 static const uint8_t gLexTable[] = {
 // 00    01    02    03    04    05    06    07
 ,    S,    S,    S,    S,    S,    S,    S,
 // 08   TAB    LF    0B    FF    CR    0E    0F
     S,   SH,    V,    S,    V,    V,    S,    S,
 // 10    11    12    13    14    15    16    17
     S,    S,    S,    S,    S,    S,    S,    S,
 // 18    19    1A    1B    1C    1D    1E    1F
     S,    S,    S,    S,    S,    S,    S,    S,
 //SPC     !     "     #     $     %     &     '
    SH,   SU,    0,   SU,   SU,   SU,   SU,    0,
 //  (     )     *     +     ,     -     .     /
     S,    S,   SU,   SU,   SU,  SUI,   SU,   SU,
 //  0     1     2     3     4     5     6     7
  SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX,
 //  8     9     :     ;     <     =     >     ?
  SUIX, SUIX,   SU,   SU,   SU,   SU,   SU,   SU,
 //  @     A     B     C     D     E     F     G
    SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
 //  H     I     J     K     L     M     N     O
  SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
 //  P     Q     R     S     T     U     V     W
  SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
 //  X     Y     Z     [     \     ]     ^     _
  SUIJ, SUIJ, SUIJ,   SU,    J,   SU,   SU, SUIJ,
 //  `     a     b     c     d     e     f     g
    SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
 //  h     i     j     k     l     m     n     o
  SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
 //  p     q     r     s     t     u     v     w
  SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
 //  x     y     z     {     |     }     ~    7F
  SUIJ, SUIJ, SUIJ,   SU,   SU,   SU,   SU,    S,
 };
 static_assert(MOZ_ARRAY_LENGTH(gLexTable) == 128,
               "gLexTable expected to cover all 128 ASCII characters");
 #undef I
 #undef J
 #undef U
 #undef S
 #undef X
 #undef SH
 #undef SU
 #undef SUI
 #undef SUIJ
 #undef SUIX
 #undef SUIJX
 /**
  * True if 'ch' is in character class 'cls', which should be one of
  * the constants above or some combination of them.  All characters
  * above U+007F are considered to be in 'cls'.  EOF is never in 'cls'.
  */
 static inline bool
 IsOpenCharClass(int32_t ch, uint8_t cls) {
   return ch >= 0 && (ch >= 128 || (gLexTable[ch] & cls) != 0);
 }
 /**
  * True if 'ch' is in character class 'cls', which should be one of
  * the constants above or some combination of them.  No characters
  * above U+007F are considered to be in 'cls'. EOF is never in 'cls'.
  */
 static inline bool
 IsClosedCharClass(int32_t ch, uint8_t cls) {
   return uint32_t(ch) < 128 && (gLexTable[ch] & cls) != 0;
 }
 /**
  * True if 'ch' is CSS whitespace, i.e. any of the ASCII characters
  * TAB, LF, FF, CR, or SPC.
  */
 static inline bool
 IsWhitespace(int32_t ch) {
   return IsClosedCharClass(ch, IS_SPACE);
 }
 /**
  * True if 'ch' is horizontal whitespace, i.e. TAB or SPC.
  */
 static inline bool
 IsHorzSpace(int32_t ch) {
   return IsClosedCharClass(ch, IS_HSPACE);
 }
 /**
  * True if 'ch' is vertical whitespace, i.e. LF, FF, or CR.  Vertical
  * whitespace requires special handling when consumed, see AdvanceLine.
  */
 static inline bool
 IsVertSpace(int32_t ch) {
   return IsClosedCharClass(ch, IS_VSPACE);
 }
 /**
  * True if 'ch' is a character that can appear in the middle of an identifier.
  * This includes U+0000 since it is handled as U+FFFD, but for purposes of
  * GatherText it should not be included in IsOpenCharClass.
  */
 static inline bool
 IsIdentChar(int32_t ch) {
   return IsOpenCharClass(ch, IS_IDCHAR) || ch == 0;
 }
 /**
  * True if 'ch' is a character that by itself begins an identifier.
  * This includes U+0000 since it is handled as U+FFFD, but for purposes of
  * GatherText it should not be included in IsOpenCharClass.
  * (This is a subset of IsIdentChar.)
  */
 static inline bool
 IsIdentStart(int32_t ch) {
   return IsOpenCharClass(ch, IS_IDSTART) || ch == 0;
 }
 /**
  * True if the two-character sequence aFirstChar+aSecondChar begins an
  * identifier.
  */
 static inline bool
 StartsIdent(int32_t aFirstChar, int32_t aSecondChar)
 {
   return IsIdentStart(aFirstChar) ||
     (aFirstChar == '-' && IsIdentStart(aSecondChar));
 }
 /**
  * True if 'ch' is a decimal digit.
  */
 static inline bool
 IsDigit(int32_t ch) {
   return (ch >= '0') && (ch <= '9');
 }
 /**
  * True if 'ch' is a hexadecimal digit.
  */
 static inline bool
 IsHexDigit(int32_t ch) {
   return IsClosedCharClass(ch, IS_HEX_DIGIT);
 }
 /**
  * Assuming that 'ch' is a decimal digit, return its numeric value.
  */
 static inline uint32_t
 DecimalDigitValue(int32_t ch)
 {
   return ch - '0';
 }
 /**
  * Assuming that 'ch' is a hexadecimal digit, return its numeric value.
  */
 static inline uint32_t
 HexDigitValue(int32_t ch)
 {
   if (IsDigit(ch)) {
     return DecimalDigitValue(ch);
   } else {
     // Note: c&7 just keeps the low three bits which causes
     // upper and lower case alphabetics to both yield their
     // "relative to 10" value for computing the hex value.
     return (ch & 0x7) + 9;
   }
 }
 /**
  * If 'ch' can be the first character of a two-character match operator
  * token, return the token type code for that token, otherwise return
  * eCSSToken_Symbol to indicate that it can't.
  */
 static inline nsCSSTokenType
 MatchOperatorType(int32_t ch)
 {
   switch (ch) {
   case '~': return eCSSToken_Includes;
   case '|': return eCSSToken_Dashmatch;
   case '^': return eCSSToken_Beginsmatch;
   case '$': return eCSSToken_Endsmatch;
   case '*': return eCSSToken_Containsmatch;
   default:  return eCSSToken_Symbol;
   }
 }
 /* Out-of-line nsCSSToken methods. */
 /**
  * Append the textual representation of |this| to |aBuffer|.
  */
 void
 nsCSSToken::AppendToString(nsString& aBuffer) const
 {
   switch (mType) {
     case eCSSToken_Ident:
       nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
       break;
     case eCSSToken_AtKeyword:
       aBuffer.Append('@');
       nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
       break;
     case eCSSToken_ID:
     case eCSSToken_Hash:
       aBuffer.Append('#');
       nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
       break;
     case eCSSToken_Function:
       nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
       aBuffer.Append('(');
       break;
     case eCSSToken_URL:
     case eCSSToken_Bad_URL:
       aBuffer.AppendLiteral("url(");
       if (mSymbol != char16_t(0)) {
         nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
       } else {
         aBuffer.Append(mIdent);
       }
       if (mType == eCSSToken_URL) {
         aBuffer.Append(char16_t(')'));
       }
       break;
     case eCSSToken_Number:
       if (mIntegerValid) {
         aBuffer.AppendInt(mInteger, 10);
       } else {
         aBuffer.AppendFloat(mNumber);
       }
       break;
     case eCSSToken_Percentage:
       aBuffer.AppendFloat(mNumber * 100.0f);
       aBuffer.Append(char16_t('%'));
       break;
     case eCSSToken_Dimension:
       if (mIntegerValid) {
         aBuffer.AppendInt(mInteger, 10);
       } else {
         aBuffer.AppendFloat(mNumber);
       }
       nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
       break;
     case eCSSToken_Bad_String:
       nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
       // remove the trailing quote character
       aBuffer.Truncate(aBuffer.Length() - 1);
       break;
     case eCSSToken_String:
       nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
       break;
     case eCSSToken_Symbol:
       aBuffer.Append(mSymbol);
       break;
     case eCSSToken_Whitespace:
       aBuffer.Append(' ');
       break;
     case eCSSToken_HTMLComment:
     case eCSSToken_URange:
       aBuffer.Append(mIdent);
       break;
     case eCSSToken_Includes:
       aBuffer.AppendLiteral("~=");
       break;
     case eCSSToken_Dashmatch:
       aBuffer.AppendLiteral("|=");
       break;
     case eCSSToken_Beginsmatch:
       aBuffer.AppendLiteral("^=");
       break;
     case eCSSToken_Endsmatch:
       aBuffer.AppendLiteral("$=");
       break;
     case eCSSToken_Containsmatch:
       aBuffer.AppendLiteral("*=");
       break;
     default:
       NS_ERROR("invalid token type");
       break;
   }
 }
 /* nsCSSScanner methods. */
 nsCSSScanner::nsCSSScanner(const nsAString& aBuffer, uint32_t aLineNumber)
   : mBuffer(aBuffer.BeginReading())
   , mOffset(0)
   , mCount(aBuffer.Length())
   , mLineNumber(aLineNumber)
   , mLineOffset(0)
   , mTokenLineNumber(aLineNumber)
   , mTokenLineOffset(0)
   , mTokenOffset(0)
   , mRecordStartOffset(0)
   , mEOFCharacters(eEOFCharacters_None)
   , mReporter(nullptr)
   , mSVGMode(false)
   , mRecording(false)
   , mSeenBadToken(false)
   , mSeenVariableReference(false)
 {
   MOZ_COUNT_CTOR(nsCSSScanner);
 }
 nsCSSScanner::~nsCSSScanner()
 {
   MOZ_COUNT_DTOR(nsCSSScanner);
 }
 void
 nsCSSScanner::StartRecording()
 {
   MOZ_ASSERT(!mRecording, "already started recording");
   mRecording = true;
   mRecordStartOffset = mOffset;
 }
 void
 nsCSSScanner::StopRecording()
 {
   MOZ_ASSERT(mRecording, "haven't started recording");
   mRecording = false;
 }
 void
 nsCSSScanner::StopRecording(nsString& aBuffer)
 {
   MOZ_ASSERT(mRecording, "haven't started recording");
   mRecording = false;
   aBuffer.Append(mBuffer + mRecordStartOffset,
                  mOffset - mRecordStartOffset);
 }
 uint32_t
 nsCSSScanner::RecordingLength() const
 {
   MOZ_ASSERT(mRecording, "haven't started recording");
   return mOffset - mRecordStartOffset;
 }
 #ifdef DEBUG
 bool
 nsCSSScanner::IsRecording() const
 {
   return mRecording;
 }
 #endif
 nsDependentSubstring
 nsCSSScanner::GetCurrentLine() const
 {
   uint32_t end = mTokenOffset;
   while (end < mCount && !IsVertSpace(mBuffer[end])) {
     end++;
   }
   return nsDependentSubstring(mBuffer + mTokenLineOffset,
                               mBuffer + end);
 }
 /**
  * Return the raw UTF-16 code unit at position |mOffset + n| within
  * the read buffer.  If that is beyond the end of the buffer, returns
  * -1 to indicate end of input.
  */
 inline int32_t
 nsCSSScanner::Peek(uint32_t n)
 {
   if (mOffset + n >= mCount) {
     return -1;
   }
   return mBuffer[mOffset + n];
 }
 /**
  * Advance |mOffset| over |n| code units.  Advance(0) is a no-op.
  * If |n| is greater than the distance to end of input, will silently
  * stop at the end.  May not be used to advance over a line boundary;
  * AdvanceLine() must be used instead.
  */
 inline void
 nsCSSScanner::Advance(uint32_t n)
 {
 #ifdef DEBUG
   while (mOffset < mCount && n > 0) {
     MOZ_ASSERT(!IsVertSpace(mBuffer[mOffset]),
                "may not Advance() over a line boundary");
     mOffset++;
     n--;
   }
 #else
   if (mOffset + n >= mCount || mOffset + n < mOffset)
     mOffset = mCount;
   else
     mOffset += n;
 #endif
 }
 /**
  * Advance |mOffset| over a line boundary.
  */
 void
 nsCSSScanner::AdvanceLine()
 {
   MOZ_ASSERT(IsVertSpace(mBuffer[mOffset]),
              "may not AdvanceLine() over a horizontal character");
   // Advance over \r\n as a unit.
   if (mBuffer[mOffset]   == '\r' && mOffset + 1 < mCount &&
       mBuffer[mOffset+1] == '\n')
     mOffset += 2;
   else
     mOffset += 1;
   // 0 is a magical line number meaning that we don't know (i.e., script)
   if (mLineNumber != 0)
     mLineNumber++;
   mLineOffset = mOffset;
 }
 /**
  * Back up |mOffset| over |n| code units.  Backup(0) is a no-op.
  * If |n| is greater than the distance to beginning of input, will
  * silently stop at the beginning.  May not be used to back up over a
  * line boundary.
  */
 void
 nsCSSScanner::Backup(uint32_t n)
 {
 #ifdef DEBUG
   while (mOffset > 0 && n > 0) {
     MOZ_ASSERT(!IsVertSpace(mBuffer[mOffset-1]),
                "may not Backup() over a line boundary");
     mOffset--;
     n--;
   }
 #else
   if (mOffset < n)
     mOffset = 0;
   else
     mOffset -= n;
 #endif
 }
 void
 nsCSSScanner::SavePosition(nsCSSScannerPosition& aState)
 {
   aState.mOffset = mOffset;
   aState.mLineNumber = mLineNumber;
   aState.mLineOffset = mLineOffset;
   aState.mTokenLineNumber = mTokenLineNumber;
   aState.mTokenLineOffset = mTokenLineOffset;
   aState.mTokenOffset = mTokenOffset;
   aState.mInitialized = true;
 }
 void
 nsCSSScanner::RestoreSavedPosition(const nsCSSScannerPosition& aState)
 {
   MOZ_ASSERT(aState.mInitialized, "have not saved state");
   if (aState.mInitialized) {
     mOffset = aState.mOffset;
     mLineNumber = aState.mLineNumber;
     mLineOffset = aState.mLineOffset;
     mTokenLineNumber = aState.mTokenLineNumber;
     mTokenLineOffset = aState.mTokenLineOffset;
     mTokenOffset = aState.mTokenOffset;
   }
 }
 /**
  * Skip over a sequence of whitespace characters (vertical or
  * horizontal) starting at the current read position.
  */
 void
 nsCSSScanner::SkipWhitespace()
 {
   for (;;) {
     int32_t ch = Peek();
     if (!IsWhitespace(ch)) { // EOF counts as non-whitespace
       break;
     }
     if (IsVertSpace(ch)) {
       AdvanceLine();
     } else {
       Advance();
     }
   }
 }
 /**
  * Skip over one CSS comment starting at the current read position.
  */
 void
 nsCSSScanner::SkipComment()
 {
   MOZ_ASSERT(Peek() == '/' && Peek(1) == '*', "should not have been called");
   Advance(2);
   for (;;) {
     int32_t ch = Peek();
     if (ch < 0) {
       mReporter->ReportUnexpectedEOF("PECommentEOF");
       SetEOFCharacters(eEOFCharacters_Asterisk | eEOFCharacters_Slash);
       return;
     }
     if (ch == '*') {
       Advance();
       ch = Peek();
       if (ch < 0) {
         mReporter->ReportUnexpectedEOF("PECommentEOF");
         SetEOFCharacters(eEOFCharacters_Slash);
         return;
       }
       if (ch == '/') {
         Advance();
         return;
       }
     } else if (IsVertSpace(ch)) {
       AdvanceLine();
     } else {
       Advance();
     }
   }
 }
 /**
  * If there is a valid escape sequence starting at the current read
  * position, consume it, decode it, append the result to |aOutput|,
  * and return true.  Otherwise, consume nothing, leave |aOutput|
  * unmodified, and return false.  If |aInString| is true, accept the
  * additional form of escape sequence allowed within string-like tokens.
  */
 bool
 nsCSSScanner::GatherEscape(nsString& aOutput, bool aInString)
 {
   MOZ_ASSERT(Peek() == '\\', "should not have been called");
   int32_t ch = Peek(1);
   if (ch < 0) {
     // If we are in a string (or a url() containing a string), we want to drop
     // the backslash on the floor.  Otherwise, we want to treat it as a U+FFFD
     // character.
     Advance();
     if (aInString) {
       SetEOFCharacters(eEOFCharacters_DropBackslash);
     } else {
       aOutput.Append(UCS2_REPLACEMENT_CHAR);
       SetEOFCharacters(eEOFCharacters_ReplacementChar);
     }
     return true;
   }
   if (IsVertSpace(ch)) {
     if (aInString) {
       // In strings (and in url() containing a string), escaped
       // newlines are completely removed, to allow splitting over
       // multiple lines.
       Advance();
       AdvanceLine();
       return true;
     }
     // Outside of strings, backslash followed by a newline is not an escape.
     return false;
   }
   if (!IsHexDigit(ch)) {
     // "Any character (except a hexadecimal digit, linefeed, carriage
     // return, or form feed) can be escaped with a backslash to remove
     // its special meaning." -- CSS2.1 section 4.1.3
     Advance(2);
     if (ch == 0) {
       aOutput.Append(UCS2_REPLACEMENT_CHAR);
     } else {
       aOutput.Append(ch);
     }
     return true;
   }
   // "[at most six hexadecimal digits following a backslash] stand
   // for the ISO 10646 character with that number, which must not be
   // zero. (It is undefined in CSS 2.1 what happens if a style sheet
   // does contain a character with Unicode codepoint zero.)"
   //   -- CSS2.1 section 4.1.3
   // At this point we know we have \ followed by at least one
   // hexadecimal digit, therefore the escape sequence is valid and we
   // can go ahead and consume the backslash.
   Advance();
   uint32_t val = 0;
   int i = 0;
   do {
     val = val * 16 + HexDigitValue(ch);
     i++;
     Advance();
     ch = Peek();
   } while (i < 6 && IsHexDigit(ch));
   // "Interpret the hex digits as a hexadecimal number. If this number is zero,
   // or is greater than the maximum allowed codepoint, return U+FFFD
   // REPLACEMENT CHARACTER" -- CSS Syntax Level 3
   if (MOZ_UNLIKELY(val == 0)) {
     aOutput.Append(UCS2_REPLACEMENT_CHAR);
   } else {
     AppendUCS4ToUTF16(ENSURE_VALID_CHAR(val), aOutput);
   }
   // Consume exactly one whitespace character after a
   // hexadecimal escape sequence.
   if (IsVertSpace(ch)) {
     AdvanceLine();
   } else if (IsHorzSpace(ch)) {
     Advance();
   }
   return true;
 }
 /**
  * Consume a run of "text" beginning with the current read position,
  * consisting of characters in the class |aClass| (which must be a
  * suitable argument to IsOpenCharClass) plus escape sequences.
  * Append the text to |aText|, after decoding escape sequences.
  *
  * Returns true if at least one character was appended to |aText|,
  * false otherwise.
  */
 bool
 nsCSSScanner::GatherText(uint8_t aClass, nsString& aText)
 {
   // This is all of the character classes currently used with
   // GatherText.  If you have a need to use this function with a
   // different class, go ahead and add it.
   MOZ_ASSERT(aClass == IS_STRING ||
              aClass == IS_IDCHAR ||
              aClass == IS_URL_CHAR,
              "possibly-inappropriate character class");
   uint32_t start = mOffset;
   bool inString = aClass == IS_STRING;
   for (;;) {
     // Consume runs of unescaped characters in one go.
     uint32_t n = mOffset;
     while (n < mCount && IsOpenCharClass(mBuffer[n], aClass)) {
       n++;
     }
     if (n > mOffset) {
       aText.Append(&mBuffer[mOffset], n - mOffset);
       mOffset = n;
     }
     if (n == mCount) {
       break;
     }
     int32_t ch = Peek();
     MOZ_ASSERT(!IsOpenCharClass(ch, aClass),
                "should not have exited the inner loop");
     if (ch == 0) {
       Advance();
       aText.Append(UCS2_REPLACEMENT_CHAR);
       continue;
     }
     if (ch != '\\') {
       break;
     }
     if (!GatherEscape(aText, inString)) {
       break;
     }
   }
   return mOffset > start;
 }
 /**
  * Scan an Ident token.  This also handles Function and URL tokens,
  * both of which begin indistinguishably from an identifier.  It can
  * produce a Symbol token when an apparent identifier actually led
  * into an invalid escape sequence.
  */
 bool
 nsCSSScanner::ScanIdent(nsCSSToken& aToken)
 {
   if (MOZ_UNLIKELY(!GatherText(IS_IDCHAR, aToken.mIdent))) {
     MOZ_ASSERT(Peek() == '\\',
                "unexpected IsIdentStart character that did not begin an ident");
     aToken.mSymbol = Peek();
     Advance();
     return true;
   }
   if (MOZ_LIKELY(Peek() != '(')) {
     aToken.mType = eCSSToken_Ident;
     return true;
   }
   Advance();
   aToken.mType = eCSSToken_Function;
   if (aToken.mIdent.LowerCaseEqualsLiteral("url")) {
     NextURL(aToken);
   } else if (aToken.mIdent.LowerCaseEqualsLiteral("var")) {
     mSeenVariableReference = true;
   }
   return true;
 }
 /**
  * Scan an AtKeyword token.  Also handles production of Symbol when
  * an '@' is not followed by an identifier.
  */
 bool
 nsCSSScanner::ScanAtKeyword(nsCSSToken& aToken)
 {
   MOZ_ASSERT(Peek() == '@', "should not have been called");
   // Fall back for when '@' isn't followed by an identifier.
   aToken.mSymbol = '@';
   Advance();
   int32_t ch = Peek();
   if (StartsIdent(ch, Peek(1))) {
     if (GatherText(IS_IDCHAR, aToken.mIdent)) {
        aToken.mType = eCSSToken_AtKeyword;
      }
   }
   return true;
 }
 /**
  * Scan a Hash token.  Handles the distinction between eCSSToken_ID
  * and eCSSToken_Hash, and handles production of Symbol when a '#'
  * is not followed by identifier characters.
  */
 bool
 nsCSSScanner::ScanHash(nsCSSToken& aToken)
 {
   MOZ_ASSERT(Peek() == '#', "should not have been called");
   // Fall back for when '#' isn't followed by identifier characters.
   aToken.mSymbol = '#';
   Advance();
   int32_t ch = Peek();
   if (IsIdentChar(ch) || ch == '\\') {
     nsCSSTokenType type =
       StartsIdent(ch, Peek(1)) ? eCSSToken_ID : eCSSToken_Hash;
     aToken.mIdent.SetLength(0);
     if (GatherText(IS_IDCHAR, aToken.mIdent)) {
       aToken.mType = type;
     }
   }
   return true;
 }
 /**
  * Scan a Number, Percentage, or Dimension token (all of which begin
  * like a Number).  Can produce a Symbol when a '.' is not followed by
  * digits, or when '+' or '-' are not followed by either a digit or a
  * '.' and then a digit.  Can also produce a HTMLComment when it
  * encounters '-->'.
  */
 bool
 nsCSSScanner::ScanNumber(nsCSSToken& aToken)
 {
   int32_t c = Peek();
 #ifdef DEBUG
   {
     int32_t c2 = Peek(1);
     int32_t c3 = Peek(2);
     MOZ_ASSERT(IsDigit(c) ||
                (IsDigit(c2) && (c == '.' || c == '+' || c == '-')) ||
                (IsDigit(c3) && (c == '+' || c == '-') && c2 == '.'),
                "should not have been called");
   }
 #endif
   // Sign of the mantissa (-1 or 1).
   int32_t sign = c == '-' ? -1 : 1;
   // Absolute value of the integer part of the mantissa.  This is a double so
   // we don't run into overflow issues for consumers that only care about our
   // floating-point value while still being able to express the full int32_t
   // range for consumers who want integers.
   double intPart = 0;
   // Fractional part of the mantissa.  This is a double so that when we convert
   // to float at the end we'll end up rounding to nearest float instead of
   // truncating down (as we would if fracPart were a float and we just
   // effectively lost the last several digits).
   double fracPart = 0;
   // Absolute value of the power of 10 that we should multiply by (only
   // relevant for numbers in scientific notation).  Has to be a signed integer,
   // because multiplication of signed by unsigned converts the unsigned to
   // signed, so if we plan to actually multiply by expSign...
   int32_t exponent = 0;
   // Sign of the exponent.
   int32_t expSign = 1;
   aToken.mHasSign = (c == '+' || c == '-');
   if (aToken.mHasSign) {
     Advance();
     c = Peek();
   }
   bool gotDot = (c == '.');
   if (!gotDot) {
     // Scan the integer part of the mantissa.
     MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
     do {
       intPart = 10*intPart + DecimalDigitValue(c);
       Advance();
       c = Peek();
     } while (IsDigit(c));
     gotDot = (c == '.') && IsDigit(Peek(1));
   }
   if (gotDot) {
     // Scan the fractional part of the mantissa.
     Advance();
     c = Peek();
     MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
     // Power of ten by which we need to divide our next digit
     double divisor = 10;
     do {
       fracPart += DecimalDigitValue(c) / divisor;
       divisor *= 10;
       Advance();
       c = Peek();
     } while (IsDigit(c));
   }
   bool gotE = false;
   if (c == 'e' || c == 'E') {
     int32_t expSignChar = Peek(1);
     int32_t nextChar = Peek(2);
     if (IsDigit(expSignChar) ||
         ((expSignChar == '-' || expSignChar == '+') && IsDigit(nextChar))) {
       gotE = true;
       if (expSignChar == '-') {
         expSign = -1;
       }
       Advance(); // consumes the E
       if (expSignChar == '-' || expSignChar == '+') {
         Advance();
         c = nextChar;
       } else {
         c = expSignChar;
       }
       MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
       do {
         exponent = 10*exponent + DecimalDigitValue(c);
         Advance();
         c = Peek();
       } while (IsDigit(c));
     }
   }
   nsCSSTokenType type = eCSSToken_Number;
   // Set mIntegerValid for all cases (except %, below) because we need
   // it for the "2n" in :nth-child(2n).
   aToken.mIntegerValid = false;
   // Time to reassemble our number.
   // Do all the math in double precision so it's truncated only once.
   double value = sign * (intPart + fracPart);
   if (gotE) {
     // Explicitly cast expSign*exponent to double to avoid issues with
     // overloaded pow() on Windows.
     value *= pow(10.0, double(expSign * exponent));
   } else if (!gotDot) {
     // Clamp values outside of integer range.
     if (sign > 0) {
       aToken.mInteger = int32_t(std::min(intPart, double(INT32_MAX)));
     } else {
       aToken.mInteger = int32_t(std::max(-intPart, double(INT32_MIN)));
     }
     aToken.mIntegerValid = true;
   }
   nsString& ident = aToken.mIdent;
   // Check for Dimension and Percentage tokens.
   if (c >= 0) {
     if (StartsIdent(c, Peek(1))) {
       if (GatherText(IS_IDCHAR, ident)) {
         type = eCSSToken_Dimension;
       }
     } else if (c == '%') {
       Advance();
       type = eCSSToken_Percentage;
       value = value / 100.0f;
       aToken.mIntegerValid = false;
     }
   }
   aToken.mNumber = value;
   aToken.mType = type;
   return true;
 }
 /**
  * Scan a string constant ('foo' or "foo").  Will always produce
  * either a String or a Bad_String token; the latter occurs when the
  * close quote is missing.  Always returns true (for convenience in Next()).
  */
 bool
 nsCSSScanner::ScanString(nsCSSToken& aToken)
 {
   int32_t aStop = Peek();
   MOZ_ASSERT(aStop == '"' || aStop == '\'', "should not have been called");
   aToken.mType = eCSSToken_String;
   aToken.mSymbol = char16_t(aStop); // Remember how it's quoted.
   Advance();
   for (;;) {
     GatherText(IS_STRING, aToken.mIdent);
     int32_t ch = Peek();
     if (ch == -1) {
       AddEOFCharacters(aStop == '"' ? eEOFCharacters_DoubleQuote :
                                       eEOFCharacters_SingleQuote);
       break; // EOF ends a string token with no error.
     }
     if (ch == aStop) {
       Advance();
       break;
     }
     // Both " and ' are excluded from IS_STRING.
     if (ch == '"' || ch == '\'') {
       aToken.mIdent.Append(ch);
       Advance();
       continue;
     }
     mSeenBadToken = true;
     aToken.mType = eCSSToken_Bad_String;
     mReporter->ReportUnexpected("SEUnterminatedString", aToken);
     break;
   }
   return true;
 }
 /**
  * Scan a unicode-range token.  These match the regular expression
  *
  *     u\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?
  *
  * However, some such tokens are "invalid".  There are three valid forms:
  *
  *     u+[0-9a-f]{x}              1 <= x <= 6
  *     u+[0-9a-f]{x}\?{y}         1 <= x+y <= 6
  *     u+[0-9a-f]{x}-[0-9a-f]{y}  1 <= x <= 6, 1 <= y <= 6
  *
  * All unicode-range tokens have their text recorded in mIdent; valid ones
  * are also decoded into mInteger and mInteger2, and mIntegerValid is set.
  * Note that this does not validate the numeric range, only the syntactic
  * form.
  */
 bool
 nsCSSScanner::ScanURange(nsCSSToken& aResult)
 {
   int32_t intro1 = Peek();
   int32_t intro2 = Peek(1);
   int32_t ch = Peek(2);
   MOZ_ASSERT((intro1 == 'u' || intro1 == 'U') &&
              intro2 == '+' &&
              (IsHexDigit(ch) || ch == '?'),
              "should not have been called");
   aResult.mIdent.Append(intro1);
   aResult.mIdent.Append(intro2);
   Advance(2);
   bool valid = true;
   bool haveQues = false;
   uint32_t low = 0;
   uint32_t high = 0;
   int i = 0;
   do {
     aResult.mIdent.Append(ch);
     if (IsHexDigit(ch)) {
       if (haveQues) {
         valid = false; // All question marks should be at the end.
       }
       low = low*16 + HexDigitValue(ch);
       high = high*16 + HexDigitValue(ch);
     } else {
       haveQues = true;
       low = low*16 + 0x0;
       high = high*16 + 0xF;
     }
     i++;
     Advance();
     ch = Peek();
   } while (i < 6 && (IsHexDigit(ch) || ch == '?'));
   if (ch == '-' && IsHexDigit(Peek(1))) {
     if (haveQues) {
       valid = false;
     }
     aResult.mIdent.Append(ch);
     Advance();
     ch = Peek();
     high = 0;
     i = 0;
     do {
       aResult.mIdent.Append(ch);
       high = high*16 + HexDigitValue(ch);
       i++;
       Advance();
       ch = Peek();
     } while (i < 6 && IsHexDigit(ch));
   }
   aResult.mInteger = low;
   aResult.mInteger2 = high;
   aResult.mIntegerValid = valid;
   aResult.mType = eCSSToken_URange;
   return true;
 }
 #ifdef DEBUG
 /* static */ void
 nsCSSScanner::AssertEOFCharactersValid(uint32_t c)
 {
   MOZ_ASSERT(c == eEOFCharacters_None ||
              c == eEOFCharacters_ReplacementChar ||
              c == eEOFCharacters_Slash ||
              c == (eEOFCharacters_Asterisk |
                    eEOFCharacters_Slash) ||
              c == eEOFCharacters_DoubleQuote ||
              c == eEOFCharacters_SingleQuote ||
              c == (eEOFCharacters_DropBackslash |
                    eEOFCharacters_DoubleQuote) ||
              c == (eEOFCharacters_DropBackslash |
                    eEOFCharacters_SingleQuote) ||
              c == eEOFCharacters_CloseParen ||
              c == (eEOFCharacters_ReplacementChar |
                    eEOFCharacters_CloseParen) ||
              c == (eEOFCharacters_DoubleQuote |
                    eEOFCharacters_CloseParen) ||
              c == (eEOFCharacters_SingleQuote |
                    eEOFCharacters_CloseParen) ||
              c == (eEOFCharacters_DropBackslash |
                    eEOFCharacters_DoubleQuote |
                    eEOFCharacters_CloseParen) ||
              c == (eEOFCharacters_DropBackslash |
                    eEOFCharacters_SingleQuote |
                    eEOFCharacters_CloseParen),
              "invalid EOFCharacters value");
 }
 #endif
 void
 nsCSSScanner::SetEOFCharacters(uint32_t aEOFCharacters)
 {
   mEOFCharacters = EOFCharacters(aEOFCharacters);
 }
 void
 nsCSSScanner::AddEOFCharacters(uint32_t aEOFCharacters)
 {
   mEOFCharacters = EOFCharacters(mEOFCharacters | aEOFCharacters);
 }
 static const char16_t kImpliedEOFCharacters[] = {
   UCS2_REPLACEMENT_CHAR, '*', '/', '"', '\'', ')', 0
 };
 /* static */ void
 nsCSSScanner::AppendImpliedEOFCharacters(EOFCharacters aEOFCharacters,
                                          nsAString& aResult)
 {
   // First, ignore eEOFCharacters_DropBackslash.
   uint32_t c = aEOFCharacters >> 1;
   // All of the remaining EOFCharacters bits represent appended characters,
   // and the bits are in the order that they need appending.
   for (const char16_t* p = kImpliedEOFCharacters; *p && c; p++, c >>= 1) {
     if (c & 1) {
       aResult.Append(*p);
     }
   }
   MOZ_ASSERT(c == 0, "too many bits in mEOFCharacters");
 }
 /**
  * Consume the part of an URL token after the initial 'url('.  Caller
  * is assumed to have consumed 'url(' already.  Will always produce
  * either an URL or a Bad_URL token.
  *
  * Exposed for use by nsCSSParser::ParseMozDocumentRule, which applies
  * the special lexical rules for URL tokens in a nonstandard context.
  */
 bool
 nsCSSScanner::NextURL(nsCSSToken& aToken)
 {
   SkipWhitespace();
   int32_t ch = Peek();
   if (ch < 0) {
     return false;
   }
   // aToken.mIdent may be "url" at this point; clear that out
   aToken.mIdent.Truncate();
   // Do we have a string?
   if (ch == '"' || ch == '\'') {
     ScanString(aToken);
     if (MOZ_UNLIKELY(aToken.mType == eCSSToken_Bad_String)) {
       aToken.mType = eCSSToken_Bad_URL;
       return true;
     }
     MOZ_ASSERT(aToken.mType == eCSSToken_String, "unexpected token type");
   } else {
     // Otherwise, this is the start of a non-quoted url (which may be empty).
     aToken.mSymbol = char16_t(0);
     GatherText(IS_URL_CHAR, aToken.mIdent);
   }
   // Consume trailing whitespace and then look for a close parenthesis.
   SkipWhitespace();
   ch = Peek();
   if (MOZ_LIKELY(ch < 0 || ch == ')')) {
     Advance();
     aToken.mType = eCSSToken_URL;
     if (ch < 0) {
       AddEOFCharacters(eEOFCharacters_CloseParen);
     }
   } else {
     mSeenBadToken = true;
     aToken.mType = eCSSToken_Bad_URL;
   }
   return true;
 }
 /**
  * Primary scanner entry point.  Consume one token and fill in
  * |aToken| accordingly.  Will skip over any number of comments first,
  * and will also skip over rather than return whitespace tokens if
  * |aSkipWS| is true.
  *
  * Returns true if it successfully consumed a token, false if EOF has
  * been reached.  Will always advance the current read position by at
  * least one character unless called when already at EOF.
  */
 bool
 nsCSSScanner::Next(nsCSSToken& aToken, bool aSkipWS)
 {
   int32_t ch;
   // do this here so we don't have to do it in dozens of other places
   aToken.mIdent.Truncate();
   aToken.mType = eCSSToken_Symbol;
   for (;;) {
     // Consume any number of comments, and possibly also whitespace tokens,
     // in between other tokens.
     mTokenOffset = mOffset;
     mTokenLineOffset = mLineOffset;
     mTokenLineNumber = mLineNumber;
     ch = Peek();
     if (IsWhitespace(ch)) {
       SkipWhitespace();
       if (!aSkipWS) {
         aToken.mType = eCSSToken_Whitespace;
         return true;
       }
       continue; // start again at the beginning
     }
     if (ch == '/' && !IsSVGMode() && Peek(1) == '*') {
       // FIXME: Editor wants comments to be preserved (bug 60290).
       SkipComment();
       continue; // start again at the beginning
     }
     break;
   }
   // EOF
   if (ch < 0) {
     return false;
   }
   // 'u' could be UNICODE-RANGE or an identifier-family token
   if (ch == 'u' || ch == 'U') {
     int32_t c2 = Peek(1);
     int32_t c3 = Peek(2);
     if (c2 == '+' && (IsHexDigit(c3) || c3 == '?')) {
       return ScanURange(aToken);
     }
     return ScanIdent(aToken);
   }
   // identifier family
   if (IsIdentStart(ch)) {
     return ScanIdent(aToken);
   }
   // number family
   if (IsDigit(ch)) {
     return ScanNumber(aToken);
   }
   if (ch == '.' && IsDigit(Peek(1))) {
     return ScanNumber(aToken);
   }
   if (ch == '+') {
     int32_t c2 = Peek(1);
     if (IsDigit(c2) || (c2 == '.' && IsDigit(Peek(2)))) {
       return ScanNumber(aToken);
     }
   }
   // '-' can start an identifier-family token, a number-family token,
   // or an HTML-comment
   if (ch == '-') {
     int32_t c2 = Peek(1);
     int32_t c3 = Peek(2);
     if (IsIdentStart(c2) || (c2 == '-' && c3 != '>')) {
       return ScanIdent(aToken);
     }
     if (IsDigit(c2) || (c2 == '.' && IsDigit(c3))) {
       return ScanNumber(aToken);
     }
     if (c2 == '-' && c3 == '>') {
       Advance(3);
       aToken.mType = eCSSToken_HTMLComment;
       aToken.mIdent.AssignLiteral("-->");
       return true;
     }
   }
   // the other HTML-comment token
   if (ch == '<' && Peek(1) == '!' && Peek(2) == '-' && Peek(3) == '-') {
     Advance(4);
     aToken.mType = eCSSToken_HTMLComment;
     aToken.mIdent.AssignLiteral("<!--");
     return true;
   }
   // AT_KEYWORD
   if (ch == '@') {
     return ScanAtKeyword(aToken);
   }
   // HASH
   if (ch == '#') {
     return ScanHash(aToken);
   }
   // STRING
   if (ch == '"' || ch == '\'') {
     return ScanString(aToken);
   }
   // Match operators: ~= |= ^= $= *=
   nsCSSTokenType opType = MatchOperatorType(ch);
   if (opType != eCSSToken_Symbol && Peek(1) == '=') {
     aToken.mType = opType;
     Advance(2);
     return true;
   }
   // Otherwise, a symbol (DELIM).
   aToken.mSymbol = ch;
   Advance();
   return true;
 }
 /* nsCSSGridTemplateAreaScanner methods. */
 nsCSSGridTemplateAreaScanner::nsCSSGridTemplateAreaScanner(const nsAString& aBuffer)
   : mBuffer(aBuffer.BeginReading())
   , mOffset(0)
   , mCount(aBuffer.Length())
 {
 }
 bool
 nsCSSGridTemplateAreaScanner::Next(nsCSSGridTemplateAreaToken& aTokenResult)
 {
   int32_t ch;
   // Skip whitespace
   do {
     if (mOffset >= mCount) {
       return false;
     }
     ch = mBuffer[mOffset];
     mOffset++;
   } while (IsWhitespace(ch));
   if (IsOpenCharClass(ch, IS_IDCHAR)) {
     // Named cell token
     uint32_t start = mOffset - 1;  // offset of |ch|
     while (mOffset < mCount && IsOpenCharClass(mBuffer[mOffset], IS_IDCHAR)) {
       mOffset++;
     }
     aTokenResult.mName.Assign(&mBuffer[start], mOffset - start);
     aTokenResult.isTrash = false;
   } else if (ch == '.') {
     // Null cell token
     aTokenResult.mName.Truncate();
     aTokenResult.isTrash = false;
   } else {
     // Trash token
     aTokenResult.isTrash = true;
   }
   return true;
 }

The Tor Browser / annotate

layout/style/nsCSSScanner.cpp@141e0f1194b1 (annotated)

layout/style/nsCSSScanner.cpp