michael@0: /**
michael@0:  * This file is taken from the below mentioned url and is under CC0 license.
michael@0:  * https://github.com/tabatkins/css-parser/blob/master/tokenizer.js
michael@0:  * Please retain this comment while updating this file from upstream.
michael@0:  */
michael@0: 
michael@0: (function (root, factory) {
michael@0:     // Universal Module Definition (UMD) to support AMD, CommonJS/Node.js,
michael@0:     // Rhino, and plain browser loading.
michael@0:     if (typeof define === 'function' && define.amd) {
michael@0:         define(['exports'], factory);
michael@0:     } else if (typeof exports !== 'undefined') {
michael@0:         factory(exports);
michael@0:     } else {
michael@0:         factory(root);
michael@0:     }
michael@0: }(this, function (exports) {
michael@0: 
michael@0: var between = function (num, first, last) { return num >= first && num <= last; }
michael@0: function digit(code) { return between(code, 0x30,0x39); }
michael@0: function hexdigit(code) { return digit(code) || between(code, 0x41,0x46) || between(code, 0x61,0x66); }
michael@0: function uppercaseletter(code) { return between(code, 0x41,0x5a); }
michael@0: function lowercaseletter(code) { return between(code, 0x61,0x7a); }
michael@0: function letter(code) { return uppercaseletter(code) || lowercaseletter(code); }
michael@0: function nonascii(code) { return code >= 0xa0; }
michael@0: function namestartchar(code) { return letter(code) || nonascii(code) || code == 0x5f; }
michael@0: function namechar(code) { return namestartchar(code) || digit(code) || code == 0x2d; }
michael@0: function nonprintable(code) { return between(code, 0,8) || between(code, 0xe,0x1f) || between(code, 0x7f,0x9f); }
michael@0: function newline(code) { return code == 0xa || code == 0xc; }
michael@0: function whitespace(code) { return newline(code) || code == 9 || code == 0x20; }
michael@0: function badescape(code) { return newline(code) || isNaN(code); }
michael@0: 
michael@0: // Note: I'm not yet acting smart enough to actually handle astral characters.
michael@0: var maximumallowedcodepoint = 0x10ffff;
michael@0: 
michael@0: function tokenize(str, options) {
michael@0:   if(options == undefined) options = {transformFunctionWhitespace:false, scientificNotation:false};
michael@0:   var i = -1;
michael@0:   var tokens = [];
michael@0:   var state = "data";
michael@0:   var code;
michael@0:   var currtoken;
michael@0: 
michael@0:   // Line number information.
michael@0:   var line = 0;
michael@0:   var column = 0;
michael@0:   // The only use of lastLineLength is in reconsume().
michael@0:   var lastLineLength = 0;
michael@0:   var incrLineno = function() {
michael@0:     line += 1;
michael@0:     lastLineLength = column;
michael@0:     column = 0;
michael@0:   };
michael@0:   var locStart = {line:line, column:column};
michael@0: 
michael@0:   var next = function(num) { if(num === undefined) num = 1; return str.charCodeAt(i+num); };
michael@0:   var consume = function(num) {
michael@0:     if(num === undefined)
michael@0:       num = 1;
michael@0:     i += num;
michael@0:     code = str.charCodeAt(i);
michael@0:     if (newline(code)) incrLineno();
michael@0:     else column += num;
michael@0:     //console.log('Consume '+i+' '+String.fromCharCode(code) + ' 0x' + code.toString(16));
michael@0:     return true;
michael@0:   };
michael@0:   var reconsume = function() {
michael@0:     i -= 1;
michael@0:     if (newline(code)) {
michael@0:       line -= 1;
michael@0:       column = lastLineLength;
michael@0:     } else {
michael@0:       column -= 1;
michael@0:     }
michael@0:     locStart.line = line;
michael@0:     locStart.column = column;
michael@0:     return true;
michael@0:   };
michael@0:   var eof = function() { return i >= str.length; };
michael@0:   var donothing = function() {};
michael@0:   var emit = function(token) {
michael@0:     if(token) {
michael@0:       token.finish();
michael@0:     } else {
michael@0:       token = currtoken.finish();
michael@0:     }
michael@0:     if (options.loc === true) {
michael@0:       token.loc = {};
michael@0:       token.loc.start = {line:locStart.line, column:locStart.column};
michael@0:       locStart = {line: line, column: column};
michael@0:       token.loc.end = locStart;
michael@0:     }
michael@0:     tokens.push(token);
michael@0:     //console.log('Emitting ' + token);
michael@0:     currtoken = undefined;
michael@0:     return true;
michael@0:   };
michael@0:   var create = function(token) { currtoken = token; return true; };
michael@0:   var parseerror = function() { console.log("Parse error at index " + i + ", processing codepoint 0x" + code.toString(16) + " in state " + state + ".");return true; };
michael@0:   var switchto = function(newstate) {
michael@0:     state = newstate;
michael@0:     //console.log('Switching to ' + state);
michael@0:     return true;
michael@0:   };
michael@0:   var consumeEscape = function() {
michael@0:     // Assume the the current character is the \
michael@0:     consume();
michael@0:     if(hexdigit(code)) {
michael@0:       // Consume 1-6 hex digits
michael@0:       var digits = [];
michael@0:       for(var total = 0; total < 6; total++) {
michael@0:         if(hexdigit(code)) {
michael@0:           digits.push(code);
michael@0:           consume();
michael@0:         } else { break; }
michael@0:       }
michael@0:       var value = parseInt(digits.map(String.fromCharCode).join(''), 16);
michael@0:       if( value > maximumallowedcodepoint ) value = 0xfffd;
michael@0:       // If the current char is whitespace, cool, we'll just eat it.
michael@0:       // Otherwise, put it back.
michael@0:       if(!whitespace(code)) reconsume();
michael@0:       return value;
michael@0:     } else {
michael@0:       return code;
michael@0:     }
michael@0:   };
michael@0: 
michael@0:   for(;;) {
michael@0:     if(i > str.length*2) return "I'm infinite-looping!";
michael@0:     consume();
michael@0:     switch(state) {
michael@0:     case "data":
michael@0:       if(whitespace(code)) {
michael@0:         emit(new WhitespaceToken);
michael@0:         while(whitespace(next())) consume();
michael@0:       }
michael@0:       else if(code == 0x22) switchto("double-quote-string");
michael@0:       else if(code == 0x23) switchto("hash");
michael@0:       else if(code == 0x27) switchto("single-quote-string");
michael@0:       else if(code == 0x28) emit(new OpenParenToken);
michael@0:       else if(code == 0x29) emit(new CloseParenToken);
michael@0:       else if(code == 0x2b) {
michael@0:         if(digit(next()) || (next() == 0x2e && digit(next(2)))) switchto("number") && reconsume();
michael@0:         else emit(new DelimToken(code));
michael@0:       }
michael@0:       else if(code == 0x2d) {
michael@0:         if(next(1) == 0x2d && next(2) == 0x3e) consume(2) && emit(new CDCToken);
michael@0:         else if(digit(next()) || (next(1) == 0x2e && digit(next(2)))) switchto("number") && reconsume();
michael@0:         else if(namestartchar(next())) switchto("identifier") && reconsume();
michael@0:         else emit(new DelimToken(code));
michael@0:       }
michael@0:       else if(code == 0x2e) {
michael@0:         if(digit(next())) switchto("number") && reconsume();
michael@0:         else emit(new DelimToken(code));
michael@0:       }
michael@0:       else if(code == 0x2f) {
michael@0:         if(next() == 0x2a) switchto("comment");
michael@0:         else emit(new DelimToken(code));
michael@0:       }
michael@0:       else if(code == 0x3a) emit(new ColonToken);
michael@0:       else if(code == 0x3b) emit(new SemicolonToken);
michael@0:       else if(code == 0x3c) {
michael@0:         if(next(1) == 0x21 && next(2) == 0x2d && next(3) == 0x2d) consume(3) && emit(new CDOToken);
michael@0:         else emit(new DelimToken(code));
michael@0:       }
michael@0:       else if(code == 0x40) switchto("at-keyword");
michael@0:       else if(code == 0x5b) emit(new OpenSquareToken);
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit(new DelimToken(code));
michael@0:         else switchto("identifier") && reconsume();
michael@0:       }
michael@0:       else if(code == 0x5d) emit(new CloseSquareToken);
michael@0:       else if(code == 0x7b) emit(new OpenCurlyToken);
michael@0:       else if(code == 0x7d) emit(new CloseCurlyToken);
michael@0:       else if(digit(code)) switchto("number") && reconsume();
michael@0:       else if(code == 0x55 || code == 0x75) {
michael@0:         if(next(1) == 0x2b && hexdigit(next(2))) consume() && switchto("unicode-range");
michael@0:         else if((next(1) == 0x52 || next(1) == 0x72) && (next(2) == 0x4c || next(2) == 0x6c) && (next(3) == 0x28)) consume(3) && switchto("url");
michael@0:         else switchto("identifier") && reconsume();
michael@0:       }
michael@0:       else if(namestartchar(code)) switchto("identifier") && reconsume();
michael@0:       else if(eof()) { emit(new EOFToken); return tokens; }
michael@0:       else emit(new DelimToken(code));
michael@0:       break;
michael@0: 
michael@0:     case "double-quote-string":
michael@0:       if(currtoken == undefined) create(new StringToken);
michael@0: 
michael@0:       if(code == 0x22) emit() && switchto("data");
michael@0:       else if(eof()) parseerror() && emit() && switchto("data");
michael@0:       else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume();
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data");
michael@0:         else if(newline(next())) consume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else currtoken.append(code);
michael@0:       break;
michael@0: 
michael@0:     case "single-quote-string":
michael@0:       if(currtoken == undefined) create(new StringToken);
michael@0: 
michael@0:       if(code == 0x27) emit() && switchto("data");
michael@0:       else if(eof()) parseerror() && emit() && switchto("data");
michael@0:       else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume();
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data");
michael@0:         else if(newline(next())) consume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else currtoken.append(code);
michael@0:       break;
michael@0: 
michael@0:     case "hash":
michael@0:       if(namechar(code)) create(new HashToken(code)) && switchto("hash-rest");
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
michael@0:         else create(new HashToken(consumeEscape())) && switchto('hash-rest');
michael@0:       }
michael@0:       else emit(new DelimToken(0x23)) && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "hash-rest":
michael@0:       if(namechar(code)) currtoken.append(code);
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "comment":
michael@0:       if(code == 0x2a) {
michael@0:         if(next() == 0x2f) consume() && switchto('data');
michael@0:         else donothing();
michael@0:       }
michael@0:       else if(eof()) parseerror() && switchto('data') && reconsume();
michael@0:       else donothing();
michael@0:       break;
michael@0: 
michael@0:     case "at-keyword":
michael@0:       if(code == 0x2d) {
michael@0:         if(namestartchar(next())) consume() && create(new AtKeywordToken([0x40,code])) && switchto('at-keyword-rest');
michael@0:         else emit(new DelimToken(0x40)) && switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(namestartchar(code)) create(new AtKeywordToken(code)) && switchto('at-keyword-rest');
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
michael@0:         else create(new AtKeywordToken(consumeEscape())) && switchto('at-keyword-rest');
michael@0:       }
michael@0:       else emit(new DelimToken(0x40)) && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "at-keyword-rest":
michael@0:       if(namechar(code)) currtoken.append(code);
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "identifier":
michael@0:       if(code == 0x2d) {
michael@0:         if(namestartchar(next())) create(new IdentifierToken(code)) && switchto('identifier-rest');
michael@0:         else switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(namestartchar(code)) create(new IdentifierToken(code)) && switchto('identifier-rest');
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && switchto("data") && reconsume();
michael@0:         else create(new IdentifierToken(consumeEscape())) && switchto('identifier-rest');
michael@0:       }
michael@0:       else switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "identifier-rest":
michael@0:       if(namechar(code)) currtoken.append(code);
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data');
michael@0:       else if(whitespace(code) && options.transformFunctionWhitespace) switchto('transform-function-whitespace');
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "transform-function-whitespace":
michael@0:       if(whitespace(code)) donothing();
michael@0:       else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data');
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "number":
michael@0:       create(new NumberToken());
michael@0: 
michael@0:       if(code == 0x2d) {
michael@0:         if(digit(next())) consume() && currtoken.append([0x2d,code]) && switchto('number-rest');
michael@0:         else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2d,0x2e,code]) && switchto('number-fraction');
michael@0:         else switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(code == 0x2b) {
michael@0:         if(digit(next())) consume() && currtoken.append([0x2b,code]) && switchto('number-rest');
michael@0:         else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2b,0x2e,code]) && switchto('number-fraction');
michael@0:         else switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(digit(code)) currtoken.append(code) && switchto('number-rest');
michael@0:       else if(code == 0x2e) {
michael@0:         if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction');
michael@0:         else switchto('data') && reconsume();
michael@0:       }
michael@0:       else switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "number-rest":
michael@0:       if(digit(code)) currtoken.append(code);
michael@0:       else if(code == 0x2e) {
michael@0:         if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction');
michael@0:         else emit() && switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume();
michael@0:       else if(code == 0x45 || code == 0x65) {
michael@0:         if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension');
michael@0:         else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation');
michael@0:         else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation');
michael@0:         else create(new DimensionToken(currtoken,code)) && switchto('dimension');
michael@0:       }
michael@0:       else if(code == 0x2d) {
michael@0:         if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension');
michael@0:         else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume();
michael@0:         else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension');
michael@0:         else emit() && switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension');
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next)) emit() && switchto('data') && reconsume();
michael@0:         else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension');
michael@0:       }
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "number-fraction":
michael@0:       currtoken.type = "number";
michael@0: 
michael@0:       if(digit(code)) currtoken.append(code);
michael@0:       else if(code == 0x2e) emit() && switchto('data') && reconsume();
michael@0:       else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume();
michael@0:       else if(code == 0x45 || code == 0x65) {
michael@0:         if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension');
michael@0:         else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation');
michael@0:         else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation');
michael@0:         else create(new DimensionToken(currtoken,code)) && switchto('dimension');
michael@0:       }
michael@0:       else if(code == 0x2d) {
michael@0:         if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension');
michael@0:         else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume();
michael@0:         else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension');
michael@0:         else emit() && switchto('data') && reconsume();
michael@0:       }
michael@0:       else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension');
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next)) emit() && switchto('data') && reconsume();
michael@0:         else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension');
michael@0:       }
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "dimension":
michael@0:       if(namechar(code)) currtoken.append(code);
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && emit() && switchto('data') && reconsume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "sci-notation":
michael@0:       if(digit(code)) currtoken.append(code);
michael@0:       else emit() && switchto('data') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "url":
michael@0:       if(code == 0x22) switchto('url-double-quote');
michael@0:       else if(code == 0x27) switchto('url-single-quote');
michael@0:       else if(code == 0x29) emit(new URLToken) && switchto('data');
michael@0:       else if(whitespace(code)) donothing();
michael@0:       else switchto('url-unquoted') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "url-double-quote":
michael@0:       if(currtoken == undefined) create(new URLToken);
michael@0: 
michael@0:       if(code == 0x22) switchto('url-end');
michael@0:       else if(newline(code)) parseerror() && switchto('bad-url');
michael@0:       else if(code == 0x5c) {
michael@0:         if(newline(next())) consume();
michael@0:         else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else currtoken.append(code);
michael@0:       break;
michael@0: 
michael@0:     case "url-single-quote":
michael@0:       if(currtoken == undefined) create(new URLToken);
michael@0: 
michael@0:       if(code == 0x27) switchto('url-end');
michael@0:       else if(newline(code)) parseerror() && switchto('bad-url');
michael@0:       else if(code == 0x5c) {
michael@0:         if(newline(next())) consume();
michael@0:         else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume();
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else currtoken.append(code);
michael@0:       break;
michael@0: 
michael@0:     case "url-end":
michael@0:       if(whitespace(code)) donothing();
michael@0:       else if(code == 0x29) emit() && switchto('data');
michael@0:       else parseerror() && switchto('bad-url') && reconsume();
michael@0:       break;
michael@0: 
michael@0:     case "url-unquoted":
michael@0:       if(currtoken == undefined) create(new URLToken);
michael@0: 
michael@0:       if(whitespace(code)) switchto('url-end');
michael@0:       else if(code == 0x29) emit() && switchto('data');
michael@0:       else if(code == 0x22 || code == 0x27 || code == 0x28 || nonprintable(code)) parseerror() && switchto('bad-url');
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) parseerror() && switchto('bad-url');
michael@0:         else currtoken.append(consumeEscape());
michael@0:       }
michael@0:       else currtoken.append(code);
michael@0:       break;
michael@0: 
michael@0:     case "bad-url":
michael@0:       if(code == 0x29) emit(new BadURLToken) && switchto('data');
michael@0:       else if(code == 0x5c) {
michael@0:         if(badescape(next())) donothing();
michael@0:         else consumeEscape()
michael@0:       }
michael@0:       else donothing();
michael@0:       break;
michael@0: 
michael@0:     case "unicode-range":
michael@0:       // We already know that the current code is a hexdigit.
michael@0: 
michael@0:       var start = [code], end = [code];
michael@0: 
michael@0:       for(var total = 1; total < 6; total++) {
michael@0:         if(hexdigit(next())) {
michael@0:           consume();
michael@0:           start.push(code);
michael@0:           end.push(code);
michael@0:         }
michael@0:         else break;
michael@0:       }
michael@0: 
michael@0:       if(next() == 0x3f) {
michael@0:         for(;total < 6; total++) {
michael@0:           if(next() == 0x3f) {
michael@0:             consume();
michael@0:             start.push("0".charCodeAt(0));
michael@0:             end.push("f".charCodeAt(0));
michael@0:           }
michael@0:           else break;
michael@0:         }
michael@0:         emit(new UnicodeRangeToken(start,end)) && switchto('data');
michael@0:       }
michael@0:       else if(next(1) == 0x2d && hexdigit(next(2))) {
michael@0:         consume();
michael@0:         consume();
michael@0:         end = [code];
michael@0:         for(var total = 1; total < 6; total++) {
michael@0:           if(hexdigit(next())) {
michael@0:             consume();
michael@0:             end.push(code);
michael@0:           }
michael@0:           else break;
michael@0:         }
michael@0:         emit(new UnicodeRangeToken(start,end)) && switchto('data');
michael@0:       }
michael@0:       else emit(new UnicodeRangeToken(start)) && switchto('data');
michael@0:       break;
michael@0: 
michael@0:     default:
michael@0:       console.log("Unknown state '" + state + "'");
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: function stringFromCodeArray(arr) {
michael@0:   return String.fromCharCode.apply(null,arr.filter(function(e){return e;}));
michael@0: }
michael@0: 
michael@0: function CSSParserToken(options) { return this; }
michael@0: CSSParserToken.prototype.finish = function() { return this; }
michael@0: CSSParserToken.prototype.toString = function() { return this.tokenType; }
michael@0: CSSParserToken.prototype.toJSON = function() { return this.toString(); }
michael@0: 
michael@0: function BadStringToken() { return this; }
michael@0: BadStringToken.prototype = new CSSParserToken;
michael@0: BadStringToken.prototype.tokenType = "BADSTRING";
michael@0: 
michael@0: function BadURLToken() { return this; }
michael@0: BadURLToken.prototype = new CSSParserToken;
michael@0: BadURLToken.prototype.tokenType = "BADURL";
michael@0: 
michael@0: function WhitespaceToken() { return this; }
michael@0: WhitespaceToken.prototype = new CSSParserToken;
michael@0: WhitespaceToken.prototype.tokenType = "WHITESPACE";
michael@0: WhitespaceToken.prototype.toString = function() { return "WS"; }
michael@0: 
michael@0: function CDOToken() { return this; }
michael@0: CDOToken.prototype = new CSSParserToken;
michael@0: CDOToken.prototype.tokenType = "CDO";
michael@0: 
michael@0: function CDCToken() { return this; }
michael@0: CDCToken.prototype = new CSSParserToken;
michael@0: CDCToken.prototype.tokenType = "CDC";
michael@0: 
michael@0: function ColonToken() { return this; }
michael@0: ColonToken.prototype = new CSSParserToken;
michael@0: ColonToken.prototype.tokenType = ":";
michael@0: 
michael@0: function SemicolonToken() { return this; }
michael@0: SemicolonToken.prototype = new CSSParserToken;
michael@0: SemicolonToken.prototype.tokenType = ";";
michael@0: 
michael@0: function OpenCurlyToken() { return this; }
michael@0: OpenCurlyToken.prototype = new CSSParserToken;
michael@0: OpenCurlyToken.prototype.tokenType = "{";
michael@0: 
michael@0: function CloseCurlyToken() { return this; }
michael@0: CloseCurlyToken.prototype = new CSSParserToken;
michael@0: CloseCurlyToken.prototype.tokenType = "}";
michael@0: 
michael@0: function OpenSquareToken() { return this; }
michael@0: OpenSquareToken.prototype = new CSSParserToken;
michael@0: OpenSquareToken.prototype.tokenType = "[";
michael@0: 
michael@0: function CloseSquareToken() { return this; }
michael@0: CloseSquareToken.prototype = new CSSParserToken;
michael@0: CloseSquareToken.prototype.tokenType = "]";
michael@0: 
michael@0: function OpenParenToken() { return this; }
michael@0: OpenParenToken.prototype = new CSSParserToken;
michael@0: OpenParenToken.prototype.tokenType = "(";
michael@0: 
michael@0: function CloseParenToken() { return this; }
michael@0: CloseParenToken.prototype = new CSSParserToken;
michael@0: CloseParenToken.prototype.tokenType = ")";
michael@0: 
michael@0: function EOFToken() { return this; }
michael@0: EOFToken.prototype = new CSSParserToken;
michael@0: EOFToken.prototype.tokenType = "EOF";
michael@0: 
michael@0: function DelimToken(code) {
michael@0:   this.value = String.fromCharCode(code);
michael@0:   return this;
michael@0: }
michael@0: DelimToken.prototype = new CSSParserToken;
michael@0: DelimToken.prototype.tokenType = "DELIM";
michael@0: DelimToken.prototype.toString = function() { return "DELIM("+this.value+")"; }
michael@0: 
michael@0: function StringValuedToken() { return this; }
michael@0: StringValuedToken.prototype = new CSSParserToken;
michael@0: StringValuedToken.prototype.append = function(val) {
michael@0:   if(val instanceof Array) {
michael@0:     for(var i = 0; i < val.length; i++) {
michael@0:       this.value.push(val[i]);
michael@0:     }
michael@0:   } else {
michael@0:     this.value.push(val);
michael@0:   }
michael@0:   return true;
michael@0: }
michael@0: StringValuedToken.prototype.finish = function() {
michael@0:   this.value = stringFromCodeArray(this.value);
michael@0:   return this;
michael@0: }
michael@0: 
michael@0: function IdentifierToken(val) {
michael@0:   this.value = [];
michael@0:   this.append(val);
michael@0: }
michael@0: IdentifierToken.prototype = new StringValuedToken;
michael@0: IdentifierToken.prototype.tokenType = "IDENT";
michael@0: IdentifierToken.prototype.toString = function() { return "IDENT("+this.value+")"; }
michael@0: 
michael@0: function FunctionToken(val) {
michael@0:   // These are always constructed by passing an IdentifierToken
michael@0:   this.value = val.finish().value;
michael@0: }
michael@0: FunctionToken.prototype = new CSSParserToken;
michael@0: FunctionToken.prototype.tokenType = "FUNCTION";
michael@0: FunctionToken.prototype.toString = function() { return "FUNCTION("+this.value+")"; }
michael@0: 
michael@0: function AtKeywordToken(val) {
michael@0:   this.value = [];
michael@0:   this.append(val);
michael@0: }
michael@0: AtKeywordToken.prototype = new StringValuedToken;
michael@0: AtKeywordToken.prototype.tokenType = "AT-KEYWORD";
michael@0: AtKeywordToken.prototype.toString = function() { return "AT("+this.value+")"; }
michael@0: 
michael@0: function HashToken(val) {
michael@0:   this.value = [];
michael@0:   this.append(val);
michael@0: }
michael@0: HashToken.prototype = new StringValuedToken;
michael@0: HashToken.prototype.tokenType = "HASH";
michael@0: HashToken.prototype.toString = function() { return "HASH("+this.value+")"; }
michael@0: 
michael@0: function StringToken(val) {
michael@0:   this.value = [];
michael@0:   this.append(val);
michael@0: }
michael@0: StringToken.prototype = new StringValuedToken;
michael@0: StringToken.prototype.tokenType = "STRING";
michael@0: StringToken.prototype.toString = function() { return "\""+this.value+"\""; }
michael@0: 
michael@0: function URLToken(val) {
michael@0:   this.value = [];
michael@0:   this.append(val);
michael@0: }
michael@0: URLToken.prototype = new StringValuedToken;
michael@0: URLToken.prototype.tokenType = "URL";
michael@0: URLToken.prototype.toString = function() { return "URL("+this.value+")"; }
michael@0: 
michael@0: function NumberToken(val) {
michael@0:   this.value = [];
michael@0:   this.append(val);
michael@0:   this.type = "integer";
michael@0: }
michael@0: NumberToken.prototype = new StringValuedToken;
michael@0: NumberToken.prototype.tokenType = "NUMBER";
michael@0: NumberToken.prototype.toString = function() {
michael@0:   if(this.type == "integer")
michael@0:     return "INT("+this.value+")";
michael@0:   return "NUMBER("+this.value+")";
michael@0: }
michael@0: NumberToken.prototype.finish = function() {
michael@0:   this.repr = stringFromCodeArray(this.value);
michael@0:   this.value = this.repr * 1;
michael@0:   if(Math.abs(this.value) % 1 != 0) this.type = "number";
michael@0:   return this;
michael@0: }
michael@0: 
michael@0: function PercentageToken(val) {
michael@0:   // These are always created by passing a NumberToken as val
michael@0:   val.finish();
michael@0:   this.value = val.value;
michael@0:   this.repr = val.repr;
michael@0: }
michael@0: PercentageToken.prototype = new CSSParserToken;
michael@0: PercentageToken.prototype.tokenType = "PERCENTAGE";
michael@0: PercentageToken.prototype.toString = function() { return "PERCENTAGE("+this.value+")"; }
michael@0: 
michael@0: function DimensionToken(val,unit) {
michael@0:   // These are always created by passing a NumberToken as the val
michael@0:   val.finish();
michael@0:   this.num = val.value;
michael@0:   this.unit = [];
michael@0:   this.repr = val.repr;
michael@0:   this.append(unit);
michael@0: }
michael@0: DimensionToken.prototype = new CSSParserToken;
michael@0: DimensionToken.prototype.tokenType = "DIMENSION";
michael@0: DimensionToken.prototype.toString = function() { return "DIM("+this.num+","+this.unit+")"; }
michael@0: DimensionToken.prototype.append = function(val) {
michael@0:   if(val instanceof Array) {
michael@0:     for(var i = 0; i < val.length; i++) {
michael@0:       this.unit.push(val[i]);
michael@0:     }
michael@0:   } else {
michael@0:     this.unit.push(val);
michael@0:   }
michael@0:   return true;
michael@0: }
michael@0: DimensionToken.prototype.finish = function() {
michael@0:   this.unit = stringFromCodeArray(this.unit);
michael@0:   this.repr += this.unit;
michael@0:   return this;
michael@0: }
michael@0: 
michael@0: function UnicodeRangeToken(start,end) {
michael@0:   // start and end are array of char codes, completely finished
michael@0:   start = parseInt(stringFromCodeArray(start),16);
michael@0:   if(end === undefined) end = start + 1;
michael@0:   else end = parseInt(stringFromCodeArray(end),16);
michael@0: 
michael@0:   if(start > maximumallowedcodepoint) end = start;
michael@0:   if(end < start) end = start;
michael@0:   if(end > maximumallowedcodepoint) end = maximumallowedcodepoint;
michael@0: 
michael@0:   this.start = start;
michael@0:   this.end = end;
michael@0:   return this;
michael@0: }
michael@0: UnicodeRangeToken.prototype = new CSSParserToken;
michael@0: UnicodeRangeToken.prototype.tokenType = "UNICODE-RANGE";
michael@0: UnicodeRangeToken.prototype.toString = function() {
michael@0:   if(this.start+1 == this.end)
michael@0:     return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+")";
michael@0:   if(this.start < this.end)
michael@0:     return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+"-"+this.end.toString(16).toUpperCase()+")";
michael@0:   return "UNICODE-RANGE()";
michael@0: }
michael@0: UnicodeRangeToken.prototype.contains = function(code) {
michael@0:   return code >= this.start && code < this.end;
michael@0: }
michael@0: 
michael@0: 
michael@0: // Exportation.
michael@0: // TODO: also export the various tokens objects?
michael@0: module.exports = tokenize;
michael@0: 
michael@0: }));