1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/browser/devtools/sourceeditor/css-tokenizer.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,717 @@
1.4 +/**
1.5 + * This file is taken from the below mentioned url and is under CC0 license.
1.6 + * https://github.com/tabatkins/css-parser/blob/master/tokenizer.js
1.7 + * Please retain this comment while updating this file from upstream.
1.8 + */
1.9 +
1.10 +(function (root, factory) {
1.11 + // Universal Module Definition (UMD) to support AMD, CommonJS/Node.js,
1.12 + // Rhino, and plain browser loading.
1.13 + if (typeof define === 'function' && define.amd) {
1.14 + define(['exports'], factory);
1.15 + } else if (typeof exports !== 'undefined') {
1.16 + factory(exports);
1.17 + } else {
1.18 + factory(root);
1.19 + }
1.20 +}(this, function (exports) {
1.21 +
1.22 +var between = function (num, first, last) { return num >= first && num <= last; }
1.23 +function digit(code) { return between(code, 0x30,0x39); }
1.24 +function hexdigit(code) { return digit(code) || between(code, 0x41,0x46) || between(code, 0x61,0x66); }
1.25 +function uppercaseletter(code) { return between(code, 0x41,0x5a); }
1.26 +function lowercaseletter(code) { return between(code, 0x61,0x7a); }
1.27 +function letter(code) { return uppercaseletter(code) || lowercaseletter(code); }
1.28 +function nonascii(code) { return code >= 0xa0; }
1.29 +function namestartchar(code) { return letter(code) || nonascii(code) || code == 0x5f; }
1.30 +function namechar(code) { return namestartchar(code) || digit(code) || code == 0x2d; }
1.31 +function nonprintable(code) { return between(code, 0,8) || between(code, 0xe,0x1f) || between(code, 0x7f,0x9f); }
1.32 +function newline(code) { return code == 0xa || code == 0xc; }
1.33 +function whitespace(code) { return newline(code) || code == 9 || code == 0x20; }
1.34 +function badescape(code) { return newline(code) || isNaN(code); }
1.35 +
1.36 +// Note: I'm not yet acting smart enough to actually handle astral characters.
1.37 +var maximumallowedcodepoint = 0x10ffff;
1.38 +
1.39 +function tokenize(str, options) {
1.40 + if(options == undefined) options = {transformFunctionWhitespace:false, scientificNotation:false};
1.41 + var i = -1;
1.42 + var tokens = [];
1.43 + var state = "data";
1.44 + var code;
1.45 + var currtoken;
1.46 +
1.47 + // Line number information.
1.48 + var line = 0;
1.49 + var column = 0;
1.50 + // The only use of lastLineLength is in reconsume().
1.51 + var lastLineLength = 0;
1.52 + var incrLineno = function() {
1.53 + line += 1;
1.54 + lastLineLength = column;
1.55 + column = 0;
1.56 + };
1.57 + var locStart = {line:line, column:column};
1.58 +
1.59 + var next = function(num) { if(num === undefined) num = 1; return str.charCodeAt(i+num); };
1.60 + var consume = function(num) {
1.61 + if(num === undefined)
1.62 + num = 1;
1.63 + i += num;
1.64 + code = str.charCodeAt(i);
1.65 + if (newline(code)) incrLineno();
1.66 + else column += num;
1.67 + //console.log('Consume '+i+' '+String.fromCharCode(code) + ' 0x' + code.toString(16));
1.68 + return true;
1.69 + };
1.70 + var reconsume = function() {
1.71 + i -= 1;
1.72 + if (newline(code)) {
1.73 + line -= 1;
1.74 + column = lastLineLength;
1.75 + } else {
1.76 + column -= 1;
1.77 + }
1.78 + locStart.line = line;
1.79 + locStart.column = column;
1.80 + return true;
1.81 + };
1.82 + var eof = function() { return i >= str.length; };
1.83 + var donothing = function() {};
1.84 + var emit = function(token) {
1.85 + if(token) {
1.86 + token.finish();
1.87 + } else {
1.88 + token = currtoken.finish();
1.89 + }
1.90 + if (options.loc === true) {
1.91 + token.loc = {};
1.92 + token.loc.start = {line:locStart.line, column:locStart.column};
1.93 + locStart = {line: line, column: column};
1.94 + token.loc.end = locStart;
1.95 + }
1.96 + tokens.push(token);
1.97 + //console.log('Emitting ' + token);
1.98 + currtoken = undefined;
1.99 + return true;
1.100 + };
1.101 + var create = function(token) { currtoken = token; return true; };
1.102 + var parseerror = function() { console.log("Parse error at index " + i + ", processing codepoint 0x" + code.toString(16) + " in state " + state + ".");return true; };
1.103 + var switchto = function(newstate) {
1.104 + state = newstate;
1.105 + //console.log('Switching to ' + state);
1.106 + return true;
1.107 + };
1.108 + var consumeEscape = function() {
1.109 + // Assume the the current character is the \
1.110 + consume();
1.111 + if(hexdigit(code)) {
1.112 + // Consume 1-6 hex digits
1.113 + var digits = [];
1.114 + for(var total = 0; total < 6; total++) {
1.115 + if(hexdigit(code)) {
1.116 + digits.push(code);
1.117 + consume();
1.118 + } else { break; }
1.119 + }
1.120 + var value = parseInt(digits.map(String.fromCharCode).join(''), 16);
1.121 + if( value > maximumallowedcodepoint ) value = 0xfffd;
1.122 + // If the current char is whitespace, cool, we'll just eat it.
1.123 + // Otherwise, put it back.
1.124 + if(!whitespace(code)) reconsume();
1.125 + return value;
1.126 + } else {
1.127 + return code;
1.128 + }
1.129 + };
1.130 +
1.131 + for(;;) {
1.132 + if(i > str.length*2) return "I'm infinite-looping!";
1.133 + consume();
1.134 + switch(state) {
1.135 + case "data":
1.136 + if(whitespace(code)) {
1.137 + emit(new WhitespaceToken);
1.138 + while(whitespace(next())) consume();
1.139 + }
1.140 + else if(code == 0x22) switchto("double-quote-string");
1.141 + else if(code == 0x23) switchto("hash");
1.142 + else if(code == 0x27) switchto("single-quote-string");
1.143 + else if(code == 0x28) emit(new OpenParenToken);
1.144 + else if(code == 0x29) emit(new CloseParenToken);
1.145 + else if(code == 0x2b) {
1.146 + if(digit(next()) || (next() == 0x2e && digit(next(2)))) switchto("number") && reconsume();
1.147 + else emit(new DelimToken(code));
1.148 + }
1.149 + else if(code == 0x2d) {
1.150 + if(next(1) == 0x2d && next(2) == 0x3e) consume(2) && emit(new CDCToken);
1.151 + else if(digit(next()) || (next(1) == 0x2e && digit(next(2)))) switchto("number") && reconsume();
1.152 + else if(namestartchar(next())) switchto("identifier") && reconsume();
1.153 + else emit(new DelimToken(code));
1.154 + }
1.155 + else if(code == 0x2e) {
1.156 + if(digit(next())) switchto("number") && reconsume();
1.157 + else emit(new DelimToken(code));
1.158 + }
1.159 + else if(code == 0x2f) {
1.160 + if(next() == 0x2a) switchto("comment");
1.161 + else emit(new DelimToken(code));
1.162 + }
1.163 + else if(code == 0x3a) emit(new ColonToken);
1.164 + else if(code == 0x3b) emit(new SemicolonToken);
1.165 + else if(code == 0x3c) {
1.166 + if(next(1) == 0x21 && next(2) == 0x2d && next(3) == 0x2d) consume(3) && emit(new CDOToken);
1.167 + else emit(new DelimToken(code));
1.168 + }
1.169 + else if(code == 0x40) switchto("at-keyword");
1.170 + else if(code == 0x5b) emit(new OpenSquareToken);
1.171 + else if(code == 0x5c) {
1.172 + if(badescape(next())) parseerror() && emit(new DelimToken(code));
1.173 + else switchto("identifier") && reconsume();
1.174 + }
1.175 + else if(code == 0x5d) emit(new CloseSquareToken);
1.176 + else if(code == 0x7b) emit(new OpenCurlyToken);
1.177 + else if(code == 0x7d) emit(new CloseCurlyToken);
1.178 + else if(digit(code)) switchto("number") && reconsume();
1.179 + else if(code == 0x55 || code == 0x75) {
1.180 + if(next(1) == 0x2b && hexdigit(next(2))) consume() && switchto("unicode-range");
1.181 + else if((next(1) == 0x52 || next(1) == 0x72) && (next(2) == 0x4c || next(2) == 0x6c) && (next(3) == 0x28)) consume(3) && switchto("url");
1.182 + else switchto("identifier") && reconsume();
1.183 + }
1.184 + else if(namestartchar(code)) switchto("identifier") && reconsume();
1.185 + else if(eof()) { emit(new EOFToken); return tokens; }
1.186 + else emit(new DelimToken(code));
1.187 + break;
1.188 +
1.189 + case "double-quote-string":
1.190 + if(currtoken == undefined) create(new StringToken);
1.191 +
1.192 + if(code == 0x22) emit() && switchto("data");
1.193 + else if(eof()) parseerror() && emit() && switchto("data");
1.194 + else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume();
1.195 + else if(code == 0x5c) {
1.196 + if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data");
1.197 + else if(newline(next())) consume();
1.198 + else currtoken.append(consumeEscape());
1.199 + }
1.200 + else currtoken.append(code);
1.201 + break;
1.202 +
1.203 + case "single-quote-string":
1.204 + if(currtoken == undefined) create(new StringToken);
1.205 +
1.206 + if(code == 0x27) emit() && switchto("data");
1.207 + else if(eof()) parseerror() && emit() && switchto("data");
1.208 + else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume();
1.209 + else if(code == 0x5c) {
1.210 + if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data");
1.211 + else if(newline(next())) consume();
1.212 + else currtoken.append(consumeEscape());
1.213 + }
1.214 + else currtoken.append(code);
1.215 + break;
1.216 +
1.217 + case "hash":
1.218 + if(namechar(code)) create(new HashToken(code)) && switchto("hash-rest");
1.219 + else if(code == 0x5c) {
1.220 + if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
1.221 + else create(new HashToken(consumeEscape())) && switchto('hash-rest');
1.222 + }
1.223 + else emit(new DelimToken(0x23)) && switchto('data') && reconsume();
1.224 + break;
1.225 +
1.226 + case "hash-rest":
1.227 + if(namechar(code)) currtoken.append(code);
1.228 + else if(code == 0x5c) {
1.229 + if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
1.230 + else currtoken.append(consumeEscape());
1.231 + }
1.232 + else emit() && switchto('data') && reconsume();
1.233 + break;
1.234 +
1.235 + case "comment":
1.236 + if(code == 0x2a) {
1.237 + if(next() == 0x2f) consume() && switchto('data');
1.238 + else donothing();
1.239 + }
1.240 + else if(eof()) parseerror() && switchto('data') && reconsume();
1.241 + else donothing();
1.242 + break;
1.243 +
1.244 + case "at-keyword":
1.245 + if(code == 0x2d) {
1.246 + if(namestartchar(next())) consume() && create(new AtKeywordToken([0x40,code])) && switchto('at-keyword-rest');
1.247 + else emit(new DelimToken(0x40)) && switchto('data') && reconsume();
1.248 + }
1.249 + else if(namestartchar(code)) create(new AtKeywordToken(code)) && switchto('at-keyword-rest');
1.250 + else if(code == 0x5c) {
1.251 + if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
1.252 + else create(new AtKeywordToken(consumeEscape())) && switchto('at-keyword-rest');
1.253 + }
1.254 + else emit(new DelimToken(0x40)) && switchto('data') && reconsume();
1.255 + break;
1.256 +
1.257 + case "at-keyword-rest":
1.258 + if(namechar(code)) currtoken.append(code);
1.259 + else if(code == 0x5c) {
1.260 + if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume();
1.261 + else currtoken.append(consumeEscape());
1.262 + }
1.263 + else emit() && switchto('data') && reconsume();
1.264 + break;
1.265 +
1.266 + case "identifier":
1.267 + if(code == 0x2d) {
1.268 + if(namestartchar(next())) create(new IdentifierToken(code)) && switchto('identifier-rest');
1.269 + else switchto('data') && reconsume();
1.270 + }
1.271 + else if(namestartchar(code)) create(new IdentifierToken(code)) && switchto('identifier-rest');
1.272 + else if(code == 0x5c) {
1.273 + if(badescape(next())) parseerror() && switchto("data") && reconsume();
1.274 + else create(new IdentifierToken(consumeEscape())) && switchto('identifier-rest');
1.275 + }
1.276 + else switchto('data') && reconsume();
1.277 + break;
1.278 +
1.279 + case "identifier-rest":
1.280 + if(namechar(code)) currtoken.append(code);
1.281 + else if(code == 0x5c) {
1.282 + if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume();
1.283 + else currtoken.append(consumeEscape());
1.284 + }
1.285 + else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data');
1.286 + else if(whitespace(code) && options.transformFunctionWhitespace) switchto('transform-function-whitespace');
1.287 + else emit() && switchto('data') && reconsume();
1.288 + break;
1.289 +
1.290 + case "transform-function-whitespace":
1.291 + if(whitespace(code)) donothing();
1.292 + else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data');
1.293 + else emit() && switchto('data') && reconsume();
1.294 + break;
1.295 +
1.296 + case "number":
1.297 + create(new NumberToken());
1.298 +
1.299 + if(code == 0x2d) {
1.300 + if(digit(next())) consume() && currtoken.append([0x2d,code]) && switchto('number-rest');
1.301 + else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2d,0x2e,code]) && switchto('number-fraction');
1.302 + else switchto('data') && reconsume();
1.303 + }
1.304 + else if(code == 0x2b) {
1.305 + if(digit(next())) consume() && currtoken.append([0x2b,code]) && switchto('number-rest');
1.306 + else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2b,0x2e,code]) && switchto('number-fraction');
1.307 + else switchto('data') && reconsume();
1.308 + }
1.309 + else if(digit(code)) currtoken.append(code) && switchto('number-rest');
1.310 + else if(code == 0x2e) {
1.311 + if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction');
1.312 + else switchto('data') && reconsume();
1.313 + }
1.314 + else switchto('data') && reconsume();
1.315 + break;
1.316 +
1.317 + case "number-rest":
1.318 + if(digit(code)) currtoken.append(code);
1.319 + else if(code == 0x2e) {
1.320 + if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction');
1.321 + else emit() && switchto('data') && reconsume();
1.322 + }
1.323 + else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume();
1.324 + else if(code == 0x45 || code == 0x65) {
1.325 + if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension');
1.326 + else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation');
1.327 + else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation');
1.328 + else create(new DimensionToken(currtoken,code)) && switchto('dimension');
1.329 + }
1.330 + else if(code == 0x2d) {
1.331 + if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension');
1.332 + else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume();
1.333 + else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension');
1.334 + else emit() && switchto('data') && reconsume();
1.335 + }
1.336 + else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension');
1.337 + else if(code == 0x5c) {
1.338 + if(badescape(next)) emit() && switchto('data') && reconsume();
1.339 + else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension');
1.340 + }
1.341 + else emit() && switchto('data') && reconsume();
1.342 + break;
1.343 +
1.344 + case "number-fraction":
1.345 + currtoken.type = "number";
1.346 +
1.347 + if(digit(code)) currtoken.append(code);
1.348 + else if(code == 0x2e) emit() && switchto('data') && reconsume();
1.349 + else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume();
1.350 + else if(code == 0x45 || code == 0x65) {
1.351 + if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension');
1.352 + else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation');
1.353 + else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation');
1.354 + else create(new DimensionToken(currtoken,code)) && switchto('dimension');
1.355 + }
1.356 + else if(code == 0x2d) {
1.357 + if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension');
1.358 + else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume();
1.359 + else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension');
1.360 + else emit() && switchto('data') && reconsume();
1.361 + }
1.362 + else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension');
1.363 + else if(code == 0x5c) {
1.364 + if(badescape(next)) emit() && switchto('data') && reconsume();
1.365 + else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension');
1.366 + }
1.367 + else emit() && switchto('data') && reconsume();
1.368 + break;
1.369 +
1.370 + case "dimension":
1.371 + if(namechar(code)) currtoken.append(code);
1.372 + else if(code == 0x5c) {
1.373 + if(badescape(next())) parseerror() && emit() && switchto('data') && reconsume();
1.374 + else currtoken.append(consumeEscape());
1.375 + }
1.376 + else emit() && switchto('data') && reconsume();
1.377 + break;
1.378 +
1.379 + case "sci-notation":
1.380 + if(digit(code)) currtoken.append(code);
1.381 + else emit() && switchto('data') && reconsume();
1.382 + break;
1.383 +
1.384 + case "url":
1.385 + if(code == 0x22) switchto('url-double-quote');
1.386 + else if(code == 0x27) switchto('url-single-quote');
1.387 + else if(code == 0x29) emit(new URLToken) && switchto('data');
1.388 + else if(whitespace(code)) donothing();
1.389 + else switchto('url-unquoted') && reconsume();
1.390 + break;
1.391 +
1.392 + case "url-double-quote":
1.393 + if(currtoken == undefined) create(new URLToken);
1.394 +
1.395 + if(code == 0x22) switchto('url-end');
1.396 + else if(newline(code)) parseerror() && switchto('bad-url');
1.397 + else if(code == 0x5c) {
1.398 + if(newline(next())) consume();
1.399 + else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume();
1.400 + else currtoken.append(consumeEscape());
1.401 + }
1.402 + else currtoken.append(code);
1.403 + break;
1.404 +
1.405 + case "url-single-quote":
1.406 + if(currtoken == undefined) create(new URLToken);
1.407 +
1.408 + if(code == 0x27) switchto('url-end');
1.409 + else if(newline(code)) parseerror() && switchto('bad-url');
1.410 + else if(code == 0x5c) {
1.411 + if(newline(next())) consume();
1.412 + else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume();
1.413 + else currtoken.append(consumeEscape());
1.414 + }
1.415 + else currtoken.append(code);
1.416 + break;
1.417 +
1.418 + case "url-end":
1.419 + if(whitespace(code)) donothing();
1.420 + else if(code == 0x29) emit() && switchto('data');
1.421 + else parseerror() && switchto('bad-url') && reconsume();
1.422 + break;
1.423 +
1.424 + case "url-unquoted":
1.425 + if(currtoken == undefined) create(new URLToken);
1.426 +
1.427 + if(whitespace(code)) switchto('url-end');
1.428 + else if(code == 0x29) emit() && switchto('data');
1.429 + else if(code == 0x22 || code == 0x27 || code == 0x28 || nonprintable(code)) parseerror() && switchto('bad-url');
1.430 + else if(code == 0x5c) {
1.431 + if(badescape(next())) parseerror() && switchto('bad-url');
1.432 + else currtoken.append(consumeEscape());
1.433 + }
1.434 + else currtoken.append(code);
1.435 + break;
1.436 +
1.437 + case "bad-url":
1.438 + if(code == 0x29) emit(new BadURLToken) && switchto('data');
1.439 + else if(code == 0x5c) {
1.440 + if(badescape(next())) donothing();
1.441 + else consumeEscape()
1.442 + }
1.443 + else donothing();
1.444 + break;
1.445 +
1.446 + case "unicode-range":
1.447 + // We already know that the current code is a hexdigit.
1.448 +
1.449 + var start = [code], end = [code];
1.450 +
1.451 + for(var total = 1; total < 6; total++) {
1.452 + if(hexdigit(next())) {
1.453 + consume();
1.454 + start.push(code);
1.455 + end.push(code);
1.456 + }
1.457 + else break;
1.458 + }
1.459 +
1.460 + if(next() == 0x3f) {
1.461 + for(;total < 6; total++) {
1.462 + if(next() == 0x3f) {
1.463 + consume();
1.464 + start.push("0".charCodeAt(0));
1.465 + end.push("f".charCodeAt(0));
1.466 + }
1.467 + else break;
1.468 + }
1.469 + emit(new UnicodeRangeToken(start,end)) && switchto('data');
1.470 + }
1.471 + else if(next(1) == 0x2d && hexdigit(next(2))) {
1.472 + consume();
1.473 + consume();
1.474 + end = [code];
1.475 + for(var total = 1; total < 6; total++) {
1.476 + if(hexdigit(next())) {
1.477 + consume();
1.478 + end.push(code);
1.479 + }
1.480 + else break;
1.481 + }
1.482 + emit(new UnicodeRangeToken(start,end)) && switchto('data');
1.483 + }
1.484 + else emit(new UnicodeRangeToken(start)) && switchto('data');
1.485 + break;
1.486 +
1.487 + default:
1.488 + console.log("Unknown state '" + state + "'");
1.489 + }
1.490 + }
1.491 +}
1.492 +
1.493 +function stringFromCodeArray(arr) {
1.494 + return String.fromCharCode.apply(null,arr.filter(function(e){return e;}));
1.495 +}
1.496 +
1.497 +function CSSParserToken(options) { return this; }
1.498 +CSSParserToken.prototype.finish = function() { return this; }
1.499 +CSSParserToken.prototype.toString = function() { return this.tokenType; }
1.500 +CSSParserToken.prototype.toJSON = function() { return this.toString(); }
1.501 +
1.502 +function BadStringToken() { return this; }
1.503 +BadStringToken.prototype = new CSSParserToken;
1.504 +BadStringToken.prototype.tokenType = "BADSTRING";
1.505 +
1.506 +function BadURLToken() { return this; }
1.507 +BadURLToken.prototype = new CSSParserToken;
1.508 +BadURLToken.prototype.tokenType = "BADURL";
1.509 +
1.510 +function WhitespaceToken() { return this; }
1.511 +WhitespaceToken.prototype = new CSSParserToken;
1.512 +WhitespaceToken.prototype.tokenType = "WHITESPACE";
1.513 +WhitespaceToken.prototype.toString = function() { return "WS"; }
1.514 +
1.515 +function CDOToken() { return this; }
1.516 +CDOToken.prototype = new CSSParserToken;
1.517 +CDOToken.prototype.tokenType = "CDO";
1.518 +
1.519 +function CDCToken() { return this; }
1.520 +CDCToken.prototype = new CSSParserToken;
1.521 +CDCToken.prototype.tokenType = "CDC";
1.522 +
1.523 +function ColonToken() { return this; }
1.524 +ColonToken.prototype = new CSSParserToken;
1.525 +ColonToken.prototype.tokenType = ":";
1.526 +
1.527 +function SemicolonToken() { return this; }
1.528 +SemicolonToken.prototype = new CSSParserToken;
1.529 +SemicolonToken.prototype.tokenType = ";";
1.530 +
1.531 +function OpenCurlyToken() { return this; }
1.532 +OpenCurlyToken.prototype = new CSSParserToken;
1.533 +OpenCurlyToken.prototype.tokenType = "{";
1.534 +
1.535 +function CloseCurlyToken() { return this; }
1.536 +CloseCurlyToken.prototype = new CSSParserToken;
1.537 +CloseCurlyToken.prototype.tokenType = "}";
1.538 +
1.539 +function OpenSquareToken() { return this; }
1.540 +OpenSquareToken.prototype = new CSSParserToken;
1.541 +OpenSquareToken.prototype.tokenType = "[";
1.542 +
1.543 +function CloseSquareToken() { return this; }
1.544 +CloseSquareToken.prototype = new CSSParserToken;
1.545 +CloseSquareToken.prototype.tokenType = "]";
1.546 +
1.547 +function OpenParenToken() { return this; }
1.548 +OpenParenToken.prototype = new CSSParserToken;
1.549 +OpenParenToken.prototype.tokenType = "(";
1.550 +
1.551 +function CloseParenToken() { return this; }
1.552 +CloseParenToken.prototype = new CSSParserToken;
1.553 +CloseParenToken.prototype.tokenType = ")";
1.554 +
1.555 +function EOFToken() { return this; }
1.556 +EOFToken.prototype = new CSSParserToken;
1.557 +EOFToken.prototype.tokenType = "EOF";
1.558 +
1.559 +function DelimToken(code) {
1.560 + this.value = String.fromCharCode(code);
1.561 + return this;
1.562 +}
1.563 +DelimToken.prototype = new CSSParserToken;
1.564 +DelimToken.prototype.tokenType = "DELIM";
1.565 +DelimToken.prototype.toString = function() { return "DELIM("+this.value+")"; }
1.566 +
1.567 +function StringValuedToken() { return this; }
1.568 +StringValuedToken.prototype = new CSSParserToken;
1.569 +StringValuedToken.prototype.append = function(val) {
1.570 + if(val instanceof Array) {
1.571 + for(var i = 0; i < val.length; i++) {
1.572 + this.value.push(val[i]);
1.573 + }
1.574 + } else {
1.575 + this.value.push(val);
1.576 + }
1.577 + return true;
1.578 +}
1.579 +StringValuedToken.prototype.finish = function() {
1.580 + this.value = stringFromCodeArray(this.value);
1.581 + return this;
1.582 +}
1.583 +
1.584 +function IdentifierToken(val) {
1.585 + this.value = [];
1.586 + this.append(val);
1.587 +}
1.588 +IdentifierToken.prototype = new StringValuedToken;
1.589 +IdentifierToken.prototype.tokenType = "IDENT";
1.590 +IdentifierToken.prototype.toString = function() { return "IDENT("+this.value+")"; }
1.591 +
1.592 +function FunctionToken(val) {
1.593 + // These are always constructed by passing an IdentifierToken
1.594 + this.value = val.finish().value;
1.595 +}
1.596 +FunctionToken.prototype = new CSSParserToken;
1.597 +FunctionToken.prototype.tokenType = "FUNCTION";
1.598 +FunctionToken.prototype.toString = function() { return "FUNCTION("+this.value+")"; }
1.599 +
1.600 +function AtKeywordToken(val) {
1.601 + this.value = [];
1.602 + this.append(val);
1.603 +}
1.604 +AtKeywordToken.prototype = new StringValuedToken;
1.605 +AtKeywordToken.prototype.tokenType = "AT-KEYWORD";
1.606 +AtKeywordToken.prototype.toString = function() { return "AT("+this.value+")"; }
1.607 +
1.608 +function HashToken(val) {
1.609 + this.value = [];
1.610 + this.append(val);
1.611 +}
1.612 +HashToken.prototype = new StringValuedToken;
1.613 +HashToken.prototype.tokenType = "HASH";
1.614 +HashToken.prototype.toString = function() { return "HASH("+this.value+")"; }
1.615 +
1.616 +function StringToken(val) {
1.617 + this.value = [];
1.618 + this.append(val);
1.619 +}
1.620 +StringToken.prototype = new StringValuedToken;
1.621 +StringToken.prototype.tokenType = "STRING";
1.622 +StringToken.prototype.toString = function() { return "\""+this.value+"\""; }
1.623 +
1.624 +function URLToken(val) {
1.625 + this.value = [];
1.626 + this.append(val);
1.627 +}
1.628 +URLToken.prototype = new StringValuedToken;
1.629 +URLToken.prototype.tokenType = "URL";
1.630 +URLToken.prototype.toString = function() { return "URL("+this.value+")"; }
1.631 +
1.632 +function NumberToken(val) {
1.633 + this.value = [];
1.634 + this.append(val);
1.635 + this.type = "integer";
1.636 +}
1.637 +NumberToken.prototype = new StringValuedToken;
1.638 +NumberToken.prototype.tokenType = "NUMBER";
1.639 +NumberToken.prototype.toString = function() {
1.640 + if(this.type == "integer")
1.641 + return "INT("+this.value+")";
1.642 + return "NUMBER("+this.value+")";
1.643 +}
1.644 +NumberToken.prototype.finish = function() {
1.645 + this.repr = stringFromCodeArray(this.value);
1.646 + this.value = this.repr * 1;
1.647 + if(Math.abs(this.value) % 1 != 0) this.type = "number";
1.648 + return this;
1.649 +}
1.650 +
1.651 +function PercentageToken(val) {
1.652 + // These are always created by passing a NumberToken as val
1.653 + val.finish();
1.654 + this.value = val.value;
1.655 + this.repr = val.repr;
1.656 +}
1.657 +PercentageToken.prototype = new CSSParserToken;
1.658 +PercentageToken.prototype.tokenType = "PERCENTAGE";
1.659 +PercentageToken.prototype.toString = function() { return "PERCENTAGE("+this.value+")"; }
1.660 +
1.661 +function DimensionToken(val,unit) {
1.662 + // These are always created by passing a NumberToken as the val
1.663 + val.finish();
1.664 + this.num = val.value;
1.665 + this.unit = [];
1.666 + this.repr = val.repr;
1.667 + this.append(unit);
1.668 +}
1.669 +DimensionToken.prototype = new CSSParserToken;
1.670 +DimensionToken.prototype.tokenType = "DIMENSION";
1.671 +DimensionToken.prototype.toString = function() { return "DIM("+this.num+","+this.unit+")"; }
1.672 +DimensionToken.prototype.append = function(val) {
1.673 + if(val instanceof Array) {
1.674 + for(var i = 0; i < val.length; i++) {
1.675 + this.unit.push(val[i]);
1.676 + }
1.677 + } else {
1.678 + this.unit.push(val);
1.679 + }
1.680 + return true;
1.681 +}
1.682 +DimensionToken.prototype.finish = function() {
1.683 + this.unit = stringFromCodeArray(this.unit);
1.684 + this.repr += this.unit;
1.685 + return this;
1.686 +}
1.687 +
1.688 +function UnicodeRangeToken(start,end) {
1.689 + // start and end are array of char codes, completely finished
1.690 + start = parseInt(stringFromCodeArray(start),16);
1.691 + if(end === undefined) end = start + 1;
1.692 + else end = parseInt(stringFromCodeArray(end),16);
1.693 +
1.694 + if(start > maximumallowedcodepoint) end = start;
1.695 + if(end < start) end = start;
1.696 + if(end > maximumallowedcodepoint) end = maximumallowedcodepoint;
1.697 +
1.698 + this.start = start;
1.699 + this.end = end;
1.700 + return this;
1.701 +}
1.702 +UnicodeRangeToken.prototype = new CSSParserToken;
1.703 +UnicodeRangeToken.prototype.tokenType = "UNICODE-RANGE";
1.704 +UnicodeRangeToken.prototype.toString = function() {
1.705 + if(this.start+1 == this.end)
1.706 + return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+")";
1.707 + if(this.start < this.end)
1.708 + return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+"-"+this.end.toString(16).toUpperCase()+")";
1.709 + return "UNICODE-RANGE()";
1.710 +}
1.711 +UnicodeRangeToken.prototype.contains = function(code) {
1.712 + return code >= this.start && code < this.end;
1.713 +}
1.714 +
1.715 +
1.716 +// Exportation.
1.717 +// TODO: also export the various tokens objects?
1.718 +module.exports = tokenize;
1.719 +
1.720 +}));
