The Tor Browser / file comparison
comparison: browser/devtools/sourceeditor/css-tokenizer.js
browser/devtools/sourceeditor/css-tokenizer.js
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:eecc2028eda4 |
|---|---|
| 1 /** | |
| 2 * This file is taken from the below mentioned url and is under CC0 license. | |
| 3 * https://github.com/tabatkins/css-parser/blob/master/tokenizer.js | |
| 4 * Please retain this comment while updating this file from upstream. | |
| 5 */ | |
| 6 | |
| 7 (function (root, factory) { | |
| 8 // Universal Module Definition (UMD) to support AMD, CommonJS/Node.js, | |
| 9 // Rhino, and plain browser loading. | |
| 10 if (typeof define === 'function' && define.amd) { | |
| 11 define(['exports'], factory); | |
| 12 } else if (typeof exports !== 'undefined') { | |
| 13 factory(exports); | |
| 14 } else { | |
| 15 factory(root); | |
| 16 } | |
| 17 }(this, function (exports) { | |
| 18 | |
| 19 var between = function (num, first, last) { return num >= first && num <= last; } | |
| 20 function digit(code) { return between(code, 0x30,0x39); } | |
| 21 function hexdigit(code) { return digit(code) || between(code, 0x41,0x46) || between(code, 0x61,0x66); } | |
| 22 function uppercaseletter(code) { return between(code, 0x41,0x5a); } | |
| 23 function lowercaseletter(code) { return between(code, 0x61,0x7a); } | |
| 24 function letter(code) { return uppercaseletter(code) || lowercaseletter(code); } | |
| 25 function nonascii(code) { return code >= 0xa0; } | |
| 26 function namestartchar(code) { return letter(code) || nonascii(code) || code == 0x5f; } | |
| 27 function namechar(code) { return namestartchar(code) || digit(code) || code == 0x2d; } | |
| 28 function nonprintable(code) { return between(code, 0,8) || between(code, 0xe,0x1f) || between(code, 0x7f,0x9f); } | |
| 29 function newline(code) { return code == 0xa || code == 0xc; } | |
| 30 function whitespace(code) { return newline(code) || code == 9 || code == 0x20; } | |
| 31 function badescape(code) { return newline(code) || isNaN(code); } | |
| 32 | |
| 33 // Note: I'm not yet acting smart enough to actually handle astral characters. | |
| 34 var maximumallowedcodepoint = 0x10ffff; | |
| 35 | |
| 36 function tokenize(str, options) { | |
| 37 if(options == undefined) options = {transformFunctionWhitespace:false, scientificNotation:false}; | |
| 38 var i = -1; | |
| 39 var tokens = []; | |
| 40 var state = "data"; | |
| 41 var code; | |
| 42 var currtoken; | |
| 43 | |
| 44 // Line number information. | |
| 45 var line = 0; | |
| 46 var column = 0; | |
| 47 // The only use of lastLineLength is in reconsume(). | |
| 48 var lastLineLength = 0; | |
| 49 var incrLineno = function() { | |
| 50 line += 1; | |
| 51 lastLineLength = column; | |
| 52 column = 0; | |
| 53 }; | |
| 54 var locStart = {line:line, column:column}; | |
| 55 | |
| 56 var next = function(num) { if(num === undefined) num = 1; return str.charCodeAt(i+num); }; | |
| 57 var consume = function(num) { | |
| 58 if(num === undefined) | |
| 59 num = 1; | |
| 60 i += num; | |
| 61 code = str.charCodeAt(i); | |
| 62 if (newline(code)) incrLineno(); | |
| 63 else column += num; | |
| 64 //console.log('Consume '+i+' '+String.fromCharCode(code) + ' 0x' + code.toString(16)); | |
| 65 return true; | |
| 66 }; | |
| 67 var reconsume = function() { | |
| 68 i -= 1; | |
| 69 if (newline(code)) { | |
| 70 line -= 1; | |
| 71 column = lastLineLength; | |
| 72 } else { | |
| 73 column -= 1; | |
| 74 } | |
| 75 locStart.line = line; | |
| 76 locStart.column = column; | |
| 77 return true; | |
| 78 }; | |
| 79 var eof = function() { return i >= str.length; }; | |
| 80 var donothing = function() {}; | |
| 81 var emit = function(token) { | |
| 82 if(token) { | |
| 83 token.finish(); | |
| 84 } else { | |
| 85 token = currtoken.finish(); | |
| 86 } | |
| 87 if (options.loc === true) { | |
| 88 token.loc = {}; | |
| 89 token.loc.start = {line:locStart.line, column:locStart.column}; | |
| 90 locStart = {line: line, column: column}; | |
| 91 token.loc.end = locStart; | |
| 92 } | |
| 93 tokens.push(token); | |
| 94 //console.log('Emitting ' + token); | |
| 95 currtoken = undefined; | |
| 96 return true; | |
| 97 }; | |
| 98 var create = function(token) { currtoken = token; return true; }; | |
| 99 var parseerror = function() { console.log("Parse error at index " + i + ", processing codepoint 0x" + code.toString(16) + " in state " + state + ".");return true; }; | |
| 100 var switchto = function(newstate) { | |
| 101 state = newstate; | |
| 102 //console.log('Switching to ' + state); | |
| 103 return true; | |
| 104 }; | |
| 105 var consumeEscape = function() { | |
| 106 // Assume the the current character is the \ | |
| 107 consume(); | |
| 108 if(hexdigit(code)) { | |
| 109 // Consume 1-6 hex digits | |
| 110 var digits = []; | |
| 111 for(var total = 0; total < 6; total++) { | |
| 112 if(hexdigit(code)) { | |
| 113 digits.push(code); | |
| 114 consume(); | |
| 115 } else { break; } | |
| 116 } | |
| 117 var value = parseInt(digits.map(String.fromCharCode).join(''), 16); | |
| 118 if( value > maximumallowedcodepoint ) value = 0xfffd; | |
| 119 // If the current char is whitespace, cool, we'll just eat it. | |
| 120 // Otherwise, put it back. | |
| 121 if(!whitespace(code)) reconsume(); | |
| 122 return value; | |
| 123 } else { | |
| 124 return code; | |
| 125 } | |
| 126 }; | |
| 127 | |
| 128 for(;;) { | |
| 129 if(i > str.length*2) return "I'm infinite-looping!"; | |
| 130 consume(); | |
| 131 switch(state) { | |
| 132 case "data": | |
| 133 if(whitespace(code)) { | |
| 134 emit(new WhitespaceToken); | |
| 135 while(whitespace(next())) consume(); | |
| 136 } | |
| 137 else if(code == 0x22) switchto("double-quote-string"); | |
| 138 else if(code == 0x23) switchto("hash"); | |
| 139 else if(code == 0x27) switchto("single-quote-string"); | |
| 140 else if(code == 0x28) emit(new OpenParenToken); | |
| 141 else if(code == 0x29) emit(new CloseParenToken); | |
| 142 else if(code == 0x2b) { | |
| 143 if(digit(next()) || (next() == 0x2e && digit(next(2)))) switchto("number") && reconsume(); | |
| 144 else emit(new DelimToken(code)); | |
| 145 } | |
| 146 else if(code == 0x2d) { | |
| 147 if(next(1) == 0x2d && next(2) == 0x3e) consume(2) && emit(new CDCToken); | |
| 148 else if(digit(next()) || (next(1) == 0x2e && digit(next(2)))) switchto("number") && reconsume(); | |
| 149 else if(namestartchar(next())) switchto("identifier") && reconsume(); | |
| 150 else emit(new DelimToken(code)); | |
| 151 } | |
| 152 else if(code == 0x2e) { | |
| 153 if(digit(next())) switchto("number") && reconsume(); | |
| 154 else emit(new DelimToken(code)); | |
| 155 } | |
| 156 else if(code == 0x2f) { | |
| 157 if(next() == 0x2a) switchto("comment"); | |
| 158 else emit(new DelimToken(code)); | |
| 159 } | |
| 160 else if(code == 0x3a) emit(new ColonToken); | |
| 161 else if(code == 0x3b) emit(new SemicolonToken); | |
| 162 else if(code == 0x3c) { | |
| 163 if(next(1) == 0x21 && next(2) == 0x2d && next(3) == 0x2d) consume(3) && emit(new CDOToken); | |
| 164 else emit(new DelimToken(code)); | |
| 165 } | |
| 166 else if(code == 0x40) switchto("at-keyword"); | |
| 167 else if(code == 0x5b) emit(new OpenSquareToken); | |
| 168 else if(code == 0x5c) { | |
| 169 if(badescape(next())) parseerror() && emit(new DelimToken(code)); | |
| 170 else switchto("identifier") && reconsume(); | |
| 171 } | |
| 172 else if(code == 0x5d) emit(new CloseSquareToken); | |
| 173 else if(code == 0x7b) emit(new OpenCurlyToken); | |
| 174 else if(code == 0x7d) emit(new CloseCurlyToken); | |
| 175 else if(digit(code)) switchto("number") && reconsume(); | |
| 176 else if(code == 0x55 || code == 0x75) { | |
| 177 if(next(1) == 0x2b && hexdigit(next(2))) consume() && switchto("unicode-range"); | |
| 178 else if((next(1) == 0x52 || next(1) == 0x72) && (next(2) == 0x4c || next(2) == 0x6c) && (next(3) == 0x28)) consume(3) && switchto("url"); | |
| 179 else switchto("identifier") && reconsume(); | |
| 180 } | |
| 181 else if(namestartchar(code)) switchto("identifier") && reconsume(); | |
| 182 else if(eof()) { emit(new EOFToken); return tokens; } | |
| 183 else emit(new DelimToken(code)); | |
| 184 break; | |
| 185 | |
| 186 case "double-quote-string": | |
| 187 if(currtoken == undefined) create(new StringToken); | |
| 188 | |
| 189 if(code == 0x22) emit() && switchto("data"); | |
| 190 else if(eof()) parseerror() && emit() && switchto("data"); | |
| 191 else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume(); | |
| 192 else if(code == 0x5c) { | |
| 193 if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data"); | |
| 194 else if(newline(next())) consume(); | |
| 195 else currtoken.append(consumeEscape()); | |
| 196 } | |
| 197 else currtoken.append(code); | |
| 198 break; | |
| 199 | |
| 200 case "single-quote-string": | |
| 201 if(currtoken == undefined) create(new StringToken); | |
| 202 | |
| 203 if(code == 0x27) emit() && switchto("data"); | |
| 204 else if(eof()) parseerror() && emit() && switchto("data"); | |
| 205 else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume(); | |
| 206 else if(code == 0x5c) { | |
| 207 if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data"); | |
| 208 else if(newline(next())) consume(); | |
| 209 else currtoken.append(consumeEscape()); | |
| 210 } | |
| 211 else currtoken.append(code); | |
| 212 break; | |
| 213 | |
| 214 case "hash": | |
| 215 if(namechar(code)) create(new HashToken(code)) && switchto("hash-rest"); | |
| 216 else if(code == 0x5c) { | |
| 217 if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume(); | |
| 218 else create(new HashToken(consumeEscape())) && switchto('hash-rest'); | |
| 219 } | |
| 220 else emit(new DelimToken(0x23)) && switchto('data') && reconsume(); | |
| 221 break; | |
| 222 | |
| 223 case "hash-rest": | |
| 224 if(namechar(code)) currtoken.append(code); | |
| 225 else if(code == 0x5c) { | |
| 226 if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume(); | |
| 227 else currtoken.append(consumeEscape()); | |
| 228 } | |
| 229 else emit() && switchto('data') && reconsume(); | |
| 230 break; | |
| 231 | |
| 232 case "comment": | |
| 233 if(code == 0x2a) { | |
| 234 if(next() == 0x2f) consume() && switchto('data'); | |
| 235 else donothing(); | |
| 236 } | |
| 237 else if(eof()) parseerror() && switchto('data') && reconsume(); | |
| 238 else donothing(); | |
| 239 break; | |
| 240 | |
| 241 case "at-keyword": | |
| 242 if(code == 0x2d) { | |
| 243 if(namestartchar(next())) consume() && create(new AtKeywordToken([0x40,code])) && switchto('at-keyword-rest'); | |
| 244 else emit(new DelimToken(0x40)) && switchto('data') && reconsume(); | |
| 245 } | |
| 246 else if(namestartchar(code)) create(new AtKeywordToken(code)) && switchto('at-keyword-rest'); | |
| 247 else if(code == 0x5c) { | |
| 248 if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume(); | |
| 249 else create(new AtKeywordToken(consumeEscape())) && switchto('at-keyword-rest'); | |
| 250 } | |
| 251 else emit(new DelimToken(0x40)) && switchto('data') && reconsume(); | |
| 252 break; | |
| 253 | |
| 254 case "at-keyword-rest": | |
| 255 if(namechar(code)) currtoken.append(code); | |
| 256 else if(code == 0x5c) { | |
| 257 if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume(); | |
| 258 else currtoken.append(consumeEscape()); | |
| 259 } | |
| 260 else emit() && switchto('data') && reconsume(); | |
| 261 break; | |
| 262 | |
| 263 case "identifier": | |
| 264 if(code == 0x2d) { | |
| 265 if(namestartchar(next())) create(new IdentifierToken(code)) && switchto('identifier-rest'); | |
| 266 else switchto('data') && reconsume(); | |
| 267 } | |
| 268 else if(namestartchar(code)) create(new IdentifierToken(code)) && switchto('identifier-rest'); | |
| 269 else if(code == 0x5c) { | |
| 270 if(badescape(next())) parseerror() && switchto("data") && reconsume(); | |
| 271 else create(new IdentifierToken(consumeEscape())) && switchto('identifier-rest'); | |
| 272 } | |
| 273 else switchto('data') && reconsume(); | |
| 274 break; | |
| 275 | |
| 276 case "identifier-rest": | |
| 277 if(namechar(code)) currtoken.append(code); | |
| 278 else if(code == 0x5c) { | |
| 279 if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume(); | |
| 280 else currtoken.append(consumeEscape()); | |
| 281 } | |
| 282 else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data'); | |
| 283 else if(whitespace(code) && options.transformFunctionWhitespace) switchto('transform-function-whitespace'); | |
| 284 else emit() && switchto('data') && reconsume(); | |
| 285 break; | |
| 286 | |
| 287 case "transform-function-whitespace": | |
| 288 if(whitespace(code)) donothing(); | |
| 289 else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data'); | |
| 290 else emit() && switchto('data') && reconsume(); | |
| 291 break; | |
| 292 | |
| 293 case "number": | |
| 294 create(new NumberToken()); | |
| 295 | |
| 296 if(code == 0x2d) { | |
| 297 if(digit(next())) consume() && currtoken.append([0x2d,code]) && switchto('number-rest'); | |
| 298 else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2d,0x2e,code]) && switchto('number-fraction'); | |
| 299 else switchto('data') && reconsume(); | |
| 300 } | |
| 301 else if(code == 0x2b) { | |
| 302 if(digit(next())) consume() && currtoken.append([0x2b,code]) && switchto('number-rest'); | |
| 303 else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2b,0x2e,code]) && switchto('number-fraction'); | |
| 304 else switchto('data') && reconsume(); | |
| 305 } | |
| 306 else if(digit(code)) currtoken.append(code) && switchto('number-rest'); | |
| 307 else if(code == 0x2e) { | |
| 308 if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction'); | |
| 309 else switchto('data') && reconsume(); | |
| 310 } | |
| 311 else switchto('data') && reconsume(); | |
| 312 break; | |
| 313 | |
| 314 case "number-rest": | |
| 315 if(digit(code)) currtoken.append(code); | |
| 316 else if(code == 0x2e) { | |
| 317 if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction'); | |
| 318 else emit() && switchto('data') && reconsume(); | |
| 319 } | |
| 320 else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume(); | |
| 321 else if(code == 0x45 || code == 0x65) { | |
| 322 if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension'); | |
| 323 else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation'); | |
| 324 else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation'); | |
| 325 else create(new DimensionToken(currtoken,code)) && switchto('dimension'); | |
| 326 } | |
| 327 else if(code == 0x2d) { | |
| 328 if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension'); | |
| 329 else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume(); | |
| 330 else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension'); | |
| 331 else emit() && switchto('data') && reconsume(); | |
| 332 } | |
| 333 else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension'); | |
| 334 else if(code == 0x5c) { | |
| 335 if(badescape(next)) emit() && switchto('data') && reconsume(); | |
| 336 else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension'); | |
| 337 } | |
| 338 else emit() && switchto('data') && reconsume(); | |
| 339 break; | |
| 340 | |
| 341 case "number-fraction": | |
| 342 currtoken.type = "number"; | |
| 343 | |
| 344 if(digit(code)) currtoken.append(code); | |
| 345 else if(code == 0x2e) emit() && switchto('data') && reconsume(); | |
| 346 else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume(); | |
| 347 else if(code == 0x45 || code == 0x65) { | |
| 348 if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension'); | |
| 349 else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation'); | |
| 350 else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation'); | |
| 351 else create(new DimensionToken(currtoken,code)) && switchto('dimension'); | |
| 352 } | |
| 353 else if(code == 0x2d) { | |
| 354 if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension'); | |
| 355 else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume(); | |
| 356 else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension'); | |
| 357 else emit() && switchto('data') && reconsume(); | |
| 358 } | |
| 359 else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension'); | |
| 360 else if(code == 0x5c) { | |
| 361 if(badescape(next)) emit() && switchto('data') && reconsume(); | |
| 362 else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension'); | |
| 363 } | |
| 364 else emit() && switchto('data') && reconsume(); | |
| 365 break; | |
| 366 | |
| 367 case "dimension": | |
| 368 if(namechar(code)) currtoken.append(code); | |
| 369 else if(code == 0x5c) { | |
| 370 if(badescape(next())) parseerror() && emit() && switchto('data') && reconsume(); | |
| 371 else currtoken.append(consumeEscape()); | |
| 372 } | |
| 373 else emit() && switchto('data') && reconsume(); | |
| 374 break; | |
| 375 | |
| 376 case "sci-notation": | |
| 377 if(digit(code)) currtoken.append(code); | |
| 378 else emit() && switchto('data') && reconsume(); | |
| 379 break; | |
| 380 | |
| 381 case "url": | |
| 382 if(code == 0x22) switchto('url-double-quote'); | |
| 383 else if(code == 0x27) switchto('url-single-quote'); | |
| 384 else if(code == 0x29) emit(new URLToken) && switchto('data'); | |
| 385 else if(whitespace(code)) donothing(); | |
| 386 else switchto('url-unquoted') && reconsume(); | |
| 387 break; | |
| 388 | |
| 389 case "url-double-quote": | |
| 390 if(currtoken == undefined) create(new URLToken); | |
| 391 | |
| 392 if(code == 0x22) switchto('url-end'); | |
| 393 else if(newline(code)) parseerror() && switchto('bad-url'); | |
| 394 else if(code == 0x5c) { | |
| 395 if(newline(next())) consume(); | |
| 396 else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume(); | |
| 397 else currtoken.append(consumeEscape()); | |
| 398 } | |
| 399 else currtoken.append(code); | |
| 400 break; | |
| 401 | |
| 402 case "url-single-quote": | |
| 403 if(currtoken == undefined) create(new URLToken); | |
| 404 | |
| 405 if(code == 0x27) switchto('url-end'); | |
| 406 else if(newline(code)) parseerror() && switchto('bad-url'); | |
| 407 else if(code == 0x5c) { | |
| 408 if(newline(next())) consume(); | |
| 409 else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume(); | |
| 410 else currtoken.append(consumeEscape()); | |
| 411 } | |
| 412 else currtoken.append(code); | |
| 413 break; | |
| 414 | |
| 415 case "url-end": | |
| 416 if(whitespace(code)) donothing(); | |
| 417 else if(code == 0x29) emit() && switchto('data'); | |
| 418 else parseerror() && switchto('bad-url') && reconsume(); | |
| 419 break; | |
| 420 | |
| 421 case "url-unquoted": | |
| 422 if(currtoken == undefined) create(new URLToken); | |
| 423 | |
| 424 if(whitespace(code)) switchto('url-end'); | |
| 425 else if(code == 0x29) emit() && switchto('data'); | |
| 426 else if(code == 0x22 || code == 0x27 || code == 0x28 || nonprintable(code)) parseerror() && switchto('bad-url'); | |
| 427 else if(code == 0x5c) { | |
| 428 if(badescape(next())) parseerror() && switchto('bad-url'); | |
| 429 else currtoken.append(consumeEscape()); | |
| 430 } | |
| 431 else currtoken.append(code); | |
| 432 break; | |
| 433 | |
| 434 case "bad-url": | |
| 435 if(code == 0x29) emit(new BadURLToken) && switchto('data'); | |
| 436 else if(code == 0x5c) { | |
| 437 if(badescape(next())) donothing(); | |
| 438 else consumeEscape() | |
| 439 } | |
| 440 else donothing(); | |
| 441 break; | |
| 442 | |
| 443 case "unicode-range": | |
| 444 // We already know that the current code is a hexdigit. | |
| 445 | |
| 446 var start = [code], end = [code]; | |
| 447 | |
| 448 for(var total = 1; total < 6; total++) { | |
| 449 if(hexdigit(next())) { | |
| 450 consume(); | |
| 451 start.push(code); | |
| 452 end.push(code); | |
| 453 } | |
| 454 else break; | |
| 455 } | |
| 456 | |
| 457 if(next() == 0x3f) { | |
| 458 for(;total < 6; total++) { | |
| 459 if(next() == 0x3f) { | |
| 460 consume(); | |
| 461 start.push("0".charCodeAt(0)); | |
| 462 end.push("f".charCodeAt(0)); | |
| 463 } | |
| 464 else break; | |
| 465 } | |
| 466 emit(new UnicodeRangeToken(start,end)) && switchto('data'); | |
| 467 } | |
| 468 else if(next(1) == 0x2d && hexdigit(next(2))) { | |
| 469 consume(); | |
| 470 consume(); | |
| 471 end = [code]; | |
| 472 for(var total = 1; total < 6; total++) { | |
| 473 if(hexdigit(next())) { | |
| 474 consume(); | |
| 475 end.push(code); | |
| 476 } | |
| 477 else break; | |
| 478 } | |
| 479 emit(new UnicodeRangeToken(start,end)) && switchto('data'); | |
| 480 } | |
| 481 else emit(new UnicodeRangeToken(start)) && switchto('data'); | |
| 482 break; | |
| 483 | |
| 484 default: | |
| 485 console.log("Unknown state '" + state + "'"); | |
| 486 } | |
| 487 } | |
| 488 } | |
| 489 | |
| 490 function stringFromCodeArray(arr) { | |
| 491 return String.fromCharCode.apply(null,arr.filter(function(e){return e;})); | |
| 492 } | |
| 493 | |
| 494 function CSSParserToken(options) { return this; } | |
| 495 CSSParserToken.prototype.finish = function() { return this; } | |
| 496 CSSParserToken.prototype.toString = function() { return this.tokenType; } | |
| 497 CSSParserToken.prototype.toJSON = function() { return this.toString(); } | |
| 498 | |
| 499 function BadStringToken() { return this; } | |
| 500 BadStringToken.prototype = new CSSParserToken; | |
| 501 BadStringToken.prototype.tokenType = "BADSTRING"; | |
| 502 | |
| 503 function BadURLToken() { return this; } | |
| 504 BadURLToken.prototype = new CSSParserToken; | |
| 505 BadURLToken.prototype.tokenType = "BADURL"; | |
| 506 | |
| 507 function WhitespaceToken() { return this; } | |
| 508 WhitespaceToken.prototype = new CSSParserToken; | |
| 509 WhitespaceToken.prototype.tokenType = "WHITESPACE"; | |
| 510 WhitespaceToken.prototype.toString = function() { return "WS"; } | |
| 511 | |
| 512 function CDOToken() { return this; } | |
| 513 CDOToken.prototype = new CSSParserToken; | |
| 514 CDOToken.prototype.tokenType = "CDO"; | |
| 515 | |
| 516 function CDCToken() { return this; } | |
| 517 CDCToken.prototype = new CSSParserToken; | |
| 518 CDCToken.prototype.tokenType = "CDC"; | |
| 519 | |
| 520 function ColonToken() { return this; } | |
| 521 ColonToken.prototype = new CSSParserToken; | |
| 522 ColonToken.prototype.tokenType = ":"; | |
| 523 | |
| 524 function SemicolonToken() { return this; } | |
| 525 SemicolonToken.prototype = new CSSParserToken; | |
| 526 SemicolonToken.prototype.tokenType = ";"; | |
| 527 | |
| 528 function OpenCurlyToken() { return this; } | |
| 529 OpenCurlyToken.prototype = new CSSParserToken; | |
| 530 OpenCurlyToken.prototype.tokenType = "{"; | |
| 531 | |
| 532 function CloseCurlyToken() { return this; } | |
| 533 CloseCurlyToken.prototype = new CSSParserToken; | |
| 534 CloseCurlyToken.prototype.tokenType = "}"; | |
| 535 | |
| 536 function OpenSquareToken() { return this; } | |
| 537 OpenSquareToken.prototype = new CSSParserToken; | |
| 538 OpenSquareToken.prototype.tokenType = "["; | |
| 539 | |
| 540 function CloseSquareToken() { return this; } | |
| 541 CloseSquareToken.prototype = new CSSParserToken; | |
| 542 CloseSquareToken.prototype.tokenType = "]"; | |
| 543 | |
| 544 function OpenParenToken() { return this; } | |
| 545 OpenParenToken.prototype = new CSSParserToken; | |
| 546 OpenParenToken.prototype.tokenType = "("; | |
| 547 | |
| 548 function CloseParenToken() { return this; } | |
| 549 CloseParenToken.prototype = new CSSParserToken; | |
| 550 CloseParenToken.prototype.tokenType = ")"; | |
| 551 | |
| 552 function EOFToken() { return this; } | |
| 553 EOFToken.prototype = new CSSParserToken; | |
| 554 EOFToken.prototype.tokenType = "EOF"; | |
| 555 | |
| 556 function DelimToken(code) { | |
| 557 this.value = String.fromCharCode(code); | |
| 558 return this; | |
| 559 } | |
| 560 DelimToken.prototype = new CSSParserToken; | |
| 561 DelimToken.prototype.tokenType = "DELIM"; | |
| 562 DelimToken.prototype.toString = function() { return "DELIM("+this.value+")"; } | |
| 563 | |
| 564 function StringValuedToken() { return this; } | |
| 565 StringValuedToken.prototype = new CSSParserToken; | |
| 566 StringValuedToken.prototype.append = function(val) { | |
| 567 if(val instanceof Array) { | |
| 568 for(var i = 0; i < val.length; i++) { | |
| 569 this.value.push(val[i]); | |
| 570 } | |
| 571 } else { | |
| 572 this.value.push(val); | |
| 573 } | |
| 574 return true; | |
| 575 } | |
| 576 StringValuedToken.prototype.finish = function() { | |
| 577 this.value = stringFromCodeArray(this.value); | |
| 578 return this; | |
| 579 } | |
| 580 | |
| 581 function IdentifierToken(val) { | |
| 582 this.value = []; | |
| 583 this.append(val); | |
| 584 } | |
| 585 IdentifierToken.prototype = new StringValuedToken; | |
| 586 IdentifierToken.prototype.tokenType = "IDENT"; | |
| 587 IdentifierToken.prototype.toString = function() { return "IDENT("+this.value+")"; } | |
| 588 | |
| 589 function FunctionToken(val) { | |
| 590 // These are always constructed by passing an IdentifierToken | |
| 591 this.value = val.finish().value; | |
| 592 } | |
| 593 FunctionToken.prototype = new CSSParserToken; | |
| 594 FunctionToken.prototype.tokenType = "FUNCTION"; | |
| 595 FunctionToken.prototype.toString = function() { return "FUNCTION("+this.value+")"; } | |
| 596 | |
| 597 function AtKeywordToken(val) { | |
| 598 this.value = []; | |
| 599 this.append(val); | |
| 600 } | |
| 601 AtKeywordToken.prototype = new StringValuedToken; | |
| 602 AtKeywordToken.prototype.tokenType = "AT-KEYWORD"; | |
| 603 AtKeywordToken.prototype.toString = function() { return "AT("+this.value+")"; } | |
| 604 | |
| 605 function HashToken(val) { | |
| 606 this.value = []; | |
| 607 this.append(val); | |
| 608 } | |
| 609 HashToken.prototype = new StringValuedToken; | |
| 610 HashToken.prototype.tokenType = "HASH"; | |
| 611 HashToken.prototype.toString = function() { return "HASH("+this.value+")"; } | |
| 612 | |
| 613 function StringToken(val) { | |
| 614 this.value = []; | |
| 615 this.append(val); | |
| 616 } | |
| 617 StringToken.prototype = new StringValuedToken; | |
| 618 StringToken.prototype.tokenType = "STRING"; | |
| 619 StringToken.prototype.toString = function() { return "\""+this.value+"\""; } | |
| 620 | |
| 621 function URLToken(val) { | |
| 622 this.value = []; | |
| 623 this.append(val); | |
| 624 } | |
| 625 URLToken.prototype = new StringValuedToken; | |
| 626 URLToken.prototype.tokenType = "URL"; | |
| 627 URLToken.prototype.toString = function() { return "URL("+this.value+")"; } | |
| 628 | |
| 629 function NumberToken(val) { | |
| 630 this.value = []; | |
| 631 this.append(val); | |
| 632 this.type = "integer"; | |
| 633 } | |
| 634 NumberToken.prototype = new StringValuedToken; | |
| 635 NumberToken.prototype.tokenType = "NUMBER"; | |
| 636 NumberToken.prototype.toString = function() { | |
| 637 if(this.type == "integer") | |
| 638 return "INT("+this.value+")"; | |
| 639 return "NUMBER("+this.value+")"; | |
| 640 } | |
| 641 NumberToken.prototype.finish = function() { | |
| 642 this.repr = stringFromCodeArray(this.value); | |
| 643 this.value = this.repr * 1; | |
| 644 if(Math.abs(this.value) % 1 != 0) this.type = "number"; | |
| 645 return this; | |
| 646 } | |
| 647 | |
| 648 function PercentageToken(val) { | |
| 649 // These are always created by passing a NumberToken as val | |
| 650 val.finish(); | |
| 651 this.value = val.value; | |
| 652 this.repr = val.repr; | |
| 653 } | |
| 654 PercentageToken.prototype = new CSSParserToken; | |
| 655 PercentageToken.prototype.tokenType = "PERCENTAGE"; | |
| 656 PercentageToken.prototype.toString = function() { return "PERCENTAGE("+this.value+")"; } | |
| 657 | |
| 658 function DimensionToken(val,unit) { | |
| 659 // These are always created by passing a NumberToken as the val | |
| 660 val.finish(); | |
| 661 this.num = val.value; | |
| 662 this.unit = []; | |
| 663 this.repr = val.repr; | |
| 664 this.append(unit); | |
| 665 } | |
| 666 DimensionToken.prototype = new CSSParserToken; | |
| 667 DimensionToken.prototype.tokenType = "DIMENSION"; | |
| 668 DimensionToken.prototype.toString = function() { return "DIM("+this.num+","+this.unit+")"; } | |
| 669 DimensionToken.prototype.append = function(val) { | |
| 670 if(val instanceof Array) { | |
| 671 for(var i = 0; i < val.length; i++) { | |
| 672 this.unit.push(val[i]); | |
| 673 } | |
| 674 } else { | |
| 675 this.unit.push(val); | |
| 676 } | |
| 677 return true; | |
| 678 } | |
| 679 DimensionToken.prototype.finish = function() { | |
| 680 this.unit = stringFromCodeArray(this.unit); | |
| 681 this.repr += this.unit; | |
| 682 return this; | |
| 683 } | |
| 684 | |
| 685 function UnicodeRangeToken(start,end) { | |
| 686 // start and end are array of char codes, completely finished | |
| 687 start = parseInt(stringFromCodeArray(start),16); | |
| 688 if(end === undefined) end = start + 1; | |
| 689 else end = parseInt(stringFromCodeArray(end),16); | |
| 690 | |
| 691 if(start > maximumallowedcodepoint) end = start; | |
| 692 if(end < start) end = start; | |
| 693 if(end > maximumallowedcodepoint) end = maximumallowedcodepoint; | |
| 694 | |
| 695 this.start = start; | |
| 696 this.end = end; | |
| 697 return this; | |
| 698 } | |
| 699 UnicodeRangeToken.prototype = new CSSParserToken; | |
| 700 UnicodeRangeToken.prototype.tokenType = "UNICODE-RANGE"; | |
| 701 UnicodeRangeToken.prototype.toString = function() { | |
| 702 if(this.start+1 == this.end) | |
| 703 return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+")"; | |
| 704 if(this.start < this.end) | |
| 705 return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+"-"+this.end.toString(16).toUpperCase()+")"; | |
| 706 return "UNICODE-RANGE()"; | |
| 707 } | |
| 708 UnicodeRangeToken.prototype.contains = function(code) { | |
| 709 return code >= this.start && code < this.end; | |
| 710 } | |
| 711 | |
| 712 | |
| 713 // Exportation. | |
| 714 // TODO: also export the various tokens objects? | |
| 715 module.exports = tokenize; | |
| 716 | |
| 717 })); |
