browser/devtools/sourceeditor/css-tokenizer.js@6474c204b198
browser/devtools/sourceeditor/css-tokenizer.js
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rw-r--r--
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
1 /**
2 * This file is taken from the below mentioned url and is under CC0 license.
3 * https://github.com/tabatkins/css-parser/blob/master/tokenizer.js
4 * Please retain this comment while updating this file from upstream.
5 */
7 (function (root, factory) {
8 // Universal Module Definition (UMD) to support AMD, CommonJS/Node.js,
9 // Rhino, and plain browser loading.
10 if (typeof define === 'function' && define.amd) {
11 define(['exports'], factory);
12 } else if (typeof exports !== 'undefined') {
13 factory(exports);
14 } else {
15 factory(root);
16 }
17 }(this, function (exports) {
19 var between = function (num, first, last) { return num >= first && num <= last; }
20 function digit(code) { return between(code, 0x30,0x39); }
21 function hexdigit(code) { return digit(code) || between(code, 0x41,0x46) || between(code, 0x61,0x66); }
22 function uppercaseletter(code) { return between(code, 0x41,0x5a); }
23 function lowercaseletter(code) { return between(code, 0x61,0x7a); }
24 function letter(code) { return uppercaseletter(code) || lowercaseletter(code); }
25 function nonascii(code) { return code >= 0xa0; }
26 function namestartchar(code) { return letter(code) || nonascii(code) || code == 0x5f; }
27 function namechar(code) { return namestartchar(code) || digit(code) || code == 0x2d; }
28 function nonprintable(code) { return between(code, 0,8) || between(code, 0xe,0x1f) || between(code, 0x7f,0x9f); }
29 function newline(code) { return code == 0xa || code == 0xc; }
30 function whitespace(code) { return newline(code) || code == 9 || code == 0x20; }
31 function badescape(code) { return newline(code) || isNaN(code); }
33 // Note: I'm not yet acting smart enough to actually handle astral characters.
34 var maximumallowedcodepoint = 0x10ffff;
36 function tokenize(str, options) {
37 if(options == undefined) options = {transformFunctionWhitespace:false, scientificNotation:false};
38 var i = -1;
39 var tokens = [];
40 var state = "data";
41 var code;
42 var currtoken;
44 // Line number information.
45 var line = 0;
46 var column = 0;
47 // The only use of lastLineLength is in reconsume().
48 var lastLineLength = 0;
49 var incrLineno = function() {
50 line += 1;
51 lastLineLength = column;
52 column = 0;
53 };
54 var locStart = {line:line, column:column};
56 var next = function(num) { if(num === undefined) num = 1; return str.charCodeAt(i+num); };
57 var consume = function(num) {
58 if(num === undefined)
59 num = 1;
60 i += num;
61 code = str.charCodeAt(i);
62 if (newline(code)) incrLineno();
63 else column += num;
64 //console.log('Consume '+i+' '+String.fromCharCode(code) + ' 0x' + code.toString(16));
65 return true;
66 };
67 var reconsume = function() {
68 i -= 1;
69 if (newline(code)) {
70 line -= 1;
71 column = lastLineLength;
72 } else {
73 column -= 1;
74 }
75 locStart.line = line;
76 locStart.column = column;
77 return true;
78 };
79 var eof = function() { return i >= str.length; };
80 var donothing = function() {};
81 var emit = function(token) {
82 if(token) {
83 token.finish();
84 } else {
85 token = currtoken.finish();
86 }
87 if (options.loc === true) {
88 token.loc = {};
89 token.loc.start = {line:locStart.line, column:locStart.column};
90 locStart = {line: line, column: column};
91 token.loc.end = locStart;
92 }
93 tokens.push(token);
94 //console.log('Emitting ' + token);
95 currtoken = undefined;
96 return true;
97 };
98 var create = function(token) { currtoken = token; return true; };
99 var parseerror = function() { console.log("Parse error at index " + i + ", processing codepoint 0x" + code.toString(16) + " in state " + state + ".");return true; };
100 var switchto = function(newstate) {
101 state = newstate;
102 //console.log('Switching to ' + state);
103 return true;
104 };
105 var consumeEscape = function() {
106 // Assume the the current character is the \
107 consume();
108 if(hexdigit(code)) {
109 // Consume 1-6 hex digits
110 var digits = [];
111 for(var total = 0; total < 6; total++) {
112 if(hexdigit(code)) {
113 digits.push(code);
114 consume();
115 } else { break; }
116 }
117 var value = parseInt(digits.map(String.fromCharCode).join(''), 16);
118 if( value > maximumallowedcodepoint ) value = 0xfffd;
119 // If the current char is whitespace, cool, we'll just eat it.
120 // Otherwise, put it back.
121 if(!whitespace(code)) reconsume();
122 return value;
123 } else {
124 return code;
125 }
126 };
128 for(;;) {
129 if(i > str.length*2) return "I'm infinite-looping!";
130 consume();
131 switch(state) {
132 case "data":
133 if(whitespace(code)) {
134 emit(new WhitespaceToken);
135 while(whitespace(next())) consume();
136 }
137 else if(code == 0x22) switchto("double-quote-string");
138 else if(code == 0x23) switchto("hash");
139 else if(code == 0x27) switchto("single-quote-string");
140 else if(code == 0x28) emit(new OpenParenToken);
141 else if(code == 0x29) emit(new CloseParenToken);
142 else if(code == 0x2b) {
143 if(digit(next()) || (next() == 0x2e && digit(next(2)))) switchto("number") && reconsume();
144 else emit(new DelimToken(code));
145 }
146 else if(code == 0x2d) {
147 if(next(1) == 0x2d && next(2) == 0x3e) consume(2) && emit(new CDCToken);
148 else if(digit(next()) || (next(1) == 0x2e && digit(next(2)))) switchto("number") && reconsume();
149 else if(namestartchar(next())) switchto("identifier") && reconsume();
150 else emit(new DelimToken(code));
151 }
152 else if(code == 0x2e) {
153 if(digit(next())) switchto("number") && reconsume();
154 else emit(new DelimToken(code));
155 }
156 else if(code == 0x2f) {
157 if(next() == 0x2a) switchto("comment");
158 else emit(new DelimToken(code));
159 }
160 else if(code == 0x3a) emit(new ColonToken);
161 else if(code == 0x3b) emit(new SemicolonToken);
162 else if(code == 0x3c) {
163 if(next(1) == 0x21 && next(2) == 0x2d && next(3) == 0x2d) consume(3) && emit(new CDOToken);
164 else emit(new DelimToken(code));
165 }
166 else if(code == 0x40) switchto("at-keyword");
167 else if(code == 0x5b) emit(new OpenSquareToken);
168 else if(code == 0x5c) {
169 if(badescape(next())) parseerror() && emit(new DelimToken(code));
170 else switchto("identifier") && reconsume();
171 }
172 else if(code == 0x5d) emit(new CloseSquareToken);
173 else if(code == 0x7b) emit(new OpenCurlyToken);
174 else if(code == 0x7d) emit(new CloseCurlyToken);
175 else if(digit(code)) switchto("number") && reconsume();
176 else if(code == 0x55 || code == 0x75) {
177 if(next(1) == 0x2b && hexdigit(next(2))) consume() && switchto("unicode-range");
178 else if((next(1) == 0x52 || next(1) == 0x72) && (next(2) == 0x4c || next(2) == 0x6c) && (next(3) == 0x28)) consume(3) && switchto("url");
179 else switchto("identifier") && reconsume();
180 }
181 else if(namestartchar(code)) switchto("identifier") && reconsume();
182 else if(eof()) { emit(new EOFToken); return tokens; }
183 else emit(new DelimToken(code));
184 break;
186 case "double-quote-string":
187 if(currtoken == undefined) create(new StringToken);
189 if(code == 0x22) emit() && switchto("data");
190 else if(eof()) parseerror() && emit() && switchto("data");
191 else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume();
192 else if(code == 0x5c) {
193 if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data");
194 else if(newline(next())) consume();
195 else currtoken.append(consumeEscape());
196 }
197 else currtoken.append(code);
198 break;
200 case "single-quote-string":
201 if(currtoken == undefined) create(new StringToken);
203 if(code == 0x27) emit() && switchto("data");
204 else if(eof()) parseerror() && emit() && switchto("data");
205 else if(newline(code)) parseerror() && emit(new BadStringToken) && switchto("data") && reconsume();
206 else if(code == 0x5c) {
207 if(badescape(next())) parseerror() && emit(new BadStringToken) && switchto("data");
208 else if(newline(next())) consume();
209 else currtoken.append(consumeEscape());
210 }
211 else currtoken.append(code);
212 break;
214 case "hash":
215 if(namechar(code)) create(new HashToken(code)) && switchto("hash-rest");
216 else if(code == 0x5c) {
217 if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
218 else create(new HashToken(consumeEscape())) && switchto('hash-rest');
219 }
220 else emit(new DelimToken(0x23)) && switchto('data') && reconsume();
221 break;
223 case "hash-rest":
224 if(namechar(code)) currtoken.append(code);
225 else if(code == 0x5c) {
226 if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
227 else currtoken.append(consumeEscape());
228 }
229 else emit() && switchto('data') && reconsume();
230 break;
232 case "comment":
233 if(code == 0x2a) {
234 if(next() == 0x2f) consume() && switchto('data');
235 else donothing();
236 }
237 else if(eof()) parseerror() && switchto('data') && reconsume();
238 else donothing();
239 break;
241 case "at-keyword":
242 if(code == 0x2d) {
243 if(namestartchar(next())) consume() && create(new AtKeywordToken([0x40,code])) && switchto('at-keyword-rest');
244 else emit(new DelimToken(0x40)) && switchto('data') && reconsume();
245 }
246 else if(namestartchar(code)) create(new AtKeywordToken(code)) && switchto('at-keyword-rest');
247 else if(code == 0x5c) {
248 if(badescape(next())) parseerror() && emit(new DelimToken(0x23)) && switchto("data") && reconsume();
249 else create(new AtKeywordToken(consumeEscape())) && switchto('at-keyword-rest');
250 }
251 else emit(new DelimToken(0x40)) && switchto('data') && reconsume();
252 break;
254 case "at-keyword-rest":
255 if(namechar(code)) currtoken.append(code);
256 else if(code == 0x5c) {
257 if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume();
258 else currtoken.append(consumeEscape());
259 }
260 else emit() && switchto('data') && reconsume();
261 break;
263 case "identifier":
264 if(code == 0x2d) {
265 if(namestartchar(next())) create(new IdentifierToken(code)) && switchto('identifier-rest');
266 else switchto('data') && reconsume();
267 }
268 else if(namestartchar(code)) create(new IdentifierToken(code)) && switchto('identifier-rest');
269 else if(code == 0x5c) {
270 if(badescape(next())) parseerror() && switchto("data") && reconsume();
271 else create(new IdentifierToken(consumeEscape())) && switchto('identifier-rest');
272 }
273 else switchto('data') && reconsume();
274 break;
276 case "identifier-rest":
277 if(namechar(code)) currtoken.append(code);
278 else if(code == 0x5c) {
279 if(badescape(next())) parseerror() && emit() && switchto("data") && reconsume();
280 else currtoken.append(consumeEscape());
281 }
282 else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data');
283 else if(whitespace(code) && options.transformFunctionWhitespace) switchto('transform-function-whitespace');
284 else emit() && switchto('data') && reconsume();
285 break;
287 case "transform-function-whitespace":
288 if(whitespace(code)) donothing();
289 else if(code == 0x28) emit(new FunctionToken(currtoken)) && switchto('data');
290 else emit() && switchto('data') && reconsume();
291 break;
293 case "number":
294 create(new NumberToken());
296 if(code == 0x2d) {
297 if(digit(next())) consume() && currtoken.append([0x2d,code]) && switchto('number-rest');
298 else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2d,0x2e,code]) && switchto('number-fraction');
299 else switchto('data') && reconsume();
300 }
301 else if(code == 0x2b) {
302 if(digit(next())) consume() && currtoken.append([0x2b,code]) && switchto('number-rest');
303 else if(next(1) == 0x2e && digit(next(2))) consume(2) && currtoken.append([0x2b,0x2e,code]) && switchto('number-fraction');
304 else switchto('data') && reconsume();
305 }
306 else if(digit(code)) currtoken.append(code) && switchto('number-rest');
307 else if(code == 0x2e) {
308 if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction');
309 else switchto('data') && reconsume();
310 }
311 else switchto('data') && reconsume();
312 break;
314 case "number-rest":
315 if(digit(code)) currtoken.append(code);
316 else if(code == 0x2e) {
317 if(digit(next())) consume() && currtoken.append([0x2e,code]) && switchto('number-fraction');
318 else emit() && switchto('data') && reconsume();
319 }
320 else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume();
321 else if(code == 0x45 || code == 0x65) {
322 if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension');
323 else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation');
324 else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation');
325 else create(new DimensionToken(currtoken,code)) && switchto('dimension');
326 }
327 else if(code == 0x2d) {
328 if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension');
329 else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume();
330 else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension');
331 else emit() && switchto('data') && reconsume();
332 }
333 else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension');
334 else if(code == 0x5c) {
335 if(badescape(next)) emit() && switchto('data') && reconsume();
336 else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension');
337 }
338 else emit() && switchto('data') && reconsume();
339 break;
341 case "number-fraction":
342 currtoken.type = "number";
344 if(digit(code)) currtoken.append(code);
345 else if(code == 0x2e) emit() && switchto('data') && reconsume();
346 else if(code == 0x25) emit(new PercentageToken(currtoken)) && switchto('data') && reconsume();
347 else if(code == 0x45 || code == 0x65) {
348 if(!options.scientificNotation) create(new DimensionToken(currtoken,code)) && switchto('dimension');
349 else if(digit(next())) consume() && currtoken.append([0x25,code]) && switchto('sci-notation');
350 else if((next(1) == 0x2b || next(1) == 0x2d) && digit(next(2))) currtoken.append([0x25,next(1),next(2)]) && consume(2) && switchto('sci-notation');
351 else create(new DimensionToken(currtoken,code)) && switchto('dimension');
352 }
353 else if(code == 0x2d) {
354 if(namestartchar(next())) consume() && create(new DimensionToken(currtoken,[0x2d,code])) && switchto('dimension');
355 else if(next(1) == 0x5c && badescape(next(2))) parseerror() && emit() && switchto('data') && reconsume();
356 else if(next(1) == 0x5c) consume() && create(new DimensionToken(currtoken, [0x2d,consumeEscape()])) && switchto('dimension');
357 else emit() && switchto('data') && reconsume();
358 }
359 else if(namestartchar(code)) create(new DimensionToken(currtoken, code)) && switchto('dimension');
360 else if(code == 0x5c) {
361 if(badescape(next)) emit() && switchto('data') && reconsume();
362 else create(new DimensionToken(currtoken,consumeEscape)) && switchto('dimension');
363 }
364 else emit() && switchto('data') && reconsume();
365 break;
367 case "dimension":
368 if(namechar(code)) currtoken.append(code);
369 else if(code == 0x5c) {
370 if(badescape(next())) parseerror() && emit() && switchto('data') && reconsume();
371 else currtoken.append(consumeEscape());
372 }
373 else emit() && switchto('data') && reconsume();
374 break;
376 case "sci-notation":
377 if(digit(code)) currtoken.append(code);
378 else emit() && switchto('data') && reconsume();
379 break;
381 case "url":
382 if(code == 0x22) switchto('url-double-quote');
383 else if(code == 0x27) switchto('url-single-quote');
384 else if(code == 0x29) emit(new URLToken) && switchto('data');
385 else if(whitespace(code)) donothing();
386 else switchto('url-unquoted') && reconsume();
387 break;
389 case "url-double-quote":
390 if(currtoken == undefined) create(new URLToken);
392 if(code == 0x22) switchto('url-end');
393 else if(newline(code)) parseerror() && switchto('bad-url');
394 else if(code == 0x5c) {
395 if(newline(next())) consume();
396 else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume();
397 else currtoken.append(consumeEscape());
398 }
399 else currtoken.append(code);
400 break;
402 case "url-single-quote":
403 if(currtoken == undefined) create(new URLToken);
405 if(code == 0x27) switchto('url-end');
406 else if(newline(code)) parseerror() && switchto('bad-url');
407 else if(code == 0x5c) {
408 if(newline(next())) consume();
409 else if(badescape(next())) parseerror() && emit(new BadURLToken) && switchto('data') && reconsume();
410 else currtoken.append(consumeEscape());
411 }
412 else currtoken.append(code);
413 break;
415 case "url-end":
416 if(whitespace(code)) donothing();
417 else if(code == 0x29) emit() && switchto('data');
418 else parseerror() && switchto('bad-url') && reconsume();
419 break;
421 case "url-unquoted":
422 if(currtoken == undefined) create(new URLToken);
424 if(whitespace(code)) switchto('url-end');
425 else if(code == 0x29) emit() && switchto('data');
426 else if(code == 0x22 || code == 0x27 || code == 0x28 || nonprintable(code)) parseerror() && switchto('bad-url');
427 else if(code == 0x5c) {
428 if(badescape(next())) parseerror() && switchto('bad-url');
429 else currtoken.append(consumeEscape());
430 }
431 else currtoken.append(code);
432 break;
434 case "bad-url":
435 if(code == 0x29) emit(new BadURLToken) && switchto('data');
436 else if(code == 0x5c) {
437 if(badescape(next())) donothing();
438 else consumeEscape()
439 }
440 else donothing();
441 break;
443 case "unicode-range":
444 // We already know that the current code is a hexdigit.
446 var start = [code], end = [code];
448 for(var total = 1; total < 6; total++) {
449 if(hexdigit(next())) {
450 consume();
451 start.push(code);
452 end.push(code);
453 }
454 else break;
455 }
457 if(next() == 0x3f) {
458 for(;total < 6; total++) {
459 if(next() == 0x3f) {
460 consume();
461 start.push("0".charCodeAt(0));
462 end.push("f".charCodeAt(0));
463 }
464 else break;
465 }
466 emit(new UnicodeRangeToken(start,end)) && switchto('data');
467 }
468 else if(next(1) == 0x2d && hexdigit(next(2))) {
469 consume();
470 consume();
471 end = [code];
472 for(var total = 1; total < 6; total++) {
473 if(hexdigit(next())) {
474 consume();
475 end.push(code);
476 }
477 else break;
478 }
479 emit(new UnicodeRangeToken(start,end)) && switchto('data');
480 }
481 else emit(new UnicodeRangeToken(start)) && switchto('data');
482 break;
484 default:
485 console.log("Unknown state '" + state + "'");
486 }
487 }
488 }
490 function stringFromCodeArray(arr) {
491 return String.fromCharCode.apply(null,arr.filter(function(e){return e;}));
492 }
494 function CSSParserToken(options) { return this; }
495 CSSParserToken.prototype.finish = function() { return this; }
496 CSSParserToken.prototype.toString = function() { return this.tokenType; }
497 CSSParserToken.prototype.toJSON = function() { return this.toString(); }
499 function BadStringToken() { return this; }
500 BadStringToken.prototype = new CSSParserToken;
501 BadStringToken.prototype.tokenType = "BADSTRING";
503 function BadURLToken() { return this; }
504 BadURLToken.prototype = new CSSParserToken;
505 BadURLToken.prototype.tokenType = "BADURL";
507 function WhitespaceToken() { return this; }
508 WhitespaceToken.prototype = new CSSParserToken;
509 WhitespaceToken.prototype.tokenType = "WHITESPACE";
510 WhitespaceToken.prototype.toString = function() { return "WS"; }
512 function CDOToken() { return this; }
513 CDOToken.prototype = new CSSParserToken;
514 CDOToken.prototype.tokenType = "CDO";
516 function CDCToken() { return this; }
517 CDCToken.prototype = new CSSParserToken;
518 CDCToken.prototype.tokenType = "CDC";
520 function ColonToken() { return this; }
521 ColonToken.prototype = new CSSParserToken;
522 ColonToken.prototype.tokenType = ":";
524 function SemicolonToken() { return this; }
525 SemicolonToken.prototype = new CSSParserToken;
526 SemicolonToken.prototype.tokenType = ";";
528 function OpenCurlyToken() { return this; }
529 OpenCurlyToken.prototype = new CSSParserToken;
530 OpenCurlyToken.prototype.tokenType = "{";
532 function CloseCurlyToken() { return this; }
533 CloseCurlyToken.prototype = new CSSParserToken;
534 CloseCurlyToken.prototype.tokenType = "}";
536 function OpenSquareToken() { return this; }
537 OpenSquareToken.prototype = new CSSParserToken;
538 OpenSquareToken.prototype.tokenType = "[";
540 function CloseSquareToken() { return this; }
541 CloseSquareToken.prototype = new CSSParserToken;
542 CloseSquareToken.prototype.tokenType = "]";
544 function OpenParenToken() { return this; }
545 OpenParenToken.prototype = new CSSParserToken;
546 OpenParenToken.prototype.tokenType = "(";
548 function CloseParenToken() { return this; }
549 CloseParenToken.prototype = new CSSParserToken;
550 CloseParenToken.prototype.tokenType = ")";
552 function EOFToken() { return this; }
553 EOFToken.prototype = new CSSParserToken;
554 EOFToken.prototype.tokenType = "EOF";
556 function DelimToken(code) {
557 this.value = String.fromCharCode(code);
558 return this;
559 }
560 DelimToken.prototype = new CSSParserToken;
561 DelimToken.prototype.tokenType = "DELIM";
562 DelimToken.prototype.toString = function() { return "DELIM("+this.value+")"; }
564 function StringValuedToken() { return this; }
565 StringValuedToken.prototype = new CSSParserToken;
566 StringValuedToken.prototype.append = function(val) {
567 if(val instanceof Array) {
568 for(var i = 0; i < val.length; i++) {
569 this.value.push(val[i]);
570 }
571 } else {
572 this.value.push(val);
573 }
574 return true;
575 }
576 StringValuedToken.prototype.finish = function() {
577 this.value = stringFromCodeArray(this.value);
578 return this;
579 }
581 function IdentifierToken(val) {
582 this.value = [];
583 this.append(val);
584 }
585 IdentifierToken.prototype = new StringValuedToken;
586 IdentifierToken.prototype.tokenType = "IDENT";
587 IdentifierToken.prototype.toString = function() { return "IDENT("+this.value+")"; }
589 function FunctionToken(val) {
590 // These are always constructed by passing an IdentifierToken
591 this.value = val.finish().value;
592 }
593 FunctionToken.prototype = new CSSParserToken;
594 FunctionToken.prototype.tokenType = "FUNCTION";
595 FunctionToken.prototype.toString = function() { return "FUNCTION("+this.value+")"; }
597 function AtKeywordToken(val) {
598 this.value = [];
599 this.append(val);
600 }
601 AtKeywordToken.prototype = new StringValuedToken;
602 AtKeywordToken.prototype.tokenType = "AT-KEYWORD";
603 AtKeywordToken.prototype.toString = function() { return "AT("+this.value+")"; }
605 function HashToken(val) {
606 this.value = [];
607 this.append(val);
608 }
609 HashToken.prototype = new StringValuedToken;
610 HashToken.prototype.tokenType = "HASH";
611 HashToken.prototype.toString = function() { return "HASH("+this.value+")"; }
613 function StringToken(val) {
614 this.value = [];
615 this.append(val);
616 }
617 StringToken.prototype = new StringValuedToken;
618 StringToken.prototype.tokenType = "STRING";
619 StringToken.prototype.toString = function() { return "\""+this.value+"\""; }
621 function URLToken(val) {
622 this.value = [];
623 this.append(val);
624 }
625 URLToken.prototype = new StringValuedToken;
626 URLToken.prototype.tokenType = "URL";
627 URLToken.prototype.toString = function() { return "URL("+this.value+")"; }
629 function NumberToken(val) {
630 this.value = [];
631 this.append(val);
632 this.type = "integer";
633 }
634 NumberToken.prototype = new StringValuedToken;
635 NumberToken.prototype.tokenType = "NUMBER";
636 NumberToken.prototype.toString = function() {
637 if(this.type == "integer")
638 return "INT("+this.value+")";
639 return "NUMBER("+this.value+")";
640 }
641 NumberToken.prototype.finish = function() {
642 this.repr = stringFromCodeArray(this.value);
643 this.value = this.repr * 1;
644 if(Math.abs(this.value) % 1 != 0) this.type = "number";
645 return this;
646 }
648 function PercentageToken(val) {
649 // These are always created by passing a NumberToken as val
650 val.finish();
651 this.value = val.value;
652 this.repr = val.repr;
653 }
654 PercentageToken.prototype = new CSSParserToken;
655 PercentageToken.prototype.tokenType = "PERCENTAGE";
656 PercentageToken.prototype.toString = function() { return "PERCENTAGE("+this.value+")"; }
658 function DimensionToken(val,unit) {
659 // These are always created by passing a NumberToken as the val
660 val.finish();
661 this.num = val.value;
662 this.unit = [];
663 this.repr = val.repr;
664 this.append(unit);
665 }
666 DimensionToken.prototype = new CSSParserToken;
667 DimensionToken.prototype.tokenType = "DIMENSION";
668 DimensionToken.prototype.toString = function() { return "DIM("+this.num+","+this.unit+")"; }
669 DimensionToken.prototype.append = function(val) {
670 if(val instanceof Array) {
671 for(var i = 0; i < val.length; i++) {
672 this.unit.push(val[i]);
673 }
674 } else {
675 this.unit.push(val);
676 }
677 return true;
678 }
679 DimensionToken.prototype.finish = function() {
680 this.unit = stringFromCodeArray(this.unit);
681 this.repr += this.unit;
682 return this;
683 }
685 function UnicodeRangeToken(start,end) {
686 // start and end are array of char codes, completely finished
687 start = parseInt(stringFromCodeArray(start),16);
688 if(end === undefined) end = start + 1;
689 else end = parseInt(stringFromCodeArray(end),16);
691 if(start > maximumallowedcodepoint) end = start;
692 if(end < start) end = start;
693 if(end > maximumallowedcodepoint) end = maximumallowedcodepoint;
695 this.start = start;
696 this.end = end;
697 return this;
698 }
699 UnicodeRangeToken.prototype = new CSSParserToken;
700 UnicodeRangeToken.prototype.tokenType = "UNICODE-RANGE";
701 UnicodeRangeToken.prototype.toString = function() {
702 if(this.start+1 == this.end)
703 return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+")";
704 if(this.start < this.end)
705 return "UNICODE-RANGE("+this.start.toString(16).toUpperCase()+"-"+this.end.toString(16).toUpperCase()+")";
706 return "UNICODE-RANGE()";
707 }
708 UnicodeRangeToken.prototype.contains = function(code) {
709 return code >= this.start && code < this.end;
710 }
713 // Exportation.
714 // TODO: also export the various tokens objects?
715 module.exports = tokenize;
717 }));
