|
1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. |
|
2 // Use of this source code is governed by a BSD-style license that can be |
|
3 // found in the LICENSE file. |
|
4 |
|
5 // This is a test harness for running javascript tests in the browser. |
|
6 // The only identifier exposed by this harness is WebGLTestHarnessModule. |
|
7 // |
|
8 // To use it make an HTML page with an iframe. Then call the harness like this |
|
9 // |
|
10 // function reportResults(type, msg, success) { |
|
11 // ... |
|
12 // return true; |
|
13 // } |
|
14 // |
|
15 // var fileListURL = '00_test_list.txt'; |
|
16 // var testHarness = new WebGLTestHarnessModule.TestHarness( |
|
17 // iframe, |
|
18 // fileListURL, |
|
19 // reportResults); |
|
20 // |
|
21 // The harness will load the fileListURL and parse it for the URLs, one URL |
|
22 // per line. URLs should be on the same domain and at the same folder level |
|
23 // or below the main html file. If any URL ends in .txt it will be parsed |
|
24 // as well so you can nest .txt files. URLs inside a .txt file should be |
|
25 // relative to that text file. |
|
26 // |
|
27 // During startup, for each page found the reportFunction will be called with |
|
28 // WebGLTestHarnessModule.TestHarness.reportType.ADD_PAGE and msg will be |
|
29 // the URL of the test. |
|
30 // |
|
31 // Each test is required to call testHarness.reportResults. This is most easily |
|
32 // accomplished by storing that value on the main window with |
|
33 // |
|
34 // window.webglTestHarness = testHarness |
|
35 // |
|
36 // and then adding these to functions to your tests. |
|
37 // |
|
38 // function reportTestResultsToHarness(success, msg) { |
|
39 // if (window.parent.webglTestHarness) { |
|
40 // window.parent.webglTestHarness.reportResults(success, msg); |
|
41 // } |
|
42 // } |
|
43 // |
|
44 // function notifyFinishedToHarness() { |
|
45 // if (window.parent.webglTestHarness) { |
|
46 // window.parent.webglTestHarness.notifyFinished(); |
|
47 // } |
|
48 // } |
|
49 // |
|
50 // This way your tests will still run without the harness and you can use |
|
51 // any testing framework you want. |
|
52 // |
|
53 // Each test should call reportTestResultsToHarness with true for success if it |
|
54 // succeeded and false if it fail followed and any message it wants to |
|
55 // associate with the test. If your testing framework supports checking for |
|
56 // timeout you can call it with success equal to undefined in that case. |
|
57 // |
|
58 // To run the tests, call testHarness.runTests(); |
|
59 // |
|
60 // For each test run, before the page is loaded the reportFunction will be |
|
61 // called with WebGLTestHarnessModule.TestHarness.reportType.START_PAGE and msg |
|
62 // will be the URL of the test. You may return false if you want the test to be |
|
63 // skipped. |
|
64 // |
|
65 // For each test completed the reportFunction will be called with |
|
66 // with WebGLTestHarnessModule.TestHarness.reportType.TEST_RESULT, |
|
67 // success = true on success, false on failure, undefined on timeout |
|
68 // and msg is any message the test choose to pass on. |
|
69 // |
|
70 // When all the tests on the page have finished your page must call |
|
71 // notifyFinishedToHarness. If notifyFinishedToHarness is not called |
|
72 // the harness will assume the test timed out. |
|
73 // |
|
74 // When all the tests on a page have finished OR the page as timed out the |
|
75 // reportFunction will be called with |
|
76 // WebGLTestHarnessModule.TestHarness.reportType.FINISH_PAGE |
|
77 // where success = true if the page has completed or undefined if the page timed |
|
78 // out. |
|
79 // |
|
80 // Finally, when all the tests have completed the reportFunction will be called |
|
81 // with WebGLTestHarnessModule.TestHarness.reportType.FINISHED_ALL_TESTS. |
|
82 // |
|
83 |
|
84 WebGLTestHarnessModule = function() { |
|
85 |
|
86 /** |
|
87 * Wrapped logging function. |
|
88 */ |
|
89 var log = function(msg) { |
|
90 if (window.console && window.console.log) { |
|
91 window.console.log(msg); |
|
92 } |
|
93 }; |
|
94 |
|
95 /** |
|
96 * Loads text from an external file. This function is synchronous. |
|
97 * @param {string} url The url of the external file. |
|
98 * @param {!function(bool, string): void} callback that is sent a bool for |
|
99 * success and the string. |
|
100 */ |
|
101 var loadTextFileAsynchronous = function(url, callback) { |
|
102 log ("loading: " + url); |
|
103 var error = 'loadTextFileSynchronous failed to load url "' + url + '"'; |
|
104 var request; |
|
105 if (window.XMLHttpRequest) { |
|
106 request = new XMLHttpRequest(); |
|
107 if (request.overrideMimeType) { |
|
108 request.overrideMimeType('text/plain'); |
|
109 } |
|
110 } else { |
|
111 throw 'XMLHttpRequest is disabled'; |
|
112 } |
|
113 try { |
|
114 request.open('GET', url, true); |
|
115 request.onreadystatechange = function() { |
|
116 if (request.readyState == 4) { |
|
117 var text = ''; |
|
118 // HTTP reports success with a 200 status. The file protocol reports |
|
119 // success with zero. HTTP does not use zero as a status code (they |
|
120 // start at 100). |
|
121 // https://developer.mozilla.org/En/Using_XMLHttpRequest |
|
122 var success = request.status == 200 || request.status == 0; |
|
123 if (success) { |
|
124 text = request.responseText; |
|
125 } |
|
126 log("loaded: " + url); |
|
127 callback(success, text); |
|
128 } |
|
129 }; |
|
130 request.send(null); |
|
131 } catch (e) { |
|
132 log("failed to load: " + url); |
|
133 callback(false, ''); |
|
134 } |
|
135 }; |
|
136 |
|
137 /** |
|
138 * Compare version strings. |
|
139 */ |
|
140 var greaterThanOrEqualToVersion = function(have, want) { |
|
141 have = have.split(" ")[0].split("."); |
|
142 want = want.split(" ")[0].split("."); |
|
143 |
|
144 //have 1.2.3 want 1.1 |
|
145 //have 1.1.1 want 1.1 |
|
146 //have 1.0.9 want 1.1 |
|
147 //have 1.1 want 1.1.1 |
|
148 |
|
149 for (var ii = 0; ii < want.length; ++ii) { |
|
150 var wantNum = parseInt(want[ii]); |
|
151 var haveNum = have[ii] ? parseInt(have[ii]) : 0 |
|
152 if (haveNum < wantNum) { |
|
153 return false; |
|
154 } |
|
155 } |
|
156 return true; |
|
157 }; |
|
158 |
|
159 /** |
|
160 * Reads a file, recursively adding files referenced inside. |
|
161 * |
|
162 * Each line of URL is parsed, comments starting with '#' or ';' |
|
163 * or '//' are stripped. |
|
164 * |
|
165 * arguments beginning with -- are extracted |
|
166 * |
|
167 * lines that end in .txt are recursively scanned for more files |
|
168 * other lines are added to the list of files. |
|
169 * |
|
170 * @param {string} url The url of the file to read. |
|
171 * @param {void function(boolean, !Array.<string>)} callback. |
|
172 * Callback that is called with true for success and an |
|
173 * array of filenames. |
|
174 * @param {Object} options. Optional options |
|
175 * |
|
176 * Options: |
|
177 * version: {string} The version of the conformance test. |
|
178 * Tests with the argument --min-version <version> will |
|
179 * be ignored version is less then <version> |
|
180 * |
|
181 */ |
|
182 var getFileList = function(url, callback, options) { |
|
183 var files = []; |
|
184 |
|
185 var copyObject = function(obj) { |
|
186 return JSON.parse(JSON.stringify(obj)); |
|
187 }; |
|
188 |
|
189 var toCamelCase = function(str) { |
|
190 return str.replace(/-([a-z])/g, function (g) { return g[1].toUpperCase() }); |
|
191 }; |
|
192 |
|
193 var globalOptions = copyObject(options); |
|
194 globalOptions.defaultVersion = "1.0"; |
|
195 |
|
196 var getFileListImpl = function(prefix, line, hierarchicalOptions, callback) { |
|
197 var files = []; |
|
198 |
|
199 var args = line.split(/\s+/); |
|
200 var nonOptions = []; |
|
201 var useTest = true; |
|
202 var testOptions = {}; |
|
203 for (var jj = 0; jj < args.length; ++jj) { |
|
204 var arg = args[jj]; |
|
205 if (arg[0] == '-') { |
|
206 if (arg[1] != '-') { |
|
207 throw ("bad option at in " + url + ":" + (ii + 1) + ": " + str); |
|
208 } |
|
209 var option = arg.substring(2); |
|
210 switch (option) { |
|
211 case 'min-version': |
|
212 ++jj; |
|
213 testOptions[toCamelCase(option)] = args[jj]; |
|
214 break; |
|
215 default: |
|
216 throw ("bad unknown option '" + option + "' at in " + url + ":" + (ii + 1) + ": " + str); |
|
217 } |
|
218 } else { |
|
219 nonOptions.push(arg); |
|
220 } |
|
221 } |
|
222 var url = prefix + nonOptions.join(" "); |
|
223 |
|
224 if (url.substr(url.length - 4) != '.txt') { |
|
225 var minVersion = testOptions.minVersion; |
|
226 if (!minVersion) { |
|
227 minVersion = hierarchicalOptions.defaultVersion; |
|
228 } |
|
229 |
|
230 if (globalOptions.minVersion) { |
|
231 useTest = greaterThanOrEqualToVersion(minVersion, globalOptions.minVersion); |
|
232 } else { |
|
233 useTest = greaterThanOrEqualToVersion(globalOptions.version, minVersion); |
|
234 } |
|
235 } |
|
236 |
|
237 if (!useTest) { |
|
238 callback(true, []); |
|
239 return; |
|
240 } |
|
241 |
|
242 if (url.substr(url.length - 4) == '.txt') { |
|
243 // If a version was explicity specified pass it down. |
|
244 if (testOptions.minVersion) { |
|
245 hierarchicalOptions.defaultVersion = testOptions.minVersion; |
|
246 } |
|
247 loadTextFileAsynchronous(url, function() { |
|
248 return function(success, text) { |
|
249 if (!success) { |
|
250 callback(false, ''); |
|
251 return; |
|
252 } |
|
253 var lines = text.split('\n'); |
|
254 var prefix = ''; |
|
255 var lastSlash = url.lastIndexOf('/'); |
|
256 if (lastSlash >= 0) { |
|
257 prefix = url.substr(0, lastSlash + 1); |
|
258 } |
|
259 var fail = false; |
|
260 var count = 1; |
|
261 var index = 0; |
|
262 for (var ii = 0; ii < lines.length; ++ii) { |
|
263 var str = lines[ii].replace(/^\s\s*/, '').replace(/\s\s*$/, ''); |
|
264 if (str.length > 4 && |
|
265 str[0] != '#' && |
|
266 str[0] != ";" && |
|
267 str.substr(0, 2) != "//") { |
|
268 ++count; |
|
269 getFileListImpl(prefix, str, copyObject(hierarchicalOptions), function(index) { |
|
270 return function(success, new_files) { |
|
271 log("got files: " + new_files.length); |
|
272 if (success) { |
|
273 files[index] = new_files; |
|
274 } |
|
275 finish(success); |
|
276 }; |
|
277 }(index++)); |
|
278 } |
|
279 } |
|
280 finish(true); |
|
281 |
|
282 function finish(success) { |
|
283 if (!success) { |
|
284 fail = true; |
|
285 } |
|
286 --count; |
|
287 log("count: " + count); |
|
288 if (!count) { |
|
289 callback(!fail, files); |
|
290 } |
|
291 } |
|
292 } |
|
293 }()); |
|
294 } else { |
|
295 files.push(url); |
|
296 callback(true, files); |
|
297 } |
|
298 }; |
|
299 |
|
300 getFileListImpl('', url, globalOptions, function(success, files) { |
|
301 // flatten |
|
302 var flat = []; |
|
303 flatten(files); |
|
304 function flatten(files) { |
|
305 for (var ii = 0; ii < files.length; ++ii) { |
|
306 var value = files[ii]; |
|
307 if (typeof(value) == "string") { |
|
308 flat.push(value); |
|
309 } else { |
|
310 flatten(value); |
|
311 } |
|
312 } |
|
313 } |
|
314 callback(success, flat); |
|
315 }); |
|
316 }; |
|
317 |
|
318 var TestFile = function(url) { |
|
319 this.url = url; |
|
320 }; |
|
321 |
|
322 var TestHarness = function(iframe, filelistUrl, reportFunc, options) { |
|
323 this.window = window; |
|
324 this.iframe = iframe; |
|
325 this.reportFunc = reportFunc; |
|
326 this.timeoutDelay = 20000; |
|
327 this.files = []; |
|
328 |
|
329 var that = this; |
|
330 getFileList(filelistUrl, function() { |
|
331 return function(success, files) { |
|
332 that.addFiles_(success, files); |
|
333 }; |
|
334 }(), options); |
|
335 |
|
336 }; |
|
337 |
|
338 TestHarness.reportType = { |
|
339 ADD_PAGE: 1, |
|
340 READY: 2, |
|
341 START_PAGE: 3, |
|
342 TEST_RESULT: 4, |
|
343 FINISH_PAGE: 5, |
|
344 FINISHED_ALL_TESTS: 6 |
|
345 }; |
|
346 |
|
347 TestHarness.prototype.addFiles_ = function(success, files) { |
|
348 if (!success) { |
|
349 this.reportFunc( |
|
350 TestHarness.reportType.FINISHED_ALL_TESTS, |
|
351 'Unable to load tests. Are you running locally?\n' + |
|
352 'You need to run from a server or configure your\n' + |
|
353 'browser to allow access to local files (not recommended).\n\n' + |
|
354 'Note: An easy way to run from a server:\n\n' + |
|
355 '\tcd path_to_tests\n' + |
|
356 '\tpython -m SimpleHTTPServer\n\n' + |
|
357 'then point your browser to ' + |
|
358 '<a href="http://localhost:8000/webgl-conformance-tests.html">' + |
|
359 'http://localhost:8000/webgl-conformance-tests.html</a>', |
|
360 false) |
|
361 return; |
|
362 } |
|
363 log("total files: " + files.length); |
|
364 for (var ii = 0; ii < files.length; ++ii) { |
|
365 log("" + ii + ": " + files[ii]); |
|
366 this.files.push(new TestFile(files[ii])); |
|
367 this.reportFunc(TestHarness.reportType.ADD_PAGE, files[ii], undefined); |
|
368 } |
|
369 this.reportFunc(TestHarness.reportType.READY, undefined, undefined); |
|
370 } |
|
371 |
|
372 TestHarness.prototype.runTests = function(opt_start, opt_count) { |
|
373 var count = opt_count || this.files.length; |
|
374 this.nextFileIndex = opt_start || 0; |
|
375 this.lastFileIndex = this.nextFileIndex + count; |
|
376 this.startNextFile(); |
|
377 }; |
|
378 |
|
379 TestHarness.prototype.setTimeout = function() { |
|
380 var that = this; |
|
381 this.timeoutId = this.window.setTimeout(function() { |
|
382 that.timeout(); |
|
383 }, this.timeoutDelay); |
|
384 }; |
|
385 |
|
386 TestHarness.prototype.clearTimeout = function() { |
|
387 this.window.clearTimeout(this.timeoutId); |
|
388 }; |
|
389 |
|
390 TestHarness.prototype.startNextFile = function() { |
|
391 if (this.nextFileIndex >= this.lastFileIndex) { |
|
392 log("done"); |
|
393 this.reportFunc(TestHarness.reportType.FINISHED_ALL_TESTS, |
|
394 '', true); |
|
395 } else { |
|
396 this.currentFile = this.files[this.nextFileIndex++]; |
|
397 log("loading: " + this.currentFile.url); |
|
398 if (this.reportFunc(TestHarness.reportType.START_PAGE, |
|
399 this.currentFile.url, undefined)) { |
|
400 this.iframe.src = this.currentFile.url; |
|
401 this.setTimeout(); |
|
402 } else { |
|
403 this.reportResults(false, "skipped"); |
|
404 this.notifyFinished(); |
|
405 } |
|
406 } |
|
407 }; |
|
408 |
|
409 TestHarness.prototype.reportResults = function (success, msg) { |
|
410 this.clearTimeout(); |
|
411 log(success ? "PASS" : "FAIL", msg); |
|
412 this.reportFunc(TestHarness.reportType.TEST_RESULT, msg, success); |
|
413 // For each result we get, reset the timeout |
|
414 this.setTimeout(); |
|
415 }; |
|
416 |
|
417 TestHarness.prototype.notifyFinished = function () { |
|
418 this.clearTimeout(); |
|
419 var url = this.currentFile ? this.currentFile.url : 'unknown'; |
|
420 log(url + ": finished"); |
|
421 this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, true); |
|
422 this.startNextFile(); |
|
423 }; |
|
424 |
|
425 TestHarness.prototype.timeout = function() { |
|
426 this.clearTimeout(); |
|
427 var url = this.currentFile ? this.currentFile.url : 'unknown'; |
|
428 log(url + ": timeout"); |
|
429 this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, undefined); |
|
430 this.startNextFile(); |
|
431 }; |
|
432 |
|
433 TestHarness.prototype.setTimeoutDelay = function(x) { |
|
434 this.timeoutDelay = x; |
|
435 }; |
|
436 |
|
437 return { |
|
438 'TestHarness': TestHarness |
|
439 }; |
|
440 |
|
441 }(); |
|
442 |
|
443 |
|
444 |