1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/xpcshell/selftest.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,786 @@
1.4 +#!/usr/bin/env python
1.5 +#
1.6 +# Any copyright is dedicated to the Public Domain.
1.7 +# http://creativecommons.org/publicdomain/zero/1.0/
1.8 +#
1.9 +
1.10 +from __future__ import with_statement
1.11 +import sys, os, unittest, tempfile, shutil
1.12 +import mozinfo
1.13 +
1.14 +from StringIO import StringIO
1.15 +from xml.etree.ElementTree import ElementTree
1.16 +
1.17 +from mozbuild.base import MozbuildObject
1.18 +build_obj = MozbuildObject.from_environment()
1.19 +
1.20 +from runxpcshelltests import XPCShellTests
1.21 +
1.22 +mozinfo.find_and_update_from_json()
1.23 +
1.24 +objdir = build_obj.topobjdir.encode("utf-8")
1.25 +xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
1.26 +if sys.platform == "win32":
1.27 + xpcshellBin += ".exe"
1.28 +
1.29 +SIMPLE_PASSING_TEST = "function run_test() { do_check_true(true); }"
1.30 +SIMPLE_FAILING_TEST = "function run_test() { do_check_true(false); }"
1.31 +
1.32 +ADD_TEST_SIMPLE = '''
1.33 +function run_test() { run_next_test(); }
1.34 +
1.35 +add_test(function test_simple() {
1.36 + do_check_true(true);
1.37 + run_next_test();
1.38 +});
1.39 +'''
1.40 +
1.41 +ADD_TEST_FAILING = '''
1.42 +function run_test() { run_next_test(); }
1.43 +
1.44 +add_test(function test_failing() {
1.45 + do_check_true(false);
1.46 + run_next_test();
1.47 +});
1.48 +'''
1.49 +
1.50 +CHILD_TEST_PASSING = '''
1.51 +function run_test () { run_next_test(); }
1.52 +
1.53 +add_test(function test_child_simple () {
1.54 + run_test_in_child("test_pass.js");
1.55 + run_next_test();
1.56 +});
1.57 +'''
1.58 +
1.59 +CHILD_TEST_FAILING = '''
1.60 +function run_test () { run_next_test(); }
1.61 +
1.62 +add_test(function test_child_simple () {
1.63 + run_test_in_child("test_fail.js");
1.64 + run_next_test();
1.65 +});
1.66 +'''
1.67 +
1.68 +CHILD_TEST_HANG = '''
1.69 +function run_test () { run_next_test(); }
1.70 +
1.71 +add_test(function test_child_simple () {
1.72 + do_test_pending("hang test");
1.73 + do_load_child_test_harness();
1.74 + sendCommand("_log('child_test_start', {_message: 'CHILD-TEST-STARTED'}); " +
1.75 + + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
1.76 + do_test_finished);
1.77 + run_next_test();
1.78 +});
1.79 +'''
1.80 +
1.81 +ADD_TASK_SINGLE = '''
1.82 +Components.utils.import("resource://gre/modules/Promise.jsm");
1.83 +
1.84 +function run_test() { run_next_test(); }
1.85 +
1.86 +add_task(function test_task() {
1.87 + yield Promise.resolve(true);
1.88 + yield Promise.resolve(false);
1.89 +});
1.90 +'''
1.91 +
1.92 +ADD_TASK_MULTIPLE = '''
1.93 +Components.utils.import("resource://gre/modules/Promise.jsm");
1.94 +
1.95 +function run_test() { run_next_test(); }
1.96 +
1.97 +add_task(function test_task() {
1.98 + yield Promise.resolve(true);
1.99 +});
1.100 +
1.101 +add_task(function test_2() {
1.102 + yield Promise.resolve(true);
1.103 +});
1.104 +'''
1.105 +
1.106 +ADD_TASK_REJECTED = '''
1.107 +Components.utils.import("resource://gre/modules/Promise.jsm");
1.108 +
1.109 +function run_test() { run_next_test(); }
1.110 +
1.111 +add_task(function test_failing() {
1.112 + yield Promise.reject(new Error("I fail."));
1.113 +});
1.114 +'''
1.115 +
1.116 +ADD_TASK_FAILURE_INSIDE = '''
1.117 +Components.utils.import("resource://gre/modules/Promise.jsm");
1.118 +
1.119 +function run_test() { run_next_test(); }
1.120 +
1.121 +add_task(function test() {
1.122 + let result = yield Promise.resolve(false);
1.123 +
1.124 + do_check_true(result);
1.125 +});
1.126 +'''
1.127 +
1.128 +ADD_TASK_RUN_NEXT_TEST = '''
1.129 +function run_test() { run_next_test(); }
1.130 +
1.131 +add_task(function () {
1.132 + Assert.ok(true);
1.133 +
1.134 + run_next_test();
1.135 +});
1.136 +'''
1.137 +
1.138 +ADD_TEST_THROW_STRING = '''
1.139 +function run_test() {do_throw("Passing a string to do_throw")};
1.140 +'''
1.141 +
1.142 +ADD_TEST_THROW_OBJECT = '''
1.143 +let error = {
1.144 + message: "Error object",
1.145 + fileName: "failure.js",
1.146 + stack: "ERROR STACK",
1.147 + toString: function() {return this.message;}
1.148 +};
1.149 +function run_test() {do_throw(error)};
1.150 +'''
1.151 +
1.152 +ADD_TEST_REPORT_OBJECT = '''
1.153 +let error = {
1.154 + message: "Error object",
1.155 + fileName: "failure.js",
1.156 + stack: "ERROR STACK",
1.157 + toString: function() {return this.message;}
1.158 +};
1.159 +function run_test() {do_report_unexpected_exception(error)};
1.160 +'''
1.161 +
1.162 +# A test for genuine JS-generated Error objects
1.163 +ADD_TEST_REPORT_REF_ERROR = '''
1.164 +function run_test() {
1.165 + let obj = {blah: 0};
1.166 + try {
1.167 + obj.noSuchFunction();
1.168 + }
1.169 + catch (error) {
1.170 + do_report_unexpected_exception(error);
1.171 + }
1.172 +};
1.173 +'''
1.174 +
1.175 +# A test for failure to load a test due to a syntax error
1.176 +LOAD_ERROR_SYNTAX_ERROR = '''
1.177 +function run_test(
1.178 +'''
1.179 +
1.180 +# A test for failure to load a test due to an error other than a syntax error
1.181 +LOAD_ERROR_OTHER_ERROR = '''
1.182 +function run_test() {
1.183 + yield "foo";
1.184 + return "foo"; // can't use return in a generator!
1.185 +};
1.186 +'''
1.187 +
1.188 +# A test for asynchronous cleanup functions
1.189 +ASYNC_CLEANUP = '''
1.190 +function run_test() {
1.191 + Components.utils.import("resource://gre/modules/Promise.jsm", this);
1.192 +
1.193 + // The list of checkpoints in the order we encounter them.
1.194 + let checkpoints = [];
1.195 +
1.196 + // Cleanup tasks, in reverse order
1.197 + do_register_cleanup(function cleanup_checkout() {
1.198 + do_check_eq(checkpoints.join(""), "1234");
1.199 + do_print("At this stage, the test has succeeded");
1.200 + do_throw("Throwing an error to force displaying the log");
1.201 + });
1.202 +
1.203 + do_register_cleanup(function sync_cleanup_2() {
1.204 + checkpoints.push(4);
1.205 + });
1.206 +
1.207 + do_register_cleanup(function async_cleanup_2() {
1.208 + let deferred = Promise.defer();
1.209 + do_execute_soon(deferred.resolve);
1.210 + return deferred.promise.then(function() {
1.211 + checkpoints.push(3);
1.212 + });
1.213 + });
1.214 +
1.215 + do_register_cleanup(function sync_cleanup() {
1.216 + checkpoints.push(2);
1.217 + });
1.218 +
1.219 + do_register_cleanup(function async_cleanup() {
1.220 + let deferred = Promise.defer();
1.221 + do_execute_soon(deferred.resolve);
1.222 + return deferred.promise.then(function() {
1.223 + checkpoints.push(1);
1.224 + });
1.225 + });
1.226 +
1.227 +}
1.228 +'''
1.229 +
1.230 +
1.231 +class XPCShellTestsTests(unittest.TestCase):
1.232 + """
1.233 + Yes, these are unit tests for a unit test harness.
1.234 + """
1.235 + def setUp(self):
1.236 + self.log = StringIO()
1.237 + self.tempdir = tempfile.mkdtemp()
1.238 + self.x = XPCShellTests(log=self.log)
1.239 +
1.240 + def tearDown(self):
1.241 + shutil.rmtree(self.tempdir)
1.242 +
1.243 + def writeFile(self, name, contents):
1.244 + """
1.245 + Write |contents| to a file named |name| in the temp directory,
1.246 + and return the full path to the file.
1.247 + """
1.248 + fullpath = os.path.join(self.tempdir, name)
1.249 + with open(fullpath, "w") as f:
1.250 + f.write(contents)
1.251 + return fullpath
1.252 +
1.253 + def writeManifest(self, tests):
1.254 + """
1.255 + Write an xpcshell.ini in the temp directory and set
1.256 + self.manifest to its pathname. |tests| is a list containing
1.257 + either strings (for test names), or tuples with a test name
1.258 + as the first element and manifest conditions as the following
1.259 + elements.
1.260 + """
1.261 + testlines = []
1.262 + for t in tests:
1.263 + testlines.append("[%s]" % (t if isinstance(t, basestring)
1.264 + else t[0]))
1.265 + if isinstance(t, tuple):
1.266 + testlines.extend(t[1:])
1.267 + self.manifest = self.writeFile("xpcshell.ini", """
1.268 +[DEFAULT]
1.269 +head =
1.270 +tail =
1.271 +
1.272 +""" + "\n".join(testlines))
1.273 +
1.274 + def assertTestResult(self, expected, shuffle=False, xunitFilename=None, verbose=False):
1.275 + """
1.276 + Assert that self.x.runTests with manifest=self.manifest
1.277 + returns |expected|.
1.278 + """
1.279 + self.assertEquals(expected,
1.280 + self.x.runTests(xpcshellBin,
1.281 + manifest=self.manifest,
1.282 + mozInfo=mozinfo.info,
1.283 + shuffle=shuffle,
1.284 + testsRootDir=self.tempdir,
1.285 + verbose=verbose,
1.286 + xunitFilename=xunitFilename,
1.287 + sequential=True),
1.288 + msg="""Tests should have %s, log:
1.289 +========
1.290 +%s
1.291 +========
1.292 +""" % ("passed" if expected else "failed", self.log.getvalue()))
1.293 +
1.294 + def _assertLog(self, s, expected):
1.295 + l = self.log.getvalue()
1.296 + self.assertEqual(expected, s in l,
1.297 + msg="""Value %s %s in log:
1.298 +========
1.299 +%s
1.300 +========""" % (s, "expected" if expected else "not expected", l))
1.301 +
1.302 + def assertInLog(self, s):
1.303 + """
1.304 + Assert that the string |s| is contained in self.log.
1.305 + """
1.306 + self._assertLog(s, True)
1.307 +
1.308 + def assertNotInLog(self, s):
1.309 + """
1.310 + Assert that the string |s| is not contained in self.log.
1.311 + """
1.312 + self._assertLog(s, False)
1.313 +
1.314 + def testPass(self):
1.315 + """
1.316 + Check that a simple test without any manifest conditions passes.
1.317 + """
1.318 + self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1.319 + self.writeManifest(["test_basic.js"])
1.320 +
1.321 + self.assertTestResult(True)
1.322 + self.assertEquals(1, self.x.testCount)
1.323 + self.assertEquals(1, self.x.passCount)
1.324 + self.assertEquals(0, self.x.failCount)
1.325 + self.assertEquals(0, self.x.todoCount)
1.326 + self.assertInLog("TEST-PASS")
1.327 + self.assertNotInLog("TEST-UNEXPECTED-FAIL")
1.328 +
1.329 + def testFail(self):
1.330 + """
1.331 + Check that a simple failing test without any manifest conditions fails.
1.332 + """
1.333 + self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
1.334 + self.writeManifest(["test_basic.js"])
1.335 +
1.336 + self.assertTestResult(False)
1.337 + self.assertEquals(1, self.x.testCount)
1.338 + self.assertEquals(0, self.x.passCount)
1.339 + self.assertEquals(1, self.x.failCount)
1.340 + self.assertEquals(0, self.x.todoCount)
1.341 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.342 + self.assertNotInLog("TEST-PASS")
1.343 +
1.344 + @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
1.345 + 'selftests with child processes fail on b2g desktop builds')
1.346 + def testChildPass(self):
1.347 + """
1.348 + Check that a simple test running in a child process passes.
1.349 + """
1.350 + self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
1.351 + self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
1.352 + self.writeManifest(["test_child_pass.js"])
1.353 +
1.354 + self.assertTestResult(True, verbose=True)
1.355 + self.assertEquals(1, self.x.testCount)
1.356 + self.assertEquals(1, self.x.passCount)
1.357 + self.assertEquals(0, self.x.failCount)
1.358 + self.assertEquals(0, self.x.todoCount)
1.359 + self.assertInLog("TEST-PASS")
1.360 + self.assertInLog("CHILD-TEST-STARTED")
1.361 + self.assertInLog("CHILD-TEST-COMPLETED")
1.362 + self.assertNotInLog("TEST-UNEXPECTED-FAIL")
1.363 +
1.364 +
1.365 + @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
1.366 + 'selftests with child processes fail on b2g desktop builds')
1.367 + def testChildFail(self):
1.368 + """
1.369 + Check that a simple failing test running in a child process fails.
1.370 + """
1.371 + self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
1.372 + self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
1.373 + self.writeManifest(["test_child_fail.js"])
1.374 +
1.375 + self.assertTestResult(False)
1.376 + self.assertEquals(1, self.x.testCount)
1.377 + self.assertEquals(0, self.x.passCount)
1.378 + self.assertEquals(1, self.x.failCount)
1.379 + self.assertEquals(0, self.x.todoCount)
1.380 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.381 + self.assertInLog("CHILD-TEST-STARTED")
1.382 + self.assertInLog("CHILD-TEST-COMPLETED")
1.383 + self.assertNotInLog("TEST-PASS")
1.384 +
1.385 + @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
1.386 + 'selftests with child processes fail on b2g desktop builds')
1.387 + def testChildHang(self):
1.388 + """
1.389 + Check that incomplete output from a child process results in a
1.390 + test failure.
1.391 + """
1.392 + self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
1.393 + self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
1.394 + self.writeManifest(["test_child_hang.js"])
1.395 +
1.396 + self.assertTestResult(False)
1.397 + self.assertEquals(1, self.x.testCount)
1.398 + self.assertEquals(0, self.x.passCount)
1.399 + self.assertEquals(1, self.x.failCount)
1.400 + self.assertEquals(0, self.x.todoCount)
1.401 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.402 + self.assertInLog("CHILD-TEST-STARTED")
1.403 + self.assertNotInLog("CHILD-TEST-COMPLETED")
1.404 + self.assertNotInLog("TEST-PASS")
1.405 +
1.406 + def testSyntaxError(self):
1.407 + """
1.408 + Check that running a test file containing a syntax error produces
1.409 + a test failure and expected output.
1.410 + """
1.411 + self.writeFile("test_syntax_error.js", '"')
1.412 + self.writeManifest(["test_syntax_error.js"])
1.413 +
1.414 + self.assertTestResult(False, verbose=True)
1.415 + self.assertEquals(1, self.x.testCount)
1.416 + self.assertEquals(0, self.x.passCount)
1.417 + self.assertEquals(1, self.x.failCount)
1.418 + self.assertEquals(0, self.x.todoCount)
1.419 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.420 + self.assertNotInLog("TEST-PASS")
1.421 +
1.422 + def testPassFail(self):
1.423 + """
1.424 + Check that running more than one test works.
1.425 + """
1.426 + self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
1.427 + self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
1.428 + self.writeManifest(["test_pass.js", "test_fail.js"])
1.429 +
1.430 + self.assertTestResult(False)
1.431 + self.assertEquals(2, self.x.testCount)
1.432 + self.assertEquals(1, self.x.passCount)
1.433 + self.assertEquals(1, self.x.failCount)
1.434 + self.assertEquals(0, self.x.todoCount)
1.435 + self.assertInLog("TEST-PASS")
1.436 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.437 +
1.438 + def testSkip(self):
1.439 + """
1.440 + Check that a simple failing test skipped in the manifest does
1.441 + not cause failure.
1.442 + """
1.443 + self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
1.444 + self.writeManifest([("test_basic.js", "skip-if = true")])
1.445 + self.assertTestResult(True)
1.446 + self.assertEquals(1, self.x.testCount)
1.447 + self.assertEquals(0, self.x.passCount)
1.448 + self.assertEquals(0, self.x.failCount)
1.449 + self.assertEquals(0, self.x.todoCount)
1.450 + self.assertNotInLog("TEST-UNEXPECTED-FAIL")
1.451 + self.assertNotInLog("TEST-PASS")
1.452 +
1.453 + def testKnownFail(self):
1.454 + """
1.455 + Check that a simple failing test marked as known-fail in the manifest
1.456 + does not cause failure.
1.457 + """
1.458 + self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
1.459 + self.writeManifest([("test_basic.js", "fail-if = true")])
1.460 + self.assertTestResult(True)
1.461 + self.assertEquals(1, self.x.testCount)
1.462 + self.assertEquals(0, self.x.passCount)
1.463 + self.assertEquals(0, self.x.failCount)
1.464 + self.assertEquals(1, self.x.todoCount)
1.465 + self.assertInLog("TEST-KNOWN-FAIL")
1.466 + # This should be suppressed because the harness doesn't include
1.467 + # the full log from the xpcshell run when things pass.
1.468 + self.assertNotInLog("TEST-UNEXPECTED-FAIL")
1.469 + self.assertNotInLog("TEST-PASS")
1.470 +
1.471 + def testUnexpectedPass(self):
1.472 + """
1.473 + Check that a simple failing test marked as known-fail in the manifest
1.474 + that passes causes an unexpected pass.
1.475 + """
1.476 + self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1.477 + self.writeManifest([("test_basic.js", "fail-if = true")])
1.478 + self.assertTestResult(False)
1.479 + self.assertEquals(1, self.x.testCount)
1.480 + self.assertEquals(0, self.x.passCount)
1.481 + self.assertEquals(1, self.x.failCount)
1.482 + self.assertEquals(0, self.x.todoCount)
1.483 + # From the outer (Python) harness
1.484 + self.assertInLog("TEST-UNEXPECTED-PASS")
1.485 + self.assertNotInLog("TEST-KNOWN-FAIL")
1.486 + # From the inner (JS) harness
1.487 + self.assertInLog("TEST-PASS")
1.488 +
1.489 + def testReturnNonzero(self):
1.490 + """
1.491 + Check that a test where xpcshell returns nonzero fails.
1.492 + """
1.493 + self.writeFile("test_error.js", "throw 'foo'")
1.494 + self.writeManifest(["test_error.js"])
1.495 +
1.496 + self.assertTestResult(False)
1.497 + self.assertEquals(1, self.x.testCount)
1.498 + self.assertEquals(0, self.x.passCount)
1.499 + self.assertEquals(1, self.x.failCount)
1.500 + self.assertEquals(0, self.x.todoCount)
1.501 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.502 + self.assertNotInLog("TEST-PASS")
1.503 +
1.504 + def testAddTestSimple(self):
1.505 + """
1.506 + Ensure simple add_test() works.
1.507 + """
1.508 + self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
1.509 + self.writeManifest(["test_add_test_simple.js"])
1.510 +
1.511 + self.assertTestResult(True)
1.512 + self.assertEquals(1, self.x.testCount)
1.513 + self.assertEquals(1, self.x.passCount)
1.514 + self.assertEquals(0, self.x.failCount)
1.515 +
1.516 + def testAddTestFailing(self):
1.517 + """
1.518 + Ensure add_test() with a failing test is reported.
1.519 + """
1.520 + self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
1.521 + self.writeManifest(["test_add_test_failing.js"])
1.522 +
1.523 + self.assertTestResult(False)
1.524 + self.assertEquals(1, self.x.testCount)
1.525 + self.assertEquals(0, self.x.passCount)
1.526 + self.assertEquals(1, self.x.failCount)
1.527 +
1.528 + def testAddTaskTestSingle(self):
1.529 + """
1.530 + Ensure add_test_task() with a single passing test works.
1.531 + """
1.532 + self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
1.533 + self.writeManifest(["test_add_task_simple.js"])
1.534 +
1.535 + self.assertTestResult(True)
1.536 + self.assertEquals(1, self.x.testCount)
1.537 + self.assertEquals(1, self.x.passCount)
1.538 + self.assertEquals(0, self.x.failCount)
1.539 +
1.540 + def testAddTaskTestMultiple(self):
1.541 + """
1.542 + Ensure multiple calls to add_test_task() work as expected.
1.543 + """
1.544 + self.writeFile("test_add_task_multiple.js",
1.545 + ADD_TASK_MULTIPLE)
1.546 + self.writeManifest(["test_add_task_multiple.js"])
1.547 +
1.548 + self.assertTestResult(True)
1.549 + self.assertEquals(1, self.x.testCount)
1.550 + self.assertEquals(1, self.x.passCount)
1.551 + self.assertEquals(0, self.x.failCount)
1.552 +
1.553 + def testAddTaskTestRejected(self):
1.554 + """
1.555 + Ensure rejected task reports as failure.
1.556 + """
1.557 + self.writeFile("test_add_task_rejected.js",
1.558 + ADD_TASK_REJECTED)
1.559 + self.writeManifest(["test_add_task_rejected.js"])
1.560 +
1.561 + self.assertTestResult(False)
1.562 + self.assertEquals(1, self.x.testCount)
1.563 + self.assertEquals(0, self.x.passCount)
1.564 + self.assertEquals(1, self.x.failCount)
1.565 +
1.566 + def testAddTaskTestFailureInside(self):
1.567 + """
1.568 + Ensure tests inside task are reported as failures.
1.569 + """
1.570 + self.writeFile("test_add_task_failure_inside.js",
1.571 + ADD_TASK_FAILURE_INSIDE)
1.572 + self.writeManifest(["test_add_task_failure_inside.js"])
1.573 +
1.574 + self.assertTestResult(False)
1.575 + self.assertEquals(1, self.x.testCount)
1.576 + self.assertEquals(0, self.x.passCount)
1.577 + self.assertEquals(1, self.x.failCount)
1.578 +
1.579 + def testAddTaskRunNextTest(self):
1.580 + """
1.581 + Calling run_next_test() from inside add_task() results in failure.
1.582 + """
1.583 + self.writeFile("test_add_task_run_next_test.js",
1.584 + ADD_TASK_RUN_NEXT_TEST)
1.585 + self.writeManifest(["test_add_task_run_next_test.js"])
1.586 +
1.587 + self.assertTestResult(False)
1.588 + self.assertEquals(1, self.x.testCount)
1.589 + self.assertEquals(0, self.x.passCount)
1.590 + self.assertEquals(1, self.x.failCount)
1.591 +
1.592 + def testMissingHeadFile(self):
1.593 + """
1.594 + Ensure that missing head file results in fatal error.
1.595 + """
1.596 + self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1.597 + self.writeManifest([("test_basic.js", "head = missing.js")])
1.598 +
1.599 + raised = False
1.600 +
1.601 + try:
1.602 + # The actual return value is never checked because we raise.
1.603 + self.assertTestResult(True)
1.604 + except Exception, ex:
1.605 + raised = True
1.606 + self.assertEquals(ex.message[0:9], "head file")
1.607 +
1.608 + self.assertTrue(raised)
1.609 +
1.610 + def testMissingTailFile(self):
1.611 + """
1.612 + Ensure that missing tail file results in fatal error.
1.613 + """
1.614 + self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1.615 + self.writeManifest([("test_basic.js", "tail = missing.js")])
1.616 +
1.617 + raised = False
1.618 +
1.619 + try:
1.620 + self.assertTestResult(True)
1.621 + except Exception, ex:
1.622 + raised = True
1.623 + self.assertEquals(ex.message[0:9], "tail file")
1.624 +
1.625 + self.assertTrue(raised)
1.626 +
1.627 + def testRandomExecution(self):
1.628 + """
1.629 + Check that random execution doesn't break.
1.630 + """
1.631 + manifest = []
1.632 + for i in range(0, 10):
1.633 + filename = "test_pass_%d.js" % i
1.634 + self.writeFile(filename, SIMPLE_PASSING_TEST)
1.635 + manifest.append(filename)
1.636 +
1.637 + self.writeManifest(manifest)
1.638 + self.assertTestResult(True, shuffle=True)
1.639 + self.assertEquals(10, self.x.testCount)
1.640 + self.assertEquals(10, self.x.passCount)
1.641 +
1.642 + def testXunitOutput(self):
1.643 + """
1.644 + Check that Xunit XML files are written.
1.645 + """
1.646 + self.writeFile("test_00.js", SIMPLE_PASSING_TEST)
1.647 + self.writeFile("test_01.js", SIMPLE_FAILING_TEST)
1.648 + self.writeFile("test_02.js", SIMPLE_PASSING_TEST)
1.649 +
1.650 + manifest = [
1.651 + "test_00.js",
1.652 + "test_01.js",
1.653 + ("test_02.js", "skip-if = true")
1.654 + ]
1.655 +
1.656 + self.writeManifest(manifest)
1.657 +
1.658 + filename = os.path.join(self.tempdir, "xunit.xml")
1.659 +
1.660 + self.assertTestResult(False, xunitFilename=filename)
1.661 +
1.662 + self.assertTrue(os.path.exists(filename))
1.663 + self.assertTrue(os.path.getsize(filename) > 0)
1.664 +
1.665 + tree = ElementTree()
1.666 + tree.parse(filename)
1.667 + suite = tree.getroot()
1.668 +
1.669 + self.assertTrue(suite is not None)
1.670 + self.assertEqual(suite.get("tests"), "3")
1.671 + self.assertEqual(suite.get("failures"), "1")
1.672 + self.assertEqual(suite.get("skip"), "1")
1.673 +
1.674 + testcases = suite.findall("testcase")
1.675 + self.assertEqual(len(testcases), 3)
1.676 +
1.677 + for testcase in testcases:
1.678 + attributes = testcase.keys()
1.679 + self.assertTrue("classname" in attributes)
1.680 + self.assertTrue("name" in attributes)
1.681 + self.assertTrue("time" in attributes)
1.682 +
1.683 + self.assertTrue(testcases[1].find("failure") is not None)
1.684 + self.assertTrue(testcases[2].find("skipped") is not None)
1.685 +
1.686 + def testDoThrowString(self):
1.687 + """
1.688 + Check that do_throw produces reasonable messages when the
1.689 + input is a string instead of an object
1.690 + """
1.691 + self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
1.692 + self.writeManifest(["test_error.js"])
1.693 +
1.694 + self.assertTestResult(False)
1.695 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.696 + self.assertInLog("Passing a string to do_throw")
1.697 + self.assertNotInLog("TEST-PASS")
1.698 +
1.699 + def testDoThrowForeignObject(self):
1.700 + """
1.701 + Check that do_throw produces reasonable messages when the
1.702 + input is a generic object with 'filename', 'message' and 'stack' attributes
1.703 + but 'object instanceof Error' returns false
1.704 + """
1.705 + self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
1.706 + self.writeManifest(["test_error.js"])
1.707 +
1.708 + self.assertTestResult(False)
1.709 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.710 + self.assertInLog("failure.js")
1.711 + self.assertInLog("Error object")
1.712 + self.assertInLog("ERROR STACK")
1.713 + self.assertNotInLog("TEST-PASS")
1.714 +
1.715 + def testDoReportForeignObject(self):
1.716 + """
1.717 + Check that do_report_unexpected_exception produces reasonable messages when the
1.718 + input is a generic object with 'filename', 'message' and 'stack' attributes
1.719 + but 'object instanceof Error' returns false
1.720 + """
1.721 + self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
1.722 + self.writeManifest(["test_error.js"])
1.723 +
1.724 + self.assertTestResult(False)
1.725 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.726 + self.assertInLog("failure.js")
1.727 + self.assertInLog("Error object")
1.728 + self.assertInLog("ERROR STACK")
1.729 + self.assertNotInLog("TEST-PASS")
1.730 +
1.731 + def testDoReportRefError(self):
1.732 + """
1.733 + Check that do_report_unexpected_exception produces reasonable messages when the
1.734 + input is a JS-generated Error
1.735 + """
1.736 + self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
1.737 + self.writeManifest(["test_error.js"])
1.738 +
1.739 + self.assertTestResult(False)
1.740 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.741 + self.assertInLog("test_error.js")
1.742 + self.assertInLog("obj.noSuchFunction is not a function")
1.743 + self.assertInLog("run_test@")
1.744 + self.assertNotInLog("TEST-PASS")
1.745 +
1.746 + def testDoReportSyntaxError(self):
1.747 + """
1.748 + Check that attempting to load a test file containing a syntax error
1.749 + generates details of the error in the log
1.750 + """
1.751 + self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
1.752 + self.writeManifest(["test_error.js"])
1.753 +
1.754 + self.assertTestResult(False)
1.755 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.756 + self.assertInLog("test_error.js")
1.757 + self.assertInLog("test_error.js contains SyntaxError")
1.758 + self.assertInLog("Diagnostic: SyntaxError: missing formal parameter at")
1.759 + self.assertInLog("test_error.js:3")
1.760 + self.assertNotInLog("TEST-PASS")
1.761 +
1.762 + def testDoReportNonSyntaxError(self):
1.763 + """
1.764 + Check that attempting to load a test file containing an error other
1.765 + than a syntax error generates details of the error in the log
1.766 + """
1.767 + self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
1.768 + self.writeManifest(["test_error.js"])
1.769 +
1.770 + self.assertTestResult(False)
1.771 + self.assertInLog("TEST-UNEXPECTED-FAIL")
1.772 + self.assertInLog("Diagnostic: TypeError: generator function run_test returns a value at")
1.773 + self.assertInLog("test_error.js:4")
1.774 + self.assertNotInLog("TEST-PASS")
1.775 +
1.776 + def testAsyncCleanup(self):
1.777 + """
1.778 + Check that do_register_cleanup handles nicely cleanup tasks that
1.779 + return a promise
1.780 + """
1.781 + self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
1.782 + self.writeManifest(["test_asyncCleanup.js"])
1.783 + self.assertTestResult(False)
1.784 + self.assertInLog("\"1234\" == \"1234\"")
1.785 + self.assertInLog("At this stage, the test has succeeded")
1.786 + self.assertInLog("Throwing an error to force displaying the log")
1.787 +
1.788 +if __name__ == "__main__":
1.789 + unittest.main()
