michael@0: #!/usr/bin/env python
michael@0: #
michael@0: # Any copyright is dedicated to the Public Domain.
michael@0: # http://creativecommons.org/publicdomain/zero/1.0/
michael@0: #
michael@0: 
michael@0: from __future__ import with_statement
michael@0: import sys, os, unittest, tempfile, shutil
michael@0: import mozinfo
michael@0: 
michael@0: from StringIO import StringIO
michael@0: from xml.etree.ElementTree import ElementTree
michael@0: 
michael@0: from mozbuild.base import MozbuildObject
michael@0: build_obj = MozbuildObject.from_environment()
michael@0: 
michael@0: from runxpcshelltests import XPCShellTests
michael@0: 
michael@0: mozinfo.find_and_update_from_json()
michael@0: 
michael@0: objdir = build_obj.topobjdir.encode("utf-8")
michael@0: xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
michael@0: if sys.platform == "win32":
michael@0:     xpcshellBin += ".exe"
michael@0: 
michael@0: SIMPLE_PASSING_TEST = "function run_test() { do_check_true(true); }"
michael@0: SIMPLE_FAILING_TEST = "function run_test() { do_check_true(false); }"
michael@0: 
michael@0: ADD_TEST_SIMPLE = '''
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_test(function test_simple() {
michael@0:   do_check_true(true);
michael@0:   run_next_test();
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TEST_FAILING = '''
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_test(function test_failing() {
michael@0:   do_check_true(false);
michael@0:   run_next_test();
michael@0: });
michael@0: '''
michael@0: 
michael@0: CHILD_TEST_PASSING = '''
michael@0: function run_test () { run_next_test(); }
michael@0: 
michael@0: add_test(function test_child_simple () {
michael@0:   run_test_in_child("test_pass.js");
michael@0:   run_next_test();
michael@0: });
michael@0: '''
michael@0: 
michael@0: CHILD_TEST_FAILING = '''
michael@0: function run_test () { run_next_test(); }
michael@0: 
michael@0: add_test(function test_child_simple () {
michael@0:   run_test_in_child("test_fail.js");
michael@0:   run_next_test();
michael@0: });
michael@0: '''
michael@0: 
michael@0: CHILD_TEST_HANG = '''
michael@0: function run_test () { run_next_test(); }
michael@0: 
michael@0: add_test(function test_child_simple () {
michael@0:   do_test_pending("hang test");
michael@0:   do_load_child_test_harness();
michael@0:   sendCommand("_log('child_test_start', {_message: 'CHILD-TEST-STARTED'}); " +
michael@0:               + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
michael@0:               do_test_finished);
michael@0:   run_next_test();
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TASK_SINGLE = '''
michael@0: Components.utils.import("resource://gre/modules/Promise.jsm");
michael@0: 
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_task(function test_task() {
michael@0:   yield Promise.resolve(true);
michael@0:   yield Promise.resolve(false);
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TASK_MULTIPLE = '''
michael@0: Components.utils.import("resource://gre/modules/Promise.jsm");
michael@0: 
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_task(function test_task() {
michael@0:   yield Promise.resolve(true);
michael@0: });
michael@0: 
michael@0: add_task(function test_2() {
michael@0:   yield Promise.resolve(true);
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TASK_REJECTED = '''
michael@0: Components.utils.import("resource://gre/modules/Promise.jsm");
michael@0: 
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_task(function test_failing() {
michael@0:   yield Promise.reject(new Error("I fail."));
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TASK_FAILURE_INSIDE = '''
michael@0: Components.utils.import("resource://gre/modules/Promise.jsm");
michael@0: 
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_task(function test() {
michael@0:   let result = yield Promise.resolve(false);
michael@0: 
michael@0:   do_check_true(result);
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TASK_RUN_NEXT_TEST = '''
michael@0: function run_test() { run_next_test(); }
michael@0: 
michael@0: add_task(function () {
michael@0:   Assert.ok(true);
michael@0: 
michael@0:   run_next_test();
michael@0: });
michael@0: '''
michael@0: 
michael@0: ADD_TEST_THROW_STRING = '''
michael@0: function run_test() {do_throw("Passing a string to do_throw")};
michael@0: '''
michael@0: 
michael@0: ADD_TEST_THROW_OBJECT = '''
michael@0: let error = {
michael@0:   message: "Error object",
michael@0:   fileName: "failure.js",
michael@0:   stack: "ERROR STACK",
michael@0:   toString: function() {return this.message;}
michael@0: };
michael@0: function run_test() {do_throw(error)};
michael@0: '''
michael@0: 
michael@0: ADD_TEST_REPORT_OBJECT = '''
michael@0: let error = {
michael@0:   message: "Error object",
michael@0:   fileName: "failure.js",
michael@0:   stack: "ERROR STACK",
michael@0:   toString: function() {return this.message;}
michael@0: };
michael@0: function run_test() {do_report_unexpected_exception(error)};
michael@0: '''
michael@0: 
michael@0: # A test for genuine JS-generated Error objects
michael@0: ADD_TEST_REPORT_REF_ERROR = '''
michael@0: function run_test() {
michael@0:   let obj = {blah: 0};
michael@0:   try {
michael@0:     obj.noSuchFunction();
michael@0:   }
michael@0:   catch (error) {
michael@0:     do_report_unexpected_exception(error);
michael@0:   }
michael@0: };
michael@0: '''
michael@0: 
michael@0: # A test for failure to load a test due to a syntax error
michael@0: LOAD_ERROR_SYNTAX_ERROR = '''
michael@0: function run_test(
michael@0: '''
michael@0: 
michael@0: # A test for failure to load a test due to an error other than a syntax error
michael@0: LOAD_ERROR_OTHER_ERROR = '''
michael@0: function run_test() {
michael@0:     yield "foo";
michael@0:     return "foo"; // can't use return in a generator!
michael@0: };
michael@0: '''
michael@0: 
michael@0: # A test for asynchronous cleanup functions
michael@0: ASYNC_CLEANUP = '''
michael@0: function run_test() {
michael@0:   Components.utils.import("resource://gre/modules/Promise.jsm", this);
michael@0: 
michael@0:   // The list of checkpoints in the order we encounter them.
michael@0:   let checkpoints = [];
michael@0: 
michael@0:   // Cleanup tasks, in reverse order
michael@0:   do_register_cleanup(function cleanup_checkout() {
michael@0:     do_check_eq(checkpoints.join(""), "1234");
michael@0:     do_print("At this stage, the test has succeeded");
michael@0:     do_throw("Throwing an error to force displaying the log");
michael@0:   });
michael@0: 
michael@0:   do_register_cleanup(function sync_cleanup_2() {
michael@0:     checkpoints.push(4);
michael@0:   });
michael@0: 
michael@0:   do_register_cleanup(function async_cleanup_2() {
michael@0:     let deferred = Promise.defer();
michael@0:     do_execute_soon(deferred.resolve);
michael@0:     return deferred.promise.then(function() {
michael@0:       checkpoints.push(3);
michael@0:     });
michael@0:   });
michael@0: 
michael@0:   do_register_cleanup(function sync_cleanup() {
michael@0:     checkpoints.push(2);
michael@0:   });
michael@0: 
michael@0:   do_register_cleanup(function async_cleanup() {
michael@0:     let deferred = Promise.defer();
michael@0:     do_execute_soon(deferred.resolve);
michael@0:     return deferred.promise.then(function() {
michael@0:       checkpoints.push(1);
michael@0:     });
michael@0:   });
michael@0: 
michael@0: }
michael@0: '''
michael@0: 
michael@0: 
michael@0: class XPCShellTestsTests(unittest.TestCase):
michael@0:     """
michael@0:     Yes, these are unit tests for a unit test harness.
michael@0:     """
michael@0:     def setUp(self):
michael@0:         self.log = StringIO()
michael@0:         self.tempdir = tempfile.mkdtemp()
michael@0:         self.x = XPCShellTests(log=self.log)
michael@0: 
michael@0:     def tearDown(self):
michael@0:         shutil.rmtree(self.tempdir)
michael@0: 
michael@0:     def writeFile(self, name, contents):
michael@0:         """
michael@0:         Write |contents| to a file named |name| in the temp directory,
michael@0:         and return the full path to the file.
michael@0:         """
michael@0:         fullpath = os.path.join(self.tempdir, name)
michael@0:         with open(fullpath, "w") as f:
michael@0:             f.write(contents)
michael@0:         return fullpath
michael@0: 
michael@0:     def writeManifest(self, tests):
michael@0:         """
michael@0:         Write an xpcshell.ini in the temp directory and set
michael@0:         self.manifest to its pathname. |tests| is a list containing
michael@0:         either strings (for test names), or tuples with a test name
michael@0:         as the first element and manifest conditions as the following
michael@0:         elements.
michael@0:         """
michael@0:         testlines = []
michael@0:         for t in tests:
michael@0:             testlines.append("[%s]" % (t if isinstance(t, basestring)
michael@0:                                        else t[0]))
michael@0:             if isinstance(t, tuple):
michael@0:                 testlines.extend(t[1:])
michael@0:         self.manifest = self.writeFile("xpcshell.ini", """
michael@0: [DEFAULT]
michael@0: head =
michael@0: tail =
michael@0: 
michael@0: """ + "\n".join(testlines))
michael@0: 
michael@0:     def assertTestResult(self, expected, shuffle=False, xunitFilename=None, verbose=False):
michael@0:         """
michael@0:         Assert that self.x.runTests with manifest=self.manifest
michael@0:         returns |expected|.
michael@0:         """
michael@0:         self.assertEquals(expected,
michael@0:                           self.x.runTests(xpcshellBin,
michael@0:                                           manifest=self.manifest,
michael@0:                                           mozInfo=mozinfo.info,
michael@0:                                           shuffle=shuffle,
michael@0:                                           testsRootDir=self.tempdir,
michael@0:                                           verbose=verbose,
michael@0:                                           xunitFilename=xunitFilename,
michael@0:                                           sequential=True),
michael@0:                           msg="""Tests should have %s, log:
michael@0: ========
michael@0: %s
michael@0: ========
michael@0: """ % ("passed" if expected else "failed", self.log.getvalue()))
michael@0: 
michael@0:     def _assertLog(self, s, expected):
michael@0:         l = self.log.getvalue()
michael@0:         self.assertEqual(expected, s in l,
michael@0:                          msg="""Value %s %s in log:
michael@0: ========
michael@0: %s
michael@0: ========""" % (s, "expected" if expected else "not expected", l))
michael@0: 
michael@0:     def assertInLog(self, s):
michael@0:         """
michael@0:         Assert that the string |s| is contained in self.log.
michael@0:         """
michael@0:         self._assertLog(s, True)
michael@0: 
michael@0:     def assertNotInLog(self, s):
michael@0:         """
michael@0:         Assert that the string |s| is not contained in self.log.
michael@0:         """
michael@0:         self._assertLog(s, False)
michael@0: 
michael@0:     def testPass(self):
michael@0:         """
michael@0:         Check that a simple test without any manifest conditions passes.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeManifest(["test_basic.js"])
michael@0: 
michael@0:         self.assertTestResult(True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(1, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-PASS")
michael@0:         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
michael@0: 
michael@0:     def testFail(self):
michael@0:         """
michael@0:         Check that a simple failing test without any manifest conditions fails.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
michael@0:         self.writeManifest(["test_basic.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
michael@0:                      'selftests with child processes fail on b2g desktop builds')
michael@0:     def testChildPass(self):
michael@0:         """
michael@0:         Check that a simple test running in a child process passes.
michael@0:         """
michael@0:         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
michael@0:         self.writeManifest(["test_child_pass.js"])
michael@0: 
michael@0:         self.assertTestResult(True, verbose=True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(1, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-PASS")
michael@0:         self.assertInLog("CHILD-TEST-STARTED")
michael@0:         self.assertInLog("CHILD-TEST-COMPLETED")
michael@0:         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
michael@0: 
michael@0: 
michael@0:     @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
michael@0:                      'selftests with child processes fail on b2g desktop builds')
michael@0:     def testChildFail(self):
michael@0:         """
michael@0:         Check that a simple failing test running in a child process fails.
michael@0:         """
michael@0:         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
michael@0:         self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
michael@0:         self.writeManifest(["test_child_fail.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("CHILD-TEST-STARTED")
michael@0:         self.assertInLog("CHILD-TEST-COMPLETED")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
michael@0:                      'selftests with child processes fail on b2g desktop builds')
michael@0:     def testChildHang(self):
michael@0:         """
michael@0:         Check that incomplete output from a child process results in a
michael@0:         test failure.
michael@0:         """
michael@0:         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
michael@0:         self.writeManifest(["test_child_hang.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("CHILD-TEST-STARTED")
michael@0:         self.assertNotInLog("CHILD-TEST-COMPLETED")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testSyntaxError(self):
michael@0:         """
michael@0:         Check that running a test file containing a syntax error produces
michael@0:         a test failure and expected output.
michael@0:         """
michael@0:         self.writeFile("test_syntax_error.js", '"')
michael@0:         self.writeManifest(["test_syntax_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False, verbose=True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testPassFail(self):
michael@0:         """
michael@0:         Check that running more than one test works.
michael@0:         """
michael@0:         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
michael@0:         self.writeManifest(["test_pass.js", "test_fail.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(2, self.x.testCount)
michael@0:         self.assertEquals(1, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-PASS")
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0: 
michael@0:     def testSkip(self):
michael@0:         """
michael@0:         Check that a simple failing test skipped in the manifest does
michael@0:         not cause failure.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
michael@0:         self.writeManifest([("test_basic.js", "skip-if = true")])
michael@0:         self.assertTestResult(True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testKnownFail(self):
michael@0:         """
michael@0:         Check that a simple failing test marked as known-fail in the manifest
michael@0:         does not cause failure.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
michael@0:         self.writeManifest([("test_basic.js", "fail-if = true")])
michael@0:         self.assertTestResult(True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0:         self.assertEquals(1, self.x.todoCount)
michael@0:         self.assertInLog("TEST-KNOWN-FAIL")
michael@0:         # This should be suppressed because the harness doesn't include
michael@0:         # the full log from the xpcshell run when things pass.
michael@0:         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testUnexpectedPass(self):
michael@0:         """
michael@0:         Check that a simple failing test marked as known-fail in the manifest
michael@0:         that passes causes an unexpected pass.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeManifest([("test_basic.js", "fail-if = true")])
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         # From the outer (Python) harness
michael@0:         self.assertInLog("TEST-UNEXPECTED-PASS")
michael@0:         self.assertNotInLog("TEST-KNOWN-FAIL")
michael@0:         # From the inner (JS) harness
michael@0:         self.assertInLog("TEST-PASS")
michael@0: 
michael@0:     def testReturnNonzero(self):
michael@0:         """
michael@0:         Check that a test where xpcshell returns nonzero fails.
michael@0:         """
michael@0:         self.writeFile("test_error.js", "throw 'foo'")
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0:         self.assertEquals(0, self.x.todoCount)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testAddTestSimple(self):
michael@0:         """
michael@0:         Ensure simple add_test() works.
michael@0:         """
michael@0:         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
michael@0:         self.writeManifest(["test_add_test_simple.js"])
michael@0: 
michael@0:         self.assertTestResult(True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(1, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0: 
michael@0:     def testAddTestFailing(self):
michael@0:         """
michael@0:         Ensure add_test() with a failing test is reported.
michael@0:         """
michael@0:         self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
michael@0:         self.writeManifest(["test_add_test_failing.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0: 
michael@0:     def testAddTaskTestSingle(self):
michael@0:         """
michael@0:         Ensure add_test_task() with a single passing test works.
michael@0:         """
michael@0:         self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
michael@0:         self.writeManifest(["test_add_task_simple.js"])
michael@0: 
michael@0:         self.assertTestResult(True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(1, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0: 
michael@0:     def testAddTaskTestMultiple(self):
michael@0:         """
michael@0:         Ensure multiple calls to add_test_task() work as expected.
michael@0:         """
michael@0:         self.writeFile("test_add_task_multiple.js",
michael@0:             ADD_TASK_MULTIPLE)
michael@0:         self.writeManifest(["test_add_task_multiple.js"])
michael@0: 
michael@0:         self.assertTestResult(True)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(1, self.x.passCount)
michael@0:         self.assertEquals(0, self.x.failCount)
michael@0: 
michael@0:     def testAddTaskTestRejected(self):
michael@0:         """
michael@0:         Ensure rejected task reports as failure.
michael@0:         """
michael@0:         self.writeFile("test_add_task_rejected.js",
michael@0:             ADD_TASK_REJECTED)
michael@0:         self.writeManifest(["test_add_task_rejected.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0: 
michael@0:     def testAddTaskTestFailureInside(self):
michael@0:         """
michael@0:         Ensure tests inside task are reported as failures.
michael@0:         """
michael@0:         self.writeFile("test_add_task_failure_inside.js",
michael@0:             ADD_TASK_FAILURE_INSIDE)
michael@0:         self.writeManifest(["test_add_task_failure_inside.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0: 
michael@0:     def testAddTaskRunNextTest(self):
michael@0:         """
michael@0:         Calling run_next_test() from inside add_task() results in failure.
michael@0:         """
michael@0:         self.writeFile("test_add_task_run_next_test.js",
michael@0:             ADD_TASK_RUN_NEXT_TEST)
michael@0:         self.writeManifest(["test_add_task_run_next_test.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertEquals(1, self.x.testCount)
michael@0:         self.assertEquals(0, self.x.passCount)
michael@0:         self.assertEquals(1, self.x.failCount)
michael@0: 
michael@0:     def testMissingHeadFile(self):
michael@0:         """
michael@0:         Ensure that missing head file results in fatal error.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeManifest([("test_basic.js", "head = missing.js")])
michael@0: 
michael@0:         raised = False
michael@0: 
michael@0:         try:
michael@0:             # The actual return value is never checked because we raise.
michael@0:             self.assertTestResult(True)
michael@0:         except Exception, ex:
michael@0:             raised = True
michael@0:             self.assertEquals(ex.message[0:9], "head file")
michael@0: 
michael@0:         self.assertTrue(raised)
michael@0: 
michael@0:     def testMissingTailFile(self):
michael@0:         """
michael@0:         Ensure that missing tail file results in fatal error.
michael@0:         """
michael@0:         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeManifest([("test_basic.js", "tail = missing.js")])
michael@0: 
michael@0:         raised = False
michael@0: 
michael@0:         try:
michael@0:             self.assertTestResult(True)
michael@0:         except Exception, ex:
michael@0:             raised = True
michael@0:             self.assertEquals(ex.message[0:9], "tail file")
michael@0: 
michael@0:         self.assertTrue(raised)
michael@0: 
michael@0:     def testRandomExecution(self):
michael@0:         """
michael@0:         Check that random execution doesn't break.
michael@0:         """
michael@0:         manifest = []
michael@0:         for i in range(0, 10):
michael@0:             filename = "test_pass_%d.js" % i
michael@0:             self.writeFile(filename, SIMPLE_PASSING_TEST)
michael@0:             manifest.append(filename)
michael@0: 
michael@0:         self.writeManifest(manifest)
michael@0:         self.assertTestResult(True, shuffle=True)
michael@0:         self.assertEquals(10, self.x.testCount)
michael@0:         self.assertEquals(10, self.x.passCount)
michael@0: 
michael@0:     def testXunitOutput(self):
michael@0:         """
michael@0:         Check that Xunit XML files are written.
michael@0:         """
michael@0:         self.writeFile("test_00.js", SIMPLE_PASSING_TEST)
michael@0:         self.writeFile("test_01.js", SIMPLE_FAILING_TEST)
michael@0:         self.writeFile("test_02.js", SIMPLE_PASSING_TEST)
michael@0: 
michael@0:         manifest = [
michael@0:             "test_00.js",
michael@0:             "test_01.js",
michael@0:             ("test_02.js", "skip-if = true")
michael@0:         ]
michael@0: 
michael@0:         self.writeManifest(manifest)
michael@0: 
michael@0:         filename = os.path.join(self.tempdir, "xunit.xml")
michael@0: 
michael@0:         self.assertTestResult(False, xunitFilename=filename)
michael@0: 
michael@0:         self.assertTrue(os.path.exists(filename))
michael@0:         self.assertTrue(os.path.getsize(filename) > 0)
michael@0: 
michael@0:         tree = ElementTree()
michael@0:         tree.parse(filename)
michael@0:         suite = tree.getroot()
michael@0: 
michael@0:         self.assertTrue(suite is not None)
michael@0:         self.assertEqual(suite.get("tests"), "3")
michael@0:         self.assertEqual(suite.get("failures"), "1")
michael@0:         self.assertEqual(suite.get("skip"), "1")
michael@0: 
michael@0:         testcases = suite.findall("testcase")
michael@0:         self.assertEqual(len(testcases), 3)
michael@0: 
michael@0:         for testcase in testcases:
michael@0:             attributes = testcase.keys()
michael@0:             self.assertTrue("classname" in attributes)
michael@0:             self.assertTrue("name" in attributes)
michael@0:             self.assertTrue("time" in attributes)
michael@0: 
michael@0:         self.assertTrue(testcases[1].find("failure") is not None)
michael@0:         self.assertTrue(testcases[2].find("skipped") is not None)
michael@0: 
michael@0:     def testDoThrowString(self):
michael@0:         """
michael@0:         Check that do_throw produces reasonable messages when the
michael@0:         input is a string instead of an object
michael@0:         """
michael@0:         self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("Passing a string to do_throw")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testDoThrowForeignObject(self):
michael@0:         """
michael@0:         Check that do_throw produces reasonable messages when the
michael@0:         input is a generic object with 'filename', 'message' and 'stack' attributes
michael@0:         but 'object instanceof Error' returns false
michael@0:         """
michael@0:         self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("failure.js")
michael@0:         self.assertInLog("Error object")
michael@0:         self.assertInLog("ERROR STACK")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testDoReportForeignObject(self):
michael@0:         """
michael@0:         Check that do_report_unexpected_exception produces reasonable messages when the
michael@0:         input is a generic object with 'filename', 'message' and 'stack' attributes
michael@0:         but 'object instanceof Error' returns false
michael@0:         """
michael@0:         self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("failure.js")
michael@0:         self.assertInLog("Error object")
michael@0:         self.assertInLog("ERROR STACK")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testDoReportRefError(self):
michael@0:         """
michael@0:         Check that do_report_unexpected_exception produces reasonable messages when the
michael@0:         input is a JS-generated Error
michael@0:         """
michael@0:         self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("test_error.js")
michael@0:         self.assertInLog("obj.noSuchFunction is not a function")
michael@0:         self.assertInLog("run_test@")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testDoReportSyntaxError(self):
michael@0:         """
michael@0:         Check that attempting to load a test file containing a syntax error
michael@0:         generates details of the error in the log
michael@0:         """
michael@0:         self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("test_error.js")
michael@0:         self.assertInLog("test_error.js contains SyntaxError")
michael@0:         self.assertInLog("Diagnostic: SyntaxError: missing formal parameter at")
michael@0:         self.assertInLog("test_error.js:3")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testDoReportNonSyntaxError(self):
michael@0:         """
michael@0:         Check that attempting to load a test file containing an error other
michael@0:         than a syntax error generates details of the error in the log
michael@0:         """
michael@0:         self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
michael@0:         self.writeManifest(["test_error.js"])
michael@0: 
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("TEST-UNEXPECTED-FAIL")
michael@0:         self.assertInLog("Diagnostic: TypeError: generator function run_test returns a value at")
michael@0:         self.assertInLog("test_error.js:4")
michael@0:         self.assertNotInLog("TEST-PASS")
michael@0: 
michael@0:     def testAsyncCleanup(self):
michael@0:         """
michael@0:         Check that do_register_cleanup handles nicely cleanup tasks that
michael@0:         return a promise
michael@0:         """
michael@0:         self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
michael@0:         self.writeManifest(["test_asyncCleanup.js"])
michael@0:         self.assertTestResult(False)
michael@0:         self.assertInLog("\"1234\" == \"1234\"")
michael@0:         self.assertInLog("At this stage, the test has succeeded")
michael@0:         self.assertInLog("Throwing an error to force displaying the log")
michael@0: 
michael@0: if __name__ == "__main__":
michael@0:     unittest.main()