michael@0: #!/usr/bin/env python
michael@0: #
michael@0: # Copyright 2008, Google Inc.
michael@0: # All rights reserved.
michael@0: #
michael@0: # Redistribution and use in source and binary forms, with or without
michael@0: # modification, are permitted provided that the following conditions are
michael@0: # met:
michael@0: #
michael@0: #     * Redistributions of source code must retain the above copyright
michael@0: # notice, this list of conditions and the following disclaimer.
michael@0: #     * Redistributions in binary form must reproduce the above
michael@0: # copyright notice, this list of conditions and the following disclaimer
michael@0: # in the documentation and/or other materials provided with the
michael@0: # distribution.
michael@0: #     * Neither the name of Google Inc. nor the names of its
michael@0: # contributors may be used to endorse or promote products derived from
michael@0: # this software without specific prior written permission.
michael@0: #
michael@0: # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
michael@0: # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
michael@0: # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
michael@0: # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
michael@0: # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
michael@0: # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
michael@0: # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
michael@0: # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
michael@0: # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
michael@0: # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
michael@0: # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
michael@0: 
michael@0: """Tests the text output of Google C++ Testing Framework.
michael@0: 
michael@0: SYNOPSIS
michael@0:        gtest_output_test.py --build_dir=BUILD/DIR --gengolden
michael@0:          # where BUILD/DIR contains the built gtest_output_test_ file.
michael@0:        gtest_output_test.py --gengolden
michael@0:        gtest_output_test.py
michael@0: """
michael@0: 
michael@0: __author__ = 'wan@google.com (Zhanyong Wan)'
michael@0: 
michael@0: import os
michael@0: import re
michael@0: import sys
michael@0: import gtest_test_utils
michael@0: 
michael@0: 
michael@0: # The flag for generating the golden file
michael@0: GENGOLDEN_FLAG = '--gengolden'
michael@0: CATCH_EXCEPTIONS_ENV_VAR_NAME = 'GTEST_CATCH_EXCEPTIONS'
michael@0: 
michael@0: IS_WINDOWS = os.name == 'nt'
michael@0: 
michael@0: # TODO(vladl@google.com): remove the _lin suffix.
michael@0: GOLDEN_NAME = 'gtest_output_test_golden_lin.txt'
michael@0: 
michael@0: PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath('gtest_output_test_')
michael@0: 
michael@0: # At least one command we exercise must not have the
michael@0: # --gtest_internal_skip_environment_and_ad_hoc_tests flag.
michael@0: COMMAND_LIST_TESTS = ({}, [PROGRAM_PATH, '--gtest_list_tests'])
michael@0: COMMAND_WITH_COLOR = ({}, [PROGRAM_PATH, '--gtest_color=yes'])
michael@0: COMMAND_WITH_TIME = ({}, [PROGRAM_PATH,
michael@0:                           '--gtest_print_time',
michael@0:                           '--gtest_internal_skip_environment_and_ad_hoc_tests',
michael@0:                           '--gtest_filter=FatalFailureTest.*:LoggingTest.*'])
michael@0: COMMAND_WITH_DISABLED = (
michael@0:     {}, [PROGRAM_PATH,
michael@0:          '--gtest_also_run_disabled_tests',
michael@0:          '--gtest_internal_skip_environment_and_ad_hoc_tests',
michael@0:          '--gtest_filter=*DISABLED_*'])
michael@0: COMMAND_WITH_SHARDING = (
michael@0:     {'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'},
michael@0:     [PROGRAM_PATH,
michael@0:      '--gtest_internal_skip_environment_and_ad_hoc_tests',
michael@0:      '--gtest_filter=PassingTest.*'])
michael@0: 
michael@0: GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME)
michael@0: 
michael@0: 
michael@0: def ToUnixLineEnding(s):
michael@0:   """Changes all Windows/Mac line endings in s to UNIX line endings."""
michael@0: 
michael@0:   return s.replace('\r\n', '\n').replace('\r', '\n')
michael@0: 
michael@0: 
michael@0: def RemoveLocations(test_output):
michael@0:   """Removes all file location info from a Google Test program's output.
michael@0: 
michael@0:   Args:
michael@0:        test_output:  the output of a Google Test program.
michael@0: 
michael@0:   Returns:
michael@0:        output with all file location info (in the form of
michael@0:        'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or
michael@0:        'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by
michael@0:        'FILE_NAME:#: '.
michael@0:   """
michael@0: 
michael@0:   return re.sub(r'.*[/\\](.+)(\:\d+|\(\d+\))\: ', r'\1:#: ', test_output)
michael@0: 
michael@0: 
michael@0: def RemoveStackTraceDetails(output):
michael@0:   """Removes all stack traces from a Google Test program's output."""
michael@0: 
michael@0:   # *? means "find the shortest string that matches".
michael@0:   return re.sub(r'Stack trace:(.|\n)*?\n\n',
michael@0:                 'Stack trace: (omitted)\n\n', output)
michael@0: 
michael@0: 
michael@0: def RemoveStackTraces(output):
michael@0:   """Removes all traces of stack traces from a Google Test program's output."""
michael@0: 
michael@0:   # *? means "find the shortest string that matches".
michael@0:   return re.sub(r'Stack trace:(.|\n)*?\n\n', '', output)
michael@0: 
michael@0: 
michael@0: def RemoveTime(output):
michael@0:   """Removes all time information from a Google Test program's output."""
michael@0: 
michael@0:   return re.sub(r'\(\d+ ms', '(? ms', output)
michael@0: 
michael@0: 
michael@0: def RemoveTypeInfoDetails(test_output):
michael@0:   """Removes compiler-specific type info from Google Test program's output.
michael@0: 
michael@0:   Args:
michael@0:        test_output:  the output of a Google Test program.
michael@0: 
michael@0:   Returns:
michael@0:        output with type information normalized to canonical form.
michael@0:   """
michael@0: 
michael@0:   # some compilers output the name of type 'unsigned int' as 'unsigned'
michael@0:   return re.sub(r'unsigned int', 'unsigned', test_output)
michael@0: 
michael@0: 
michael@0: def NormalizeToCurrentPlatform(test_output):
michael@0:   """Normalizes platform specific output details for easier comparison."""
michael@0: 
michael@0:   if IS_WINDOWS:
michael@0:     # Removes the color information that is not present on Windows.
michael@0:     test_output = re.sub('\x1b\\[(0;3\d)?m', '', test_output)
michael@0:     # Changes failure message headers into the Windows format.
michael@0:     test_output = re.sub(r': Failure\n', r': error: ', test_output)
michael@0:     # Changes file(line_number) to file:line_number.
michael@0:     test_output = re.sub(r'((\w|\.)+)\((\d+)\):', r'\1:\3:', test_output)
michael@0: 
michael@0:   return test_output
michael@0: 
michael@0: 
michael@0: def RemoveTestCounts(output):
michael@0:   """Removes test counts from a Google Test program's output."""
michael@0: 
michael@0:   output = re.sub(r'\d+ tests?, listed below',
michael@0:                   '? tests, listed below', output)
michael@0:   output = re.sub(r'\d+ FAILED TESTS',
michael@0:                   '? FAILED TESTS', output)
michael@0:   output = re.sub(r'\d+ tests? from \d+ test cases?',
michael@0:                   '? tests from ? test cases', output)
michael@0:   output = re.sub(r'\d+ tests? from ([a-zA-Z_])',
michael@0:                   r'? tests from \1', output)
michael@0:   return re.sub(r'\d+ tests?\.', '? tests.', output)
michael@0: 
michael@0: 
michael@0: def RemoveMatchingTests(test_output, pattern):
michael@0:   """Removes output of specified tests from a Google Test program's output.
michael@0: 
michael@0:   This function strips not only the beginning and the end of a test but also
michael@0:   all output in between.
michael@0: 
michael@0:   Args:
michael@0:     test_output:       A string containing the test output.
michael@0:     pattern:           A regex string that matches names of test cases or
michael@0:                        tests to remove.
michael@0: 
michael@0:   Returns:
michael@0:     Contents of test_output with tests whose names match pattern removed.
michael@0:   """
michael@0: 
michael@0:   test_output = re.sub(
michael@0:       r'.*\[ RUN      \] .*%s(.|\n)*?\[(  FAILED  |       OK )\] .*%s.*\n' % (
michael@0:           pattern, pattern),
michael@0:       '',
michael@0:       test_output)
michael@0:   return re.sub(r'.*%s.*\n' % pattern, '', test_output)
michael@0: 
michael@0: 
michael@0: def NormalizeOutput(output):
michael@0:   """Normalizes output (the output of gtest_output_test_.exe)."""
michael@0: 
michael@0:   output = ToUnixLineEnding(output)
michael@0:   output = RemoveLocations(output)
michael@0:   output = RemoveStackTraceDetails(output)
michael@0:   output = RemoveTime(output)
michael@0:   return output
michael@0: 
michael@0: 
michael@0: def GetShellCommandOutput(env_cmd):
michael@0:   """Runs a command in a sub-process, and returns its output in a string.
michael@0: 
michael@0:   Args:
michael@0:     env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra
michael@0:              environment variables to set, and element 1 is a string with
michael@0:              the command and any flags.
michael@0: 
michael@0:   Returns:
michael@0:     A string with the command's combined standard and diagnostic output.
michael@0:   """
michael@0: 
michael@0:   # Spawns cmd in a sub-process, and gets its standard I/O file objects.
michael@0:   # Set and save the environment properly.
michael@0:   environ = os.environ.copy()
michael@0:   environ.update(env_cmd[0])
michael@0:   p = gtest_test_utils.Subprocess(env_cmd[1], env=environ, capture_stderr=False)
michael@0: 
michael@0:   return p.output
michael@0: 
michael@0: 
michael@0: def GetCommandOutput(env_cmd):
michael@0:   """Runs a command and returns its output with all file location
michael@0:   info stripped off.
michael@0: 
michael@0:   Args:
michael@0:     env_cmd:  The shell command. A 2-tuple where element 0 is a dict of extra
michael@0:               environment variables to set, and element 1 is a string with
michael@0:               the command and any flags.
michael@0:   """
michael@0: 
michael@0:   # Disables exception pop-ups on Windows.
michael@0:   environ, cmdline = env_cmd
michael@0:   environ = dict(environ)  # Ensures we are modifying a copy.
michael@0:   environ[CATCH_EXCEPTIONS_ENV_VAR_NAME] = '1'
michael@0:   return NormalizeOutput(GetShellCommandOutput((environ, cmdline)))
michael@0: 
michael@0: 
michael@0: def GetOutputOfAllCommands():
michael@0:   """Returns concatenated output from several representative commands."""
michael@0: 
michael@0:   return (GetCommandOutput(COMMAND_WITH_COLOR) +
michael@0:           GetCommandOutput(COMMAND_WITH_TIME) +
michael@0:           GetCommandOutput(COMMAND_WITH_DISABLED) +
michael@0:           GetCommandOutput(COMMAND_WITH_SHARDING))
michael@0: 
michael@0: 
michael@0: test_list = GetShellCommandOutput(COMMAND_LIST_TESTS)
michael@0: SUPPORTS_DEATH_TESTS = 'DeathTest' in test_list
michael@0: SUPPORTS_TYPED_TESTS = 'TypedTest' in test_list
michael@0: SUPPORTS_THREADS = 'ExpectFailureWithThreadsTest' in test_list
michael@0: SUPPORTS_STACK_TRACES = False
michael@0: 
michael@0: CAN_GENERATE_GOLDEN_FILE = (SUPPORTS_DEATH_TESTS and
michael@0:                             SUPPORTS_TYPED_TESTS and
michael@0:                             SUPPORTS_THREADS)
michael@0: 
michael@0: 
michael@0: class GTestOutputTest(gtest_test_utils.TestCase):
michael@0:   def RemoveUnsupportedTests(self, test_output):
michael@0:     if not SUPPORTS_DEATH_TESTS:
michael@0:       test_output = RemoveMatchingTests(test_output, 'DeathTest')
michael@0:     if not SUPPORTS_TYPED_TESTS:
michael@0:       test_output = RemoveMatchingTests(test_output, 'TypedTest')
michael@0:       test_output = RemoveMatchingTests(test_output, 'TypedDeathTest')
michael@0:       test_output = RemoveMatchingTests(test_output, 'TypeParamDeathTest')
michael@0:     if not SUPPORTS_THREADS:
michael@0:       test_output = RemoveMatchingTests(test_output,
michael@0:                                         'ExpectFailureWithThreadsTest')
michael@0:       test_output = RemoveMatchingTests(test_output,
michael@0:                                         'ScopedFakeTestPartResultReporterTest')
michael@0:       test_output = RemoveMatchingTests(test_output,
michael@0:                                         'WorksConcurrently')
michael@0:     if not SUPPORTS_STACK_TRACES:
michael@0:       test_output = RemoveStackTraces(test_output)
michael@0: 
michael@0:     return test_output
michael@0: 
michael@0:   def testOutput(self):
michael@0:     output = GetOutputOfAllCommands()
michael@0: 
michael@0:     golden_file = open(GOLDEN_PATH, 'rb')
michael@0:     # A mis-configured source control system can cause \r appear in EOL
michael@0:     # sequences when we read the golden file irrespective of an operating
michael@0:     # system used. Therefore, we need to strip those \r's from newlines
michael@0:     # unconditionally.
michael@0:     golden = ToUnixLineEnding(golden_file.read())
michael@0:     golden_file.close()
michael@0: 
michael@0:     # We want the test to pass regardless of certain features being
michael@0:     # supported or not.
michael@0: 
michael@0:     # We still have to remove type name specifics in all cases.
michael@0:     normalized_actual = RemoveTypeInfoDetails(output)
michael@0:     normalized_golden = RemoveTypeInfoDetails(golden)
michael@0: 
michael@0:     if CAN_GENERATE_GOLDEN_FILE:
michael@0:       self.assertEqual(normalized_golden, normalized_actual)
michael@0:     else:
michael@0:       normalized_actual = NormalizeToCurrentPlatform(
michael@0:           RemoveTestCounts(normalized_actual))
michael@0:       normalized_golden = NormalizeToCurrentPlatform(
michael@0:           RemoveTestCounts(self.RemoveUnsupportedTests(normalized_golden)))
michael@0: 
michael@0:       # This code is very handy when debugging golden file differences:
michael@0:       if os.getenv('DEBUG_GTEST_OUTPUT_TEST'):
michael@0:         open(os.path.join(
michael@0:             gtest_test_utils.GetSourceDir(),
michael@0:             '_gtest_output_test_normalized_actual.txt'), 'wb').write(
michael@0:                 normalized_actual)
michael@0:         open(os.path.join(
michael@0:             gtest_test_utils.GetSourceDir(),
michael@0:             '_gtest_output_test_normalized_golden.txt'), 'wb').write(
michael@0:                 normalized_golden)
michael@0: 
michael@0:       self.assertEqual(normalized_golden, normalized_actual)
michael@0: 
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:   if sys.argv[1:] == [GENGOLDEN_FLAG]:
michael@0:     if CAN_GENERATE_GOLDEN_FILE:
michael@0:       output = GetOutputOfAllCommands()
michael@0:       golden_file = open(GOLDEN_PATH, 'wb')
michael@0:       golden_file.write(output)
michael@0:       golden_file.close()
michael@0:     else:
michael@0:       message = (
michael@0:           """Unable to write a golden file when compiled in an environment
michael@0: that does not support all the required features (death tests, typed tests,
michael@0: and multiple threads).  Please generate the golden file using a binary built
michael@0: with those features enabled.""")
michael@0: 
michael@0:       sys.stderr.write(message)
michael@0:       sys.exit(1)
michael@0:   else:
michael@0:     gtest_test_utils.Main()