1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/media/webrtc/trunk/testing/gtest/test/gtest_shuffle_test.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,326 @@
1.4 +#!/usr/bin/env python
1.5 +#
1.6 +# Copyright 2009 Google Inc. All Rights Reserved.
1.7 +#
1.8 +# Redistribution and use in source and binary forms, with or without
1.9 +# modification, are permitted provided that the following conditions are
1.10 +# met:
1.11 +#
1.12 +# * Redistributions of source code must retain the above copyright
1.13 +# notice, this list of conditions and the following disclaimer.
1.14 +# * Redistributions in binary form must reproduce the above
1.15 +# copyright notice, this list of conditions and the following disclaimer
1.16 +# in the documentation and/or other materials provided with the
1.17 +# distribution.
1.18 +# * Neither the name of Google Inc. nor the names of its
1.19 +# contributors may be used to endorse or promote products derived from
1.20 +# this software without specific prior written permission.
1.21 +#
1.22 +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
1.23 +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
1.24 +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
1.25 +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
1.26 +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
1.27 +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
1.28 +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
1.29 +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
1.30 +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1.31 +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
1.32 +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.33 +
1.34 +"""Verifies that test shuffling works."""
1.35 +
1.36 +__author__ = 'wan@google.com (Zhanyong Wan)'
1.37 +
1.38 +import os
1.39 +import gtest_test_utils
1.40 +
1.41 +# Command to run the gtest_shuffle_test_ program.
1.42 +COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')
1.43 +
1.44 +# The environment variables for test sharding.
1.45 +TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
1.46 +SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
1.47 +
1.48 +TEST_FILTER = 'A*.A:A*.B:C*'
1.49 +
1.50 +ALL_TESTS = []
1.51 +ACTIVE_TESTS = []
1.52 +FILTERED_TESTS = []
1.53 +SHARDED_TESTS = []
1.54 +
1.55 +SHUFFLED_ALL_TESTS = []
1.56 +SHUFFLED_ACTIVE_TESTS = []
1.57 +SHUFFLED_FILTERED_TESTS = []
1.58 +SHUFFLED_SHARDED_TESTS = []
1.59 +
1.60 +
1.61 +def AlsoRunDisabledTestsFlag():
1.62 + return '--gtest_also_run_disabled_tests'
1.63 +
1.64 +
1.65 +def FilterFlag(test_filter):
1.66 + return '--gtest_filter=%s' % (test_filter,)
1.67 +
1.68 +
1.69 +def RepeatFlag(n):
1.70 + return '--gtest_repeat=%s' % (n,)
1.71 +
1.72 +
1.73 +def ShuffleFlag():
1.74 + return '--gtest_shuffle'
1.75 +
1.76 +
1.77 +def RandomSeedFlag(n):
1.78 + return '--gtest_random_seed=%s' % (n,)
1.79 +
1.80 +
1.81 +def RunAndReturnOutput(extra_env, args):
1.82 + """Runs the test program and returns its output."""
1.83 +
1.84 + environ_copy = os.environ.copy()
1.85 + environ_copy.update(extra_env)
1.86 +
1.87 + return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy,
1.88 + capture_stderr=False).output
1.89 +
1.90 +
1.91 +def GetTestsForAllIterations(extra_env, args):
1.92 + """Runs the test program and returns a list of test lists.
1.93 +
1.94 + Args:
1.95 + extra_env: a map from environment variables to their values
1.96 + args: command line flags to pass to gtest_shuffle_test_
1.97 +
1.98 + Returns:
1.99 + A list where the i-th element is the list of tests run in the i-th
1.100 + test iteration.
1.101 + """
1.102 +
1.103 + test_iterations = []
1.104 + for line in RunAndReturnOutput(extra_env, args).split('\n'):
1.105 + if line.startswith('----'):
1.106 + tests = []
1.107 + test_iterations.append(tests)
1.108 + elif line.strip():
1.109 + tests.append(line.strip()) # 'TestCaseName.TestName'
1.110 +
1.111 + return test_iterations
1.112 +
1.113 +
1.114 +def GetTestCases(tests):
1.115 + """Returns a list of test cases in the given full test names.
1.116 +
1.117 + Args:
1.118 + tests: a list of full test names
1.119 +
1.120 + Returns:
1.121 + A list of test cases from 'tests', in their original order.
1.122 + Consecutive duplicates are removed.
1.123 + """
1.124 +
1.125 + test_cases = []
1.126 + for test in tests:
1.127 + test_case = test.split('.')[0]
1.128 + if not test_case in test_cases:
1.129 + test_cases.append(test_case)
1.130 +
1.131 + return test_cases
1.132 +
1.133 +
1.134 +def CalculateTestLists():
1.135 + """Calculates the list of tests run under different flags."""
1.136 +
1.137 + if not ALL_TESTS:
1.138 + ALL_TESTS.extend(
1.139 + GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
1.140 +
1.141 + if not ACTIVE_TESTS:
1.142 + ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
1.143 +
1.144 + if not FILTERED_TESTS:
1.145 + FILTERED_TESTS.extend(
1.146 + GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
1.147 +
1.148 + if not SHARDED_TESTS:
1.149 + SHARDED_TESTS.extend(
1.150 + GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
1.151 + SHARD_INDEX_ENV_VAR: '1'},
1.152 + [])[0])
1.153 +
1.154 + if not SHUFFLED_ALL_TESTS:
1.155 + SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
1.156 + {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
1.157 +
1.158 + if not SHUFFLED_ACTIVE_TESTS:
1.159 + SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
1.160 + {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
1.161 +
1.162 + if not SHUFFLED_FILTERED_TESTS:
1.163 + SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
1.164 + {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
1.165 +
1.166 + if not SHUFFLED_SHARDED_TESTS:
1.167 + SHUFFLED_SHARDED_TESTS.extend(
1.168 + GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
1.169 + SHARD_INDEX_ENV_VAR: '1'},
1.170 + [ShuffleFlag(), RandomSeedFlag(1)])[0])
1.171 +
1.172 +
1.173 +class GTestShuffleUnitTest(gtest_test_utils.TestCase):
1.174 + """Tests test shuffling."""
1.175 +
1.176 + def setUp(self):
1.177 + CalculateTestLists()
1.178 +
1.179 + def testShufflePreservesNumberOfTests(self):
1.180 + self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
1.181 + self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
1.182 + self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
1.183 + self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
1.184 +
1.185 + def testShuffleChangesTestOrder(self):
1.186 + self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
1.187 + self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
1.188 + self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
1.189 + SHUFFLED_FILTERED_TESTS)
1.190 + self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
1.191 + SHUFFLED_SHARDED_TESTS)
1.192 +
1.193 + def testShuffleChangesTestCaseOrder(self):
1.194 + self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
1.195 + GetTestCases(SHUFFLED_ALL_TESTS))
1.196 + self.assert_(
1.197 + GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
1.198 + GetTestCases(SHUFFLED_ACTIVE_TESTS))
1.199 + self.assert_(
1.200 + GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
1.201 + GetTestCases(SHUFFLED_FILTERED_TESTS))
1.202 + self.assert_(
1.203 + GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
1.204 + GetTestCases(SHUFFLED_SHARDED_TESTS))
1.205 +
1.206 + def testShuffleDoesNotRepeatTest(self):
1.207 + for test in SHUFFLED_ALL_TESTS:
1.208 + self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
1.209 + '%s appears more than once' % (test,))
1.210 + for test in SHUFFLED_ACTIVE_TESTS:
1.211 + self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
1.212 + '%s appears more than once' % (test,))
1.213 + for test in SHUFFLED_FILTERED_TESTS:
1.214 + self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
1.215 + '%s appears more than once' % (test,))
1.216 + for test in SHUFFLED_SHARDED_TESTS:
1.217 + self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
1.218 + '%s appears more than once' % (test,))
1.219 +
1.220 + def testShuffleDoesNotCreateNewTest(self):
1.221 + for test in SHUFFLED_ALL_TESTS:
1.222 + self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
1.223 + for test in SHUFFLED_ACTIVE_TESTS:
1.224 + self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
1.225 + for test in SHUFFLED_FILTERED_TESTS:
1.226 + self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
1.227 + for test in SHUFFLED_SHARDED_TESTS:
1.228 + self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
1.229 +
1.230 + def testShuffleIncludesAllTests(self):
1.231 + for test in ALL_TESTS:
1.232 + self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
1.233 + for test in ACTIVE_TESTS:
1.234 + self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
1.235 + for test in FILTERED_TESTS:
1.236 + self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
1.237 + for test in SHARDED_TESTS:
1.238 + self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
1.239 +
1.240 + def testShuffleLeavesDeathTestsAtFront(self):
1.241 + non_death_test_found = False
1.242 + for test in SHUFFLED_ACTIVE_TESTS:
1.243 + if 'DeathTest.' in test:
1.244 + self.assert_(not non_death_test_found,
1.245 + '%s appears after a non-death test' % (test,))
1.246 + else:
1.247 + non_death_test_found = True
1.248 +
1.249 + def _VerifyTestCasesDoNotInterleave(self, tests):
1.250 + test_cases = []
1.251 + for test in tests:
1.252 + [test_case, _] = test.split('.')
1.253 + if test_cases and test_cases[-1] != test_case:
1.254 + test_cases.append(test_case)
1.255 + self.assertEqual(1, test_cases.count(test_case),
1.256 + 'Test case %s is not grouped together in %s' %
1.257 + (test_case, tests))
1.258 +
1.259 + def testShuffleDoesNotInterleaveTestCases(self):
1.260 + self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
1.261 + self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
1.262 + self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
1.263 + self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
1.264 +
1.265 + def testShuffleRestoresOrderAfterEachIteration(self):
1.266 + # Get the test lists in all 3 iterations, using random seed 1, 2,
1.267 + # and 3 respectively. Google Test picks a different seed in each
1.268 + # iteration, and this test depends on the current implementation
1.269 + # picking successive numbers. This dependency is not ideal, but
1.270 + # makes the test much easier to write.
1.271 + [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
1.272 + GetTestsForAllIterations(
1.273 + {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
1.274 +
1.275 + # Make sure running the tests with random seed 1 gets the same
1.276 + # order as in iteration 1 above.
1.277 + [tests_with_seed1] = GetTestsForAllIterations(
1.278 + {}, [ShuffleFlag(), RandomSeedFlag(1)])
1.279 + self.assertEqual(tests_in_iteration1, tests_with_seed1)
1.280 +
1.281 + # Make sure running the tests with random seed 2 gets the same
1.282 + # order as in iteration 2 above. Success means that Google Test
1.283 + # correctly restores the test order before re-shuffling at the
1.284 + # beginning of iteration 2.
1.285 + [tests_with_seed2] = GetTestsForAllIterations(
1.286 + {}, [ShuffleFlag(), RandomSeedFlag(2)])
1.287 + self.assertEqual(tests_in_iteration2, tests_with_seed2)
1.288 +
1.289 + # Make sure running the tests with random seed 3 gets the same
1.290 + # order as in iteration 3 above. Success means that Google Test
1.291 + # correctly restores the test order before re-shuffling at the
1.292 + # beginning of iteration 3.
1.293 + [tests_with_seed3] = GetTestsForAllIterations(
1.294 + {}, [ShuffleFlag(), RandomSeedFlag(3)])
1.295 + self.assertEqual(tests_in_iteration3, tests_with_seed3)
1.296 +
1.297 + def testShuffleGeneratesNewOrderInEachIteration(self):
1.298 + [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
1.299 + GetTestsForAllIterations(
1.300 + {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
1.301 +
1.302 + self.assert_(tests_in_iteration1 != tests_in_iteration2,
1.303 + tests_in_iteration1)
1.304 + self.assert_(tests_in_iteration1 != tests_in_iteration3,
1.305 + tests_in_iteration1)
1.306 + self.assert_(tests_in_iteration2 != tests_in_iteration3,
1.307 + tests_in_iteration2)
1.308 +
1.309 + def testShuffleShardedTestsPreservesPartition(self):
1.310 + # If we run M tests on N shards, the same M tests should be run in
1.311 + # total, regardless of the random seeds used by the shards.
1.312 + [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
1.313 + SHARD_INDEX_ENV_VAR: '0'},
1.314 + [ShuffleFlag(), RandomSeedFlag(1)])
1.315 + [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
1.316 + SHARD_INDEX_ENV_VAR: '1'},
1.317 + [ShuffleFlag(), RandomSeedFlag(20)])
1.318 + [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
1.319 + SHARD_INDEX_ENV_VAR: '2'},
1.320 + [ShuffleFlag(), RandomSeedFlag(25)])
1.321 + sorted_sharded_tests = tests1 + tests2 + tests3
1.322 + sorted_sharded_tests.sort()
1.323 + sorted_active_tests = []
1.324 + sorted_active_tests.extend(ACTIVE_TESTS)
1.325 + sorted_active_tests.sort()
1.326 + self.assertEqual(sorted_active_tests, sorted_sharded_tests)
1.327 +
1.328 +if __name__ == '__main__':
1.329 + gtest_test_utils.Main()
