The Tor Browser / file comparison
comparison: media/webrtc/trunk/testing/gtest/test/gtest_shuffle_test.py
media/webrtc/trunk/testing/gtest/test/gtest_shuffle_test.py
- branch
- TOR_BUG_3246
- changeset 5
- 4ab42b5ab56c
equal
deleted
inserted
replaced
| -1:000000000000 | 0:aee6ffc0c732 |
|---|---|
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2009 Google Inc. All Rights Reserved. | |
| 4 # | |
| 5 # Redistribution and use in source and binary forms, with or without | |
| 6 # modification, are permitted provided that the following conditions are | |
| 7 # met: | |
| 8 # | |
| 9 # * Redistributions of source code must retain the above copyright | |
| 10 # notice, this list of conditions and the following disclaimer. | |
| 11 # * Redistributions in binary form must reproduce the above | |
| 12 # copyright notice, this list of conditions and the following disclaimer | |
| 13 # in the documentation and/or other materials provided with the | |
| 14 # distribution. | |
| 15 # * Neither the name of Google Inc. nor the names of its | |
| 16 # contributors may be used to endorse or promote products derived from | |
| 17 # this software without specific prior written permission. | |
| 18 # | |
| 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 30 | |
| 31 """Verifies that test shuffling works.""" | |
| 32 | |
| 33 __author__ = 'wan@google.com (Zhanyong Wan)' | |
| 34 | |
| 35 import os | |
| 36 import gtest_test_utils | |
| 37 | |
| 38 # Command to run the gtest_shuffle_test_ program. | |
| 39 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') | |
| 40 | |
| 41 # The environment variables for test sharding. | |
| 42 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' | |
| 43 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' | |
| 44 | |
| 45 TEST_FILTER = 'A*.A:A*.B:C*' | |
| 46 | |
| 47 ALL_TESTS = [] | |
| 48 ACTIVE_TESTS = [] | |
| 49 FILTERED_TESTS = [] | |
| 50 SHARDED_TESTS = [] | |
| 51 | |
| 52 SHUFFLED_ALL_TESTS = [] | |
| 53 SHUFFLED_ACTIVE_TESTS = [] | |
| 54 SHUFFLED_FILTERED_TESTS = [] | |
| 55 SHUFFLED_SHARDED_TESTS = [] | |
| 56 | |
| 57 | |
| 58 def AlsoRunDisabledTestsFlag(): | |
| 59 return '--gtest_also_run_disabled_tests' | |
| 60 | |
| 61 | |
| 62 def FilterFlag(test_filter): | |
| 63 return '--gtest_filter=%s' % (test_filter,) | |
| 64 | |
| 65 | |
| 66 def RepeatFlag(n): | |
| 67 return '--gtest_repeat=%s' % (n,) | |
| 68 | |
| 69 | |
| 70 def ShuffleFlag(): | |
| 71 return '--gtest_shuffle' | |
| 72 | |
| 73 | |
| 74 def RandomSeedFlag(n): | |
| 75 return '--gtest_random_seed=%s' % (n,) | |
| 76 | |
| 77 | |
| 78 def RunAndReturnOutput(extra_env, args): | |
| 79 """Runs the test program and returns its output.""" | |
| 80 | |
| 81 environ_copy = os.environ.copy() | |
| 82 environ_copy.update(extra_env) | |
| 83 | |
| 84 return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy, | |
| 85 capture_stderr=False).output | |
| 86 | |
| 87 | |
| 88 def GetTestsForAllIterations(extra_env, args): | |
| 89 """Runs the test program and returns a list of test lists. | |
| 90 | |
| 91 Args: | |
| 92 extra_env: a map from environment variables to their values | |
| 93 args: command line flags to pass to gtest_shuffle_test_ | |
| 94 | |
| 95 Returns: | |
| 96 A list where the i-th element is the list of tests run in the i-th | |
| 97 test iteration. | |
| 98 """ | |
| 99 | |
| 100 test_iterations = [] | |
| 101 for line in RunAndReturnOutput(extra_env, args).split('\n'): | |
| 102 if line.startswith('----'): | |
| 103 tests = [] | |
| 104 test_iterations.append(tests) | |
| 105 elif line.strip(): | |
| 106 tests.append(line.strip()) # 'TestCaseName.TestName' | |
| 107 | |
| 108 return test_iterations | |
| 109 | |
| 110 | |
| 111 def GetTestCases(tests): | |
| 112 """Returns a list of test cases in the given full test names. | |
| 113 | |
| 114 Args: | |
| 115 tests: a list of full test names | |
| 116 | |
| 117 Returns: | |
| 118 A list of test cases from 'tests', in their original order. | |
| 119 Consecutive duplicates are removed. | |
| 120 """ | |
| 121 | |
| 122 test_cases = [] | |
| 123 for test in tests: | |
| 124 test_case = test.split('.')[0] | |
| 125 if not test_case in test_cases: | |
| 126 test_cases.append(test_case) | |
| 127 | |
| 128 return test_cases | |
| 129 | |
| 130 | |
| 131 def CalculateTestLists(): | |
| 132 """Calculates the list of tests run under different flags.""" | |
| 133 | |
| 134 if not ALL_TESTS: | |
| 135 ALL_TESTS.extend( | |
| 136 GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) | |
| 137 | |
| 138 if not ACTIVE_TESTS: | |
| 139 ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) | |
| 140 | |
| 141 if not FILTERED_TESTS: | |
| 142 FILTERED_TESTS.extend( | |
| 143 GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) | |
| 144 | |
| 145 if not SHARDED_TESTS: | |
| 146 SHARDED_TESTS.extend( | |
| 147 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 148 SHARD_INDEX_ENV_VAR: '1'}, | |
| 149 [])[0]) | |
| 150 | |
| 151 if not SHUFFLED_ALL_TESTS: | |
| 152 SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( | |
| 153 {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
| 154 | |
| 155 if not SHUFFLED_ACTIVE_TESTS: | |
| 156 SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( | |
| 157 {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
| 158 | |
| 159 if not SHUFFLED_FILTERED_TESTS: | |
| 160 SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( | |
| 161 {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) | |
| 162 | |
| 163 if not SHUFFLED_SHARDED_TESTS: | |
| 164 SHUFFLED_SHARDED_TESTS.extend( | |
| 165 GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 166 SHARD_INDEX_ENV_VAR: '1'}, | |
| 167 [ShuffleFlag(), RandomSeedFlag(1)])[0]) | |
| 168 | |
| 169 | |
| 170 class GTestShuffleUnitTest(gtest_test_utils.TestCase): | |
| 171 """Tests test shuffling.""" | |
| 172 | |
| 173 def setUp(self): | |
| 174 CalculateTestLists() | |
| 175 | |
| 176 def testShufflePreservesNumberOfTests(self): | |
| 177 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) | |
| 178 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) | |
| 179 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) | |
| 180 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) | |
| 181 | |
| 182 def testShuffleChangesTestOrder(self): | |
| 183 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) | |
| 184 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) | |
| 185 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, | |
| 186 SHUFFLED_FILTERED_TESTS) | |
| 187 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, | |
| 188 SHUFFLED_SHARDED_TESTS) | |
| 189 | |
| 190 def testShuffleChangesTestCaseOrder(self): | |
| 191 self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), | |
| 192 GetTestCases(SHUFFLED_ALL_TESTS)) | |
| 193 self.assert_( | |
| 194 GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), | |
| 195 GetTestCases(SHUFFLED_ACTIVE_TESTS)) | |
| 196 self.assert_( | |
| 197 GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), | |
| 198 GetTestCases(SHUFFLED_FILTERED_TESTS)) | |
| 199 self.assert_( | |
| 200 GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), | |
| 201 GetTestCases(SHUFFLED_SHARDED_TESTS)) | |
| 202 | |
| 203 def testShuffleDoesNotRepeatTest(self): | |
| 204 for test in SHUFFLED_ALL_TESTS: | |
| 205 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), | |
| 206 '%s appears more than once' % (test,)) | |
| 207 for test in SHUFFLED_ACTIVE_TESTS: | |
| 208 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), | |
| 209 '%s appears more than once' % (test,)) | |
| 210 for test in SHUFFLED_FILTERED_TESTS: | |
| 211 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), | |
| 212 '%s appears more than once' % (test,)) | |
| 213 for test in SHUFFLED_SHARDED_TESTS: | |
| 214 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), | |
| 215 '%s appears more than once' % (test,)) | |
| 216 | |
| 217 def testShuffleDoesNotCreateNewTest(self): | |
| 218 for test in SHUFFLED_ALL_TESTS: | |
| 219 self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) | |
| 220 for test in SHUFFLED_ACTIVE_TESTS: | |
| 221 self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) | |
| 222 for test in SHUFFLED_FILTERED_TESTS: | |
| 223 self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) | |
| 224 for test in SHUFFLED_SHARDED_TESTS: | |
| 225 self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) | |
| 226 | |
| 227 def testShuffleIncludesAllTests(self): | |
| 228 for test in ALL_TESTS: | |
| 229 self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) | |
| 230 for test in ACTIVE_TESTS: | |
| 231 self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) | |
| 232 for test in FILTERED_TESTS: | |
| 233 self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) | |
| 234 for test in SHARDED_TESTS: | |
| 235 self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) | |
| 236 | |
| 237 def testShuffleLeavesDeathTestsAtFront(self): | |
| 238 non_death_test_found = False | |
| 239 for test in SHUFFLED_ACTIVE_TESTS: | |
| 240 if 'DeathTest.' in test: | |
| 241 self.assert_(not non_death_test_found, | |
| 242 '%s appears after a non-death test' % (test,)) | |
| 243 else: | |
| 244 non_death_test_found = True | |
| 245 | |
| 246 def _VerifyTestCasesDoNotInterleave(self, tests): | |
| 247 test_cases = [] | |
| 248 for test in tests: | |
| 249 [test_case, _] = test.split('.') | |
| 250 if test_cases and test_cases[-1] != test_case: | |
| 251 test_cases.append(test_case) | |
| 252 self.assertEqual(1, test_cases.count(test_case), | |
| 253 'Test case %s is not grouped together in %s' % | |
| 254 (test_case, tests)) | |
| 255 | |
| 256 def testShuffleDoesNotInterleaveTestCases(self): | |
| 257 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) | |
| 258 self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) | |
| 259 self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) | |
| 260 self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) | |
| 261 | |
| 262 def testShuffleRestoresOrderAfterEachIteration(self): | |
| 263 # Get the test lists in all 3 iterations, using random seed 1, 2, | |
| 264 # and 3 respectively. Google Test picks a different seed in each | |
| 265 # iteration, and this test depends on the current implementation | |
| 266 # picking successive numbers. This dependency is not ideal, but | |
| 267 # makes the test much easier to write. | |
| 268 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( | |
| 269 GetTestsForAllIterations( | |
| 270 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) | |
| 271 | |
| 272 # Make sure running the tests with random seed 1 gets the same | |
| 273 # order as in iteration 1 above. | |
| 274 [tests_with_seed1] = GetTestsForAllIterations( | |
| 275 {}, [ShuffleFlag(), RandomSeedFlag(1)]) | |
| 276 self.assertEqual(tests_in_iteration1, tests_with_seed1) | |
| 277 | |
| 278 # Make sure running the tests with random seed 2 gets the same | |
| 279 # order as in iteration 2 above. Success means that Google Test | |
| 280 # correctly restores the test order before re-shuffling at the | |
| 281 # beginning of iteration 2. | |
| 282 [tests_with_seed2] = GetTestsForAllIterations( | |
| 283 {}, [ShuffleFlag(), RandomSeedFlag(2)]) | |
| 284 self.assertEqual(tests_in_iteration2, tests_with_seed2) | |
| 285 | |
| 286 # Make sure running the tests with random seed 3 gets the same | |
| 287 # order as in iteration 3 above. Success means that Google Test | |
| 288 # correctly restores the test order before re-shuffling at the | |
| 289 # beginning of iteration 3. | |
| 290 [tests_with_seed3] = GetTestsForAllIterations( | |
| 291 {}, [ShuffleFlag(), RandomSeedFlag(3)]) | |
| 292 self.assertEqual(tests_in_iteration3, tests_with_seed3) | |
| 293 | |
| 294 def testShuffleGeneratesNewOrderInEachIteration(self): | |
| 295 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( | |
| 296 GetTestsForAllIterations( | |
| 297 {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) | |
| 298 | |
| 299 self.assert_(tests_in_iteration1 != tests_in_iteration2, | |
| 300 tests_in_iteration1) | |
| 301 self.assert_(tests_in_iteration1 != tests_in_iteration3, | |
| 302 tests_in_iteration1) | |
| 303 self.assert_(tests_in_iteration2 != tests_in_iteration3, | |
| 304 tests_in_iteration2) | |
| 305 | |
| 306 def testShuffleShardedTestsPreservesPartition(self): | |
| 307 # If we run M tests on N shards, the same M tests should be run in | |
| 308 # total, regardless of the random seeds used by the shards. | |
| 309 [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 310 SHARD_INDEX_ENV_VAR: '0'}, | |
| 311 [ShuffleFlag(), RandomSeedFlag(1)]) | |
| 312 [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 313 SHARD_INDEX_ENV_VAR: '1'}, | |
| 314 [ShuffleFlag(), RandomSeedFlag(20)]) | |
| 315 [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', | |
| 316 SHARD_INDEX_ENV_VAR: '2'}, | |
| 317 [ShuffleFlag(), RandomSeedFlag(25)]) | |
| 318 sorted_sharded_tests = tests1 + tests2 + tests3 | |
| 319 sorted_sharded_tests.sort() | |
| 320 sorted_active_tests = [] | |
| 321 sorted_active_tests.extend(ACTIVE_TESTS) | |
| 322 sorted_active_tests.sort() | |
| 323 self.assertEqual(sorted_active_tests, sorted_sharded_tests) | |
| 324 | |
| 325 if __name__ == '__main__': | |
| 326 gtest_test_utils.Main() |
