The Tor Browser: media/webrtc/trunk/testing/gtest/test/gtest_shuffle

     1 #!/usr/bin/env python

2 #

     3 # Copyright 2009 Google Inc. All Rights Reserved.

4 #

     5 # Redistribution and use in source and binary forms, with or without

     6 # modification, are permitted provided that the following conditions are

     7 # met:

8 #

     9 #     * Redistributions of source code must retain the above copyright

    10 # notice, this list of conditions and the following disclaimer.

    11 #     * Redistributions in binary form must reproduce the above

    12 # copyright notice, this list of conditions and the following disclaimer

    13 # in the documentation and/or other materials provided with the

    14 # distribution.

    15 #     * Neither the name of Google Inc. nor the names of its

    16 # contributors may be used to endorse or promote products derived from

    17 # this software without specific prior written permission.

    18 #

    19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

    20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

    21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

    22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

    23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

    24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

    25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

    26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

    27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

    28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

    29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

    31 """Verifies that test shuffling works."""

    33 __author__ = 'wan@google.com (Zhanyong Wan)'

    35 import os

    36 import gtest_test_utils

    38 # Command to run the gtest_shuffle_test_ program.

    39 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')

    41 # The environment variables for test sharding.

    42 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'

    43 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'

    45 TEST_FILTER = 'A*.A:A*.B:C*'

    47 ALL_TESTS = []

    48 ACTIVE_TESTS = []

    49 FILTERED_TESTS = []

    50 SHARDED_TESTS = []

    52 SHUFFLED_ALL_TESTS = []

    53 SHUFFLED_ACTIVE_TESTS = []

    54 SHUFFLED_FILTERED_TESTS = []

    55 SHUFFLED_SHARDED_TESTS = []

    58 def AlsoRunDisabledTestsFlag():

    59   return '--gtest_also_run_disabled_tests'

    62 def FilterFlag(test_filter):

    63   return '--gtest_filter=%s' % (test_filter,)

    66 def RepeatFlag(n):

    67   return '--gtest_repeat=%s' % (n,)

    70 def ShuffleFlag():

    71   return '--gtest_shuffle'

    74 def RandomSeedFlag(n):

    75   return '--gtest_random_seed=%s' % (n,)

    78 def RunAndReturnOutput(extra_env, args):

    79   """Runs the test program and returns its output."""

    81   environ_copy = os.environ.copy()

    82   environ_copy.update(extra_env)

    84   return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy,

    85                                      capture_stderr=False).output

    88 def GetTestsForAllIterations(extra_env, args):

    89   """Runs the test program and returns a list of test lists.

    91   Args:

    92     extra_env: a map from environment variables to their values

    93     args: command line flags to pass to gtest_shuffle_test_

    95   Returns:

    96     A list where the i-th element is the list of tests run in the i-th

    97     test iteration.

    98   """

   100   test_iterations = []

   101   for line in RunAndReturnOutput(extra_env, args).split('\n'):

   102     if line.startswith('----'):

   103       tests = []

   104       test_iterations.append(tests)

   105     elif line.strip():

   106       tests.append(line.strip())  # 'TestCaseName.TestName'

   108   return test_iterations

   111 def GetTestCases(tests):

   112   """Returns a list of test cases in the given full test names.

   114   Args:

   115     tests: a list of full test names

   117   Returns:

   118     A list of test cases from 'tests', in their original order.

   119     Consecutive duplicates are removed.

   120   """

   122   test_cases = []

   123   for test in tests:

   124     test_case = test.split('.')[0]

   125     if not test_case in test_cases:

   126       test_cases.append(test_case)

   128   return test_cases

   131 def CalculateTestLists():

   132   """Calculates the list of tests run under different flags."""

   134   if not ALL_TESTS:

   135     ALL_TESTS.extend(

   136         GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])

   138   if not ACTIVE_TESTS:

   139     ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])

   141   if not FILTERED_TESTS:

   142     FILTERED_TESTS.extend(

   143         GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])

   145   if not SHARDED_TESTS:

   146     SHARDED_TESTS.extend(

   147         GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',

   148                                   SHARD_INDEX_ENV_VAR: '1'},

   149                                  [])[0])

   151   if not SHUFFLED_ALL_TESTS:

   152     SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(

   153         {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])

   155   if not SHUFFLED_ACTIVE_TESTS:

   156     SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(

   157         {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])

   159   if not SHUFFLED_FILTERED_TESTS:

   160     SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(

   161         {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])

   163   if not SHUFFLED_SHARDED_TESTS:

   164     SHUFFLED_SHARDED_TESTS.extend(

   165         GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',

   166                                   SHARD_INDEX_ENV_VAR: '1'},

   167                                  [ShuffleFlag(), RandomSeedFlag(1)])[0])

   170 class GTestShuffleUnitTest(gtest_test_utils.TestCase):

   171   """Tests test shuffling."""

   173   def setUp(self):

   174     CalculateTestLists()

   176   def testShufflePreservesNumberOfTests(self):

   177     self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))

   178     self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))

   179     self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))

   180     self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))

   182   def testShuffleChangesTestOrder(self):

   183     self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)

   184     self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)

   185     self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,

   186                  SHUFFLED_FILTERED_TESTS)

   187     self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,

   188                  SHUFFLED_SHARDED_TESTS)

   190   def testShuffleChangesTestCaseOrder(self):

   191     self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),

   192                  GetTestCases(SHUFFLED_ALL_TESTS))

   193     self.assert_(

   194         GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),

   195         GetTestCases(SHUFFLED_ACTIVE_TESTS))

   196     self.assert_(

   197         GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),

   198         GetTestCases(SHUFFLED_FILTERED_TESTS))

   199     self.assert_(

   200         GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),

   201         GetTestCases(SHUFFLED_SHARDED_TESTS))

   203   def testShuffleDoesNotRepeatTest(self):

   204     for test in SHUFFLED_ALL_TESTS:

   205       self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),

   206                        '%s appears more than once' % (test,))

   207     for test in SHUFFLED_ACTIVE_TESTS:

   208       self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),

   209                        '%s appears more than once' % (test,))

   210     for test in SHUFFLED_FILTERED_TESTS:

   211       self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),

   212                        '%s appears more than once' % (test,))

   213     for test in SHUFFLED_SHARDED_TESTS:

   214       self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),

   215                        '%s appears more than once' % (test,))

   217   def testShuffleDoesNotCreateNewTest(self):

   218     for test in SHUFFLED_ALL_TESTS:

   219       self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))

   220     for test in SHUFFLED_ACTIVE_TESTS:

   221       self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))

   222     for test in SHUFFLED_FILTERED_TESTS:

   223       self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))

   224     for test in SHUFFLED_SHARDED_TESTS:

   225       self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))

   227   def testShuffleIncludesAllTests(self):

   228     for test in ALL_TESTS:

   229       self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))

   230     for test in ACTIVE_TESTS:

   231       self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))

   232     for test in FILTERED_TESTS:

   233       self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))

   234     for test in SHARDED_TESTS:

   235       self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))

   237   def testShuffleLeavesDeathTestsAtFront(self):

   238     non_death_test_found = False

   239     for test in SHUFFLED_ACTIVE_TESTS:

   240       if 'DeathTest.' in test:

   241         self.assert_(not non_death_test_found,

   242                      '%s appears after a non-death test' % (test,))

   243       else:

   244         non_death_test_found = True

   246   def _VerifyTestCasesDoNotInterleave(self, tests):

   247     test_cases = []

   248     for test in tests:

   249       [test_case, _] = test.split('.')

   250       if test_cases and test_cases[-1] != test_case:

   251         test_cases.append(test_case)

   252         self.assertEqual(1, test_cases.count(test_case),

   253                          'Test case %s is not grouped together in %s' %

   254                          (test_case, tests))

   256   def testShuffleDoesNotInterleaveTestCases(self):

   257     self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)

   258     self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)

   259     self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)

   260     self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)

   262   def testShuffleRestoresOrderAfterEachIteration(self):

   263     # Get the test lists in all 3 iterations, using random seed 1, 2,

   264     # and 3 respectively.  Google Test picks a different seed in each

   265     # iteration, and this test depends on the current implementation

   266     # picking successive numbers.  This dependency is not ideal, but

   267     # makes the test much easier to write.

   268     [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (

   269         GetTestsForAllIterations(

   270             {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))

   272     # Make sure running the tests with random seed 1 gets the same

   273     # order as in iteration 1 above.

   274     [tests_with_seed1] = GetTestsForAllIterations(

   275         {}, [ShuffleFlag(), RandomSeedFlag(1)])

   276     self.assertEqual(tests_in_iteration1, tests_with_seed1)

   278     # Make sure running the tests with random seed 2 gets the same

   279     # order as in iteration 2 above.  Success means that Google Test

   280     # correctly restores the test order before re-shuffling at the

   281     # beginning of iteration 2.

   282     [tests_with_seed2] = GetTestsForAllIterations(

   283         {}, [ShuffleFlag(), RandomSeedFlag(2)])

   284     self.assertEqual(tests_in_iteration2, tests_with_seed2)

   286     # Make sure running the tests with random seed 3 gets the same

   287     # order as in iteration 3 above.  Success means that Google Test

   288     # correctly restores the test order before re-shuffling at the

   289     # beginning of iteration 3.

   290     [tests_with_seed3] = GetTestsForAllIterations(

   291         {}, [ShuffleFlag(), RandomSeedFlag(3)])

   292     self.assertEqual(tests_in_iteration3, tests_with_seed3)

   294   def testShuffleGeneratesNewOrderInEachIteration(self):

   295     [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (

   296         GetTestsForAllIterations(

   297             {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))

   299     self.assert_(tests_in_iteration1 != tests_in_iteration2,

   300                  tests_in_iteration1)

   301     self.assert_(tests_in_iteration1 != tests_in_iteration3,

   302                  tests_in_iteration1)

   303     self.assert_(tests_in_iteration2 != tests_in_iteration3,

   304                  tests_in_iteration2)

   306   def testShuffleShardedTestsPreservesPartition(self):

   307     # If we run M tests on N shards, the same M tests should be run in

   308     # total, regardless of the random seeds used by the shards.

   309     [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',

   310                                          SHARD_INDEX_ENV_VAR: '0'},

   311                                         [ShuffleFlag(), RandomSeedFlag(1)])

   312     [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',

   313                                          SHARD_INDEX_ENV_VAR: '1'},

   314                                         [ShuffleFlag(), RandomSeedFlag(20)])

   315     [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',

   316                                          SHARD_INDEX_ENV_VAR: '2'},

   317                                         [ShuffleFlag(), RandomSeedFlag(25)])

   318     sorted_sharded_tests = tests1 + tests2 + tests3

   319     sorted_sharded_tests.sort()

   320     sorted_active_tests = []

   321     sorted_active_tests.extend(ACTIVE_TESTS)

   322     sorted_active_tests.sort()

   323     self.assertEqual(sorted_active_tests, sorted_sharded_tests)

   325 if __name__ == '__main__':

   326   gtest_test_utils.Main()

The Tor Browser / file revision

media/webrtc/trunk/testing/gtest/test/gtest_shuffle_test.py@4ab42b5ab56c

media/webrtc/trunk/testing/gtest/test/gtest_shuffle_test.py