From 5848b3a1d70f09aaf598d794569ad758183f41db Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Fri, 9 Jul 2021 16:12:17 -0400
Subject: [PATCH] Fixed #32914 -- Prevented test --shuffle from skipping test
methods.
"test --shuffle" skipped test methods when test classes were mixed.
This changes runner.py's reorder_tests() to group by TestCase class.
Regression in 90ba716bf060ee7fef79dc230b0b20644839069f.
---
django/test/runner.py | 25 +++++++++++++++++--------
tests/test_runner/tests.py | 26 ++++++++++++++++++++++++++
2 files changed, 43 insertions(+), 8 deletions(-)
diff --git a/django/test/runner.py b/django/test/runner.py
index b22b8c3bf2..3846b3be01 100644
--- a/django/test/runner.py
+++ b/django/test/runner.py
@@ -11,6 +11,7 @@ import random
import sys
import textwrap
import unittest
+from collections import defaultdict
from contextlib import contextmanager
from importlib import import_module
from io import StringIO
@@ -986,21 +987,27 @@ def reorder_test_bin(tests, shuffler=None, reverse=False):
def reorder_tests(tests, classes, reverse=False, shuffler=None):
"""
- Reorder an iterable of tests by test type, removing any duplicates.
+ Reorder an iterable of tests, grouping by the given TestCase classes.
+
+ This function also removes any duplicates and reorders so that tests of the
+ same type are consecutive.
The result is returned as an iterator. `classes` is a sequence of types.
- All tests of type classes[0] are placed first, then tests of type
- classes[1], etc. Tests with no match in classes are placed last.
+ Tests that are instances of `classes[0]` are grouped first, followed by
+ instances of `classes[1]`, etc. Tests that are not instances of any of the
+ classes are grouped last.
- If `reverse` is True, sort tests within classes in opposite order but
- don't reverse test classes.
+ If `reverse` is True, the tests within each `classes` group are reversed,
+ but without reversing the order of `classes` itself.
The `shuffler` argument is an optional instance of this module's `Shuffler`
class. If provided, tests will be shuffled within each `classes` group, but
keeping tests with other tests of their TestCase class. Reversing is
applied after shuffling to allow reversing the same random order.
"""
- bins = [OrderedSet() for i in range(len(classes) + 1)]
+ # Each bin maps TestCase class to OrderedSet of tests. This permits tests
+ # to be grouped by TestCase class even if provided non-consecutively.
+ bins = [defaultdict(OrderedSet) for i in range(len(classes) + 1)]
*class_bins, last_bin = bins
for test in tests:
@@ -1009,9 +1016,11 @@ def reorder_tests(tests, classes, reverse=False, shuffler=None):
break
else:
test_bin = last_bin
- test_bin.add(test)
+ test_bin[type(test)].add(test)
- for tests in bins:
+ for test_bin in bins:
+ # Call list() since reorder_test_bin()'s input must support reversed().
+ tests = list(itertools.chain.from_iterable(test_bin.values()))
yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse)
diff --git a/tests/test_runner/tests.py b/tests/test_runner/tests.py
index 77a756a1de..3a535f3e1e 100644
--- a/tests/test_runner/tests.py
+++ b/tests/test_runner/tests.py
@@ -181,6 +181,19 @@ class TestSuiteTests(SimpleTestCase):
'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1',
])
+ def test_reorder_tests_same_type_consecutive(self):
+ """Tests of the same type are made consecutive."""
+ tests = self.make_tests()
+ # Move the last item to the front.
+ tests.insert(0, tests.pop())
+ self.assertTestNames(tests, expected=[
+ 'Tests2.test2', 'Tests1.test1', 'Tests1.test2', 'Tests2.test1',
+ ])
+ reordered_tests = reorder_tests(tests, classes=[])
+ self.assertTestNames(reordered_tests, expected=[
+ 'Tests2.test2', 'Tests2.test1', 'Tests1.test1', 'Tests1.test2',
+ ])
+
def test_reorder_tests_random(self):
tests = self.make_tests()
# Choose a seed that shuffles both the classes and methods.
@@ -191,6 +204,19 @@ class TestSuiteTests(SimpleTestCase):
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
])
+ def test_reorder_tests_random_mixed_classes(self):
+ tests = self.make_tests()
+ # Move the last item to the front.
+ tests.insert(0, tests.pop())
+ shuffler = Shuffler(seed=9)
+ self.assertTestNames(tests, expected=[
+ 'Tests2.test2', 'Tests1.test1', 'Tests1.test2', 'Tests2.test1',
+ ])
+ reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
+ self.assertTestNames(reordered_tests, expected=[
+ 'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+ ])
+
def test_reorder_tests_reverse_with_duplicates(self):
class Tests1(unittest.TestCase):
def test1(self):