Refactored qs.add_q() and utils/tree.py

The sql/query.py add_q method did a lot of where/having tree hacking to
get complex queries to work correctly. The logic was refactored so that
it should be simpler to understand. The new logic should also produce
leaner WHERE conditions.

The changes cascade somewhat, as some other parts of Django (like
add_filter() and WhereNode) expect boolean trees in certain format or
they fail to work. So to fix the add_q() one must fix utils/tree.py,
some things in add_filter(), WhereNode and so on.

This commit also fixed add_filter to see negate clauses up the path.
A query like .exclude(Q(reversefk__in=a_list)) didn't work similarly to
.filter(~Q(reversefk__in=a_list)). The reason for this is that only
the immediate parent negate clauses were seen by add_filter, and thus a
tree like AND: (NOT AND: (AND: condition)) will not be handled
correctly, as there is one intermediary AND node in the tree. The
example tree is generated by .exclude(~Q(reversefk__in=a_list)).

Still, aggregation lost connectors in OR cases, and F() objects and
aggregates in same filter clause caused GROUP BY problems on some
databases.

Fixed #17600, fixed #13198, fixed #17025, fixed #17000, fixed #11293.

tests/aggregation_regress/tests.py

@@ -10,6 +10,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.db.models import Count, Max, Avg, Sum, StdDev, Variance, F, Q
 from django.test import TestCase, Approximate, skipUnlessDBFeature
 from django.utils import six
+from django.utils.unittest import expectedFailure
 
 from .models import (Author, Book, Publisher, Clues, Entries, HardbackBook,
         ItemTag, WithManualPK)
@@ -472,7 +473,7 @@ class AggregationTests(TestCase):
         # Regression for #15709 - Ensure each group_by field only exists once
         # per query
         qs = Book.objects.values('publisher').annotate(max_pages=Max('pages')).order_by()
-        grouping, gb_params = qs.query.get_compiler(qs.db).get_grouping([])
+        grouping, gb_params = qs.query.get_compiler(qs.db).get_grouping([], [])
         self.assertEqual(len(grouping), 1)
 
     def test_duplicate_alias(self):
@@ -847,14 +848,14 @@ class AggregationTests(TestCase):
 
         # The name of the explicitly provided annotation name in this case
         # poses no problem
-        qs = Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2)
+        qs = Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2).order_by('name')
         self.assertQuerysetEqual(
             qs,
             ['Peter Norvig'],
             lambda b: b.name
         )
         # Neither in this case
-        qs = Author.objects.annotate(book_count=Count('book')).filter(book_count=2)
+        qs = Author.objects.annotate(book_count=Count('book')).filter(book_count=2).order_by('name')
         self.assertQuerysetEqual(
             qs,
             ['Peter Norvig'],
@@ -862,7 +863,7 @@ class AggregationTests(TestCase):
         )
         # This case used to fail because the ORM couldn't resolve the
         # automatically generated annotation name `book__count`
-        qs = Author.objects.annotate(Count('book')).filter(book__count=2)
+        qs = Author.objects.annotate(Count('book')).filter(book__count=2).order_by('name')
         self.assertQuerysetEqual(
             qs,
             ['Peter Norvig'],
@@ -1020,3 +1021,83 @@ class AggregationTests(TestCase):
                 ('The Definitive Guide to Django: Web Development Done Right', 0)
             ]
         )
+
+    def test_negated_aggregation(self):
+        expected_results = Author.objects.exclude(
+            pk__in=Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2)
+        ).order_by('name')
+        expected_results = [a.name for a in expected_results]
+        qs = Author.objects.annotate(book_cnt=Count('book')).exclude(
+            Q(book_cnt=2), Q(book_cnt=2)).order_by('name')
+        self.assertQuerysetEqual(
+            qs,
+            expected_results,
+            lambda b: b.name
+        )
+        expected_results = Author.objects.exclude(
+            pk__in=Author.objects.annotate(book_cnt=Count('book')).filter(book_cnt=2)
+        ).order_by('name')
+        expected_results = [a.name for a in expected_results]
+        qs = Author.objects.annotate(book_cnt=Count('book')).exclude(Q(book_cnt=2)|Q(book_cnt=2)).order_by('name')
+        self.assertQuerysetEqual(
+            qs,
+            expected_results,
+            lambda b: b.name
+        )
+
+    def test_name_filters(self):
+        qs = Author.objects.annotate(Count('book')).filter(
+            Q(book__count__exact=2)|Q(name='Adrian Holovaty')
+        ).order_by('name')
+        self.assertQuerysetEqual(
+            qs,
+            ['Adrian Holovaty', 'Peter Norvig'],
+            lambda b: b.name
+        )
+
+    def test_name_expressions(self):
+        # Test that aggregates are spotted corretly from F objects.
+        # Note that Adrian's age is 34 in the fixtures, and he has one book
+        # so both conditions match one author.
+        qs = Author.objects.annotate(Count('book')).filter(
+            Q(name='Peter Norvig')|Q(age=F('book__count') + 33)
+        ).order_by('name')
+        self.assertQuerysetEqual(
+            qs,
+            ['Adrian Holovaty', 'Peter Norvig'],
+            lambda b: b.name
+        )
+
+    def test_ticket_11293(self):
+        q1 = Q(price__gt=50)
+        q2 = Q(authors__count__gt=1)
+        query = Book.objects.annotate(Count('authors')).filter(
+            q1 | q2).order_by('pk')
+        self.assertQuerysetEqual(
+            query, [1, 4, 5, 6],
+            lambda b: b.pk)
+
+    def test_ticket_11293_q_immutable(self):
+        """
+        Check that splitting a q object to parts for where/having doesn't alter
+        the original q-object.
+        """
+        q1 = Q(isbn='')
+        q2 = Q(authors__count__gt=1)
+        query = Book.objects.annotate(Count('authors'))
+        query.filter(q1 | q2)
+        self.assertEqual(len(q2.children), 1)
+
+    def test_fobj_group_by(self):
+        """
+        Check that an F() object referring to related column works correctly
+        in group by.
+        """
+        qs = Book.objects.annotate(
+            acount=Count('authors')
+        ).filter(
+            acount=F('publisher__num_awards')
+        )
+        self.assertQuerysetEqual(
+            qs, ['Sams Teach Yourself Django in 24 Hours'],
+            lambda b: b.name)