mirror of
https://github.com/django/django.git
synced 2025-10-24 06:06:09 +00:00
Fixed #17001 -- Custom querysets for prefetch_related.
This patch introduces the Prefetch object which allows customizing prefetch operations. This enables things like filtering prefetched relations, calling select_related from a prefetched relation, or prefetching the same relation multiple times with different querysets. When a Prefetch instance specifies a to_attr argument, the result is stored in a list rather than a QuerySet. This has the fortunate consequence of being significantly faster. The preformance improvement is due to the fact that we save the costly creation of a QuerySet instance. Thanks @akaariai for the original patch and @bmispelon and @timgraham for the reviews.
This commit is contained in:
Loic Bistuer
committed by
Anssi Kääriäinen
Anssi Kääriäinen
parent
b1b04df065
commit
f51c1f5900
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db import connection
|
||||
from django.db.models import Prefetch
|
||||
from django.test import TestCase
|
||||
from django.test.utils import override_settings
|
||||
from django.utils import six
|
||||
@@ -13,9 +14,7 @@ from .models import (Author, Book, Reader, Qualification, Teacher, Department,
|
||||
|
||||
|
||||
class PrefetchRelatedTests(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
self.book1 = Book.objects.create(title="Poems")
|
||||
self.book2 = Book.objects.create(title="Jane Eyre")
|
||||
self.book3 = Book.objects.create(title="Wuthering Heights")
|
||||
@@ -207,6 +206,292 @@ class PrefetchRelatedTests(TestCase):
|
||||
self.assertTrue("name" in str(cm.exception))
|
||||
|
||||
|
||||
class CustomPrefetchTests(TestCase):
|
||||
@classmethod
|
||||
def traverse_qs(cls, obj_iter, path):
|
||||
"""
|
||||
Helper method that returns a list containing a list of the objects in the
|
||||
obj_iter. Then for each object in the obj_iter, the path will be
|
||||
recursively travelled and the found objects are added to the return value.
|
||||
"""
|
||||
ret_val = []
|
||||
|
||||
if hasattr(obj_iter, 'all'):
|
||||
obj_iter = obj_iter.all()
|
||||
|
||||
try:
|
||||
iter(obj_iter)
|
||||
except TypeError:
|
||||
obj_iter = [obj_iter]
|
||||
|
||||
for obj in obj_iter:
|
||||
rel_objs = []
|
||||
for part in path:
|
||||
if not part:
|
||||
continue
|
||||
rel_objs.extend(cls.traverse_qs(getattr(obj, part[0]), [part[1:]]))
|
||||
ret_val.append((obj, rel_objs))
|
||||
return ret_val
|
||||
|
||||
def setUp(self):
|
||||
self.person1 = Person.objects.create(name="Joe")
|
||||
self.person2 = Person.objects.create(name="Mary")
|
||||
self.house1 = House.objects.create(address="123 Main St", owner=self.person1)
|
||||
self.house2 = House.objects.create(address="45 Side St", owner=self.person1)
|
||||
self.house3 = House.objects.create(address="6 Downing St", owner=self.person2)
|
||||
self.house4 = House.objects.create(address="7 Regents St", owner=self.person2)
|
||||
self.room1_1 = Room.objects.create(name="Dining room", house=self.house1)
|
||||
self.room1_2 = Room.objects.create(name="Lounge", house=self.house1)
|
||||
self.room1_3 = Room.objects.create(name="Kitchen", house=self.house1)
|
||||
self.room2_1 = Room.objects.create(name="Dining room", house=self.house2)
|
||||
self.room2_2 = Room.objects.create(name="Lounge", house=self.house2)
|
||||
self.room2_3 = Room.objects.create(name="Kitchen", house=self.house2)
|
||||
self.room3_1 = Room.objects.create(name="Dining room", house=self.house3)
|
||||
self.room3_2 = Room.objects.create(name="Lounge", house=self.house3)
|
||||
self.room3_3 = Room.objects.create(name="Kitchen", house=self.house3)
|
||||
self.room4_1 = Room.objects.create(name="Dining room", house=self.house4)
|
||||
self.room4_2 = Room.objects.create(name="Lounge", house=self.house4)
|
||||
self.room4_3 = Room.objects.create(name="Kitchen", house=self.house4)
|
||||
self.person1.houses.add(self.house1, self.house2)
|
||||
self.person2.houses.add(self.house3, self.house4)
|
||||
|
||||
def test_traverse_qs(self):
|
||||
qs = Person.objects.prefetch_related('houses')
|
||||
related_objs_normal = [list(p.houses.all()) for p in qs],
|
||||
related_objs_from_traverse = [[inner[0] for inner in o[1]]
|
||||
for o in self.traverse_qs(qs, [['houses']])]
|
||||
self.assertEqual(related_objs_normal, (related_objs_from_traverse,))
|
||||
|
||||
def test_ambiguous(self):
|
||||
# Ambiguous.
|
||||
with self.assertRaises(ValueError):
|
||||
self.traverse_qs(
|
||||
Person.objects.prefetch_related('houses__rooms', Prefetch('houses', queryset=House.objects.all())),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
|
||||
with self.assertRaises(AttributeError):
|
||||
self.traverse_qs(
|
||||
Person.objects.prefetch_related('houses_list__rooms', Prefetch('houses', queryset=House.objects.all(), to_attr='houses_lst')),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
|
||||
# Not ambiguous.
|
||||
self.traverse_qs(
|
||||
Person.objects.prefetch_related('houses__rooms', 'houses'),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
|
||||
self.traverse_qs(
|
||||
Person.objects.prefetch_related('houses__rooms', Prefetch('houses', queryset=House.objects.all(), to_attr='houses_lst')),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
|
||||
def test_m2m(self):
|
||||
# Control lookups.
|
||||
with self.assertNumQueries(2):
|
||||
lst1 = self.traverse_qs(
|
||||
Person.objects.prefetch_related('houses'),
|
||||
[['houses']]
|
||||
)
|
||||
|
||||
# Test lookups.
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = self.traverse_qs(
|
||||
Person.objects.prefetch_related(Prefetch('houses')),
|
||||
[['houses']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = self.traverse_qs(
|
||||
Person.objects.prefetch_related(Prefetch('houses', to_attr='houses_lst')),
|
||||
[['houses_lst']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
|
||||
def test_reverse_m2m(self):
|
||||
# Control lookups.
|
||||
with self.assertNumQueries(2):
|
||||
lst1 = self.traverse_qs(
|
||||
House.objects.prefetch_related('occupants'),
|
||||
[['occupants']]
|
||||
)
|
||||
|
||||
# Test lookups.
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = self.traverse_qs(
|
||||
House.objects.prefetch_related(Prefetch('occupants')),
|
||||
[['occupants']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = self.traverse_qs(
|
||||
House.objects.prefetch_related(Prefetch('occupants', to_attr='occupants_lst')),
|
||||
[['occupants_lst']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
|
||||
def test_m2m_through_fk(self):
|
||||
# Control lookups.
|
||||
with self.assertNumQueries(3):
|
||||
lst1 = self.traverse_qs(
|
||||
Room.objects.prefetch_related('house__occupants'),
|
||||
[['house', 'occupants']]
|
||||
)
|
||||
|
||||
# Test lookups.
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
Room.objects.prefetch_related(Prefetch('house__occupants')),
|
||||
[['house', 'occupants']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
Room.objects.prefetch_related(Prefetch('house__occupants', to_attr='occupants_lst')),
|
||||
[['house', 'occupants_lst']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
|
||||
def test_m2m_through_gfk(self):
|
||||
TaggedItem.objects.create(tag="houses", content_object=self.house1)
|
||||
TaggedItem.objects.create(tag="houses", content_object=self.house2)
|
||||
|
||||
# Control lookups.
|
||||
with self.assertNumQueries(3):
|
||||
lst1 = self.traverse_qs(
|
||||
TaggedItem.objects.filter(tag='houses').prefetch_related('content_object__rooms'),
|
||||
[['content_object', 'rooms']]
|
||||
)
|
||||
|
||||
# Test lookups.
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
TaggedItem.objects.prefetch_related(
|
||||
Prefetch('content_object'),
|
||||
Prefetch('content_object__rooms', to_attr='rooms_lst')
|
||||
),
|
||||
[['content_object', 'rooms_lst']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
|
||||
def test_o2m_through_m2m(self):
|
||||
# Control lookups.
|
||||
with self.assertNumQueries(3):
|
||||
lst1 = self.traverse_qs(
|
||||
Person.objects.prefetch_related('houses', 'houses__rooms'),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
|
||||
# Test lookups.
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
Person.objects.prefetch_related(Prefetch('houses'), 'houses__rooms'),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
Person.objects.prefetch_related(Prefetch('houses'), Prefetch('houses__rooms')),
|
||||
[['houses', 'rooms']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
Person.objects.prefetch_related(Prefetch('houses', to_attr='houses_lst'), 'houses_lst__rooms'),
|
||||
[['houses_lst', 'rooms']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = self.traverse_qs(
|
||||
Person.objects.prefetch_related(
|
||||
Prefetch('houses', to_attr='houses_lst'),
|
||||
Prefetch('houses_lst__rooms', to_attr='rooms_lst')
|
||||
),
|
||||
[['houses_lst', 'rooms_lst']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
|
||||
def test_generic_rel(self):
|
||||
bookmark = Bookmark.objects.create(url='http://www.djangoproject.com/')
|
||||
TaggedItem.objects.create(content_object=bookmark, tag='django')
|
||||
TaggedItem.objects.create(content_object=bookmark, favorite=bookmark, tag='python')
|
||||
|
||||
# Control lookups.
|
||||
with self.assertNumQueries(4):
|
||||
lst1 = self.traverse_qs(
|
||||
Bookmark.objects.prefetch_related('tags', 'tags__content_object', 'favorite_tags'),
|
||||
[['tags', 'content_object'], ['favorite_tags']]
|
||||
)
|
||||
|
||||
# Test lookups.
|
||||
with self.assertNumQueries(4):
|
||||
lst2 = self.traverse_qs(
|
||||
Bookmark.objects.prefetch_related(
|
||||
Prefetch('tags', to_attr='tags_lst'),
|
||||
Prefetch('tags_lst__content_object'),
|
||||
Prefetch('favorite_tags'),
|
||||
),
|
||||
[['tags_lst', 'content_object'], ['favorite_tags']]
|
||||
)
|
||||
self.assertEqual(lst1, lst2)
|
||||
|
||||
def test_custom_qs(self):
|
||||
# Test basic.
|
||||
with self.assertNumQueries(2):
|
||||
lst1 = list(Person.objects.prefetch_related('houses'))
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = list(Person.objects.prefetch_related(
|
||||
Prefetch('houses', queryset=House.objects.all(), to_attr='houses_lst')))
|
||||
self.assertEqual(
|
||||
self.traverse_qs(lst1, [['houses']]),
|
||||
self.traverse_qs(lst2, [['houses_lst']])
|
||||
)
|
||||
|
||||
# Test queryset filtering.
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = list(Person.objects.prefetch_related(
|
||||
Prefetch('houses', queryset=House.objects.filter(pk__in=[self.house1.pk, self.house3.pk]), to_attr='houses_lst')))
|
||||
self.assertEqual(len(lst2[0].houses_lst), 1)
|
||||
self.assertEqual(lst2[0].houses_lst[0], self.house1)
|
||||
self.assertEqual(len(lst2[1].houses_lst), 1)
|
||||
self.assertEqual(lst2[1].houses_lst[0], self.house3)
|
||||
|
||||
# Test flattened.
|
||||
with self.assertNumQueries(3):
|
||||
lst1 = list(Person.objects.prefetch_related('houses__rooms'))
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = list(Person.objects.prefetch_related(
|
||||
Prefetch('houses__rooms', queryset=Room.objects.all(), to_attr='rooms_lst')))
|
||||
self.assertEqual(
|
||||
self.traverse_qs(lst1, [['houses', 'rooms']]),
|
||||
self.traverse_qs(lst2, [['houses', 'rooms_lst']])
|
||||
)
|
||||
|
||||
# Test inner select_related.
|
||||
with self.assertNumQueries(3):
|
||||
lst1 = list(Person.objects.prefetch_related('houses__owner'))
|
||||
with self.assertNumQueries(2):
|
||||
lst2 = list(Person.objects.prefetch_related(
|
||||
Prefetch('houses', queryset=House.objects.select_related('owner'))))
|
||||
self.assertEqual(
|
||||
self.traverse_qs(lst1, [['houses', 'owner']]),
|
||||
self.traverse_qs(lst2, [['houses', 'owner']])
|
||||
)
|
||||
|
||||
# Test inner prefetch.
|
||||
inner_rooms_qs = Room.objects.filter(pk__in=[self.room1_1.pk, self.room1_2.pk])
|
||||
houses_qs_prf = House.objects.prefetch_related(
|
||||
Prefetch('rooms', queryset=inner_rooms_qs, to_attr='rooms_lst'))
|
||||
with self.assertNumQueries(3):
|
||||
lst2 = list(Person.objects.prefetch_related(
|
||||
Prefetch('houses', queryset=houses_qs_prf.filter(pk=self.house1.pk), to_attr='houses_lst')))
|
||||
|
||||
self.assertEqual(len(lst2[0].houses_lst[0].rooms_lst), 2)
|
||||
self.assertEqual(lst2[0].houses_lst[0].rooms_lst[0], self.room1_1)
|
||||
self.assertEqual(lst2[0].houses_lst[0].rooms_lst[1], self.room1_2)
|
||||
self.assertEqual(len(lst2[1].houses_lst), 0)
|
||||
|
||||
class DefaultManagerTests(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -627,6 +912,45 @@ class MultiDbTests(TestCase):
|
||||
|
||||
self.assertEqual(ages, "50, 49")
|
||||
|
||||
def test_using_is_honored_custom_qs(self):
|
||||
B = Book.objects.using('other')
|
||||
A = Author.objects.using('other')
|
||||
book1 = B.create(title="Poems")
|
||||
book2 = B.create(title="Sense and Sensibility")
|
||||
|
||||
A.create(name="Charlotte Bronte", first_book=book1)
|
||||
A.create(name="Jane Austen", first_book=book2)
|
||||
|
||||
# Implicit hinting
|
||||
with self.assertNumQueries(2, using='other'):
|
||||
prefetch = Prefetch('first_time_authors', queryset=Author.objects.all())
|
||||
books = "".join("%s (%s)\n" %
|
||||
(b.title, ", ".join(a.name for a in b.first_time_authors.all()))
|
||||
for b in B.prefetch_related(prefetch))
|
||||
self.assertEqual(books,
|
||||
"Poems (Charlotte Bronte)\n"
|
||||
"Sense and Sensibility (Jane Austen)\n")
|
||||
|
||||
# Explicit using on the same db.
|
||||
with self.assertNumQueries(2, using='other'):
|
||||
prefetch = Prefetch('first_time_authors', queryset=Author.objects.using('other'))
|
||||
books = "".join("%s (%s)\n" %
|
||||
(b.title, ", ".join(a.name for a in b.first_time_authors.all()))
|
||||
for b in B.prefetch_related(prefetch))
|
||||
self.assertEqual(books,
|
||||
"Poems (Charlotte Bronte)\n"
|
||||
"Sense and Sensibility (Jane Austen)\n")
|
||||
|
||||
# Explicit using on a different db.
|
||||
with self.assertNumQueries(1, using='default'), self.assertNumQueries(1, using='other'):
|
||||
prefetch = Prefetch('first_time_authors', queryset=Author.objects.using('default'))
|
||||
books = "".join("%s (%s)\n" %
|
||||
(b.title, ", ".join(a.name for a in b.first_time_authors.all()))
|
||||
for b in B.prefetch_related(prefetch))
|
||||
self.assertEqual(books,
|
||||
"Poems ()\n"
|
||||
"Sense and Sensibility ()\n")
|
||||
|
||||
|
||||
class Ticket19607Tests(TestCase):
|
||||
|
||||
|
||||
Reference in New Issue
Block a user