From 870b0a1f8689aa8fd4806623262228b8ac3a88a5 Mon Sep 17 00:00:00 2001
From: Loic Bistuer <loic.bistuer@gmail.com>
Date: Mon, 19 May 2014 11:14:55 +0700
Subject: [PATCH] Fixed the ordering of prefetch lookups so that latter lookups
can refer to former lookups.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Thanks Anssi Kääriäinen and Tim Graham for the reviews. Refs #17001 and #22650.
---
django/db/models/query.py | 36 +++++++++++++++------------------
tests/prefetch_related/tests.py | 9 ++++++---
2 files changed, 22 insertions(+), 23 deletions(-)
diff --git a/django/db/models/query.py b/django/db/models/query.py
index 53f87e80e5..f2defa2100 100644
--- a/django/db/models/query.py
+++ b/django/db/models/query.py
@@ -2,8 +2,8 @@
The main QuerySet implementation. This provides the public API for the ORM.
"""
+from collections import deque
import copy
-import itertools
import sys
from django.conf import settings
@@ -1680,6 +1680,9 @@ class Prefetch(object):
return self.prefetch_to == other.prefetch_to
return False
+ def __hash__(self):
+ return hash(self.__class__) ^ hash(self.prefetch_to)
+
def normalize_prefetch_lookups(lookups, prefix=None):
"""
@@ -1713,11 +1716,12 @@ def prefetch_related_objects(result_cache, related_lookups):
# ensure we don't do duplicate work.
done_queries = {} # dictionary of things like 'foo__bar': [results]
- auto_lookups = [] # we add to this as we go through.
+ auto_lookups = set() # we add to this as we go through.
followed_descriptors = set() # recursion protection
- all_lookups = itertools.chain(related_lookups, auto_lookups)
- for lookup in all_lookups:
+ all_lookups = deque(related_lookups)
+ while all_lookups:
+ lookup = all_lookups.popleft()
if lookup.prefetch_to in done_queries:
if lookup.queryset:
raise ValueError("'%s' lookup was already seen with a different queryset. "
@@ -1788,7 +1792,9 @@ def prefetch_related_objects(result_cache, related_lookups):
# the new lookups from relationships we've seen already.
if not (lookup in auto_lookups and descriptor in followed_descriptors):
done_queries[prefetch_to] = obj_list
- auto_lookups.extend(normalize_prefetch_lookups(additional_lookups, prefetch_to))
+ new_lookups = normalize_prefetch_lookups(additional_lookups, prefetch_to)
+ auto_lookups.update(new_lookups)
+ all_lookups.extendleft(new_lookups)
followed_descriptors.add(descriptor)
else:
# Either a singly related object that has already been fetched
@@ -1827,7 +1833,6 @@ def get_prefetcher(instance, attr):
a boolean that is True if the attribute has already been fetched)
"""
prefetcher = None
- attr_found = False
is_fetched = False
# For singly related objects, we have to avoid getting the attribute
@@ -1835,16 +1840,7 @@ def get_prefetcher(instance, attr):
# on the class, in order to get the descriptor object.
rel_obj_descriptor = getattr(instance.__class__, attr, None)
if rel_obj_descriptor is None:
- try:
- rel_obj = getattr(instance, attr)
- attr_found = True
- # If we are following a lookup path which leads us through a previous
- # fetch from a custom Prefetch then we might end up into a list
- # instead of related qs. This means the objects are already fetched.
- if isinstance(rel_obj, list):
- is_fetched = True
- except AttributeError:
- pass
+ attr_found = hasattr(instance, attr)
else:
attr_found = True
if rel_obj_descriptor:
@@ -1889,10 +1885,10 @@ def prefetch_one_level(instances, prefetcher, lookup, level):
rel_qs, rel_obj_attr, instance_attr, single, cache_name = (
prefetcher.get_prefetch_queryset(instances, lookup.get_current_queryset(level)))
- # We have to handle the possibility that the default manager itself added
- # prefetch_related lookups to the QuerySet we just got back. We don't want to
- # trigger the prefetch_related functionality by evaluating the query.
- # Rather, we need to merge in the prefetch_related lookups.
+ # We have to handle the possibility that the QuerySet we just got back
+ # contains some prefetch_related lookups. We don't want to trigger the
+ # prefetch_related functionality by evaluating the query. Rather, we need
+ # to merge in the prefetch_related lookups.
additional_lookups = getattr(rel_qs, '_prefetch_related_lookups', [])
if additional_lookups:
# Don't need to clone because the manager should have given us a fresh
diff --git a/tests/prefetch_related/tests.py b/tests/prefetch_related/tests.py
index 5aee9519c3..a698cd8395 100644
--- a/tests/prefetch_related/tests.py
+++ b/tests/prefetch_related/tests.py
@@ -195,7 +195,7 @@ class PrefetchRelatedTests(TestCase):
def test_reverse_one_to_one_then_m2m(self):
"""
Test that we can follow a m2m relation after going through
- the select_related reverse of a o2o.
+ the select_related reverse of an o2o.
"""
qs = Author.objects.prefetch_related('bio__books').select_related('bio')
@@ -559,13 +559,16 @@ class CustomPrefetchTests(TestCase):
inner_rooms_qs = Room.objects.filter(pk__in=[self.room1_1.pk, self.room1_2.pk])
houses_qs_prf = House.objects.prefetch_related(
Prefetch('rooms', queryset=inner_rooms_qs, to_attr='rooms_lst'))
- with self.assertNumQueries(3):
+ with self.assertNumQueries(4):
lst2 = list(Person.objects.prefetch_related(
- Prefetch('houses', queryset=houses_qs_prf.filter(pk=self.house1.pk), to_attr='houses_lst')))
+ Prefetch('houses', queryset=houses_qs_prf.filter(pk=self.house1.pk), to_attr='houses_lst'),
+ Prefetch('houses_lst__rooms_lst__main_room_of')
+ ))
self.assertEqual(len(lst2[0].houses_lst[0].rooms_lst), 2)
self.assertEqual(lst2[0].houses_lst[0].rooms_lst[0], self.room1_1)
self.assertEqual(lst2[0].houses_lst[0].rooms_lst[1], self.room1_2)
+ self.assertEqual(lst2[0].houses_lst[0].rooms_lst[0].main_room_of, self.house1)
self.assertEqual(len(lst2[1].houses_lst), 0)
# Test ReverseSingleRelatedObjectDescriptor.