Fixed #21169 -- Reworked RelatedManager methods use default filtering

The `remove()` and `clear()` methods of the related managers created by
`ForeignKey`, `GenericForeignKey`, and `ManyToManyField` suffered from a
number of issues. Some operations ran multiple data modifying queries without
wrapping them in a transaction, and some operations didn't respect default
filtering when it was present (i.e. when the default manager on the related
model implemented a custom `get_queryset()`).

Fixing the issues introduced some backward incompatible changes:

- The implementation of `remove()` for `ForeignKey` related managers changed
  from a series of `Model.save()` calls to a single `QuerySet.update()` call.
  The change means that `pre_save` and `post_save` signals aren't called anymore.

- The `remove()` and `clear()` methods for `GenericForeignKey` related
  managers now perform bulk delete so `Model.delete()` isn't called anymore.

- The `remove()` and `clear()` methods for `ManyToManyField` related
  managers perform nested queries when filtering is involved, which may
  or may not be an issue depending on the database and the data itself.

Refs. #3871, #21174.

Thanks Anssi Kääriäinen and Tim Graham for the reviews.

django/db/models/fields/related.py

@@ -2,7 +2,7 @@ from operator import attrgetter
 
 from django.db import connection, connections, router, transaction
 from django.db.backends import utils
-from django.db.models import signals
+from django.db.models import signals, Q
 from django.db.models.fields import (AutoField, Field, IntegerField,
     PositiveIntegerField, PositiveSmallIntegerField, FieldDoesNotExist)
 from django.db.models.related import RelatedObject, PathInfo
@@ -464,14 +464,21 @@ def create_foreign_related_manager(superclass, rel_field, rel_model):
         # remove() and clear() are only provided if the ForeignKey can have a value of null.
         if rel_field.null:
             def remove(self, *objs):
+                # If there aren't any objects, there is nothing to do.
+                if not objs:
+                    return
+
                 val = rel_field.get_foreign_related_value(self.instance)
+
+                old_ids = set()
                 for obj in objs:
                     # Is obj actually part of this descriptor set?
                     if rel_field.get_local_related_value(obj) == val:
-                        setattr(obj, rel_field.name, None)
-                        obj.save()
+                        old_ids.add(obj.pk)
                     else:
                         raise rel_field.rel.to.DoesNotExist("%r is not related to %r." % (obj, self.instance))
+
+                self.filter(pk__in=old_ids).update(**{rel_field.name: None})
             remove.alters_data = True
 
             def clear(self):
@@ -536,6 +543,7 @@ def create_many_related_manager(superclass, rel):
             self.instance = instance
             self.symmetrical = symmetrical
             self.source_field = source_field
+            self.target_field = through._meta.get_field(target_field_name)
             self.source_field_name = source_field_name
             self.target_field_name = target_field_name
             self.reverse = reverse
@@ -572,6 +580,19 @@ def create_many_related_manager(superclass, rel):
             )
         do_not_call_in_templates = True
 
+        def _build_clear_filters(self, qs):
+            filters = Q(**{
+                self.source_field_name: self.related_val,
+                '%s__in' % self.target_field_name: qs
+            })
+
+            if self.symmetrical:
+                filters |= Q(**{
+                    self.target_field_name: self.related_val,
+                    '%s__in' % self.source_field_name: qs
+                })
+            return filters
+
         def get_queryset(self):
             try:
                 return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
@@ -625,18 +646,20 @@ def create_many_related_manager(superclass, rel):
 
             def remove(self, *objs):
                 self._remove_items(self.source_field_name, self.target_field_name, *objs)
-
-                # If this is a symmetrical m2m relation to self, remove the mirror entry in the m2m table
-                if self.symmetrical:
-                    self._remove_items(self.target_field_name, self.source_field_name, *objs)
             remove.alters_data = True
 
         def clear(self):
-            self._clear_items(self.source_field_name)
+            db = router.db_for_write(self.through, instance=self.instance)
 
-            # If this is a symmetrical m2m relation to self, clear the mirror entry in the m2m table
-            if self.symmetrical:
-                self._clear_items(self.target_field_name)
+            signals.m2m_changed.send(sender=self.through, action="pre_clear",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=None, using=db)
+            filters = self._build_clear_filters(self.using(db))
+            self.through._default_manager.using(db).filter(filters).delete()
+
+            signals.m2m_changed.send(sender=self.through, action="post_clear",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=None, using=db)
         clear.alters_data = True
 
         def create(self, **kwargs):
@@ -722,55 +745,33 @@ def create_many_related_manager(superclass, rel):
             # *objs - objects to remove
 
             # If there aren't any objects, there is nothing to do.
-            if objs:
-                # Check that all the objects are of the right type
-                old_ids = set()
-                for obj in objs:
-                    if isinstance(obj, self.model):
-                        fk_val = self.through._meta.get_field(
-                            target_field_name).get_foreign_related_value(obj)[0]
-                        old_ids.add(fk_val)
-                    else:
-                        old_ids.add(obj)
-                # Work out what DB we're operating on
-                db = router.db_for_write(self.through, instance=self.instance)
-                # Send a signal to the other end if need be.
-                if self.reverse or source_field_name == self.source_field_name:
-                    # Don't send the signal when we are deleting the
-                    # duplicate data row for symmetrical reverse entries.
-                    signals.m2m_changed.send(sender=self.through, action="pre_remove",
-                        instance=self.instance, reverse=self.reverse,
-                        model=self.model, pk_set=old_ids, using=db)
-                # Remove the specified objects from the join table
-                self.through._default_manager.using(db).filter(**{
-                    source_field_name: self.related_val[0],
-                    '%s__in' % target_field_name: old_ids
-                }).delete()
-                if self.reverse or source_field_name == self.source_field_name:
-                    # Don't send the signal when we are deleting the
-                    # duplicate data row for symmetrical reverse entries.
-                    signals.m2m_changed.send(sender=self.through, action="post_remove",
-                        instance=self.instance, reverse=self.reverse,
-                        model=self.model, pk_set=old_ids, using=db)
+            if not objs:
+                return
+
+            # Check that all the objects are of the right type
+            old_ids = set()
+            for obj in objs:
+                if isinstance(obj, self.model):
+                    fk_val = self.target_field.get_foreign_related_value(obj)[0]
+                    old_ids.add(fk_val)
+                else:
+                    old_ids.add(obj)
 
-        def _clear_items(self, source_field_name):
             db = router.db_for_write(self.through, instance=self.instance)
-            # source_field_name: the PK colname in join table for the source object
-            if self.reverse or source_field_name == self.source_field_name:
-                # Don't send the signal when we are clearing the
-                # duplicate data rows for symmetrical reverse entries.
-                signals.m2m_changed.send(sender=self.through, action="pre_clear",
-                    instance=self.instance, reverse=self.reverse,
-                    model=self.model, pk_set=None, using=db)
-            self.through._default_manager.using(db).filter(**{
-                source_field_name: self.related_val
-            }).delete()
-            if self.reverse or source_field_name == self.source_field_name:
-                # Don't send the signal when we are clearing the
-                # duplicate data rows for symmetrical reverse entries.
-                signals.m2m_changed.send(sender=self.through, action="post_clear",
-                    instance=self.instance, reverse=self.reverse,
-                    model=self.model, pk_set=None, using=db)
+
+            # Send a signal to the other end if need be.
+            signals.m2m_changed.send(sender=self.through, action="pre_remove",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=old_ids, using=db)
+
+            old_vals_qs = self.using(db).filter(**{
+                '%s__in' % self.target_field.related_field.attname: old_ids})
+            filters = self._build_clear_filters(old_vals_qs)
+            self.through._default_manager.using(db).filter(filters).delete()
+
+            signals.m2m_changed.send(sender=self.through, action="post_remove",
+                instance=self.instance, reverse=self.reverse,
+                model=self.model, pk_set=old_ids, using=db)
 
     return ManyRelatedManager