id	summary	reporter	owner	description	type	status	component	version	severity	resolution	keywords	cc	stage	has_patch	needs_docs	needs_tests	needs_better_patch	easy	ui_ux
36282	Prefetched relations are not used from parent classes in all cases	Take Weiland	Take Weiland	"When using `prefetch_related` to prefetch relations, Django makes some attempt to use prefetched data present in parent models, but it is very limited. It is only implemented for `ForeignKey` and only works for the immediate parent.

Assuming this model hierarchy:

{{{
class Related(models.Model):
    pass


class GrandParent(models.Model):
    name = models.CharField(max_length=50)
    gp_fk = models.ForeignKey(Related, null=True, on_delete=models.CASCADE, related_name='gp_fk_rel')
    gp_m2m = models.ManyToManyField(Related, related_name='gp_m2m_rel')


class Parent(GrandParent):
    pass


class Child(Parent):
    pass
}}}

I have written these test cases:

{{{
class PrefetchRelatedWorksWithInheritance(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.related1 = Related.objects.create()
        cls.related2 = Related.objects.create()
        cls.related3 = Related.objects.create()

        cls.child = Child.objects.create(
            gp_fk=cls.related1,
        )
        cls.m2m_child = Child.objects.create()
        cls.m2m_child.gp_m2m.set([cls.related1, cls.related2, cls.related3])

    def test_parent_fk_available_in_child(self):
        qs = GrandParent.objects.select_related('parent').prefetch_related(
            'gp_fk'
        ).filter(pk=self.child.pk)
        with self.assertNumQueries(2):
            results = list(qs)
            self.assertEqual(len(results), 1)
            # Works, Parent can look into its GrandParent and find the prefetched data
            self.assertEqual(results[0].parent.gp_fk, self.related1)

    def test_grandparent_fk_available_in_child(self):
        qs = GrandParent.objects.select_related('parent', 'parent__child').prefetch_related(
            'gp_fk'
        ).filter(pk=self.child.pk)
        with self.assertNumQueries(2):
            results = list(qs)
            self.assertEqual(len(results), 1)
            # Causes extra query, Child only looks in Parent, not in GrandParent
            self.assertEqual(results[0].parent.child.gp_fk, self.related1)

    def test_parent_m2m_available_in_child(self):
        qs = GrandParent.objects.select_related('parent').prefetch_related(
            'gp_m2m'
        ).filter(pk=self.m2m_child.pk)
        with self.assertNumQueries(2):
            results = list(qs)
            self.assertEqual(len(results), 1)
            # Causes extra query, M2M never looks in its parents
            self.assertEqual(set(results[0].parent.gp_m2m.all()), {self.related1, self.related2, self.related3})

    def test_grandparent_m2m_available_in_child(self):
        qs = GrandParent.objects.select_related('parent', 'parent__child').prefetch_related(
            'gp_m2m'
        ).filter(pk=self.m2m_child.pk)
        with self.assertNumQueries(2):
            results = list(qs)
            self.assertEqual(len(results), 1)
            # Causes extra query, M2M never looks in its parents
            self.assertEqual(set(results[0].parent.child.gp_m2m.all()), {self.related1, self.related2, self.related3})

}}}

Only the first of the tests passes.
For ForeignKeys the reason is here: https://github.com/django/django/blob/c3a23aa02faa1cf1d32e43d66858e793cd9ecac4/django/db/models/fields/related_descriptors.py#L229-L240
This only looks into the immediate parent, not any grandparents.

For ManyToManyField the culprit is here: https://github.com/django/django/blob/c3a23aa02faa1cf1d32e43d66858e793cd9ecac4/django/db/models/fields/related_descriptors.py#L1094-L1098
It only looks in the instance's _prefetched_objects_cache, when it could walk up to its parents, just like ForwardManyToOneDescriptor does.

I stumbled upon this when implementing support for [https://github.com/jazzband/django-model-utils/pull/639 prefetch_related in Django-Model-Utils' InheritanceManager]. It automatically returns subclasses form a custom QuerySet iterable. But those subclass-instances then do not see the prefetched data of their parent, so in my fix I had to manually copy the caches, so that Django would find them.

I would be willing to work on a patch to make this work out of the box in Django."	Bug	closed	Database layer (models, ORM)	5.1	Normal	fixed	mti prefetch_related		Ready for checkin	1	0	0	0	0	0