FieldError when using selected_related on ForeignObject together with defer

[Markus Holtermann]

A queryset Model1.objects.select_related("data", "model2").defer("data__field") where model2 is a ForeignObject (not ForeignKey!), results in django.core.exceptions.FieldError: Field Model1.model2 cannot be both deferred and traversed using select_related at the same time.

# models.py
from django.db import models

class JSONFieldNullable(models.Model):
    json_field = models.JSONField(blank=True, null=True)

    class Meta:
        required_db_features = {"supports_json_field"}


class Model2(models.Model):
    code = models.BigIntegerField(
        primary_key=True, serialize=False, verbose_name="Code", db_column="id"
    )


class Model1(models.Model):
    data = models.ForeignKey(JSONFieldNullable, on_delete=models.CASCADE)
    model2_code = models.CharField(max_length=10)
    model2 = models.ForeignObject(
        Model2,
        from_fields=["model2_code"],
        to_fields=["code"],
        on_delete=models.DO_NOTHING,
        related_name="+",
    )


# tests.py
from django.test import TestCase

from .models import JSONFieldNullable, Model1, Model2


class Tests(TestCase):
    @classmethod
    def setUpTestData(cls):
        data = JSONFieldNullable.objects.create(json_field={"a": "b"})
        model2 = Model2.objects.create(code=123)
        Model1.objects.create(data=data, model2_code="123")

    def test1(self):
        # 1. SELECT ... FROM queries_model1
        #    INNER JOIN queries_jsonfieldnullable
        #    INNER JOIN queries_model2
        with self.assertNumQueries(1):
            queried = [
                (
                    x.id,
                    x.model2_code,
                    x.model2,
                    x.model2.code,
                    x.data.id,
                    x.data.json_field,
                )
                for x in Model1.objects.select_related("data", "model2")
            ]

    def test2(self):
        # 1. SELECT ... FROM queries_model1
        #    INNER JOIN queries_jsonfieldnullable
        #    INNER JOIN queries_model2
        # 2. SELECT ... FROM queries_jsonfieldnullable
        #    WHERE id = ...
        with self.assertNumQueries(2):
            queried = [
                (
                    x.id,
                    x.model2_code,
                    x.model2,
                    x.model2.code,
                    x.data.id,
                    x.data.json_field,
                )
                for x in Model1.objects.select_related("data", "model2").defer(
                    "data__json_field"
                )
            ]

    def test3(self):
        # 1. SELECT ... FROM queries_model1
        #    INNER JOIN queries_jsonfieldnullable
        # 2. SELECT ... FROM queries_model2
        #    WHERE id = ...
        # 3. SELECT ... FROM queries_jsonfieldnullable
        #    WHERE id = ...
        with self.assertNumQueries(3):
            queried = [
                (
                    x.id,
                    x.model2_code,
                    x.model2,
                    x.model2.code,
                    x.data.id,
                    x.data.json_field,
                )
                for x in Model1.objects.select_related("data").defer("data__json_field")
            ]

# Traceback
======================================================================
ERROR: test2 (queries.test_foreignobject_select_related.Tests.test2)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/markus/Coding/django/tests/queries/test_foreignobject_select_related.py", line 46, in test2
    for x in Model1.objects.select_related("data", "model2").defer(
             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        "data__json_field"
        ^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/markus/Coding/django/django/db/models/query.py", line 386, in __iter__
    self._fetch_all()
    ~~~~~~~~~~~~~~~^^
  File "/home/markus/Coding/django/django/db/models/query.py", line 1954, in _fetch_all
    self._result_cache = list(self._iterable_class(self))
                         ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/markus/Coding/django/django/db/models/query.py", line 93, in __iter__
    results = compiler.execute_sql(
        chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
    )
  File "/home/markus/Coding/django/django/db/models/sql/compiler.py", line 1610, in execute_sql
    sql, params = self.as_sql()
                  ~~~~~~~~~~~^^
  File "/home/markus/Coding/django/django/db/models/sql/compiler.py", line 766, in as_sql
    extra_select, order_by, group_by = self.pre_sql_setup(
                                       ~~~~~~~~~~~~~~~~~~^
        with_col_aliases=with_col_aliases or bool(combinator),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/markus/Coding/django/django/db/models/sql/compiler.py", line 85, in pre_sql_setup
    self.setup_query(with_col_aliases=with_col_aliases)
    ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/markus/Coding/django/django/db/models/sql/compiler.py", line 74, in setup_query
    self.select, self.klass_info, self.annotation_col_map = self.get_select(
                                                            ~~~~~~~~~~~~~~~^
        with_col_aliases=with_col_aliases,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/markus/Coding/django/django/db/models/sql/compiler.py", line 299, in get_select
    related_klass_infos = self.get_related_selections(select, select_mask)
  File "/home/markus/Coding/django/django/db/models/sql/compiler.py", line 1241, in get_related_selections
    if not select_related_descend(f, restricted, requested, select_mask):
           ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/markus/Coding/django/django/db/models/query_utils.py", line 438, in select_related_descend
    raise FieldError(
    ...<2 lines>...
    )
django.core.exceptions.FieldError: Field Model1.model2 cannot be both deferred and traversed using select_related at the same time.

I understand that ForeignObject is private API (in terms of the deprecation policy per the ​comment in the composite primary keys docs), but it seems to me the given tests should still pass.

[Jacob Walls]

Thanks. Seems to be an issue with assuming that fields need to be in the select mask for select_relat'ing them to be useful. (Tacking on a .values("model2") sails through.)

[Natalia Bidart]

Replying to Vishy Algo:

Thank you, could you please confirm which of the 3 opened PRs for this ticket is yours? none of the GH handles matches the one you are using in this platform.

[Vishy Algo]

Replying to Natalia Bidart:

Replying to Vishy Algo:

Thank you, could you please confirm which of the 3 opened PRs for this ticket is yours? none of the GH handles matches the one you are using in this platform.

PR: ​https://github.com/django/django/pull/20716