Grouping on JSONField subkey throws tuple index out of range error

[Marc DEBUREAUX]

EDIT: After further investigation, it seems the issue only happens if the JSONField key is the first item of the group clause, it works fine if it is alone or if another non-JSON field group clause is positioned first.

Have this simple model:

class Link(models.Model):
  item = models.ForeignKey("Item", on_delete=models.CASCADE)
  extra = models.JSONField(blank=True, null=True)

And have this simple query:

Link.objects.all().values("extra__key", "item_id").annotate(count=Count("id"))

This throws the following stacktrace error since 5.2 release:

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
File ~/python3.12/site-packages/IPython/core/formatters.py:770, in PlainTextFormatter.__call__(self, obj)
    763 stream = StringIO()
    764 printer = pretty.RepresentationPrinter(stream, self.verbose,
    765     self.max_width, self.newline,
    766     max_seq_length=self.max_seq_length,
    767     singleton_pprinters=self.singleton_printers,
    768     type_pprinters=self.type_printers,
    769     deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
    771 printer.flush()
    772 return stream.getvalue()

File ~/python3.12/site-packages/IPython/lib/pretty.py:411, in RepresentationPrinter.pretty(self, obj)
    400                         return meth(obj, self, cycle)
    401                 if (
    402                     cls is not object
    403                     # check if cls defines __repr__
   (...)    409                     and callable(_safe_getattr(cls, "__repr__", None))
    410                 ):
--> 411                     return _repr_pprint(obj, self, cycle)
    413     return _default_pprint(obj, self, cycle)
    414 finally:

File ~/python3.12/site-packages/IPython/lib/pretty.py:786, in _repr_pprint(obj, p, cycle)
    784 """A pprint that just redirects to the normal repr function."""
    785 # Find newlines and replace them with p.break_()
--> 786 output = repr(obj)
    787 lines = output.splitlines()
    788 with p.group():

File ~/python3.12/site-packages/django/db/models/query.py:360, in QuerySet.__repr__(self)
    359 def __repr__(self):
--> 360     data = list(self[: REPR_OUTPUT_SIZE + 1])
    361     if len(data) > REPR_OUTPUT_SIZE:
    362         data[-1] = "...(remaining elements truncated)..."

File ~/python3.12/site-packages/django/db/models/query.py:384, in QuerySet.__iter__(self)
    369 def __iter__(self):
    370     """
    371     The queryset iterator protocol uses three nested iterators in the
    372     default case:
   (...)    382            - Responsible for turning the rows into model objects.
    383     """
--> 384     self._fetch_all()
    385     return iter(self._result_cache)

File ~/python3.12/site-packages/django/db/models/query.py:1935, in QuerySet._fetch_all(self)
   1933 def _fetch_all(self):
   1934     if self._result_cache is None:
-> 1935         self._result_cache = list(self._iterable_class(self))
   1936     if self._prefetch_related_lookups and not self._prefetch_done:
   1937         self._prefetch_related_objects()

File ~/python3.12/site-packages/django/db/models/query.py:216, in ValuesIterable.__iter__(self)
    210     names = [
    211         *query.extra_select,
    212         *query.values_select,
    213         *query.annotation_select,
    214     ]
    215 indexes = range(len(names))
--> 216 for row in compiler.results_iter(
    217     chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
    218 ):
    219     yield {names[i]: row[i] for i in indexes}

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1571, in SQLCompiler.results_iter(self, results, tuple_expected, chunked_fetch, chunk_size)
   1569 """Return an iterator over the results from executing this query."""
   1570 if results is None:
-> 1571     results = self.execute_sql(
   1572         MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
   1573     )
   1574 fields = [s[0] for s in self.select[0 : self.col_count]]
   1575 converters = self.get_converters(fields)

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1609, in SQLCompiler.execute_sql(self, result_type, chunked_fetch, chunk_size)
   1607 result_type = result_type or NO_RESULTS
   1608 try:
-> 1609     sql, params = self.as_sql()
   1610     if not sql:
   1611         raise EmptyResultSet

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:765, in SQLCompiler.as_sql(self, with_limits, with_col_aliases)
    763 try:
    764     combinator = self.query.combinator
--> 765     extra_select, order_by, group_by = self.pre_sql_setup(
    766         with_col_aliases=with_col_aliases or bool(combinator),
    767     )
    768     for_update_part = None
    769     # Is a LIMIT/OFFSET clause needed?

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:85, in SQLCompiler.pre_sql_setup(self, with_col_aliases)
     79 def pre_sql_setup(self, with_col_aliases=False):
     80     """
     81     Do any necessary class setup immediately prior to producing SQL. This
     82     is for things that can't necessarily be done in __init__ because we
     83     might not have all the pieces in place at that time.
     84     """
---> 85     self.setup_query(with_col_aliases=with_col_aliases)
     86     order_by = self.get_order_by()
     87     self.where, self.having, self.qualify = self.query.where.split_having_qualify(
     88         must_group_by=self.query.group_by is not None
     89     )

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:74, in SQLCompiler.setup_query(self, with_col_aliases)
     72 if all(self.query.alias_refcount[a] == 0 for a in self.query.alias_map):
     73     self.query.get_initial_alias()
---> 74 self.select, self.klass_info, self.annotation_col_map = self.get_select(
     75     with_col_aliases=with_col_aliases,
     76 )
     77 self.col_count = len(self.select)

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:286, in SQLCompiler.get_select(self, with_col_aliases)
    284 # Reference to a column.
    285 elif isinstance(expression, int):
--> 286     expression = cols[expression]
    287 # ColPairs cannot be aliased.
    288 if isinstance(expression, ColPairs):

[Marc DEBUREAUX]

It oddly works if the JSON subkey is the only group clause though, but crash when used along with any other group clause.

[Simon Charette]

Duplicate of #36292 (Annotating an aggregate function over a group including annotations or transforms followed by a column references crashes with IndexError) fixed by 543e17c4405dfdac4f18759fc78b190406d14239 in Django 5.2.1 meant to be released on May 7th.