diff --git a/AUTHORS b/AUTHORS index 31a3300..c484692 100644 --- a/AUTHORS +++ b/AUTHORS @@ -540,6 +540,7 @@ answer newbie questions, and generally made Django that much better: Gasper Zejn Jarek Zgoda Cheng Zhang + Jeffrey Gelens A big THANK YOU goes to: diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 1c3bc7e..df371bd 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -341,6 +341,9 @@ class BaseDatabaseFeatures(object): supports_stddev = None can_introspect_foreign_keys = None + # Support for the DISTINCT ON clause + can_distinct_on_fields = False + def __init__(self, connection): self.connection = connection @@ -494,6 +497,17 @@ class BaseDatabaseOperations(object): """ raise NotImplementedError('Full-text search is not implemented for this database backend') + def distinct(self, db_table, fields): + """ + Returns an SQL DISTINCT clause which removes duplicate rows from the + result set. If any fields are given, only the given fields are being + checked for duplicates. + """ + if fields: + raise NotImplementedError('DISTINCT ON fields is not supported by this database backend') + else: + return 'DISTINCT' + def last_executed_query(self, cursor, sql, params): """ Returns a string of the query last executed by the given cursor, with diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index 6ed59a6..ec75bae 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -71,6 +71,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_defer_constraint_checks = True has_select_for_update = True has_select_for_update_nowait = True + can_distinct_on_fields = True class DatabaseWrapper(BaseDatabaseWrapper): diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py index d535ee3..6bc9bd3 100644 --- a/django/db/backends/postgresql_psycopg2/operations.py +++ b/django/db/backends/postgresql_psycopg2/operations.py @@ -173,6 +173,14 @@ class DatabaseOperations(BaseDatabaseOperations): return 63 + def distinct(self, db_table, fields): + if fields: + table_name = self.quote_name(db_table) + fields = [table_name + "." + self.quote_name(field) for field in fields] + return 'DISTINCT ON (%s)' % ', '.join(fields) + else: + return 'DISTINCT' + def last_executed_query(self, cursor, sql, params): # http://initd.org/psycopg/docs/cursor.html#cursor.query # The query attribute is a Psycopg extension to the DB API 2.0. diff --git a/django/db/models/query.py b/django/db/models/query.py index af62061..9a36310 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -665,12 +665,14 @@ class QuerySet(object): obj.query.add_ordering(*field_names) return obj - def distinct(self, true_or_false=True): + def distinct(self, *field_names): """ Returns a new QuerySet instance that will select only distinct results. """ obj = self._clone() - obj.query.distinct = true_or_false + obj.query.add_distinct_fields(field_names) + obj.query.distinct = True + return obj def extra(self, select=None, where=None, params=None, tables=None, @@ -1090,7 +1092,7 @@ class EmptyQuerySet(QuerySet): """ return self - def distinct(self, true_or_false=True): + def distinct(self, fields=None): """ Always returns EmptyQuerySet. """ diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 05c19f3..fac3695 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -74,8 +74,12 @@ class SQLCompiler(object): params.extend(val[1]) result = ['SELECT'] + if self.query.distinct: - result.append('DISTINCT') + distinct_sql = self.connection.ops.distinct( + self.query.model._meta.db_table, self.query.distinct_fields) + result.append(distinct_sql) + result.append(', '.join(out_cols + self.query.ordering_aliases)) result.append('FROM') diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 110e317..bc0c7f7 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -125,6 +125,7 @@ class Query(object): self.order_by = [] self.low_mark, self.high_mark = 0, None # Used for offset/limit self.distinct = False + self.distinct_fields = None self.select_for_update = False self.select_for_update_nowait = False self.select_related = False @@ -256,6 +257,7 @@ class Query(object): obj.order_by = self.order_by[:] obj.low_mark, obj.high_mark = self.low_mark, self.high_mark obj.distinct = self.distinct + obj.distinct_fields = self.distinct_fields obj.select_for_update = self.select_for_update obj.select_for_update_nowait = self.select_for_update_nowait obj.select_related = self.select_related @@ -384,7 +386,7 @@ class Query(object): Performs a COUNT() query using the current filter constraints. """ obj = self.clone() - if len(self.select) > 1 or self.aggregate_select: + if len(self.select) > 1 or self.aggregate_select or (self.distinct and self.distinct_fields): # If a select clause exists, then the query has already started to # specify the columns that are to be returned. # In this case, we need to use a subquery to evaluate the count. @@ -1557,6 +1559,15 @@ class Query(object): self.select = [] self.select_fields = [] + def add_distinct_fields(self, field_names): + self.distinct_fields = [] + options = self.get_meta() + + for name in field_names: + field, source, opts, join_list, last, _ = self.setup_joins( + name.split(LOOKUP_SEP), options, self.get_initial_alias(), False) + self.distinct_fields.append(field.column) + def add_fields(self, field_names, allow_m2m=True): """ Adds the given (model) fields to the select set. The field names are diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index a1bd5cc..857db2f 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -345,7 +345,7 @@ undefined afterward). distinct ~~~~~~~~ -.. method:: distinct() +.. method:: distinct(*fields) Returns a new ``QuerySet`` that uses ``SELECT DISTINCT`` in its SQL query. This eliminates duplicate rows from the query results. @@ -375,6 +375,16 @@ query spans multiple tables, it's possible to get duplicate results when a ``values()`` together, be careful when ordering by fields not in the ``values()`` call. +.. versionadded:: 1.4 + ``distinct()`` takes optional positional arguments ``*fields``, which specify + field names to which the ``DISTINCT`` should be limited. This translates to + a ``SELECT DISTINCT ON`` SQL query. Note that this ``DISTINCT ON`` query is + only available in PostgreSQL. + +.. note:: + When optional ``*fields`` are given, you will have to add an :meth:`order_by` + call with the same field names as the leftmost arguments. + values ~~~~~~ diff --git a/tests/regressiontests/queries/models.py b/tests/regressiontests/queries/models.py index d1e5e6e..9cf3a09 100644 --- a/tests/regressiontests/queries/models.py +++ b/tests/regressiontests/queries/models.py @@ -208,6 +208,9 @@ class Celebrity(models.Model): name = models.CharField("Name", max_length=20) greatest_fan = models.ForeignKey("Fan", null=True, unique=True) + def __unicode__(self): + return self.name + class TvChef(Celebrity): pass @@ -317,3 +320,20 @@ class ObjectC(models.Model): def __unicode__(self): return self.name + + +class Staff(models.Model): + name = models.CharField(max_length=50) + organisation = models.CharField(max_length=100) + tags = models.ManyToManyField(Tag, through='StaffTag') + + def __unicode__(self): + return self.name + +class StaffTag(models.Model): + staff = models.ForeignKey(Staff) + tag = models.ForeignKey(Tag) + + def __unicode__(self): + return u"%s -> %s" % (self.tag, self.staff) + diff --git a/tests/regressiontests/queries/tests.py b/tests/regressiontests/queries/tests.py index 31856ba..4903a22 100644 --- a/tests/regressiontests/queries/tests.py +++ b/tests/regressiontests/queries/tests.py @@ -15,7 +15,7 @@ from models import (Annotation, Article, Author, Celebrity, Child, Cover, Detail DumbCategory, ExtraInfo, Fan, Item, LeafA, LoopX, LoopZ, ManagedModel, Member, NamedCategory, Note, Number, Plaything, PointerA, Ranking, Related, Report, ReservedName, Tag, TvChef, Valid, X, Food, Eaten, Node, ObjectA, ObjectB, - ObjectC) + ObjectC, Staff, StaffTag) class BaseQuerysetTest(TestCase): @@ -1606,6 +1606,21 @@ class ConditionalTests(BaseQuerysetTest): t4 = Tag.objects.create(name='t4', parent=t3) t5 = Tag.objects.create(name='t5', parent=t3) + p1_o1 = Staff.objects.create(name="p1", organisation="o1") + p2_o1 = Staff.objects.create(name="p2", organisation="o1") + p3_o1 = Staff.objects.create(name="p3", organisation="o1") + p1_o2 = Staff.objects.create(name="p1", organisation="o2") + + StaffTag.objects.create(staff=p1_o1, tag=t1) + StaffTag.objects.create(staff=p1_o1, tag=t1) + + celeb1 = Celebrity.objects.create(name="c1") + celeb2 = Celebrity.objects.create(name="c2") + + self.fan1 = Fan.objects.create(fan_of=celeb1) + self.fan2 = Fan.objects.create(fan_of=celeb1) + self.fan3 = Fan.objects.create(fan_of=celeb2) + # In Python 2.6 beta releases, exceptions raised in __len__ are swallowed # (Python issue 1242657), so these cases return an empty list, rather than # raising an exception. Not a lot we can do about that, unfortunately, due to @@ -1677,6 +1692,52 @@ class ConditionalTests(BaseQuerysetTest): 2500 ) + @skipUnlessDBFeature('can_distinct_on_fields') + def test_ticket6422(self): + # (qset, expected) tuples + qsets = ( + ( + Staff.objects.distinct().order_by('name'), + ['', '', '', ''], + ), + ( + Staff.objects.distinct('name').order_by('name'), + ['', '', ''], + ), + ( + Staff.objects.distinct('organisation').order_by('organisation', 'name'), + ['', ''], + ), + ( + Staff.objects.distinct('name', 'organisation').order_by('name', 'organisation'), + ['', '', '', ''], + ), + ( + Celebrity.objects.filter(fan__in=[self.fan1, self.fan2, self.fan3]).\ + distinct('name').order_by('name'), + ['', ''], + ), + ( + StaffTag.objects.distinct('staff','tag'), + [' p1>'], + ), + ( + Tag.objects.order_by('parent__pk').distinct('parent'), + ['', '', ''], + ) + ) + + for qset, expected in qsets: + self.assertQuerysetEqual(qset, expected) + self.assertEqual(qset.count(), len(expected)) + + # and check the fieldlookup + self.assertRaises( + FieldError, + lambda: Staff.objects.distinct('shrubbery') + ) + + class UnionTests(unittest.TestCase): """ Tests for the union of two querysets. Bug #12252.