diff --git a/AUTHORS b/AUTHORS --- a/AUTHORS +++ b/AUTHORS @@ -202,6 +202,7 @@ Marc Garcia Andy Gayton geber@datacollect.com + Jeffrey Gelens Baishampayan Ghose Joshua Ginsberg Dimitris Glezos diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -376,6 +376,9 @@ supports_stddev = None can_introspect_foreign_keys = None + # Support for the DISTINCT ON clause + can_distinct_on_fields = False + def __init__(self, connection): self.connection = connection @@ -529,6 +532,17 @@ """ raise NotImplementedError('Full-text search is not implemented for this database backend') + def distinct(self, fields): + """ + Returns an SQL DISTINCT clause which removes duplicate rows from the + result set. If any fields are given, only the given fields are being + checked for duplicates. + """ + if fields: + raise NotImplementedError('DISTINCT ON fields is not supported by this database backend') + else: + return 'DISTINCT' + def last_executed_query(self, cursor, sql, params): """ Returns a string of the query last executed by the given cursor, with diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -76,6 +76,7 @@ has_select_for_update_nowait = True has_bulk_insert = True supports_tablespaces = True + can_distinct_on_fields = True class DatabaseWrapper(BaseDatabaseWrapper): vendor = 'postgresql' diff --git a/django/db/backends/postgresql_psycopg2/operations.py b/django/db/backends/postgresql_psycopg2/operations.py --- a/django/db/backends/postgresql_psycopg2/operations.py +++ b/django/db/backends/postgresql_psycopg2/operations.py @@ -179,6 +179,20 @@ return 63 + def distinct(self, fields): + if fields: + fields_sql = [] + + for field in fields: + fields_sql.append( + self.quote_name(field.model._meta.db_table) + "." + \ + self.quote_name(field.column) + ) + + return 'DISTINCT ON (%s)' % ', '.join(fields_sql) + else: + return 'DISTINCT' + def last_executed_query(self, cursor, sql, params): # http://initd.org/psycopg/docs/cursor.html#cursor.query # The query attribute is a Psycopg extension to the DB API 2.0. diff --git a/django/db/models/query.py b/django/db/models/query.py --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -738,12 +738,14 @@ obj.query.add_ordering(*field_names) return obj - def distinct(self, true_or_false=True): + def distinct(self, *field_names): """ Returns a new QuerySet instance that will select only distinct results. """ obj = self._clone() - obj.query.distinct = true_or_false + obj.query.add_distinct_fields(field_names) + obj.query.distinct = True + return obj def extra(self, select=None, where=None, params=None, tables=None, @@ -1166,7 +1168,7 @@ """ return self - def distinct(self, true_or_false=True): + def distinct(self, fields=None): """ Always returns EmptyQuerySet. """ diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -76,8 +76,10 @@ params.extend(val[1]) result = ['SELECT'] + if self.query.distinct: - result.append('DISTINCT') + result.append(self.connection.ops.distinct(self.query.distinct_fields)) + result.append(', '.join(out_cols + self.query.ordering_aliases)) result.append('FROM') diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -126,6 +126,7 @@ self.order_by = [] self.low_mark, self.high_mark = 0, None # Used for offset/limit self.distinct = False + self.distinct_fields = None self.select_for_update = False self.select_for_update_nowait = False self.select_related = False @@ -264,6 +265,7 @@ obj.order_by = self.order_by[:] obj.low_mark, obj.high_mark = self.low_mark, self.high_mark obj.distinct = self.distinct + obj.distinct_fields = self.distinct_fields obj.select_for_update = self.select_for_update obj.select_for_update_nowait = self.select_for_update_nowait obj.select_related = self.select_related @@ -392,7 +394,7 @@ Performs a COUNT() query using the current filter constraints. """ obj = self.clone() - if len(self.select) > 1 or self.aggregate_select: + if len(self.select) > 1 or self.aggregate_select or (self.distinct and self.distinct_fields): # If a select clause exists, then the query has already started to # specify the columns that are to be returned. # In this case, we need to use a subquery to evaluate the count. @@ -1595,6 +1597,15 @@ self.select = [] self.select_fields = [] + def add_distinct_fields(self, field_names): + self.distinct_fields = [] + options = self.get_meta() + + for name in field_names: + field, source, opts, join_list, last, _ = self.setup_joins( + name.split(LOOKUP_SEP), options, self.get_initial_alias(), False) + self.distinct_fields.append(field) + def add_fields(self, field_names, allow_m2m=True): """ Adds the given (model) fields to the select set. The field names are diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -345,7 +345,7 @@ distinct ~~~~~~~~ -.. method:: distinct() +.. method:: distinct([*fields]) Returns a new ``QuerySet`` that uses ``SELECT DISTINCT`` in its SQL query. This eliminates duplicate rows from the query results. @@ -374,6 +374,39 @@ :meth:`values()` together, be careful when ordering by fields not in the :meth:`values()` call. +.. versionadded:: 1.4 + +The possibility to pass positional arguments (``*fields``) is new in Django 1.4. +They are names of fields to which the ``DISTINCT`` should be limited. This +translates to a ``SELECT DISTINCT ON`` SQL query. + +.. note:: + Note that the ability to specify field names is only available in PostgreSQL. + +.. note:: + When fields names are given, you will have to add an :meth:`order_by` + call with the same field names as the leftmost arguments. + +Examples:: + + >>> Author.objects.distinct() + [...] + + >>> Entry.objects.order_by('pub_date').distinct('pub_date') + [...] + + >>> Entry.objects.order_by('blog').distinct('blog') + [...] + + >>> Entry.objects.order_by('author', 'pub_date').distinct('author', 'pub_date') + [...] + + >>> Entry.objects.order_by('blog__name', 'mod_date').distinct('blog__name', 'mod_date') + [...] + + >>> Entry.objects.order_by('author', 'pub_date').distinct('author') + [...] + values ~~~~~~ diff --git a/tests/regressiontests/queries/models.py b/tests/regressiontests/queries/models.py --- a/tests/regressiontests/queries/models.py +++ b/tests/regressiontests/queries/models.py @@ -209,6 +209,9 @@ name = models.CharField("Name", max_length=20) greatest_fan = models.ForeignKey("Fan", null=True, unique=True) + def __unicode__(self): + return self.name + class TvChef(Celebrity): pass @@ -344,3 +347,17 @@ def __unicode__(self): return "one2one " + self.new_name +class Staff(models.Model): + name = models.CharField(max_length=50) + organisation = models.CharField(max_length=100) + tags = models.ManyToManyField(Tag, through='StaffTag') + + def __unicode__(self): + return self.name + +class StaffTag(models.Model): + staff = models.ForeignKey(Staff) + tag = models.ForeignKey(Tag) + + def __unicode__(self): + return u"%s -> %s" % (self.tag, self.staff) diff --git a/tests/regressiontests/queries/tests.py b/tests/regressiontests/queries/tests.py --- a/tests/regressiontests/queries/tests.py +++ b/tests/regressiontests/queries/tests.py @@ -18,7 +18,7 @@ ManagedModel, Member, NamedCategory, Note, Number, Plaything, PointerA, Ranking, Related, Report, ReservedName, Tag, TvChef, Valid, X, Food, Eaten, Node, ObjectA, ObjectB, ObjectC, CategoryItem, SimpleCategory, - SpecialCategory, OneToOneCategory) + SpecialCategory, OneToOneCategory, Staff, StaffTag) class BaseQuerysetTest(TestCase): @@ -1739,6 +1739,21 @@ t4 = Tag.objects.create(name='t4', parent=t3) t5 = Tag.objects.create(name='t5', parent=t3) + p1_o1 = Staff.objects.create(name="p1", organisation="o1") + p2_o1 = Staff.objects.create(name="p2", organisation="o1") + p3_o1 = Staff.objects.create(name="p3", organisation="o1") + p1_o2 = Staff.objects.create(name="p1", organisation="o2") + + StaffTag.objects.create(staff=p1_o1, tag=t1) + StaffTag.objects.create(staff=p1_o1, tag=t1) + + celeb1 = Celebrity.objects.create(name="c1") + celeb2 = Celebrity.objects.create(name="c2") + + self.fan1 = Fan.objects.create(fan_of=celeb1) + self.fan2 = Fan.objects.create(fan_of=celeb1) + self.fan3 = Fan.objects.create(fan_of=celeb2) + # In Python 2.6 beta releases, exceptions raised in __len__ are swallowed # (Python issue 1242657), so these cases return an empty list, rather than # raising an exception. Not a lot we can do about that, unfortunately, due to @@ -1810,6 +1825,57 @@ 2500 ) + @skipUnlessDBFeature('can_distinct_on_fields') + def test_ticket6422(self): + """QuerySet.distinct('field', ...) works""" + # (qset, expected) tuples + qsets = ( + ( + Staff.objects.distinct().order_by('name'), + ['', '', '', ''], + ), + ( + Staff.objects.distinct('name').order_by('name'), + ['', '', ''], + ), + ( + Staff.objects.distinct('organisation').order_by('organisation', 'name'), + ['', ''], + ), + ( + Staff.objects.distinct('name', 'organisation').order_by('name', 'organisation'), + ['', '', '', ''], + ), + ( + Celebrity.objects.filter(fan__in=[self.fan1, self.fan2, self.fan3]).\ + distinct('name').order_by('name'), + ['', ''], + ), + ( + StaffTag.objects.distinct('staff','tag'), + [' p1>'], + ), + ( + Tag.objects.order_by('parent__pk', 'pk').distinct('parent'), + ['', '', ''], + ), + ( + StaffTag.objects.select_related('staff').distinct('staff__name').order_by('staff__name'), + [' p1>'], + ), + ) + + for qset, expected in qsets: + self.assertQuerysetEqual(qset, expected) + self.assertEqual(qset.count(), len(expected)) + + # and check the fieldlookup + self.assertRaises( + FieldError, + lambda: Staff.objects.distinct('shrubbery') + ) + + class UnionTests(unittest.TestCase): """ Tests for the union of two querysets. Bug #12252.