Ticket #5420: queryset_fields_trunk.diff

django/db/models/manager.py

@@ -101 +101 @@
-    def values(self, *args, **kwargs):
-        return self.get_query_set().values(*args, **kwargs)
-
+    def fields(self, *args, **kwargs):
+        return self.get_query_set().fields(*args, **kwargs)
+
-class ManagerDescriptor(object):
-    # This class ensures managers aren't accessible via model instances.
-    # For example, Poll.objects works, but poll_obj.objects raises AttributeError.

django/db/models/query.py

@@ -91 +91 @@
-        self._order_by = None        # Ordering, e.g. ('date', '-name'). If None, use model's ordering.
-        self._select_related = False # Whether to fill cache for related objects.
-        self._max_related_depth = 0  # Maximum "depth" for select_related
+        self._fields = {}            # Subset of fields of this and related models to select
+        self._modelize = True        # Convert rows to models or return dictionaries
-        self._distinct = False       # Whether the query should use SELECT DISTINCT.
-        self._select = {}            # Dictionary of attname -> SQL.
-        self._where = []             # List of extra WHERE clauses to use.
@@ -189 +191 @@
-        cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
-
-        fill_cache = self._select_related
-self.model._meta.fields
+_get_model_fields(self.model._meta, "", self._fields, self._select)
-        index_end = len(fields)
-        has_resolve_columns = hasattr(self, 'resolve_columns')
-        while 1:
@@ -201 +203 @@
-                    row = self.resolve_columns(row, fields)
-                if fill_cache:
-                    obj, index_end = get_cached_row(klass=self.model, row=row,
-
+
+
+
-                else:
-
+
+
-                for i, k in enumerate(extra_select):
-
+
+
+
+
-                yield obj
-
-    def count(self):
@@ -357 +365 @@
-    # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
-    ##################################################
-
-    def values(self, *fields):
-        return self._clone(klass=ValuesQuerySet, _fields=fields)
-
-    def dates(self, field_name, kind, order='ASC'):
-        """
-        Returns a list of datetime objects representing all available dates
@@ -415 +420 @@
-        "Returns a new QuerySet instance with '_select_related' modified."
-        return self._clone(_select_related=true_or_false, _max_related_depth=depth)
-
+    def fields(self, *fields, **related_fields):
+        """Returns a new QuerySet instance with '_fields' modified.
+        owner_fields is stored inside _fields with empty string key."""
+        assert fields or related_fields, "fields() don't have sense without arguments"
+        _fields, _select_related = _merge_fields(self.model, fields, related_fields)
+        return self._clone(_modelize=True, _fields=_fields,
+                           _select_related=_select_related or self._select_related)
+
+    def values(self, *fields, **related_fields):
+        """Returns a new QuerySet instance with '_fields' and '_modelize'."""
+        if not (fields or related_fields):
+            return self._clone(_modelize=False, _fields={}) # process all fields from master model
+        _fields, _select_related = _merge_fields(self.model, fields, related_fields)
+        return self._clone(_modelize=False, _fields=_fields,
+                           _select_related=_select_related or self._select_related)
+
-    def order_by(self, *field_names):
-        "Returns a new QuerySet instance with the ordering changed."
-        assert self._limit is None and self._offset is None, \
@@ -448 +469 @@
-        c._order_by = self._order_by
-        c._select_related = self._select_related
-        c._max_related_depth = self._max_related_depth
+        c._fields = self._fields
+        c._modelize = self._modelize
-        c._distinct = self._distinct
-        c._select = self._select.copy()
-        c._where = self._where[:]
@@ -488 +511 @@
-        opts = self.model._meta
-
-        # Construct the fundamental parts of the query: SELECT X FROM Y WHERE Z.
-
+
+
-        tables = [quote_only_if_word(t) for t in self._tables]
-        joins = SortedDict()
-        where = self._where[:]
@@ -505 +529 @@
-            fill_table_cache(opts, select, tables, where,
-                             old_prefix=opts.db_table,
-                             cache_tables_seen=[opts.db_table],
-
+, fields=self._fields
-
-        # Add any additional SELECTs.
-        if self._select:
@@ -571 +595 @@
-else:
-    QuerySet = _QuerySet
-
-class ValuesQuerySet(QuerySet):
-    def __init__(self, *args, **kwargs):
-        super(ValuesQuerySet, self).__init__(*args, **kwargs)
-        # select_related isn't supported in values().
-        self._select_related = False
-
-    def iterator(self):
-        try:
-            select, sql, params = self._get_sql_clause()
-        except EmptyResultSet:
-            raise StopIteration
-
-        qn = connection.ops.quote_name
-
-        # self._select is a dictionary, and dictionaries' key order is
-        # undefined, so we convert it to a list of tuples.
-        extra_select = self._select.items()
-
-        # Construct two objects -- fields and field_names.
-        # fields is a list of Field objects to fetch.
-        # field_names is a list of field names, which will be the keys in the
-        # resulting dictionaries.
-        if self._fields:
-            if not extra_select:
-                fields = [self.model._meta.get_field(f, many_to_many=False) for f in self._fields]
-                field_names = self._fields
-            else:
-                fields = []
-                field_names = []
-                for f in self._fields:
-                    if f in [field.name for field in self.model._meta.fields]:
-                        fields.append(self.model._meta.get_field(f, many_to_many=False))
-                        field_names.append(f)
-                    elif not self._select.has_key(f):
-                        raise FieldDoesNotExist('%s has no field named %r' % (self.model._meta.object_name, f))
-        else: # Default to all fields.
-            fields = self.model._meta.fields
-            field_names = [f.attname for f in fields]
-
-        columns = [f.column for f in fields]
-        select = ['%s.%s' % (qn(self.model._meta.db_table), qn(c)) for c in columns]
-        if extra_select:
-            select.extend(['(%s) AS %s' % (quote_only_if_word(s[1]), qn(s[0])) for s in extra_select])
-            field_names.extend([f[0] for f in extra_select])
-
-        cursor = connection.cursor()
-        cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params)
-
-        has_resolve_columns = hasattr(self, 'resolve_columns')
-        while 1:
-            rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)
-            if not rows:
-                raise StopIteration
-            for row in rows:
-                if has_resolve_columns:
-                    row = self.resolve_columns(row, fields)
-                yield dict(zip(field_names, row))
-
-    def _clone(self, klass=None, **kwargs):
-        c = super(ValuesQuerySet, self)._clone(klass, **kwargs)
-        c._fields = self._fields[:]
-        return c
-
-class DateQuerySet(QuerySet):
-    def iterator(self):
-        from django.db.backends.util import typecast_timestamp
@@ -829 +790 @@
-            raise NotImplementedError
-    raise TypeError, "Got invalid lookup_type: %s" % repr(lookup_type)
-
-
+, depth_path="", fields={}, modelize=True
-    """Helper function that recursively returns an object with cache filled"""
-
-    # If we've got a max_depth set and we've exceeded that depth, bail now.
-    if max_depth and cur_depth > max_depth:
-        return None
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-            if cached_row:
-                rel_obj, index_end = cached_row
-
+
+
+
+
-    return obj, index_end
-
-
+
+
-    """
-    Helper function that recursively populates the select, tables and where (in
-    place) for select_related queries.
-    """
-
-    # If we've got a max_depth set and we've exceeded that depth, bail now.
-    if max_depth and cur_depth > max_depth:
-        return None
-
-    qn = connection.ops.quote_name
-opts.fields
-
+_get_model_fields(opts, depth_path, fields)
+ and _get_fields(f, depth_path, fields)
-            db_table = f.rel.to._meta.db_table
-            if db_table not in cache_tables_seen:
-                tables.append(qn(db_table))
@@ -869 +838 @@
-            cache_tables_seen.append(db_table)
-            where.append('%s.%s = %s.%s' % \
-                (qn(old_prefix), qn(f.column), qn(db_table), qn(f.rel.get_related_field().column)))
-
-
+
+
+
+
-
+def _lookup(depth_path, fname):
+    return depth_path and "%s__%s" % (depth_path, fname) or fname
+
+def _split_lookup(lookup):
+    """Right split lookup on parent and child."""
+    if LOOKUP_SEPARATOR in lookup:
+        return lookup.rsplit(LOOKUP_SEPARATOR, 1)
+    else:
+        return '', lookup
+
+def _merge_fields(klass, fields, related_fields):
+    """Resolves child-parent relations and checks is select_related required
+    or not."""
+    _select_related = False
+    if related_fields or [f for f in klass._meta.fields if f.name in fields and f.rel]:
+        _select_related = True
+
+    related_fields[''] = fields
+    # check that each child primary key is in parent field subset
+    for lookup in related_fields.iterkeys():
+        if lookup:                  # root lookup is empty string
+            parent, child = _split_lookup(lookup)
+            if not related_fields.has_key(parent):
+                continue            # a grandparent may have required field
+            parent_fields = related_fields[parent]
+            if child not in parent_fields:
+                parent_fields = list(parent_fields) + [child]
+                related_fields[parent] = parent_fields
+
+    return related_fields, _select_related
+
+def _get_fields(field, depth_path, fields):
+    """Returns fields of field.rel model that need to be selected from db."""
+    assert field.rel and field.rel.to, "ForeignKey is required."
+    return _get_model_fields(field.rel.to._meta, _lookup(depth_path, field.name), fields)
+
+def _get_model_fields(meta, lookup, fields, extra_select=[]):
+    if fields:                    # was used fields(...), so return only specified fields
+        if fields.has_key(lookup):      # we have implicit instruction for the lookup
+            fnames = fields[lookup]
+            if fnames:
+                # check that each field name is valid field name for this model and extra select
+                # actually this is optional step, but tests/modeltests/lookup.py require it
+                valid_fnames = [f.name for f in meta.fields]
+                if extra_select:
+                    valid_fnames += extra_select.keys()
+                for fname in fnames:
+                    if not fname in valid_fnames:
+                        raise FieldDoesNotExist("%s has no field named '%s'" %\
+                                                (meta.object_name, fname))
+            return [f for f in meta.fields if f.name in fnames]
+        else:                           # checking parent
+            while lookup:
+                lookup, child = _split_lookup(lookup)
+                if fields.has_key(lookup):
+                    # we have child defined in parent fields instead of
+                    # dedicated lookup for this child, so return all fields
+                    # for example BlogPost.objects.fields(('headline', 'blog')) should
+                    # return only BlogPost.headline, all Blog fields and all Blog children
+                    # fields (limited to _max_related_depth of course)
+                    return child in fields[lookup] and meta.fields or ()
+    else:
+        return meta.fields
+
+def _init_model(klass, fields, values):
+    """If klass, return model instance, else return dictionary."""
+    obj = dict(zip([f.attname for f in fields], values))
+    if klass:
+        obj = klass(**obj)
+    return obj
+
-def parse_lookup(kwarg_items, opts):
-    # Helper function that handles converting API kwargs
-    # (e.g. "name__exact": "tom") to SQL.

tests/modeltests/fields/models.py

@@ +1 @@
+"""
+47. Tests for fields()
+
+``fields()`` allows to select only fields subset from a model and its related
+models.  It change the behaviour of methods that return model
+instances. Affects querysets with or without select_related mode. If fields()
+isn't called for queryset, queryset behavior isn't chaged, i.e. all fields of
+all selected models are returned.
+"""
+
+from django.db import models
+
+class Author(models.Model):
+    name = models.CharField(max_length=50)
+    about = models.TextField()
+    def __unicode__(self):
+        """When self.about isn't loaded it will be shown as []."""
+        return u"%s [%s]" % (self.name, self.about)
+
+class Blog(models.Model):
+    name = models.CharField(max_length=50)
+    about = models.TextField()
+    author = models.ForeignKey(Author)
+
+    def __unicode__(self):
+        return u"%s [%s]" % (self.name, self.about)
+
+class Entry(models.Model):
+    blog = models.ForeignKey(Blog)
+    headline = models.CharField(max_length=50)
+    content = models.TextField()
+
+    def __unicode__(self):
+        return u"%s [%s]" % (self.headline, self.content)
+
+def create_world():
+    """Helper to create test models."""
+    etch = Author.objects.create(name="Etch", about="About Etch")
+    lenny = Author.objects.create(name="Lenny", about="About Lenny")
+    etch_blog = Blog.objects.create(name="Etch's blog", about="All about Etch", author=etch)
+    lenny_blog = Blog.objects.create(name="Lenny's blog", about="All about Lenny", author=lenny)
+    for blog in Blog.objects.select_related():
+        for i in xrange(5):
+            headline = "%s_%d" % (blog.author.name, i)
+            Entry.objects.create(headline=headline, content="huge content", blog=blog)
+
+
+__test__ = {'API_TESTS':"""
+
+>>> from django.conf import settings
+>>> old_debug = settings.DEBUG
+>>> settings.DEBUG = True
+>>> from django import db
+>>> create_world()
+>>> Author.objects.count(), Blog.objects.count(), Entry.objects.count()
+(2, 2, 10)
+
+# load only `id` and `headline` of a Entry
+>>> db.reset_queries()
+>>> p = Entry.objects.fields('id', 'headline').get(headline='Lenny_0')
+>>> p, p.id, p.content
+(<Entry: Lenny_0 []>, 6, '')
+>>> len(db.connection.queries)
+1
+
+# load all fields of all related objects
+>>> db.reset_queries()
+>>> p = Entry.objects.select_related().get(headline="Etch_1")
+>>> p, p.blog, p.blog.author
+(<Entry: Etch_1 [huge content]>, <Blog: Etch's blog [All about Etch]>, <Author: Etch [About Etch]>)
+>>> len(db.connection.queries)
+1
+
+# load only Entry.headline, Blog.name and Author.name and don't load
+huge TextFields for Lenny's blog posts (typical task when it's need to show an object list).
+>>> db.reset_queries()
+>>> posts = Entry.objects.fields('headline', blog=('name',), blog__author=('name',)).filter(blog__author__name="Lenny")[:1]
+>>> [(p, p.blog, p.blog.author) for p in posts]
+[(<Entry: Lenny_0 []>, <Blog: Lenny's blog []>, <Author: Lenny []>)]
+>>> p = posts[0]
+>>> p.content == p.blog.about == p.blog.author.about == ''
+True
+>>> len(db.connection.queries)
+1
+
+# load all Entry fields and only Blog.headline and Author.name (don't load `about` fields)
+>>> db.reset_queries()
+>>> p = Entry.objects.fields('id','headline', 'content', blog=('name',), blog__author=('name',)).select_related().get(headline="Etch_1")
+>>> p, p.blog, p.blog.author
+(<Entry: Etch_1 [huge content]>, <Blog: Etch's blog []>, <Author: Etch []>)
+>>> print p.content
+huge content
+>>> (p.blog.about, p.blog.author.about)
+('', '')
+>>> len(db.connection.queries)
+1
+
+# don't load any field from Blog, but load Blog.author.name
+>>> db.reset_queries()
+>>> p = Entry.objects.fields('id','headline', 'content', blog=(), blog__author=('name',)).get(headline="Lenny_1")
+>>> p, p.blog, p.blog.author
+(<Entry: Lenny_1 [huge content]>, <Blog:  []>, <Author: Lenny []>)
+>>> len(db.connection.queries)
+1
+
+# select Entry.headline, all fields from Blog and only Author.name
+>>> db.reset_queries()
+>>> p = Entry.objects.fields('headline', 'blog', blog__author=('name',)).get(headline="Lenny_2")
+>>> p, p.blog, p.blog.author
+(<Entry: Lenny_2 []>, <Blog: Lenny's blog [All about Lenny]>, <Author: Lenny []>)
+>>> len(db.connection.queries)
+1
+
+# check count()
+>>> db.reset_queries()
+>>> Entry.objects.fields('id', 'blog', blog__author=('name',)).select_related().filter(headline__contains='1').count()
+2
+>>> len(db.connection.queries)
+1
+
+# select all field from Entry, Blog and don't select Author fields because
+# select_related(depth=1) don't touch Author table and so field limitations from
+# fields() aren't used
+>>> db.reset_queries()
+>>> p = Entry.objects.fields('id','headline', 'content', 'blog', blog__author=('name',)).select_related(depth=1).get(headline="Lenny_3")
+>>> p, p.blog, p.blog.author
+(<Entry: Lenny_3 [huge content]>, <Blog: Lenny's blog [All about Lenny]>, <Author: Lenny [About Lenny]>)
+>>> len(db.connection.queries)
+2
+
+# select Blog.name and Author.name only and check extra()
+>>> db.reset_queries()
+>>> b = Blog.objects.fields('name', author=('name',)).extra(select={'post_count': "SELECT COUNT(*) FROM fields_entry t WHERE t.blog_id=fields_blog.id"}).get(name="Lenny\'s blog")
+>>> b, b.author, b.post_count
+(<Blog: Lenny's blog []>, <Author: Lenny []>, 5)
+>>> len(db.connection.queries)
+1
+
+# check extra() with collection
+>>> db.reset_queries()
+>>> blogs = Blog.objects.fields('id','name', author=('name',)).select_related().extra(select={'post_count': "SELECT COUNT(*) FROM fields_entry t WHERE t.blog_id=fields_blog.id"})
+>>> [(b, b.author, b.post_count) for b in blogs]
+[(<Blog: Etch's blog []>, <Author: Etch []>, 5), (<Blog: Lenny's blog []>, <Author: Lenny []>, 5)]
+>>> len(db.connection.queries)
+1
+
+# load Blog.name, Author.name and check extra()
+>>> db.reset_queries()
+>>> b = Blog.objects.fields('name', author=('name',)).select_related().extra(select={'post_count': "SELECT COUNT(*) FROM fields_entry t WHERE t.blog_id=fields_blog.id"}).get(name="Lenny\'s blog")
+>>> b, b.author, b.post_count
+(<Blog: Lenny's blog []>, <Author: Lenny []>, 5)
+>>> len(db.connection.queries)
+1
+
+# load only Author.name without any fields from Blog
+>>> db.reset_queries()
+>>> b1 = Blog.objects.fields(author=('name',)).select_related().get(name="Etch\'s blog")
+>>> b1, b1.author
+(<Blog:  []>, <Author: Etch []>)
+>>> len(db.connection.queries)
+1
+
+# dive even deeper in absolute beauty, load only child fields without parent fields
+>>> db.reset_queries()
+>>> p3 = Entry.objects.fields(blog=(), blog__author=('id','name',)).select_related().get(headline="Etch_3")
+>>> p3, p3.blog, p3.blog.author, p3.blog.author.id
+(<Entry:  []>, <Blog:  []>, <Author: Etch []>, 1)
+>>> len(db.connection.queries)
+1
+
+# check old-style values()
+>>> d = Entry.objects.filter(blog__author__name='Lenny').values('id', 'headline')[0]
+>>> print d['id'], d['headline']
+6 Lenny_0
+>>> l = Entry.objects.values('id', 'headline').extra(select={'id_plus_1': 'id + 1'})[:2]
+>>> [(d['id'], d['id_plus_1'], str(d['headline'])) for d in l]
+[(1, 2, 'Etch_0'), (2, 3, 'Etch_1')]
+
+# test models for docs/db-api.txt examples
+>>> joe = Author.objects.create(name="Joe")
+>>> blog = Blog.objects.create(id=1, name='Beatles Blog', about='All the latest Beatles news.', author=joe)
+>>> entry = Entry.objects.create(headline='Joe about Beatles', content='Beatles was ...', blog=blog)
+
+# load all fields from Entry and related models
+>>> e = Entry.objects.select_related().get(headline='Joe about Beatles')
+>>> e.headline, e.content, e.blog.name
+(u'Joe about Beatles', u'Beatles was ...', u'Beatles Blog')
+
+# don't load huge text field from Entry and only load name from related Blog
+>>> e = Entry.objects.fields('headline', blog=('name',)).get(headline='Joe about Beatles')
+>>> e.headline, e.content, e.blog.name
+(u'Joe about Beatles', '', u'Beatles Blog')
+
+>>> db.reset_queries()
+>>> Entry.objects.values('headline', blog=('name',)).filter(headline__contains='Beatles')
+[{'headline': u'Joe about Beatles', 'blog': {'name': u'Beatles Blog'}, 'blog_id': 1}]
+>>> len(db.connection.queries)
+1
+>>> Entry.objects.values('headline', blog=('id', 'name',)).filter(headline__contains='Beatles')
+[{'headline': u'Joe about Beatles', 'blog': {'id': 1, 'name': u'Beatles Blog'}, 'blog_id': 1}]
+
+# Reset DEBUG to where we found it.
+>>> settings.DEBUG = old_debug
+"""}

AUTHORS

@@ -357 +357 @@
-    ymasuda@ethercube.com
-    Jarek Zgoda <jarek.zgoda@gmail.com>
-    Cheng Zhang
+    Dima Dogadaylo <http://www.mysoftparade.com/>
-
-A big THANK YOU goes to:
-

docs/db-api.txt

@@ -536 +536 @@
-However, if your query spans multiple tables, it's possible to get duplicate
-results when a ``QuerySet`` is evaluated. That's when you'd use ``distinct()``.
-
-values(*
+fields(*fields, **related_
-~~~~~~~~~~~~~~~~~~~
-
-
-
+
-
+Returns a ``QuerySet`` that allows to load only some of fields from this and
+related models.
+
+This example demonstrates effect of particular field loading::
+
+    # load all fields from Entry and related models
+    >>> e = Entry.objects.select_related().get(headline='Joe about Beatles')
+    >>> e.headline, e.body_text, e.blog.name
+    (u'Joe about Beatles', u'Beatles was ...', u'Beatles Blog')
+
+    # don't load huge text field from Entry and only load name from related Blog
+    >>> e = Entry.objects.fields('headline', blog=('name',)).get(headline='Joe about Beatles')
+    >>> e.headline, e.body_text, e.blog.name
+    (u'Joe about Beatles', '', u'Beatles Blog')
+
+``fields()`` takes optional positional arguments, ``*fields``, which specify
+field names of this model for to which the ``SELECT`` should be limited. 
+
+``fields()`` also takes optional keyword arguments, ``**related_fields``,
+which specify field names of related models to which the ``SELECT`` should be
+limited. Each argument in ``**related_fields`` affects corresponding related
+model in the same way like ``*fields`` affects master model.
+
+You can reference related objects for any desired depth with standard Django
+lookup syntax, for example:
+
+    Entry.objects.fields('headline', author__address__country=('name'))
+
+The method ``fields()`` is extremelly usefull when you have models that
+contain large text or binary fields and you don't want to load these heavy
+fields. For example you need to show latest 10 blog posts and you only need
+headlines of these blog posts and their author names, with ``fields`` you can
+load only ``headline`` from ``Entry`` table and only ``name`` from ``Author``
+table (other fields of these 2 models will not be loaded and will have default
+values). 
+
+With ``fields()`` you have full control on what fields will be loaded from
+database, take this power with care. For example suppose ``Entry`` model
+references ``Author`` model, that references ``Address`` model that have
+foreign key to ``Country`` model. With call like
+    Entry.objects.fields('headline', author__address__country=('name'))
+, you can load ``Entry.headline``, don't load fields from ``Author`` and
+``Address`` and load only name from ``Country`` assotiated with blog entry's
+author.
+
+``values(*fields, **related_fields)``
+~~~~~~~~~~~~~~~~~~~
+
+**New in Django development version**
+
+Returns a ``QuerySet`` that evaluates to a list of dictionaries instead of
+model-instance objects.
+
-Each of those dictionaries represents an object, with the keys corresponding to
-the attribute names of model objects.
-
@@ -569 +621 @@
-    >>> Blog.objects.values('id', 'name')
-    [{'id': 1, 'name': 'Beatles Blog'}]
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-from a small number of the available fields and you won't need the
-functionality of a model instance object. It's more efficient to select only
-the fields you need to use.
-
-
-
+
+
+
+
-whatever. Yes, that means these two calls are identical::
-
-    Blog.objects.values().order_by('id')