Ticket #16937: prefetch_4.diff

django/contrib/contenttypes/generic.py

@@ -225 +225 @@
             content_type = content_type,
             content_type_field_name = self.field.content_type_field_name,
             object_id_field_name = self.field.object_id_field_name,
-            core_filters = {
-                '%s__pk' % self.field.content_type_field_name: content_type.id,
-                '%s__exact' % self.field.object_id_field_name: instance._get_pk_val(),
-            }
-
+            prefetch_cache_name = self.field.attname,
         )
 
         return manager
@@ -250 +246 @@
     """
 
     class GenericRelatedObjectManager(superclass):
-        def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
+        def __init__(self, model=None, instance=None, symmetrical=None,
                      source_col_name=None, target_col_name=None, content_type=None,
-                     content_type_field_name=None, object_id_field_name=None):
+                     content_type_field_name=None, object_id_field_name=None,
+                     prefetch_cache_name=None):
 
             super(GenericRelatedObjectManager, self).__init__()
-            self.core_filters = core_filters
             self.model = model
             self.content_type = content_type
             self.symmetrical = symmetrical
@@ -264 +260 @@
             self.target_col_name = target_col_name
             self.content_type_field_name = content_type_field_name
             self.object_id_field_name = object_id_field_name
+            self.prefetch_cache_name = prefetch_cache_name
             self.pk_val = self.instance._get_pk_val()
+            self.core_filters = {
+                '%s__pk' % content_type_field_name: content_type.id,
+                '%s__exact' % object_id_field_name: instance._get_pk_val(),
+            }
 
         def get_query_set(self):
             db = self._db or router.db_for_read(self.model, instance=self.instance)
             return super(GenericRelatedObjectManager, self).get_query_set().using(db).filter(**self.core_filters)
 
+        def get_prefetch_query_set(self, instances):
+            if not instances:
+                return self.model._default_manager.none()
+
+            db = self._db or router.db_for_read(self.model, instance=instances[0])
+            query = {
+                '%s__pk' % self.content_type_field_name: self.content_type.id,
+                '%s__in' % self.object_id_field_name:
+                    [obj._get_pk_val() for obj in instances]
+                }
+            return super(GenericRelatedObjectManager, self).get_query_set().using(db).filter(**query)
+
+        def select_matching_instances(self, obj, related_objects):
+            pk_val = obj._get_pk_val()
+            return [rel_obj for rel_obj in related_objects
+                    if getattr(rel_obj, self.object_id_field_name) == pk_val]
+
+        def all(self):
+            try:
+                return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
+            except (AttributeError, KeyError):
+                return super(GenericRelatedObjectManager, self).all()
+
+
         def add(self, *objs):
             for obj in objs:
                 if not isinstance(obj, self.model):

django/db/models/fields/related.py

@@ -435 +435 @@
                 db = self._db or router.db_for_read(self.model, instance=self.instance)
                 return super(RelatedManager, self).get_query_set().using(db).filter(**(self.core_filters))
 
+            def get_prefetch_query_set(self, instances):
+                """
+                Return a queryset that does the bulk lookup needed
+                by prefetch_related functionality.
+                """
+                if not instances:
+                    return self.model._default_manager.none()
+
+                db = self._db or router.db_for_read(self.model, instance=instances[0])
+                query = {'%s__%s__in' % (rel_field.name, attname):
+                             [getattr(obj, attname) for obj in instances]}
+                return super(RelatedManager, self).get_query_set().using(db).filter(**query)
+
+            def select_matching_instances(self, obj, related_objects):
+                field_val = getattr(obj, attname)
+                other_attname = rel_field.get_attname()
+                return [rel_obj for rel_obj in related_objects
+                        if getattr(rel_obj, other_attname) == field_val]
+
+            def all(self):
+                try:
+                    return self.instance._prefetched_objects_cache[rel_field.related_query_name()]
+                except (AttributeError, KeyError):
+                    return super(RelatedManager, self).all()
+
             def add(self, *objs):
                 for obj in objs:
                     if not isinstance(obj, self.model):
@@ -482 +507 @@
     """Creates a manager that subclasses 'superclass' (which is a Manager)
     and adds behavior for many-to-many related objects."""
     class ManyRelatedManager(superclass):
-        def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
+        def __init__(self, model=None, query_field_name=None, instance=None, symmetrical=None,
                      source_field_name=None, target_field_name=None, reverse=False,
-                     through=None):
+                     through=None, prefetch_cache_name=None):
             super(ManyRelatedManager, self).__init__()
             self.model = model
-            self.core_filters = core_filters
+            self.query_field_name = query_field_name
+            self.core_filters = {'%s__pk' % query_field_name: instance._get_pk_val()}
             self.instance = instance
             self.symmetrical = symmetrical
             self.source_field_name = source_field_name
             self.target_field_name = target_field_name
             self.reverse = reverse
             self.through = through
+            self.prefetch_cache_name = prefetch_cache_name
             self._pk_val = self.instance.pk
             if self._pk_val is None:
                 raise ValueError("%r instance needs to have a primary key value before a many-to-many relationship can be used." % instance.__class__.__name__)
@@ -502 +529 @@
             db = self._db or router.db_for_read(self.instance.__class__, instance=self.instance)
             return super(ManyRelatedManager, self).get_query_set().using(db)._next_is_sticky().filter(**(self.core_filters))
 
+        def get_prefetch_query_set(self, instances):
+            if not instances:
+                return self.model._default_manager.none()
+
+            from django.db import connections
+
+            db = self._db or router.db_for_read(self.model, instance=instances[0])
+            query = {'%s__pk__in' % self.query_field_name:
+                         [obj._get_pk_val() for obj in instances]}
+            qs = super(ManyRelatedManager, self).get_query_set().using(db)._next_is_sticky().filter(**query)
+
+            # M2M: need to annotate the query in order to get the PK of the
+            # primary model that the secondary model was actually related to.
+
+            # We know that there will already be a join on the join table, so we
+            # can just add the select.
+            join_table = self.through._meta.db_table
+            pk_col = "%s_id" % self.source_field_name
+            connection = connections[db]
+            qn = connection.ops.quote_name
+            qs = qs.extra(select={'_prefetch_related_pk':
+                                      '%s.%s' % (qn(join_table), qn(pk_col))})
+            return qs
+
+        def select_matching_instances(self, obj, related_objects):
+            pk_val = obj._get_pk_val()
+            return [rel_obj for rel_obj in related_objects
+                    if rel_obj._prefetch_related_pk == pk_val]
+
+        def all(self):
+            try:
+                return self.instance._prefetched_objects_cache[self.prefetch_cache_name]
+            except (AttributeError, KeyError):
+                return super(ManyRelatedManager, self).all()
+
         # If the ManyToMany relation has an intermediary model,
         # the add and remove methods do not exist.
         if rel.through._meta.auto_created:
@@ -683 +745 @@
 
         manager = self.related_manager_cls(
             model=rel_model,
-            core_filters={'%s__pk' % self.related.field.name: instance._get_pk_val()},
+            query_field_name=self.related.field.name,
+            prefetch_cache_name=self.related.field.related_query_name(),
             instance=instance,
             symmetrical=False,
             source_field_name=self.related.field.m2m_reverse_field_name(),
@@ -739 +802 @@
 
         manager = self.related_manager_cls(
             model=self.field.rel.to,
-            core_filters={'%s__pk' % self.field.related_query_name(): instance._get_pk_val()},
+            query_field_name=self.field.related_query_name(),
+            prefetch_cache_name=self.field.name,
             instance=instance,
             symmetrical=self.field.rel.symmetrical,
             source_field_name=self.field.m2m_field_name(),

django/db/models/manager.py

@@ -172 +172 @@
     def select_related(self, *args, **kwargs):
         return self.get_query_set().select_related(*args, **kwargs)
 
+    def prefetch_related(self, *args, **kwargs):
+        return self.get_query_set().prefetch_related(*args, **kwargs)
+
     def values(self, *args, **kwargs):
         return self.get_query_set().values(*args, **kwargs)
 

django/db/models/query.py

@@ -36 +36 @@
         self._iter = None
         self._sticky_filter = False
         self._for_write = False
+        self._prefetch_related = set()
+        self._prefetch_done = False
 
     ########################
     # PYTHON MAGIC METHODS #
@@ -81 +83 @@
                 self._result_cache = list(self.iterator())
         elif self._iter:
             self._result_cache.extend(self._iter)
+        if self._prefetch_related and not self._prefetch_done:
+            self._prefetch_related_objects()
         return len(self._result_cache)
 
     def __iter__(self):
+        if self._prefetch_related:
+            # We need all the results in order to be able to do the prefetch
+            # in one go. To minimize code duplication, we use the __len__
+            # code path which also forces this, and also does the prefetch
+            len(self)
+
         if self._result_cache is None:
             self._iter = self.iterator()
             self._result_cache = []
@@ -106 +116 @@
                 self._fill_cache()
 
     def __nonzero__(self):
+        if self._prefetch_related:
+            # We need all the results in order to be able to do the prefetch
+            # in one go. To minimize code duplication, we use the __len__
+            # code path which also forces this, and also does the prefetch
+            len(self)
+
         if self._result_cache is not None:
             return bool(self._result_cache)
         try:
@@ -526 +542 @@
             return self.query.has_results(using=self.db)
         return bool(self._result_cache)
 
+    def _prefetch_related_objects(self):
+        # This method can only be called once the result cache has been filled.
+        prefetch_related_objects(self._result_cache, self._prefetch_related)
+        self._prefetch_done = True
+
     ##################################################
     # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS #
     ##################################################
@@ -649 +670 @@
             obj.query.max_depth = depth
         return obj
 
+    def prefetch_related(self, *fields):
+        """
+        Returns a new QuerySet instance that will prefetch Many-To-One
+        and Many-To-Many related objects when the QuerySet is evaluated.
+
+        The fields specified must be attributes that return a RelatedManager of
+        some kind when used on instances of the evaluated QuerySet.
+
+        These RelatedManagers will be modified so that their 'all()' method will
+        return a QuerySet whose cache is already filled with objects that were
+        looked up in a single batch, rather than one query per object in the
+        current QuerySet.
+
+        When prefetch_related() is called more than once, the list of fields to
+        prefetch is added to. If prefetch_related() is called with no arguments
+        the list is cleared.
+        """
+        if fields == (None,):
+            new_fields = set()
+        else:
+            new_fields = self._prefetch_related.union(set(fields))
+        return self._clone(_prefetch_related=new_fields)
+
     def dup_select_related(self, other):
         """
         Copies the related selection status from the QuerySet 'other' to the
@@ -798 +842 @@
             query.filter_is_sticky = True
         c = klass(model=self.model, query=query, using=self._db)
         c._for_write = self._for_write
+        c._prefetch_related = self._prefetch_related
         c.__dict__.update(kwargs)
         if setup and hasattr(c, '_setup_query'):
             c._setup_query()
@@ -1484 +1529 @@
     query = sql.InsertQuery(model)
     query.insert_values(fields, objs, raw=raw)
     return query.get_compiler(using=using).execute_sql(return_id)
+
+
+def prefetch_related_objects(result_cache, fields):
+    """
+    Populates prefetched objects caches for a list of results
+    from a QuerySet
+    """
+    from django.db.models.sql.constants import LOOKUP_SEP
+
+    if len(result_cache) == 0:
+        return # nothing to do
+
+    model = result_cache[0].__class__
+
+    # We need to be able to dynamically add to the list of prefetch_related
+    # fields that we look up (see below).  So we need some book keeping to
+    # ensure we don't do duplicate work.
+    done_fields = set() # list of fields like foo__bar__baz
+    done_lookups = {}   # dictionary of things like 'foo__bar': [results]
+    fields = list(fields)
+
+    # We may expand fields, so need a loop that allows for that
+    i = 0
+    while i < len(fields):
+        # 'field' can span several relationships, and so represent multiple
+        # lookups.
+        field = fields[i]
+
+        if field in done_fields:
+            # We've done exactly this already, skip the whole thing
+            i += 1
+            continue
+        done_fields.add(field)
+
+        # Top level, the list of objects to decorate is the the result cache
+        # from the primary QuerySet. It won't be for deeper levels.
+        obj_list = result_cache
+
+        attrs = field.split(LOOKUP_SEP)
+        for level, attr in enumerate(attrs):
+            # Prepare main instances
+            if len(obj_list) == 0:
+                break
+
+            good_objects = True
+            for obj in obj_list:
+                if not hasattr(obj, '_prefetched_objects_cache'):
+                    try:
+                        obj._prefetched_objects_cache = {}
+                    except AttributeError:
+                        # Must be in a QuerySet subclass that is not returning
+                        # Model instances, either in Django or 3rd
+                        # party. prefetch_related() doesn't make sense, so quit
+                        # now.
+                        good_objects = False
+                        break
+            if not good_objects:
+                break
+
+            # Descend down tree
+            try:
+                rel_obj = getattr(obj_list[0], attr)
+            except AttributeError:
+                raise AttributeError("Cannot find '%s' on %s object, '%s' is an invalid "
+                                     "parameter to prefetch_related()" %
+                                     (attr, obj_list[0].__class__.__name__, field))
+
+            can_prefetch = hasattr(rel_obj, 'get_prefetch_query_set')
+            if level == len(attrs) - 1 and not can_prefetch:
+                # Last one, this *must* resolve to a related manager.
+                raise ValueError("'%s' does not resolve to a supported 'many related"
+                                 " manager' for model %s - this is an invalid"
+                                 " parameter to prefetch_related()."
+                                 % (field, model.__name__))
+
+            if can_prefetch:
+                # Check we didn't do this already
+                lookup = LOOKUP_SEP.join(attrs[0:level+1])
+                if lookup in done_lookups:
+                    obj_list = done_lookups[lookup]
+                else:
+                    relmanager = rel_obj
+                    obj_list, additional_prf = _prefetch_one_level(obj_list, relmanager, attr)
+                    for f in additional_prf:
+                        new_prf = LOOKUP_SEP.join([lookup, f])
+                        fields.append(new_prf)
+                    done_lookups[lookup] = obj_list
+            else:
+                # Assume we've got some singly related object. We replace
+                # the current list of parent objects with that list.
+                obj_list = [getattr(obj, attr) for obj in obj_list]
+
+        i += 1
+
+
+def _prefetch_one_level(instances, relmanager, attname):
+    """
+    Runs prefetches on all instances using the manager relmanager,
+    assigning results to queryset against instance.attname.
+
+    The prefetched objects are returned, along with any additional
+    prefetches that must be done due to prefetch_related fields
+    found from default managers.
+    """
+    mainqs = relmanager.get_prefetch_query_set(instances)
+    # We have to handle the possibility that the default manager itself added
+    # prefetch_related fields to the QuerySet we just got back. We don't want to
+    # trigger the prefetch_related functionality by evaluating the query.
+    # Rather, we need to merge in the prefetch_related fields.
+    additional_prf = list(getattr(mainqs, '_prefetch_related', []))
+    if additional_prf:
+        mainqs = mainqs.prefetch_related(None)
+    all_related_objects = list(mainqs)
+    for obj in instances:
+        qs = getattr(obj, attname).all()
+        qs._result_cache = relmanager.select_matching_instances(obj, all_related_objects)
+        # We don't want the individual qs doing prefetch_related now, since we
+        # have merged this into the current work.
+        qs._prefetch_done = True
+        obj._prefetched_objects_cache[attname] = qs
+    return all_related_objects, additional_prf

docs/ref/models/querysets.txt

@@ -690 +690 @@
 A :class:`~django.db.models.OneToOneField` is not traversed in the reverse
 direction if you are performing a depth-based ``select_related()`` call.
 
+prefetch_related
+~~~~~~~~~~~~~~~~
+
+.. method:: prefetch_related(*fields)
+
+.. versionadded:: 1.4
+
+Returns a ``QuerySet`` that will automatically retrieve, in a single batch,
+related many-to-many and many-to-one objects for the specified fields.
+
+This is similar to ``select_related`` for the 'many related objects' case, but
+note that ``prefetch_related`` causes a separate query to be issued for each set
+of related objects that you request, unlike ``select_related`` which modifies
+the original query with joins in order to get the related objects. With
+``prefetch_related``, the additional queries are done as soon as the QuerySet
+begins to be evaluated.
+
+For example, suppose you have these models::
+
+    class Topping(models.Model):
+        name = models.CharField(max_length=30)
+
+    class Pizza(models.Model):
+        name = models.CharField(max_length=50)
+        toppings = models.ManyToManyField(Topping)
+
+        def __unicode__(self):
+            return u"%s (%s)" % (self.name, u", ".join([topping.name
+                                                        for topping in self.toppings.all()]))
+
+and run this code::
+
+    >>> Pizza.objects.all()
+    [u"Hawaiian (ham, pineapple)", u"Seafood (prawns, smoked salmon)"...
+
+The problem with this code is that it will run a query on the Toppings table for
+**every** item in the Pizza ``QuerySet``.  Using ``prefetch_related``, this can
+be reduced to two:
+
+    >>> pizzas = Pizza.objects.all().prefetch_related('toppings')
+
+All the relevant toppings will be fetched in a single query, and used to make
+``QuerySets`` that have a pre-filled cache of the relevant results. These
+``QuerySets`` are then used in the ``self.toppings.all()`` calls.
+
+Please note that use of ``prefetch_related`` will mean that the additional
+queries run will **always** be executed - even if you never use the related
+objects - and it always fully populates the result cache on the primary
+``QuerySet`` (which can sometimes be avoided in other cases).
+
+Also remember that, as always with QuerySets, any subsequent chained methods
+will ignore previously cached results, and retrieve data using a fresh database
+query. So, if you write the following:
+
+    >>> pizzas = Pizza.objects.prefetch_related('toppings')
+    >>> [list(pizza.topppings.filter(spicy=True) for pizza in pizzas]
+
+...then the fact that `pizza.toppings.all()` has been prefetched will not help
+you - in fact it hurts performance, since you have done a database query that
+you haven't used. So use this feature with caution!
+
+The fields that must be supplied to this method can be any attributes on the
+model instances which represent related queries that return multiple
+objects. This includes attributes representing the 'many' side of ``ForeignKey``
+relationships, forward and reverse ``ManyToManyField`` attributes, and also any
+``GenericRelations``.
+
+You can also use the normal join syntax to do related fields of related
+fields. Suppose we have an additional model to the example above::
+
+    class Restaurant(models.Model):
+        pizzas = models.ManyToMany(Pizza, related_name='restaurants')
+        best_pizza = models.ForeignKey(Pizza, related_name='championed_by')
+
+The following are all legal:
+
+    >>> Restaurant.objects.prefetch_related('pizzas__toppings')
+
+This will prefetch all pizzas belonging to restaurants, and all toppings
+belonging to those pizzas. This will result in a total of 3 database queries -
+one for the restaurants, one for the pizzas, and one for the toppings.
+
+    >>> Restaurant.objects.select_related('best_pizza').prefetch_related('best_pizza__toppings')
+
+This will fetch the best pizza and all the toppings for the best pizza for each
+restaurant. This will be done in 2 database queries - one for the restaurants
+and 'best pizzas' combined (achieved through use of ``select_related``), and one
+for the toppings.
+
+Chaining ``prefetch_related`` calls will accumulate the fields that should have
+this behavior applied. To clear any ``prefetch_related`` behavior, pass `None`
+as a parameter::
+
+   >>> unprefetch = qs.prefetch_related(None)
+
+When using ``prefetch_related``, one difference is that, in some circumstances,
+objects created by a query can be shared between the different objects that they
+are related to i.e. a single Python model instance can appear at more than one
+point in the tree of objects that are returned. Normally this behavior will not
+be a problem, and will in fact save both memory and CPU time.
+
 extra
 ~~~~~
 

new file tests/modeltests/prefetch_related/models.py

@@ +1 @@
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes import generic
+from django.db import models
+
+## Basic tests
+
+class Author(models.Model):
+    name = models.CharField(max_length=50)
+    first_book = models.ForeignKey('Book', related_name='first_time_authors')
+
+    def __unicode__(self):
+        return self.name
+
+    class Meta:
+        ordering = ['id']
+
+
+class Book(models.Model):
+    title = models.CharField(max_length=255)
+    authors = models.ManyToManyField(Author, related_name='books')
+
+    def __unicode__(self):
+        return self.title
+
+    class Meta:
+        ordering = ['id']
+
+
+class Reader(models.Model):
+    name = models.CharField(max_length=50)
+    books_read = models.ManyToManyField(Book, related_name='read_by')
+
+    def __unicode__(self):
+        return self.name
+
+    class Meta:
+        ordering = ['id']
+
+
+## Models for default manager tests
+
+class Qualification(models.Model):
+    name = models.CharField(max_length=10)
+
+    class Meta:
+        ordering = ['id']
+
+
+class TeacherManager(models.Manager):
+    def get_query_set(self):
+        return super(TeacherManager, self).get_query_set().prefetch_related('qualifications')
+
+
+class Teacher(models.Model):
+    name = models.CharField(max_length=50)
+    qualifications = models.ManyToManyField(Qualification)
+
+    objects = TeacherManager()
+
+    def __unicode__(self):
+        return "%s (%s)" % (self.name, ", ".join(q.name for q in self.qualifications.all()))
+
+    class Meta:
+        ordering = ['id']
+
+
+class Department(models.Model):
+    name = models.CharField(max_length=50)
+    teachers = models.ManyToManyField(Teacher)
+
+    class Meta:
+        ordering = ['id']
+
+
+## Generic relation tests
+
+class TaggedItem(models.Model):
+    tag = models.SlugField()
+    content_type = models.ForeignKey(ContentType, related_name="taggeditem_set2")
+    object_id = models.PositiveIntegerField()
+    content_object = generic.GenericForeignKey('content_type', 'object_id')
+
+    def __unicode__(self):
+        return self.tag
+
+
+class Bookmark(models.Model):
+    url = models.URLField()
+    tags = generic.GenericRelation(TaggedItem)

new file tests/modeltests/prefetch_related/tests.py

@@ +1 @@
+from django.contrib.contenttypes.models import ContentType
+from django.test import TestCase
+
+from models import Author, Book, Reader, Qualification, Teacher, Department, TaggedItem, Bookmark
+
+
+class PrefetchRelatedTests(TestCase):
+
+    def setUp(self):
+
+        self.book1 = Book.objects.create(title="Poems")
+        self.book2 = Book.objects.create(title="Jane Eyre")
+        self.book3 = Book.objects.create(title="Wuthering Heights")
+        self.book4 = Book.objects.create(title="Sense and Sensibility")
+
+        self.author1 = Author.objects.create(name="Charlotte",
+                                             first_book=self.book1)
+        self.author2 = Author.objects.create(name="Anne",
+                                             first_book=self.book1)
+        self.author3 = Author.objects.create(name="Emily",
+                                             first_book=self.book1)
+        self.author4 = Author.objects.create(name="Jane",
+                                             first_book=self.book4)
+
+        self.book1.authors.add(self.author1, self.author2, self.author3)
+        self.book2.authors.add(self.author1)
+        self.book3.authors.add(self.author3)
+        self.book4.authors.add(self.author4)
+
+        self.reader1 = Reader.objects.create(name="Amy")
+        self.reader2 = Reader.objects.create(name="Belinda")
+
+        self.reader1.books_read.add(self.book1, self.book4)
+        self.reader2.books_read.add(self.book2, self.book4)
+
+    def test_m2m_forward(self):
+        with self.assertNumQueries(2):
+            lists = [list(b.authors.all()) for b in Book.objects.prefetch_related('authors')]
+
+        normal_lists = [list(b.authors.all()) for b in Book.objects.all()]
+        self.assertEqual(lists, normal_lists)
+
+
+    def test_m2m_reverse(self):
+        with self.assertNumQueries(2):
+            lists = [list(a.books.all()) for a in Author.objects.prefetch_related('books')]
+
+        normal_lists = [list(a.books.all()) for a in Author.objects.all()]
+        self.assertEqual(lists, normal_lists)
+
+    def test_foreignkey_reverse(self):
+        with self.assertNumQueries(2):
+            lists = [list(b.first_time_authors.all())
+                     for b in Book.objects.prefetch_related('first_time_authors')]
+
+        self.assertQuerysetEqual(self.book2.authors.all(), [u"<Author: Charlotte>"])
+
+    def test_survives_clone(self):
+        with self.assertNumQueries(2):
+            lists = [list(b.first_time_authors.all())
+                     for b in Book.objects.prefetch_related('first_time_authors').exclude(id=1000)]
+
+    def test_len(self):
+        with self.assertNumQueries(2):
+            qs = Book.objects.prefetch_related('first_time_authors')
+            length = len(qs)
+            lists = [list(b.first_time_authors.all())
+                     for b in qs]
+
+    def test_bool(self):
+        with self.assertNumQueries(2):
+            qs = Book.objects.prefetch_related('first_time_authors')
+            x = bool(qs)
+            lists = [list(b.first_time_authors.all())
+                     for b in qs]
+
+    def test_clear(self):
+        """
+        Test that we can clear the behavior by calling prefetch_related()
+        """
+        with self.assertNumQueries(5):
+            with_prefetch = Author.objects.prefetch_related('books')
+            without_prefetch = with_prefetch.prefetch_related(None)
+            lists = [list(a.books.all()) for a in without_prefetch]
+
+    def test_m2m_then_m2m(self):
+        """
+        Test we can follow a m2m and another m2m
+        """
+        with self.assertNumQueries(3):
+            qs = Author.objects.prefetch_related('books__read_by')
+            lists = [[[unicode(r) for r in b.read_by.all()]
+                      for b in a.books.all()]
+                     for a in qs]
+            self.assertEqual(lists,
+            [
+                [[u"Amy"], [u"Belinda"]],  # Charlotte - Poems, Jane Eyre
+                [[u"Amy"]],                # Anne - Poems
+                [[u"Amy"], []],            # Emily - Poems, Wuthering Heights
+                [[u"Amy", u"Belinda"]],    # Jane - Sense and Sense
+            ])
+
+    def test_get(self):
+        """
+        Test that objects retrieved with .get() get the prefetch behaviour
+        """
+        # Need a double
+        with self.assertNumQueries(3):
+            author = Author.objects.prefetch_related('books__read_by').get(name="Charlotte")
+            lists = [[unicode(r) for r in b.read_by.all()]
+                      for b in author.books.all()]
+            self.assertEqual(lists, [[u"Amy"], [u"Belinda"]])  # Poems, Jane Eyre
+
+    def test_foreign_key_then_m2m(self):
+        """
+        Test we can follow an m2m relation after a relation like ForeignKey
+        that doesn't have many objects
+        """
+
+        with self.assertNumQueries(2):
+            qs = Author.objects.select_related('first_book').prefetch_related('first_book__read_by')
+            lists = [[unicode(r) for r in a.first_book.read_by.all()]
+                     for a in qs]
+            self.assertEqual(lists, [[u"Amy"],
+                                     [u"Amy"],
+                                     [u"Amy"],
+                                     [u"Amy", "Belinda"]])
+
+    def test_reuse(self):
+        # Check re-use of objects.
+        qs1 = Reader.objects.all()
+        qs2 = Reader.objects.prefetch_related('books_read__first_time_authors')
+
+        authors1 = [a for r in qs1
+                    for b in r.books_read.all()
+                    for a in b.first_time_authors.all()]
+        authors2 = [a for r in qs2
+                    for b in r.books_read.all()
+                    for a in b.first_time_authors.all()]
+
+        # The second prefetch_related lookup is a reverse foreign key. This
+        # means the query for it can only be something like
+        # "first_time_authors__pk__in = [...]" and cannot return more rows then
+        # the number of Author objects in the database.  This means that these
+        # objects will be reused (since in our data we've arranged for there
+        # len(authors1) > Author.objects.count())
+
+        total_authors = Author.objects.count()
+        self.assertEqual(len(authors1), len(authors2))
+        self.assertTrue(len(authors1) > total_authors)
+        self.assertTrue(len(set(map(id, authors1))) > len(set(map(id, authors2))))
+        self.assertEqual(total_authors, len(set(map(id, authors2))))
+
+    def test_attribute_error(self):
+        qs = Reader.objects.all().prefetch_related('books_read__xyz')
+        with self.assertRaises(AttributeError) as cm:
+            list(qs)
+
+        self.assertTrue('prefetch_related' in cm.exception.message)
+
+    def test_invalid_final_lookup(self):
+        qs = Book.objects.prefetch_related('authors__first_book')
+        with self.assertRaises(ValueError) as cm:
+            list(qs)
+
+        self.assertTrue('prefetch_related' in cm.exception.message)
+        self.assertTrue("first_book" in cm.exception.message)
+
+
+class DefaultManagerTests(TestCase):
+
+    def setUp(self):
+        self.qual1 = Qualification.objects.create(name="BA")
+        self.qual2 = Qualification.objects.create(name="BSci")
+        self.qual3 = Qualification.objects.create(name="MA")
+        self.qual4 = Qualification.objects.create(name="PhD")
+
+        self.teacher1 = Teacher.objects.create(name="Mr Cleese")
+        self.teacher2 = Teacher.objects.create(name="Mr Idle")
+        self.teacher3 = Teacher.objects.create(name="Mr Chapman")
+
+        self.teacher1.qualifications.add(self.qual1, self.qual2, self.qual3, self.qual4)
+        self.teacher2.qualifications.add(self.qual1)
+        self.teacher3.qualifications.add(self.qual2)
+
+        self.dept1 = Department.objects.create(name="English")
+        self.dept2 = Department.objects.create(name="Physics")
+
+        self.dept1.teachers.add(self.teacher1, self.teacher2)
+        self.dept2.teachers.add(self.teacher1, self.teacher3)
+
+    def test_m2m_then_m2m(self):
+        with self.assertNumQueries(3):
+            # When we prefetch the teachers, and force the query, we don't want
+            # the default manager on teachers to immediately get all the related
+            # qualifications, since this will do one query per teacher.
+            qs = Department.objects.prefetch_related('teachers')
+            depts = "".join(["%s department: %s\n" %
+                             (dept.name, ", ".join(unicode(t) for t in dept.teachers.all()))
+                             for dept in qs])
+
+            self.assertEqual(depts,
+                             "English department: Mr Cleese (BA, BSci, MA, PhD), Mr Idle (BA)\n"
+                             "Physics department: Mr Cleese (BA, BSci, MA, PhD), Mr Chapman (BSci)\n")
+
+
+class GenericRelationTests(TestCase):
+
+    def test_traverse_GFK(self):
+        """
+        Test that we can traverse a 'content_object' with prefetch_related()
+        """
+        # In fact, there is no special support for this in prefetch_related code
+        # - we can traverse any object that will lead us to objects that have
+        # related managers.
+
+        book1 = Book.objects.create(title="Winnie the Pooh")
+        book2 = Book.objects.create(title="Do you like green eggs and spam?")
+
+        reader1 = Reader.objects.create(name="me")
+        reader2 = Reader.objects.create(name="you")
+
+        book1.read_by.add(reader1)
+        book2.read_by.add(reader2)
+
+        TaggedItem.objects.create(tag="awesome", content_object=book1)
+        TaggedItem.objects.create(tag="awesome", content_object=book2)
+
+        ct = ContentType.objects.get_for_model(Book)
+
+        # We get 4 queries - 1 for main query, 2 for each access to
+        # 'content_object' because these can't be handled by select_related, and
+        # 1 for the 'read_by' relation.
+        with self.assertNumQueries(4):
+            # If we limit to books, we know that they will have 'read_by'
+            # attributes, so the following makes sense:
+            qs = TaggedItem.objects.select_related('content_type').prefetch_related('content_object__read_by').filter(tag='awesome').filter(content_type=ct, tag='awesome')
+            readers_of_awesome_books = [r.name for tag in qs
+                                        for r in tag.content_object.read_by.all()]
+            self.assertEqual(readers_of_awesome_books, ["me", "you"])
+
+
+    def test_generic_relation(self):
+        b = Bookmark.objects.create(url='http://www.djangoproject.com/')
+        t1 = TaggedItem.objects.create(content_object=b, tag='django')
+        t2 = TaggedItem.objects.create(content_object=b, tag='python')
+
+        with self.assertNumQueries(2):
+            tags = [t.tag for b in Bookmark.objects.prefetch_related('tags')
+                    for t in b.tags.all()]
+            self.assertEqual(sorted(tags), ["django", "python"])