Context Navigation

Back to Ticket #5420

Ticket #5420: lazy.diff

File lazy.diff, 48.8 KB (added by Jesse Young, 16 years ago)
implementation of lazy fields

django/db/models/sql/query.py

         self.join_map = {}
         self.rev_join_map = {}  # Reverse of join_map.
         self.quote_cache = {}
-        self.default_cols = True
         self.default_ordering = True
         self.standard_ordering = True
         self.ordering_aliases = []
 …
         self.distinct = False
         self.select_related = False
         self.related_select_cols = []
+        self.custom_fields = False
         # Arbitrary maximum limit for select_related. Prevents infinite
         # recursion. Can be changed by the depth parameter to select_related().
 …
         obj.join_map = self.join_map.copy()
         obj.rev_join_map = self.rev_join_map.copy()
         obj.quote_cache = {}
-        obj.default_cols = self.default_cols
         obj.default_ordering = self.default_ordering
         obj.standard_ordering = self.standard_ordering
         obj.ordering_aliases = []
 …
         obj.distinct = self.distinct
         obj.select_related = self.select_related
         obj.related_select_cols = []
+        obj.custom_fields = self.custom_fields
         obj.max_depth = self.max_depth
         obj.extra_select = self.extra_select.copy()
         obj.extra_tables = self.extra_tables
 …
             obj._setup_query()
         return obj
+    def get_base_fields(self):
+        """
+        Returns a list of fields on the base object (i.e., not related objects)
+        that are retrieved by this query. In the result row, these columns are
+        row[len(self.extra_select):len(self.extra_select)+len(self.get_base_fields())]
+        """
+        if self.select_fields:
+            return self.select_fields
+        else:
+            return self.model._meta.default_select_fields
     def results_iter(self):
         """
         Returns an iterator over the results from executing this query.
 …
                         # We only set this up here because
                         # related_select_fields isn't populated until
                         # execute_sql() has been called.
+                        if self.select_fields:
+                            fields = self.select_fields + self.related_select_fields
+                        else:
+                            fields = self.model._meta.fields
+                        fields = self.get_base_fields()
+                        if self.related_select_fields:
+                            fields = fields + self.related_select_fields
                     row = self.resolve_columns(row, fields)
                 yield row
 …
         If 'with_aliases' is true, any column names that are duplicated
         (without the table names) are given unique aliases. This is needed in
         some cases to avoid ambiguitity with nested queries.
+        some cases to avoid ambiguitity with nested queries.
         """
         qn = self.quote_name_unless_alias
         qn2 = self.connection.ops.quote_name
 …
             col_aliases = aliases.copy()
         else:
             col_aliases = set()
         if self.select:
             for col in self.select:
                 if isinstance(col, (list, tuple)):
+                    r = '%s.%s' % (qn(col[0]), qn(col[1]))
+                    if with_aliases and col[1] in col_aliases:
+                        c_alias = 'Col%d' % len(col_aliases)
+                        result.append('%s AS %s' % (r, c_alias))
+                        aliases.add(c_alias)
+                        col_aliases.add(c_alias)
+                    else:
+                        result.append(r)
+                        aliases.add(r)
+                        col_aliases.add(col[1])
+                    result.append(self._get_column(
+                        table_alias=col[0],field=col[1],
+                        with_aliases=with_aliases,
+                        col_aliases=col_aliases,
+                        aliases=aliases))
                 else:
                     result.append(col.as_sql(quote_func=qn))
                     if hasattr(col, 'alias'):
                         aliases.add(col.alias)
                         col_aliases.add(col.alias)
+        elif self.default_cols:
+            cols, new_aliases = self.get_default_columns(with_aliases,
+                    col_aliases)
+            result.extend(cols)
+            aliases.update(new_aliases)
+        for table, col in self.related_select_cols:
+            r = '%s.%s' % (qn(table), qn(col))
+            if with_aliases and col in col_aliases:
+                c_alias = 'Col%d' % len(col_aliases)
+                result.append('%s AS %s' % (r, c_alias))
+                aliases.add(c_alias)
+                col_aliases.add(c_alias)
+            else:
+                result.append(r)
+                aliases.add(r)
+                col_aliases.add(col)
+        else:
+            result.extend(self.get_base_columns(with_aliases, col_aliases, aliases))
+        for table, field in self.related_select_cols:
+            result.append(self._get_column(table, field,
+                with_aliases, col_aliases, aliases))
         self._select_aliases = aliases
         return result
+    def get_default_columns(self, with_aliases=False, col_aliases=None,
+            start_alias=None, opts=None, as_pairs=False):
+    def _get_table_alias(self, model, seen, root_pk):
         """
+        Computes the default columns for selecting every field in the base
+        model.
+        Gets a table alias for the given model. Seen is a dict
+        mapping already-used models to table aliases, and may be updated
+        as a side effect.
+        """
+        try:
+            return seen[model]
+        except KeyError:
+            alias = self.join((seen[None], model._meta.db_table,
+                    root_pk, model._meta.pk.column))
+            seen[model] = alias
+            return alias
+    def get_base_columns(self, with_aliases, col_aliases, aliases):
+        """
+        Computes the columns to select on the base model. By default, this includes all
+        non-lazy fields. However, this list can be modified by calling toggle_fields.
+        Returns a list of strings, quoted appropriately for use in SQL
+        directly, as well as a set of aliases used in the select statement (if
+        'as_pairs' is True, returns a list of (alias, col_name) pairs instead
+        of strings as the first component and None as the second component).
+        """
+        result = []
+        if opts is None:
+            opts = self.model._meta
+        if start_alias:
+            table_alias = start_alias
+        else:
+            table_alias = self.tables[0]
+        Returns a list of strings, quoted appropriately for use in SQL directly.
+        As a side effect, updates the aliases and col_aliases sets passed in as an argument.
+        """
+        result = []
+        opts = self.model._meta
         root_pk = opts.pk.column
+        seen = {None: table_alias}
+        qn = self.quote_name_unless_alias
+        qn2 = self.connection.ops.quote_name
+        seen = {None: self.tables[0]}
         aliases = set()
+        for field, model in opts.get_fields_with_model():
+            try:
+                alias = seen[model]
+            except KeyError:
+                alias = self.join((table_alias, model._meta.db_table,
+                        root_pk, model._meta.pk.column))
+                seen[model] = alias
+            if as_pairs:
+                result.append((alias, field.column))
+                continue
+            if with_aliases and field.column in col_aliases:
+                c_alias = 'Col%d' % len(col_aliases)
+                result.append('%s.%s AS %s' % (qn(alias),
+                    qn2(field.column), c_alias))
+                col_aliases.add(c_alias)
+                aliases.add(c_alias)
+            else:
+                r = '%s.%s' % (qn(alias), qn2(field.column))
+                result.append(r)
+                aliases.add(r)
+                if with_aliases:
+                    col_aliases.add(field.column)
+        if as_pairs:
+            return result, None
+        return result, aliases
+        if self.custom_fields:
+            fields_with_model = [(f, opts.get_field_by_name(f.name)[1]) for f in self.select_fields]
+        else:
+            fields_with_model = opts.get_default_select_fields_with_model()
+        for field, model in fields_with_model:
+            table_alias = self._get_table_alias(model, seen, root_pk)
+            result.append(self._get_column(table_alias, field,
+                                           with_aliases, col_aliases, aliases))
+        return result
+    def get_related_columns(self, start_alias, opts):
+        """
+        Computes the default columns for selecting every non-lazy field for a related
+        model (described by opts). The related field columns currently cannot be
+        customized per-query.
+        Returns a list of (alias, field) pairs, where alias is a string quoted
+        appropriately for use in SQL directly.
+        """
+        result = []
+        seen = {None: start_alias}
+        root_pk = opts.pk.column
+        for field, model in opts.get_default_select_fields_with_model():
+            result.append((self._get_table_alias(model, seen, root_pk), field))
+        return result
+    def _get_field_select(self, table_alias, field):
+        """
+        Returns a string with a SQL column name, quoted appropriately for use
+        in a SQL select statement.
+        """
+        return '%s.%s' % (self.quote_name_unless_alias(table_alias),
+                          self.connection.ops.quote_name(field.column))
+    def _get_column(self, table_alias, field, with_aliases, col_aliases, aliases):
+        """
+        Returns a string for a SQL select statement, containing a column name
+        as well as a column alias if necessary.
+        The col_aliases and aliases sets are modified as a side effect.
+        """
+        r = self._get_field_select(table_alias, field)
+        if with_aliases and column_name in col_aliases:
+            c_alias = 'Col%d' % len(col_aliases)
+            col_aliases.add(c_alias)
+            aliases.add(c_alias)
+            return '%s AS %s' % (r, c_alias)
+        else:
+            aliases.add(r)
+            if with_aliases:
+                col_aliases.add(column_name)
+            return r
     def get_from_clause(self):
         """
         Returns a list of strings that are joined together to go after the
 …
             self.tables[pos] = new_alias
         self.change_aliases(change_map)
+    def toggle_fields(self, fetch_only, lazy, fetch):
+        self.custom_fields = True
+        opts = self.model._meta
+        if fetch_only is not None:
+            self.select_fields = []
+            if opts.pk.attname not in fetch_only:
+                self.select_fields.append(opts.pk)
+            for field_name in fetch_only:
+                self.select_fields.append(opts.get_field(field_name))
+        if not self.select_fields:
+            self.select_fields = opts.default_select_fields[:]
+        if fetch is not None:
+            for field_name in fetch:
+                self.select_fields.append(opts.get_field(field_name))
+        if lazy is not None:
+            self.select_fields = [f for f in self.select_fields if f.name not in lazy]
     def get_initial_alias(self):
         """
         Returns the first alias for this query, after increasing its reference
 …
                     f.rel.get_related_field().column),
                     exclusions=used.union(avoid), promote=promote)
             used.add(alias)
             self.related_select_cols.extend(self.get_default_columns(
                 start_alias=alias, opts=f.rel.to._meta, as_pairs=True)[0])
             self.related_select_fields.extend(f.rel.to._meta.fields)
+            new_related_cols = self.get_related_columns(start_alias=alias, opts=f.rel.to._meta)
+            self.related_select_cols.extend(new_related_cols)
+            self.related_select_fields.extend(f.rel.to._meta.default_select_fields)
             if restricted:
                 next = requested.get(f.name, {})
             else:
 …
         dupe_set = set()
         exclusions = set()
         extra_filters = []
+        last_pos = len(names) - 1
         for pos, name in enumerate(names):
             try:
                 exclusions.add(int_alias)
 …
                                 dupe_multis, exclusions, nullable=True,
                                 reuse=can_reuse)
                         joins.extend([int_alias, alias])
+                elif field.rel:
+                    # One-to-one or many-to-one field
+                if not field.rel or pos == last_pos:
+                    # Not a one-to-one or many-to-one field.
+                    #
+                    # Except, we do include one-to-one or many-to-one fields if their name is at the
+                    # end of the list.E.g., if names = ["fk_id"], we don't want to do any joins because
+                    # fk_id is stored in that row directly.
+                    target = field
+                    break
+                else:
+                    # One-to-one or many-to-one field, except as described above.
                     if cached_data:
                         (table, from_col, to_col, opts, target) = cached_data
                     else:
 …
                     alias = self.join((alias, table, from_col, to_col),
                             exclusions=exclusions, nullable=field.null)
                     joins.append(alias)
-                else:
-                    # Non-relation fields.
-                    target = field
-                    break
             else:
                 orig_field = field
                 field = field.field
 …
                         name.split(LOOKUP_SEP), opts, alias, False, allow_m2m,
                         True)
                 final_alias = joins[-1]
-                col = target.column
-                if len(joins) > 1:
-                    join = self.alias_map[final_alias]
-                    if col == join[RHS_JOIN_COL]:
-                        self.unref_alias(final_alias)
-                        final_alias = join[LHS_ALIAS]
-                        col = join[LHS_JOIN_COL]
-                        joins = joins[:-1]
                 self.promote_alias_chain(joins[1:])
                 self.select.append((final_alias, col))
+                self.select.append((final_alias, target))
                 self.select_fields.append(field)
         except MultiJoin:
             raise FieldError("Invalid field name: '%s'" % name)

django/db/models/base.py

 import django.db.models.manager     # Imported to register signal handler.
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned, FieldError
 from django.db.models.fields import AutoField
+from django.db.models.fields import AutoField, LazyDescriptor
 from django.db.models.fields.related import OneToOneRel, ManyToOneRel, OneToOneField
 from django.db.models.query import delete_objects, Q, CollectedObjects
 from django.db.models.options import Options
 …
     """
     def __new__(cls, name, bases, attrs):
         super_new = super(ModelBase, cls).__new__
+        #
+        # The LazyModel subclass passes _lazy_fields in attrs. In this case we quickly
+        # create a new Model subclass which is identical to its parent class
+        # (sharing e.g. the same _meta object), except for its _lazy_fields set.
+        # This allows certain fields of the Model to be lazily-loaded
+        # when they are first used, and only saved to the database if they're changed.
+        #
+        lazy_fields = attrs.get('_lazy_fields', None)
+        if lazy_fields is not None:
+            return super_new(cls, bases[0].__name__, bases, attrs)
         parents = [b for b in bases if isinstance(b, ModelBase)]
         if not parents:
             # If this isn't a subclass of Model, don't do anything special.
 …
         if getattr(new_class, '_default_manager', None):
             new_class._default_manager = None
+        # make sure subclasses don't share _lazy_fields sets
+        new_class._lazy_fields = None
         # Bail out early if we have already created this class.
         m = get_model(new_class._meta.app_label, name, False)
         if m is not None:
 …
         for field in fields_iter:
             rel_obj = None
+            use_default = False
             if kwargs:
                 if isinstance(field.rel, ManyToOneRel):
                     try:
 …
                             # Object instance wasn't passed in -- must be an ID.
                             val = kwargs.pop(field.attname)
                         except KeyError:
                             val = field.get_default()
+                            use_default = True
                     else:
                         # Object instance was passed in. Special case: You can
                         # pass in "None" for related objects if it's allowed.
                         if rel_obj is None and field.null:
                             val = None
                 else:
+                    val = kwargs.pop(field.attname, field.get_default())
+                    try:
+                        val = kwargs.pop(field.attname)
+                    except KeyError:
+                        use_default = True
             else:
+                val = field.get_default()
+                use_default = True
+            if use_default:
+                if self._lazy_fields is not None and (field.attname in self._lazy_fields):
+                    continue
+                val = field.get_default()
             # If we got passed a related instance, set it using the field.name
             # instead of field.attname (e.g. "user" instead of "user_id") so
             # that the object gets properly cached (and type checked) by the
 …
             if kwargs:
                 raise TypeError, "'%s' is an invalid keyword argument for this function" % kwargs.keys()[0]
         signals.post_init.send(sender=self.__class__, instance=self)
+    @classmethod
+    def _new_from_select_values(cls, vals, fields):
+        """
+        Creates a new instance of this model class.
+        The class will be instantiated as if calling __init__ with kwargs
+        fields[i]=vals[i] for all i (i.e., fields and vals are parallel lists).
+        However, it may call __init__ using positional arguments when all
+        fields are provided (i.e., there are no lazy fields).
+        """
+        # we use pointer-equality to make the test faster in the normal case
+        all_fields = fields is cls._meta.fields
+        if all_fields:
+            obj = cls(*vals) # the comments for __init__ indicate this is faster
+        else:
+            obj = cls(**dict(zip([f.attname for f in fields], vals)))
+        #
+        # We may have set a default-lazy field in the constructor; if so, we
+        # clear its dirty bit since that value came straight from a SELECT.
+        # However, we optimize the case where there are no lazy fields, to avoid
+        # making object instantiation slower in the normal case.
+        #
+        if cls._lazy_fields is not None:
+            for field in cls._meta.fields:
+                field.mark_unchanged(obj)
+        return obj
+    @classmethod
+    def _get_lazy_subclass(cls, non_lazy_fields):
+        """
+        Returns a subclass which is identical to the given class, except that
+        every field not in non_lazy_fields becomes lazy. However, any field
+        that was already lazy in the given class stays lazy in the subclass.
+        """
+        if non_lazy_fields is cls._meta.fields:
+            return cls
+        non_lazy_names = set()
+        for field in non_lazy_fields:
+            non_lazy_names.add(field.name)
+        new_lazy_fields = [f for f in cls._meta.fields if f.name not in non_lazy_names
+                                                       and not cls.is_lazy_field(f)]
+        if new_lazy_fields:
+            # Any field which is lazy in the base class remains lazy in the
+            # subclass, even if it's in non_lazy_fields. The reasoning for this
+            # is that default-lazy fields likely have a lot of data, so we
+            # probably still want to check if it's dirty when we call save().
+            lazy_fields = set([f.attname for f in new_lazy_fields])
+            if cls._lazy_fields is not None:
+                lazy_fields.update(cls._lazy_fields)
+            class LazyModel(cls):
+                _lazy_fields = lazy_fields
+            for field in new_lazy_fields:
+                setattr(LazyModel, field.attname, LazyDescriptor(field))
+            return LazyModel
+        else:
+            return cls
+    @classmethod
+    def is_lazy_field(cls, field):
+        return cls._lazy_fields is not None and (field.attname in cls._lazy_fields)
     def __repr__(self):
         return smart_str(u'<%s: %s>' % (self.__class__.__name__, unicode(self)))
 …
                 self.save_base(raw, parent)
                 setattr(self, field.attname, self._get_pk_val(parent._meta))
         non_pks = [f for f in meta.local_fields if not f.primary_key]
+        non_pks = [f for f in meta.local_fields if (not f.primary_key) and f.write_on_save(self)]
         # First, try an UPDATE. If that doesn't update anything, do an INSERT.
         pk_val = self._get_pk_val(meta)
 …
             if not pk_set:
                 if force_update:
                     raise ValueError("Cannot force an update in save() with no primary key.")
                 values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True))) for f in meta.local_fields if not isinstance(f, AutoField)]
+                values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True))) for f in meta.local_fields if not isinstance(f, AutoField) and f.write_on_save(self)]
             else:
                 values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True))) for f in meta.local_fields]
+                values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True))) for f in meta.local_fields if f.write_on_save(self)]
             if meta.order_with_respect_to:
                 field = meta.order_with_respect_to
 …
             if update_pk:
                 setattr(self, meta.pk.attname, result)
         transaction.commit_unless_managed()
+        for f in non_pks:
+            f.mark_unchanged(self)
         if signal:
             signals.post_save.send(sender=self.__class__, instance=self,
 …
             return
         for related in self._meta.get_all_related_objects():
+            if related.opts.is_view:
+                continue
             rel_opts_name = related.get_accessor_name()
             if isinstance(related.field.rel, OneToOneRel):
                 try:

django/db/models/manager.py

     def reverse(self, *args, **kwargs):
         return self.get_query_set().reverse(*args, **kwargs)
+    def toggle_fields(self, *args, **kwargs):
+        return self.get_query_set().toggle_fields(*args, **kwargs)
     def _insert(self, values, **kwargs):
         return insert_query(self.model, values, **kwargs)

django/db/models/options.py

         self.pk = None
         self.has_auto_field, self.auto_field = False, None
         self.one_to_one_field = None
+        self.abstract = False
+        self.has_lazy_field = False
+        self.abstract = False
         self.parents = SortedDict()
         self.duplicate_targets = {}
         # Managers that have been inherited from abstract base classes. These
 …
         if hasattr(self, '_name_map'):
             del self._name_map
+        if field.lazy:
+            self.has_lazy_field = True
     def add_virtual_field(self, field):
         self.virtual_fields.append(field)
 …
         return self._field_name_cache
     fields = property(_fields)
+    def _default_select_fields(self):
+        try:
+            return self._default_field_name_cache
+        except AttributeError:
+            self._fill_fields_cache()
+            return self._default_field_name_cache
+    default_select_fields = property(_default_select_fields)
+    def get_default_select_fields_with_model(self):
+        """
+        Returns a sequence of (field, model) pairs for only those fields which
+        are fetched by default on a SELECT query. The "model"
+        element is None for fields on the current model. Mostly of use when
+        constructing queries so that we know which model a field belongs to.
+        """
+        try:
+            self._default_field_cache
+        except AttributeError:
+            self._fill_fields_cache()
+        return self._default_field_cache
     def get_fields_with_model(self):
         """
         Returns a sequence of (field, model) pairs for all fields. The "model"
 …
         cache.extend([(f, None) for f in self.local_fields])
         self._field_cache = tuple(cache)
         self._field_name_cache = [x for x, _ in cache]
+        if self.has_lazy_field:
+            self._default_field_cache = tuple([(field, model) for field, model in cache if not field.lazy])
+            self._default_field_name_cache = [x for x, _ in self._default_field_cache]
+        else:
+            self._default_field_cache = self._field_cache
+            self._default_field_name_cache = self._field_name_cache
     def _many_to_many(self):
         try:

django/db/models/fields/init.py

             db_index=False, rel=None, default=NOT_PROVIDED, editable=True,
             serialize=True, unique_for_date=None, unique_for_month=None,
             unique_for_year=None, choices=None, help_text='', db_column=None,
             db_tablespace=None, auto_created=False):
+            db_tablespace=None, auto_created=False, lazy=False):
         self.name = name
         self.verbose_name = verbose_name
         self.primary_key = primary_key
 …
         self.db_column = db_column
         self.db_tablespace = db_tablespace or settings.DEFAULT_INDEX_TABLESPACE
         self.auto_created = auto_created
+        self.lazy = lazy
         # Set db_index to True if the field has a relationship and doesn't explicitly set db_index.
         self.db_index = db_index
 …
             self.creation_counter = Field.creation_counter
             Field.creation_counter += 1
+    def write_on_save(self, instance):
+        return not instance.is_lazy_field(self) or getattr(instance, self.get_changed_name(), False)
+    def mark_unchanged(self, instance):
+        if instance.is_lazy_field(self):
+            setattr(instance, self.get_changed_name(), False)
     def __cmp__(self, other):
         # This is needed because bisect does not take a comparison function.
         return cmp(self.creation_counter, other.creation_counter)
 …
         if self.choices:
             setattr(cls, 'get_%s_display' % self.name, curry(cls._get_FIELD_display, field=self))
+        if self.lazy:
+            if cls._lazy_fields is None:
+                cls._lazy_fields = set()
+            cls._lazy_fields.add(self.attname)
+            setattr(cls, self.attname, LazyDescriptor(self))
     def get_attname(self):
         return self.name
 …
     def get_cache_name(self):
         return '_%s_cache' % self.name
+    def get_changed_name(self):
+        return '_%s_changed' % self.attname
     def get_internal_type(self):
         return self.__class__.__name__
 …
     def __init__(self, verbose_name=None, name=None, schema_path=None, **kwargs):
         self.schema_path = schema_path
         Field.__init__(self, verbose_name, name, **kwargs)
+class LazyDescriptor(object):
+    """
+    A descriptor for a field that is lazy-loaded from the database the first time it is accessed.
+    Useful for large fields that are infrequently accessed by Django.
+    """
+    def __init__(self, lazy_field):
+        self.field = lazy_field
+        # we use a different cache_name than field.get_cache_name() to allow lazy foreign key fields
+        self.cache_name = '_%s_lcache' % lazy_field.attname
+        if self.field.primary_key:
+            raise TypeError, "A model's primary key field cannot be lazy."
+    def __get__(self, instance, instance_type=None):
+        try:
+            return getattr(instance, self.cache_name)
+        except AttributeError:
+            pkVal = instance._get_pk_val()
+            if pkVal is None:
+                val = self.field.get_default()
+            else:
+                results = instance.__class__._default_manager.filter(pk=pkVal).values(self.field.attname)
+                val = results[0][self.field.attname] # could be error if deleted
+            setattr(instance, self.cache_name, val)
+            return val
+    def __set__(self, instance, value):
+        setattr(instance, self.cache_name, value)
+        setattr(instance, self.field.get_changed_name(), True)

django/db/models/query.py

         max_depth = self.query.max_depth
         extra_select = self.query.extra_select.keys()
         index_start = len(extra_select)
+        fields = self.query.get_base_fields()
+        # Get a subclass which knows which fields are lazy and which are not.
+        # In the case where all fields are non-lazy, this is just self.model.
+        lazy_model = self.model._get_lazy_subclass(fields)
         for row in self.query.results_iter():
             if fill_cache:
                 obj, _ = get_cached_row(self.model, row, index_start,
                         max_depth, requested=requested)
+                obj, _ = get_cached_row(lazy_model, row, index_start,
+                        max_depth, requested=requested, fields=fields)
             else:
                 obj = self.model(*row[index_start:])
+                obj = lazy_model._new_from_select_values(row[index_start:], fields)
             for i, k in enumerate(extra_select):
                 setattr(obj, k, row[i])
             yield obj
+    def toggle_fields(self, fetch_only=None, lazy=None, fetch=None):
+        """
+        Change the fields that are retrieved by this query.
+        """
+        assert self.query.can_filter(), "Cannot change a query once a slice has been taken"
+        clone = self._clone()
+        clone.query.toggle_fields(fetch_only, lazy, fetch)
+        return clone
     def count(self):
         """
         Performs a SELECT COUNT() and returns the number of records as an
 …
                     else:
                         field_names.append(f)
         else:
             # Default to all fields.
             field_names = [f.attname for f in self.model._meta.fields]
+            # Default to all non-lazy fields.
+            field_names = [f.attname for f in self.model._meta.default_select_fields]
         self.query.add_fields(field_names, False)
-        self.query.default_cols = False
         self.field_names = field_names
     def _clone(self, klass=None, setup=False, **kwargs):
 …
 def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0,
                    requested=None):
+                   requested=None, fields=None):
     """
     Helper function that recursively returns an object with the specified
     related attributes already populated.
 …
         # We've recursed deeply enough; stop now.
         return None
+    if fields is None:
+        fields = klass._meta.default_select_fields
     restricted = requested is not None
     index_end = index_start + len(klass._meta.fields)
     fields = row[index_start:index_end]
+    index_end = index_start + len(fields)
+    values = row[index_start:index_end]
     if not [x for x in fields if x is not None]:
         # If we only have a list of Nones, there was not related object.
         obj = None
     else:
         obj = klass(*fields)
+        obj = klass._new_from_select_values(values, fields)
     for f in klass._meta.fields:
         if not select_related_descend(f, restricted, requested):
             continue

django/contrib/gis/db/models/sql/query.py

         obj.extra_select_fields = self.extra_select_fields.copy()
         return obj
     def get_columns(self, with_aliases=False):
+    def _get_field_select(self, table_alias, field):
         """
+        Return the list of columns to use in the select statement. If no
+        columns have been specified, returns all columns relating to fields in
+        the model.
+        If 'with_aliases' is true, any column names that are duplicated
+        (without the table names) are given unique aliases. This is needed in
+        some cases to avoid ambiguitity with nested queries.
+        This routine is overridden from Query to handle customized selection of
+        geometry columns.
+        """
+        qn = self.quote_name_unless_alias
+        qn2 = self.connection.ops.quote_name
+        result = ['(%s) AS %s' % (self.get_extra_select_format(alias) % col[0], qn2(alias))
+                  for alias, col in self.extra_select.iteritems()]
+        aliases = set(self.extra_select.keys())
+        if with_aliases:
+            col_aliases = aliases.copy()
+        Returns the SELECT SQL string for the given field.  Figures out
+        if any custom selection SQL is needed for the column  The `alias`
+        keyword may be used to manually specify the database table where
+        the column exists, if not in the model associated with this
+        `GeoQuery`.
+        """
+        if field in self.custom_select:
+            return self.get_select_format(field) % self.custom_select[field]
         else:
+            col_aliases = set()
+        if self.select:
+            # This loop customized for GeoQuery.
+            for col, field in izip(self.select, self.select_fields):
+                if isinstance(col, (list, tuple)):
+                    r = self.get_field_select(field, col[0])
+                    if with_aliases and col[1] in col_aliases:
+                        c_alias = 'Col%d' % len(col_aliases)
+                        result.append('%s AS %s' % (r, c_alias))
+                        aliases.add(c_alias)
+                        col_aliases.add(c_alias)
+                    else:
+                        result.append(r)
+                        aliases.add(r)
+                        col_aliases.add(col[1])
+                else:
+                    result.append(col.as_sql(quote_func=qn))
+                    if hasattr(col, 'alias'):
+                        aliases.add(col.alias)
+                        col_aliases.add(col.alias)
+        elif self.default_cols:
+            cols, new_aliases = self.get_default_columns(with_aliases,
+                    col_aliases)
+            result.extend(cols)
+            aliases.update(new_aliases)
+        # This loop customized for GeoQuery.
+        if not self.aggregate:
+            for (table, col), field in izip(self.related_select_cols, self.related_select_fields):
+                r = self.get_field_select(field, table)
+                if with_aliases and col in col_aliases:
+                    c_alias = 'Col%d' % len(col_aliases)
+                    result.append('%s AS %s' % (r, c_alias))
+                    aliases.add(c_alias)
+                    col_aliases.add(c_alias)
+                else:
+                    result.append(r)
+                    aliases.add(r)
+                    col_aliases.add(col)
+            return super(GeoQuery, self)._get_field_select(table_alias, field)
-        self._select_aliases = aliases
-        return result
-    def get_default_columns(self, with_aliases=False, col_aliases=None,
-                            start_alias=None, opts=None, as_pairs=False):
-        """
-        Computes the default columns for selecting every field in the base
-        model.
-        Returns a list of strings, quoted appropriately for use in SQL
-        directly, as well as a set of aliases used in the select statement.
-        This routine is overridden from Query to handle customized selection of
-        geometry columns.
-        """
-        result = []
-        if opts is None:
-            opts = self.model._meta
-        if start_alias:
-            table_alias = start_alias
-        else:
-            table_alias = self.tables[0]
-        root_pk = self.model._meta.pk.column
-        seen = {None: table_alias}
-        aliases = set()
-        for field, model in opts.get_fields_with_model():
-            try:
-                alias = seen[model]
-            except KeyError:
-                alias = self.join((table_alias, model._meta.db_table,
-                        root_pk, model._meta.pk.column))
-                seen[model] = alias
-            if as_pairs:
-                result.append((alias, field.column))
-                continue
-            # This part of the function is customized for GeoQuery. We
-            # see if there was any custom selection specified in the
-            # dictionary, and set up the selection format appropriately.
-            field_sel = self.get_field_select(field, alias)
-            if with_aliases and field.column in col_aliases:
-                c_alias = 'Col%d' % len(col_aliases)
-                result.append('%s AS %s' % (field_sel, c_alias))
-                col_aliases.add(c_alias)
-                aliases.add(c_alias)
-            else:
-                r = field_sel
-                result.append(r)
-                aliases.add(r)
-                if with_aliases:
-                    col_aliases.add(field.column)
-        if as_pairs:
-            return result, None
-        return result, aliases
     def get_ordering(self):
         """
         This routine is overridden to disable ordering for aggregate
 …
             sel_fmt = sel_fmt % self.custom_select[alias]
         return sel_fmt
-    def get_field_select(self, fld, alias=None):
-        """
-        Returns the SELECT SQL string for the given field.  Figures out
-        if any custom selection SQL is needed for the column  The `alias`
-        keyword may be used to manually specify the database table where
-        the column exists, if not in the model associated with this
-        `GeoQuery`.
-        """
-        sel_fmt = self.get_select_format(fld)
-        if fld in self.custom_select:
-            field_sel = sel_fmt % self.custom_select[fld]
-        else:
-            field_sel = sel_fmt % self._field_column(fld, alias)
-        return field_sel
     def get_select_format(self, fld):
         """
         Returns the selection format string, depending on the requirements

tests/regressiontests/lazy_fields/models.py

+from django.db import models
+class LazyObject3(models.Model):
+    lazy = models.IntegerField(lazy=True)
+    normal = models.IntegerField()
+class LazyObject2(models.Model):
+    normal = models.IntegerField()
+    fk3 = models.ForeignKey(LazyObject3)
+class LazyObject(models.Model):
+    normal = models.IntegerField()
+    lazy = models.IntegerField(lazy=True)
+    fk3 = models.ForeignKey(LazyObject3)
+    fk2 = models.ForeignKey(LazyObject2)
+    lazy2 = models.IntegerField(lazy=True, default=42)
+def checkUnchanged(obj, fieldName):
+    field = obj._meta.get_field(fieldName)
+    assert not field.write_on_save(obj)
+def checkChanged(obj, fieldName):
+    field = obj._meta.get_field(fieldName)
+    assert field.write_on_save(obj)
+def checkLazyField(obj, fieldName, value, cached=False):
+    if not cached:
+        assert not hasattr(obj,'_%s_lcache' % fieldName)
+    else:
+        assert getattr(obj, '_%s_lcache' % fieldName) == value
+    assert getattr(obj, fieldName) == value
+    assert getattr(obj, '_%s_lcache' % fieldName) == value
+def check3(_o3, correct3, eager=[]):
+    checkLazyField(_o3, 'lazy', correct3.lazy, cached=('lazy' in eager))
+    assert _o3.normal == correct3.normal
+    checkUnchanged(_o3, 'lazy')
+    _o3.lazy = 11
+    checkChanged(_o3, 'lazy')
+    checkLazyField(_o3, 'lazy', 11, cached=True)
+def check2(_o2, correct2, relatedDepth=0):
+    if relatedDepth > 0:
+        assert hasattr(_o2, '_fk3_cache')
+    else:
+        assert not hasattr(_o2, '_fk3_cache')
+    check3(_o2.fk3, correct2.fk3)
+    assert hasattr(_o2, '_fk3_cache')
+    assert _o2.normal == correct2.normal
+def check1(_o1, correct1, eager=[], relatedDepth=0):
+    if relatedDepth > 0:
+        assert hasattr(_o1, '_fk2_cache')
+        assert hasattr(_o1, '_fk3_cache')
+        if relatedDepth > 1:
+            assert hasattr(_o1._fk2_cache,'_fk3_cache')
+        else:
+            assert not hasattr(_o1._fk2_cache,'_fk3_cache')
+    else:
+        assert not hasattr(_o1, '_fk2_cache')
+        assert not hasattr(_o1, '_fk3_cache')
+    check2(_o1.fk2, correct1.fk2, relatedDepth=relatedDepth - 1)
+    assert hasattr(_o1, '_fk2_cache')
+    check3(_o1.fk3, correct1.fk3)
+    assert hasattr(_o1, '_fk3_cache')
+    assert _o1.normal == correct1.normal
+    checkLazyField(_o1, 'lazy', correct1.lazy, cached=('lazy' in eager))
+    checkUnchanged(_o1, 'lazy')
+    checkUnchanged(_o1, 'lazy2')
+    checkLazyField(_o1, 'lazy2', correct1.lazy2, cached=('lazy2' in eager))
+def checkAll(correct1, correct2, correct3):
+    "Test operation of lazy fields using toggle_fields and/or select_related"
+    check3(LazyObject3.objects.get(lazy=correct3.lazy), correct3)
+    check3(LazyObject3.objects.toggle_fields(fetch=['lazy']).get(lazy=correct3.lazy), correct3, eager=['lazy'])
+    check3(LazyObject3.objects.get(normal=correct3.normal), correct3)
+    check2(LazyObject2.objects.get(normal=correct2.normal), correct2)
+    check2(LazyObject2.objects.select_related().get(normal=correct2.normal), correct2, relatedDepth=1)
+    check1(LazyObject.objects.get(normal=correct1.normal), correct1)
+    check1(LazyObject.objects.toggle_fields(fetch=['lazy']).get(normal=correct1.normal), correct1, eager=['lazy'])
+    check1(LazyObject.objects.toggle_fields(fetch=['lazy','lazy2']).get(normal=correct1.normal), correct1, eager=['lazy','lazy2'])
+    check1(LazyObject.objects.select_related(depth=1).get(normal=correct1.normal), correct1, relatedDepth=1)
+    check1(LazyObject.objects.select_related(depth=2).get(normal=correct1.normal), correct1, relatedDepth=2)
+    check1(LazyObject.objects.toggle_fields(fetch=['lazy']).select_related(depth=1).get(normal=correct1.normal), correct1, eager=['lazy'], relatedDepth=1)
+def test():
+    LazyObject.objects.all().delete()
+    LazyObject2.objects.all().delete()
+    LazyObject3.objects.all().delete()
+    # desynchronize auto-increment ids of the 3 object types
+    LazyObject3.objects.create(lazy=999,normal=999)
+    o3_x = LazyObject3.objects.create(lazy=999,normal=999)
+    LazyObject2.objects.create(normal=999,fk3=o3_x)
+    class orig3:
+        lazy=1
+        normal=2
+    class orig2:
+        normal=3
+        fk3=orig3
+    class orig1:
+        normal=4
+        lazy=5
+        lazy2=6
+        fk2=orig2
+        fk3=orig3
+    # create the test objects
+    o3 = LazyObject3.objects.create(lazy=orig3.lazy,normal=orig3.normal)
+    o2 = LazyObject2.objects.create(normal=orig2.normal,fk3 =o3)
+    o1 = LazyObject.objects.create(normal=orig1.normal,lazy=orig1.lazy,fk3=o3,fk2=o2,lazy2=orig1.lazy2)
+    # test default values of lazy fields
+    empty_o1 = LazyObject()
+    checkLazyField(empty_o1, 'lazy', None, cached=False)
+    checkLazyField(empty_o1, 'lazy2', 42, cached=False)
+    # test toggle_fields on an object with existing default-lazy fields
+    lazy_o1 = LazyObject.objects.toggle_fields(lazy=['normal'], fetch=['lazy2']).get(lazy=orig1.lazy)
+    checkLazyField(lazy_o1, 'normal', orig1.normal, cached=False)
+    checkLazyField(lazy_o1, 'lazy', orig1.lazy, cached=False)
+    checkLazyField(lazy_o1, 'lazy2', orig1.lazy2, cached=True)
+    # test toggle_fields on an object that doesn't have any default-lazy fields
+    lazy_o2 = LazyObject2.objects.toggle_fields(lazy=['normal']).get(fk3__lazy=orig2.fk3.lazy)
+    checkLazyField(lazy_o2, 'normal', orig2.normal, cached=False)
+    # test toggle_fields with fetch_only, and lazying a foreign key
+    lazy_o1 = LazyObject.objects.toggle_fields(fetch_only=['lazy','fk3']).get(lazy=orig1.lazy)
+    checkLazyField(lazy_o1, 'normal', orig1.normal, cached=False)
+    checkLazyField(lazy_o1, 'lazy', orig1.lazy, cached=True)
+    checkLazyField(lazy_o1, 'lazy2', orig1.lazy2, cached=False)
+    checkLazyField(lazy_o1, 'fk2_id', o2.id, cached=False)
+    assert lazy_o1.fk2.normal == orig2.normal
+    assert lazy_o1.fk2_id == o2.id # make sure lazy fk2_id and fk2 cache don't overwrite each other.
+    assert lazy_o1.fk3_id == o3.id
+    # test saving lazy fields
+    checkAll(orig1, orig2, orig3)
+    orig3_a = orig3()
+    orig3_a.normal = 9
+    o3.normal = 9
+    o3.save()
+    orig2_a = orig2()
+    orig2_a.normal = 8
+    orig2_a.fk3 = orig3_a
+    o2.normal = 8
+    o2.save()
+    orig1_a = orig1()
+    orig1_a.lazy = 7
+    orig1_a.fk3 = orig3_a
+    orig1_a.fk2 = orig2_a
+    o1.lazy = 7
+    o1.save()
+    checkAll(orig1_a, orig2_a, orig3_a)
+    orig1_b = orig1_a
+    orig1_b.lazy = 10
+    o1.lazy = 10
+    orig1_b.lazy2 = 11
+    o1.lazy2 = 11
+    o1.save()
+    checkAll(orig1_b, orig2_a, orig3_a)
+    return "OK"
+__test__ = {'API_TESTS':"""
+>>> test()
+"OK"
+"""}

Download in other formats:

Original Format

Issues

Context Navigation

Ticket #5420: lazy.diff

django/db/models/sql/query.py

django/db/models/base.py

django/db/models/manager.py

django/db/models/options.py

django/db/models/fields/init.py

django/db/models/query.py

django/contrib/gis/db/models/sql/query.py

tests/regressiontests/lazy_fields/models.py

Download in other formats:

Django Links

Learn More

Get Involved

Get Help

Follow Us

Support Us