Code

Ticket #17030: deferred_init.diff

File deferred_init.diff, 18.4 KB (added by akaariai, 3 years ago)
Line 
1diff --git a/django/db/models/base.py b/django/db/models/base.py
2index b5ce39e..daa96d3 100644
3--- a/django/db/models/base.py
4+++ b/django/db/models/base.py
5@@ -287,85 +287,89 @@ class Model(object):
6         # The reason for the kwargs check is that standard iterator passes in by
7         # args, and instantiation for iteration is 33% faster.
8         args_len = len(args)
9-        if args_len > len(self._meta.fields):
10-            # Daft, but matches old exception sans the err msg.
11-            raise IndexError("Number of args exceeds number of fields")
12-
13-        fields_iter = iter(self._meta.fields)
14-        if not kwargs:
15-            # The ordering of the izip calls matter - izip throws StopIteration
16-            # when an iter throws it. So if the first iter throws it, the second
17-            # is *not* consumed. We rely on this, so don't change the order
18-            # without changing the logic.
19-            for val, field in izip(args, fields_iter):
20-                setattr(self, field.attname, val)
21+
22+        # Deferred models have a different set of init fields - this can return
23+        # different set of attnames than the attnames of all the fields in the
24+        # model.
25+        init_attnames = self._meta.get_init_attnames()
26+        # This is the common case - the case we have when loading from the DB
27+        # Make it fast by special casing.
28+        if args_len == len(init_attnames):
29+            [setattr(self, attname, val) for val, attname
30+                 in izip(args, init_attnames)]
31         else:
32-            # Slower, kwargs-ready version.
33-            for val, field in izip(args, fields_iter):
34-                setattr(self, field.attname, val)
35-                kwargs.pop(field.name, None)
36-                # Maintain compatibility with existing calls.
37-                if isinstance(field.rel, ManyToOneRel):
38-                    kwargs.pop(field.attname, None)
39-
40-        # Now we're left with the unprocessed fields that *must* come from
41-        # keywords, or default.
42-
43-        for field in fields_iter:
44-            is_related_object = False
45-            # This slightly odd construct is so that we can access any
46-            # data-descriptor object (DeferredAttribute) without triggering its
47-            # __get__ method.
48-            if (field.attname not in kwargs and
49-                    isinstance(self.__class__.__dict__.get(field.attname), DeferredAttribute)):
50-                # This field will be populated on request.
51-                continue
52-            if kwargs:
53-                if isinstance(field.rel, ManyToOneRel):
54-                    try:
55-                        # Assume object instance was passed in.
56-                        rel_obj = kwargs.pop(field.name)
57-                        is_related_object = True
58-                    except KeyError:
59+            if args_len > len(self._meta.fields):
60+                # Daft, but matches old exception sans the err msg.
61+                raise IndexError("Number of args exceeds number of fields")
62+
63+            fields_iter = iter(self._meta.fields)
64+            if not kwargs:
65+                # The ordering of the izip calls matter - izip throws StopIteration
66+                # when an iter throws it. So if the first iter throws it, the second
67+                # is *not* consumed. We rely on this, so don't change the order
68+                # without changing the logic.
69+                for val, field in izip(args, fields_iter):
70+                    setattr(self, field.attname, val)
71+            else:
72+                # Slower, kwargs-ready version.
73+                for val, field in izip(args, fields_iter):
74+                    setattr(self, field.attname, val)
75+                    kwargs.pop(field.name, None)
76+                    # Maintain compatibility with existing calls.
77+                    if isinstance(field.rel, ManyToOneRel):
78+                        kwargs.pop(field.attname, None)
79+
80+            # Now we're left with the unprocessed fields that *must* come from
81+            # keywords, or default.
82+
83+            for field in fields_iter:
84+                is_related_object = False
85+                if kwargs:
86+                    if isinstance(field.rel, ManyToOneRel):
87+                        try:
88+                            # Assume object instance was passed in.
89+                            rel_obj = kwargs.pop(field.name)
90+                            is_related_object = True
91+                        except KeyError:
92+                            try:
93+                                # Object instance wasn't passed in -- must be an ID.
94+                                val = kwargs.pop(field.attname)
95+                            except KeyError:
96+                                val = field.get_default()
97+                        else:
98+                            # Object instance was passed in. Special case: You can
99+                            # pass in "None" for related objects if it's allowed.
100+                            if rel_obj is None and field.null:
101+                                val = None
102+                    else:
103                         try:
104-                            # Object instance wasn't passed in -- must be an ID.
105                             val = kwargs.pop(field.attname)
106                         except KeyError:
107+                            # This is done with an exception rather than the
108+                            # default argument on pop because we don't want
109+                            # get_default() to be evaluated, and then not used.
110+                            # Refs #12057.
111                             val = field.get_default()
112-                    else:
113-                        # Object instance was passed in. Special case: You can
114-                        # pass in "None" for related objects if it's allowed.
115-                        if rel_obj is None and field.null:
116-                            val = None
117                 else:
118-                    try:
119-                        val = kwargs.pop(field.attname)
120-                    except KeyError:
121-                        # This is done with an exception rather than the
122-                        # default argument on pop because we don't want
123-                        # get_default() to be evaluated, and then not used.
124-                        # Refs #12057.
125-                        val = field.get_default()
126-            else:
127-                val = field.get_default()
128-            if is_related_object:
129-                # If we are passed a related instance, set it using the
130-                # field.name instead of field.attname (e.g. "user" instead of
131-                # "user_id") so that the object gets properly cached (and type
132-                # checked) by the RelatedObjectDescriptor.
133-                setattr(self, field.name, rel_obj)
134-            else:
135-                setattr(self, field.attname, val)
136-
137-        if kwargs:
138-            for prop in kwargs.keys():
139-                try:
140-                    if isinstance(getattr(self.__class__, prop), property):
141-                        setattr(self, prop, kwargs.pop(prop))
142-                except AttributeError:
143-                    pass
144+                    val = field.get_default()
145+                if is_related_object:
146+                    # If we are passed a related instance, set it using the
147+                    # field.name instead of field.attname (e.g. "user" instead of
148+                    # "user_id") so that the object gets properly cached (and type
149+                    # checked) by the RelatedObjectDescriptor.
150+                    setattr(self, field.name, rel_obj)
151+                else:
152+                    setattr(self, field.attname, val)
153+
154             if kwargs:
155-                raise TypeError("'%s' is an invalid keyword argument for this function" % kwargs.keys()[0])
156+                for prop in kwargs.keys():
157+                    try:
158+                        if isinstance(getattr(self.__class__, prop), property):
159+                            setattr(self, prop, kwargs.pop(prop))
160+                    except AttributeError:
161+                        pass
162+                if kwargs:
163+                    raise TypeError("'%s' is an invalid keyword argument for this function" % kwargs.keys()[0])
164         super(Model, self).__init__()
165         signals.post_init.send(sender=self.__class__, instance=self)
166 
167diff --git a/django/db/models/loading.py b/django/db/models/loading.py
168index c344686..44ff2f3 100644
169--- a/django/db/models/loading.py
170+++ b/django/db/models/loading.py
171@@ -213,7 +213,7 @@ class AppCache(object):
172             self._populate()
173         if only_installed and app_label not in self.app_labels:
174             return None
175-        return self.app_models.get(app_label, SortedDict()).get(model_name.lower())
176+        return self.app_models.get(app_label, {}).get(model_name.lower())
177 
178     def register_models(self, app_label, *models):
179         """
180diff --git a/django/db/models/options.py b/django/db/models/options.py
181index 0cd52a3..82b7f6a 100644
182--- a/django/db/models/options.py
183+++ b/django/db/models/options.py
184@@ -44,6 +44,9 @@ class Options(object):
185         self.parents = SortedDict()
186         self.duplicate_targets = {}
187         self.auto_created = False
188+        # Deferred models want to use only a portion of all the fields.
189+        # This is a list of fields.attnames we want to load.
190+        self.only_load = []
191 
192         # To handle various inheritance situations, we need to track where
193         # managers came from (concrete or abstract base classes).
194@@ -105,6 +108,8 @@ class Options(object):
195             self.db_table = "%s_%s" % (self.app_label, self.module_name)
196             self.db_table = truncate_name(self.db_table, connection.ops.max_name_length())
197 
198+         
199+
200     def _prepare(self, model):
201         if self.order_with_respect_to:
202             self.order_with_respect_to = self.get_field(self.order_with_respect_to)
203@@ -164,6 +169,7 @@ class Options(object):
204             if hasattr(self, '_field_cache'):
205                 del self._field_cache
206                 del self._field_name_cache
207+                del self._init_attname_cache
208 
209         if hasattr(self, '_name_map'):
210             del self._name_map
211@@ -231,6 +237,18 @@ class Options(object):
212             self._fill_fields_cache()
213         return self._field_cache
214 
215+    def get_init_attnames(self):
216+        """
217+        Returns a sequence of attribute names for model initialization. Note
218+        that for deferred models this list contains just the loaded field
219+        attribute names, not all of the model's attnames.
220+        """
221+        try:
222+            self._init_attname_cache
223+        except AttributeError:
224+            self._fill_fields_cache()
225+        return self._init_attname_cache
226+   
227     def _fill_fields_cache(self):
228         cache = []
229         for parent in self.parents:
230@@ -242,6 +260,26 @@ class Options(object):
231         cache.extend([(f, None) for f in self.local_fields])
232         self._field_cache = tuple(cache)
233         self._field_name_cache = [x for x, _ in cache]
234+        if self.only_load:
235+            self._init_attname_cache = tuple(
236+                [x.attname for x, _ in cache
237+                 if x.attname in self.only_load]
238+            )
239+        else:
240+            self._init_attname_cache = tuple([x.attname for x, _ in cache])
241+
242+    def set_loaded_fields(self, defer):
243+        """
244+        Deferred model class creation will call this method. This will set
245+        the deferred_fields list and then delete the _init_attname_cache.
246+        Next access to get_init_fields() will reload that cache.
247+        """
248+        # Due to some strange things in select_related query.py iterator
249+        # we can be called with a list of defer fields which can be either
250+        # attnames or or field names. TODO: Fix this (in query.py)
251+        self.only_load = [f.attname for f in self.fields
252+                          if f.name not in defer and f.attname not in defer]
253+        del self._init_attname_cache
254 
255     def _many_to_many(self):
256         try:
257diff --git a/django/db/models/query.py b/django/db/models/query.py
258index be42d02..83dedd4 100644
259--- a/django/db/models/query.py
260+++ b/django/db/models/query.py
261@@ -265,37 +265,29 @@ class QuerySet(object):
262         index_start = len(extra_select)
263         aggregate_start = index_start + len(load_fields or self.model._meta.fields)
264 
265-        skip = None
266         if load_fields and not fill_cache:
267             # Some fields have been deferred, so we have to initialise
268             # via keyword arguments.
269             skip = set()
270-            init_list = []
271             for field in fields:
272                 if field.name not in load_fields:
273                     skip.add(field.attname)
274-                else:
275-                    init_list.append(field.attname)
276             model_cls = deferred_class_factory(self.model, skip)
277-
278+        else:
279+            model_cls = self.model
280         # Cache db and model outside the loop
281         db = self.db
282-        model = self.model
283         compiler = self.query.get_compiler(using=db)
284         if fill_cache:
285-            klass_info = get_klass_info(model, max_depth=max_depth,
286+            klass_info = get_klass_info(self.model, max_depth=max_depth,
287                                         requested=requested, only_load=only_load)
288         for row in compiler.results_iter():
289             if fill_cache:
290                 obj, _ = get_cached_row(row, index_start, db, klass_info,
291                                         offset=len(aggregate_select))
292             else:
293-                if skip:
294-                    row_data = row[index_start:aggregate_start]
295-                    obj = model_cls(**dict(zip(init_list, row_data)))
296-                else:
297-                    # Omit aggregates in object creation.
298-                    obj = model(*row[index_start:aggregate_start])
299+                # Omit aggregates in object creation.
300+                obj = model_cls(*row[index_start:aggregate_start])
301 
302                 # Store the source database of the object
303                 obj._state.db = db
304@@ -1257,6 +1249,8 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None,
305     else:
306         load_fields = None
307 
308+    # TODO - Due to removal of special handling of deferred model's __init__
309+    # we could probably do some cleanup here.
310     if load_fields:
311         # Handle deferred fields.
312         skip = set()
313@@ -1345,10 +1339,7 @@ def get_cached_row(row, index_start, using,  klass_info, offset=0):
314     if fields == (None,) * field_count:
315         obj = None
316     else:
317-        if field_names:
318-            obj = klass(**dict(zip(field_names, fields)))
319-        else:
320-            obj = klass(*fields)
321+        obj = klass(*fields)
322 
323     # If an object was retrieved, set the database state.
324     if obj:
325@@ -1461,12 +1452,13 @@ class RawQuerySet(object):
326             model_cls = deferred_class_factory(self.model, skip)
327         else:
328             model_cls = self.model
329-            # All model's fields are present in the query. So, it is possible
330-            # to use *args based model instantation. For each field of the model,
331-            # record the query column position matching that field.
332-            model_init_field_pos = []
333-            for field in self.model._meta.fields:
334-                model_init_field_pos.append(model_init_field_names[field.attname])
335+        # For each field of the model, record the query column position matching
336+        # that field. Note that we must use the get_init_attnames() method of
337+        # the above fetched model_cls, because if it is a deferred model class
338+        # its __init__ will expect the field in get_init_attnames order.
339+        model_init_field_pos = []
340+        for attname in model_cls._meta.get_init_attnames():
341+            model_init_field_pos.append(model_init_field_names[attname])
342         if need_resolv_columns:
343             fields = [self.model_fields.get(c, None) for c in self.columns]
344         # Begin looping through the query values.
345@@ -1474,14 +1466,8 @@ class RawQuerySet(object):
346             if need_resolv_columns:
347                 values = compiler.resolve_columns(values, fields)
348             # Associate fields to values
349-            if skip:
350-                model_init_kwargs = {}
351-                for attname, pos in model_init_field_names.iteritems():
352-                    model_init_kwargs[attname] = values[pos]
353-                instance = model_cls(**model_init_kwargs)
354-            else:
355-                model_init_args = [values[pos] for pos in model_init_field_pos]
356-                instance = model_cls(*model_init_args)
357+            model_init_args = [values[pos] for pos in model_init_field_pos]
358+            instance = model_cls(*model_init_args)
359             if annotation_fields:
360                 for column, pos in annotation_fields:
361                     setattr(instance, column, values[pos])
362diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py
363index a56ab5c..a3299eb 100644
364--- a/django/db/models/query_utils.py
365+++ b/django/db/models/query_utils.py
366@@ -9,6 +9,7 @@ circular import difficulties.
367 import weakref
368 
369 from django.db.backends import util
370+from django.db.models.loading import get_model
371 from django.utils import tree
372 
373 
374@@ -146,23 +147,29 @@ def deferred_class_factory(model, attrs):
375     being replaced with DeferredAttribute objects. The "pk_value" ties the
376     deferred attributes to a particular instance of the model.
377     """
378-    class Meta:
379-        proxy = True
380-        app_label = model._meta.app_label
381-
382     # The app_cache wants a unique name for each model, otherwise the new class
383     # won't be created (we get an old one back). Therefore, we generate the
384     # name using the passed in attrs. It's OK to reuse an existing class
385     # object if the attrs are identical.
386     name = "%s_Deferred_%s" % (model.__name__, '_'.join(sorted(list(attrs))))
387     name = util.truncate_name(name, 80, 32)
388+    deferred_model = get_model(model._meta.app_label, name)
389+    if deferred_model:
390+        return deferred_model
391+
392+    class Meta:
393+        proxy = True
394+        app_label = model._meta.app_label
395+
396 
397     overrides = dict([(attr, DeferredAttribute(attr, model))
398             for attr in attrs])
399     overrides["Meta"] = Meta
400     overrides["__module__"] = model.__module__
401     overrides["_deferred"] = True
402-    return type(name, (model,), overrides)
403+    deferred_model = type(name, (model,), overrides)
404+    deferred_model._meta.set_loaded_fields(attrs)
405+    return deferred_model
406 
407 # The above function is also used to unpickle model instances with deferred
408 # fields.