Ticket #17030: deferred_init.diff

File deferred_init.diff, 18.4 KB (added by akaariai, 3 years ago)
  • django/db/models/base.py

    diff --git a/django/db/models/base.py b/django/db/models/base.py
    index b5ce39e..daa96d3 100644
    a b class Model(object): 
    287287        # The reason for the kwargs check is that standard iterator passes in by
    288288        # args, and instantiation for iteration is 33% faster.
    289289        args_len = len(args)
    290         if args_len > len(self._meta.fields):
    291             # Daft, but matches old exception sans the err msg.
    292             raise IndexError("Number of args exceeds number of fields")
    293 
    294         fields_iter = iter(self._meta.fields)
    295         if not kwargs:
    296             # The ordering of the izip calls matter - izip throws StopIteration
    297             # when an iter throws it. So if the first iter throws it, the second
    298             # is *not* consumed. We rely on this, so don't change the order
    299             # without changing the logic.
    300             for val, field in izip(args, fields_iter):
    301                 setattr(self, field.attname, val)
     290
     291        # Deferred models have a different set of init fields - this can return
     292        # different set of attnames than the attnames of all the fields in the
     293        # model.
     294        init_attnames = self._meta.get_init_attnames()
     295        # This is the common case - the case we have when loading from the DB
     296        # Make it fast by special casing.
     297        if args_len == len(init_attnames):
     298            [setattr(self, attname, val) for val, attname
     299                 in izip(args, init_attnames)]
    302300        else:
    303             # Slower, kwargs-ready version.
    304             for val, field in izip(args, fields_iter):
    305                 setattr(self, field.attname, val)
    306                 kwargs.pop(field.name, None)
    307                 # Maintain compatibility with existing calls.
    308                 if isinstance(field.rel, ManyToOneRel):
    309                     kwargs.pop(field.attname, None)
    310 
    311         # Now we're left with the unprocessed fields that *must* come from
    312         # keywords, or default.
    313 
    314         for field in fields_iter:
    315             is_related_object = False
    316             # This slightly odd construct is so that we can access any
    317             # data-descriptor object (DeferredAttribute) without triggering its
    318             # __get__ method.
    319             if (field.attname not in kwargs and
    320                     isinstance(self.__class__.__dict__.get(field.attname), DeferredAttribute)):
    321                 # This field will be populated on request.
    322                 continue
    323             if kwargs:
    324                 if isinstance(field.rel, ManyToOneRel):
    325                     try:
    326                         # Assume object instance was passed in.
    327                         rel_obj = kwargs.pop(field.name)
    328                         is_related_object = True
    329                     except KeyError:
     301            if args_len > len(self._meta.fields):
     302                # Daft, but matches old exception sans the err msg.
     303                raise IndexError("Number of args exceeds number of fields")
     304
     305            fields_iter = iter(self._meta.fields)
     306            if not kwargs:
     307                # The ordering of the izip calls matter - izip throws StopIteration
     308                # when an iter throws it. So if the first iter throws it, the second
     309                # is *not* consumed. We rely on this, so don't change the order
     310                # without changing the logic.
     311                for val, field in izip(args, fields_iter):
     312                    setattr(self, field.attname, val)
     313            else:
     314                # Slower, kwargs-ready version.
     315                for val, field in izip(args, fields_iter):
     316                    setattr(self, field.attname, val)
     317                    kwargs.pop(field.name, None)
     318                    # Maintain compatibility with existing calls.
     319                    if isinstance(field.rel, ManyToOneRel):
     320                        kwargs.pop(field.attname, None)
     321 
     322            # Now we're left with the unprocessed fields that *must* come from
     323            # keywords, or default.
     324
     325            for field in fields_iter:
     326                is_related_object = False
     327                if kwargs:
     328                    if isinstance(field.rel, ManyToOneRel):
     329                        try:
     330                            # Assume object instance was passed in.
     331                            rel_obj = kwargs.pop(field.name)
     332                            is_related_object = True
     333                        except KeyError:
     334                            try:
     335                                # Object instance wasn't passed in -- must be an ID.
     336                                val = kwargs.pop(field.attname)
     337                            except KeyError:
     338                                val = field.get_default()
     339                        else:
     340                            # Object instance was passed in. Special case: You can
     341                            # pass in "None" for related objects if it's allowed.
     342                            if rel_obj is None and field.null:
     343                                val = None
     344                    else:
    330345                        try:
    331                             # Object instance wasn't passed in -- must be an ID.
    332346                            val = kwargs.pop(field.attname)
    333347                        except KeyError:
     348                            # This is done with an exception rather than the
     349                            # default argument on pop because we don't want
     350                            # get_default() to be evaluated, and then not used.
     351                            # Refs #12057.
    334352                            val = field.get_default()
    335                     else:
    336                         # Object instance was passed in. Special case: You can
    337                         # pass in "None" for related objects if it's allowed.
    338                         if rel_obj is None and field.null:
    339                             val = None
    340353                else:
    341                     try:
    342                         val = kwargs.pop(field.attname)
    343                     except KeyError:
    344                         # This is done with an exception rather than the
    345                         # default argument on pop because we don't want
    346                         # get_default() to be evaluated, and then not used.
    347                         # Refs #12057.
    348                         val = field.get_default()
    349             else:
    350                 val = field.get_default()
    351             if is_related_object:
    352                 # If we are passed a related instance, set it using the
    353                 # field.name instead of field.attname (e.g. "user" instead of
    354                 # "user_id") so that the object gets properly cached (and type
    355                 # checked) by the RelatedObjectDescriptor.
    356                 setattr(self, field.name, rel_obj)
    357             else:
    358                 setattr(self, field.attname, val)
    359 
    360         if kwargs:
    361             for prop in kwargs.keys():
    362                 try:
    363                     if isinstance(getattr(self.__class__, prop), property):
    364                         setattr(self, prop, kwargs.pop(prop))
    365                 except AttributeError:
    366                     pass
     354                    val = field.get_default()
     355                if is_related_object:
     356                    # If we are passed a related instance, set it using the
     357                    # field.name instead of field.attname (e.g. "user" instead of
     358                    # "user_id") so that the object gets properly cached (and type
     359                    # checked) by the RelatedObjectDescriptor.
     360                    setattr(self, field.name, rel_obj)
     361                else:
     362                    setattr(self, field.attname, val)
     363
    367364            if kwargs:
    368                 raise TypeError("'%s' is an invalid keyword argument for this function" % kwargs.keys()[0])
     365                for prop in kwargs.keys():
     366                    try:
     367                        if isinstance(getattr(self.__class__, prop), property):
     368                            setattr(self, prop, kwargs.pop(prop))
     369                    except AttributeError:
     370                        pass
     371                if kwargs:
     372                    raise TypeError("'%s' is an invalid keyword argument for this function" % kwargs.keys()[0])
    369373        super(Model, self).__init__()
    370374        signals.post_init.send(sender=self.__class__, instance=self)
    371375
  • django/db/models/loading.py

    diff --git a/django/db/models/loading.py b/django/db/models/loading.py
    index c344686..44ff2f3 100644
    a b class AppCache(object): 
    213213            self._populate()
    214214        if only_installed and app_label not in self.app_labels:
    215215            return None
    216         return self.app_models.get(app_label, SortedDict()).get(model_name.lower())
     216        return self.app_models.get(app_label, {}).get(model_name.lower())
    217217
    218218    def register_models(self, app_label, *models):
    219219        """
  • django/db/models/options.py

    diff --git a/django/db/models/options.py b/django/db/models/options.py
    index 0cd52a3..82b7f6a 100644
    a b class Options(object): 
    4444        self.parents = SortedDict()
    4545        self.duplicate_targets = {}
    4646        self.auto_created = False
     47        # Deferred models want to use only a portion of all the fields.
     48        # This is a list of fields.attnames we want to load.
     49        self.only_load = []
    4750
    4851        # To handle various inheritance situations, we need to track where
    4952        # managers came from (concrete or abstract base classes).
    class Options(object): 
    105108            self.db_table = "%s_%s" % (self.app_label, self.module_name)
    106109            self.db_table = truncate_name(self.db_table, connection.ops.max_name_length())
    107110
     111         
     112
    108113    def _prepare(self, model):
    109114        if self.order_with_respect_to:
    110115            self.order_with_respect_to = self.get_field(self.order_with_respect_to)
    class Options(object): 
    164169            if hasattr(self, '_field_cache'):
    165170                del self._field_cache
    166171                del self._field_name_cache
     172                del self._init_attname_cache
    167173
    168174        if hasattr(self, '_name_map'):
    169175            del self._name_map
    class Options(object): 
    231237            self._fill_fields_cache()
    232238        return self._field_cache
    233239
     240    def get_init_attnames(self):
     241        """
     242        Returns a sequence of attribute names for model initialization. Note
     243        that for deferred models this list contains just the loaded field
     244        attribute names, not all of the model's attnames.
     245        """
     246        try:
     247            self._init_attname_cache
     248        except AttributeError:
     249            self._fill_fields_cache()
     250        return self._init_attname_cache
     251   
    234252    def _fill_fields_cache(self):
    235253        cache = []
    236254        for parent in self.parents:
    class Options(object): 
    242260        cache.extend([(f, None) for f in self.local_fields])
    243261        self._field_cache = tuple(cache)
    244262        self._field_name_cache = [x for x, _ in cache]
     263        if self.only_load:
     264            self._init_attname_cache = tuple(
     265                [x.attname for x, _ in cache
     266                 if x.attname in self.only_load]
     267            )
     268        else:
     269            self._init_attname_cache = tuple([x.attname for x, _ in cache])
     270
     271    def set_loaded_fields(self, defer):
     272        """
     273        Deferred model class creation will call this method. This will set
     274        the deferred_fields list and then delete the _init_attname_cache.
     275        Next access to get_init_fields() will reload that cache.
     276        """
     277        # Due to some strange things in select_related query.py iterator
     278        # we can be called with a list of defer fields which can be either
     279        # attnames or or field names. TODO: Fix this (in query.py)
     280        self.only_load = [f.attname for f in self.fields
     281                          if f.name not in defer and f.attname not in defer]
     282        del self._init_attname_cache
    245283
    246284    def _many_to_many(self):
    247285        try:
  • django/db/models/query.py

    diff --git a/django/db/models/query.py b/django/db/models/query.py
    index be42d02..83dedd4 100644
    a b class QuerySet(object): 
    265265        index_start = len(extra_select)
    266266        aggregate_start = index_start + len(load_fields or self.model._meta.fields)
    267267
    268         skip = None
    269268        if load_fields and not fill_cache:
    270269            # Some fields have been deferred, so we have to initialise
    271270            # via keyword arguments.
    272271            skip = set()
    273             init_list = []
    274272            for field in fields:
    275273                if field.name not in load_fields:
    276274                    skip.add(field.attname)
    277                 else:
    278                     init_list.append(field.attname)
    279275            model_cls = deferred_class_factory(self.model, skip)
    280 
     276        else:
     277            model_cls = self.model
    281278        # Cache db and model outside the loop
    282279        db = self.db
    283         model = self.model
    284280        compiler = self.query.get_compiler(using=db)
    285281        if fill_cache:
    286             klass_info = get_klass_info(model, max_depth=max_depth,
     282            klass_info = get_klass_info(self.model, max_depth=max_depth,
    287283                                        requested=requested, only_load=only_load)
    288284        for row in compiler.results_iter():
    289285            if fill_cache:
    290286                obj, _ = get_cached_row(row, index_start, db, klass_info,
    291287                                        offset=len(aggregate_select))
    292288            else:
    293                 if skip:
    294                     row_data = row[index_start:aggregate_start]
    295                     obj = model_cls(**dict(zip(init_list, row_data)))
    296                 else:
    297                     # Omit aggregates in object creation.
    298                     obj = model(*row[index_start:aggregate_start])
     289                # Omit aggregates in object creation.
     290                obj = model_cls(*row[index_start:aggregate_start])
    299291
    300292                # Store the source database of the object
    301293                obj._state.db = db
    def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, 
    12571249    else:
    12581250        load_fields = None
    12591251
     1252    # TODO - Due to removal of special handling of deferred model's __init__
     1253    # we could probably do some cleanup here.
    12601254    if load_fields:
    12611255        # Handle deferred fields.
    12621256        skip = set()
    def get_cached_row(row, index_start, using, klass_info, offset=0): 
    13451339    if fields == (None,) * field_count:
    13461340        obj = None
    13471341    else:
    1348         if field_names:
    1349             obj = klass(**dict(zip(field_names, fields)))
    1350         else:
    1351             obj = klass(*fields)
     1342        obj = klass(*fields)
    13521343
    13531344    # If an object was retrieved, set the database state.
    13541345    if obj:
    class RawQuerySet(object): 
    14611452            model_cls = deferred_class_factory(self.model, skip)
    14621453        else:
    14631454            model_cls = self.model
    1464             # All model's fields are present in the query. So, it is possible
    1465             # to use *args based model instantation. For each field of the model,
    1466             # record the query column position matching that field.
    1467             model_init_field_pos = []
    1468             for field in self.model._meta.fields:
    1469                 model_init_field_pos.append(model_init_field_names[field.attname])
     1455        # For each field of the model, record the query column position matching
     1456        # that field. Note that we must use the get_init_attnames() method of
     1457        # the above fetched model_cls, because if it is a deferred model class
     1458        # its __init__ will expect the field in get_init_attnames order.
     1459        model_init_field_pos = []
     1460        for attname in model_cls._meta.get_init_attnames():
     1461            model_init_field_pos.append(model_init_field_names[attname])
    14701462        if need_resolv_columns:
    14711463            fields = [self.model_fields.get(c, None) for c in self.columns]
    14721464        # Begin looping through the query values.
    class RawQuerySet(object): 
    14741466            if need_resolv_columns:
    14751467                values = compiler.resolve_columns(values, fields)
    14761468            # Associate fields to values
    1477             if skip:
    1478                 model_init_kwargs = {}
    1479                 for attname, pos in model_init_field_names.iteritems():
    1480                     model_init_kwargs[attname] = values[pos]
    1481                 instance = model_cls(**model_init_kwargs)
    1482             else:
    1483                 model_init_args = [values[pos] for pos in model_init_field_pos]
    1484                 instance = model_cls(*model_init_args)
     1469            model_init_args = [values[pos] for pos in model_init_field_pos]
     1470            instance = model_cls(*model_init_args)
    14851471            if annotation_fields:
    14861472                for column, pos in annotation_fields:
    14871473                    setattr(instance, column, values[pos])
  • django/db/models/query_utils.py

    diff --git a/django/db/models/query_utils.py b/django/db/models/query_utils.py
    index a56ab5c..a3299eb 100644
    a b circular import difficulties. 
    99import weakref
    1010
    1111from django.db.backends import util
     12from django.db.models.loading import get_model
    1213from django.utils import tree
    1314
    1415
    def deferred_class_factory(model, attrs): 
    146147    being replaced with DeferredAttribute objects. The "pk_value" ties the
    147148    deferred attributes to a particular instance of the model.
    148149    """
    149     class Meta:
    150         proxy = True
    151         app_label = model._meta.app_label
    152 
    153150    # The app_cache wants a unique name for each model, otherwise the new class
    154151    # won't be created (we get an old one back). Therefore, we generate the
    155152    # name using the passed in attrs. It's OK to reuse an existing class
    156153    # object if the attrs are identical.
    157154    name = "%s_Deferred_%s" % (model.__name__, '_'.join(sorted(list(attrs))))
    158155    name = util.truncate_name(name, 80, 32)
     156    deferred_model = get_model(model._meta.app_label, name)
     157    if deferred_model:
     158        return deferred_model
     159
     160    class Meta:
     161        proxy = True
     162        app_label = model._meta.app_label
     163
    159164
    160165    overrides = dict([(attr, DeferredAttribute(attr, model))
    161166            for attr in attrs])
    162167    overrides["Meta"] = Meta
    163168    overrides["__module__"] = model.__module__
    164169    overrides["_deferred"] = True
    165     return type(name, (model,), overrides)
     170    deferred_model = type(name, (model,), overrides)
     171    deferred_model._meta.set_loaded_fields(attrs)
     172    return deferred_model
    166173
    167174# The above function is also used to unpickle model instances with deferred
    168175# fields.
Back to Top