Ticket #17: instance-caching-3.patch

File instance-caching-3.patch, 11.2 KB (added by (removed), 8 years ago)

instance caching v3; add appropriate cache cleaning to Model.delete, and tweak queries delete_objects to prune the cache while it's wiping pk instances. Only potential issue I'm aware of at this point is if there are multiple pk's for a record, which is screwy behaviour for django in general.

  • django/core/serializers/python.py

    === modified file 'django/core/serializers/python.py'
     
    9494            else:
    9595                data[field.name] = field.to_python(field_value)
    9696               
     97        data["disable_inst_caching"] = True
    9798        yield base.DeserializedObject(Model(**data), m2m_data)
    9899
    99100def _get_model(model_identifier):
  • django/core/serializers/xml_serializer.py

    === modified file 'django/core/serializers/xml_serializer.py'
     
    176176                    value = field.to_python(getInnerText(field_node).strip().encode(self.encoding))
    177177                data[field.name] = value
    178178       
     179        data["disable_inst_caching"] = True
    179180        # Return a DeserializedObject so that the m2m data has a place to live.
    180181        return base.DeserializedObject(Model(**data), m2m_data)
    181182       
     
    232233            inner_text.extend(getInnerText(child))
    233234        else:
    234235           pass
    235     return "".join(inner_text)
    236  No newline at end of file
     236    return "".join(inner_text)
  • django/db/models/base.py

    === modified file 'django/db/models/base.py'
     
    1414from django.utils.functional import curry
    1515from django.conf import settings
    1616from itertools import izip
     17from weakref import WeakValueDictionary
    1718import types
    1819import sys
    1920import os
     
    7677        # registered version.
    7778        return get_model(new_class._meta.app_label, name, False)
    7879
     80    def __call__(cls, *args, **kwargs):
     81        if not kwargs.pop("disable_inst_caching", False) and cls._meta.has_auto_field:
     82            key = cls._get_cache_key(args, kwargs)
     83            if key is not None:
     84                obj = cls.__instance_cache__.get(key)
     85                if obj is None:
     86                    obj = super(ModelBase, cls).__call__(*args, **kwargs)
     87                    cls.__instance_cache__[key] = obj
     88            else:
     89                obj = super(ModelBase, cls).__call__(*args, **kwargs)
     90        else:
     91            obj = super(ModelBase, cls).__call__(*args, **kwargs)
     92        return obj
     93
     94
    7995class Model(object):
    8096    __metaclass__ = ModelBase
    8197
     
    94110    def __ne__(self, other):
    95111        return not self.__eq__(other)
    96112
     113    def _get_cache_key(cls, args, kwargs):
     114        # this should be calculated *once*, but isn't atm
     115        pk_position = cls._meta.fields.index(cls._meta.pk)
     116        if len(args) > pk_position:
     117            return args[pk_position]
     118        pk = cls._meta.pk
     119        if pk.name in kwargs:
     120            return kwargs[pk.name]
     121        elif pk.attname in kwargs:
     122            return kwargs[pk.attname]
     123        return None
     124    _get_cache_key = classmethod(_get_cache_key)
     125
     126    def get_cached_instance(cls, id):
     127        return cls.__instance_cache__.get(id)
     128    get_cached_instance = classmethod(get_cached_instance)
     129
    97130    def __init__(self, *args, **kwargs):
    98131        dispatcher.send(signal=signals.pre_init, sender=self.__class__, args=args, kwargs=kwargs)
    99132       
     
    194227        if hasattr(cls, 'get_absolute_url'):
    195228            cls.get_absolute_url = curry(get_absolute_url, opts, cls.get_absolute_url)
    196229
     230        cls.__instance_cache__ = WeakValueDictionary()
     231
    197232        dispatcher.send(signal=signals.class_prepared, sender=cls)
    198233
    199234    _prepare = classmethod(_prepare)
     
    251286                setattr(self, self._meta.pk.attname, backend.get_last_insert_id(cursor, self._meta.db_table, self._meta.pk.column))
    252287        transaction.commit_unless_managed()
    253288
     289        # if we're a new instance that hasn't been written in; save ourself.
     290        if self._meta.has_auto_field:
     291            self.__instance_cache__[self._get_pk_val()] = self
     292
    254293        # Run any post-save hooks.
    255294        dispatcher.send(signal=signals.post_save, sender=self.__class__, instance=self)
    256295
     
    311350        self._collect_sub_objects(seen_objs)
    312351
    313352        # Actually delete the objects
     353        if self._meta.has_auto_field:
     354            pk = self._get_pk_val()
     355            if pk is not None:
     356                self.__instance_cache__.pop(pk, None)
    314357        delete_objects(seen_objs)
    315358
    316359    delete.alters_data = True
  • django/db/models/fields/related.py

    === modified file 'django/db/models/fields/related.py'
     
    163163                if self.field.null:
    164164                    return None
    165165                raise self.field.rel.to.DoesNotExist
    166             other_field = self.field.rel.get_related_field()
    167             if other_field.rel:
    168                 params = {'%s__pk' % self.field.rel.field_name: val}
    169             else:
    170                 params = {'%s__exact' % self.field.rel.field_name: val}
    171             rel_obj = self.field.rel.to._default_manager.get(**params)
     166            rel_obj = self.field.rel.to.get_cached_instance(val)
     167            if rel_obj is None:
     168                other_field = self.field.rel.get_related_field()
     169                if other_field.rel:
     170                    params = {'%s__pk' % self.field.rel.field_name: val}
     171                else:
     172                    params = {'%s__exact' % self.field.rel.field_name: val}
     173                rel_obj = self.field.rel.to._default_manager.get(**params)
    172174            setattr(instance, cache_name, rel_obj)
    173175            return rel_obj
    174176
  • django/db/models/query.py

    === modified file 'django/db/models/query.py'
     
    10481048    for cls in ordered_classes:
    10491049        seen_objs[cls] = seen_objs[cls].items()
    10501050        seen_objs[cls].sort()
     1051        clean_inst_cache = cls.__instance_cache__.pop
    10511052
    10521053        # Pre notify all instances to be deleted
    10531054        for pk_val, instance in seen_objs[cls]:
    10541055            dispatcher.send(signal=signals.pre_delete, sender=cls, instance=instance)
    10551056
    10561057        pk_list = [pk for pk,instance in seen_objs[cls]]
     1058        # we wipe the cache now; it's *possible* some form of a __get__ lookup may reintroduce an item after
     1059        # the fact with the same pk (extremely unlikely)
     1060        for x in pk_list:
     1061            clean_inst_cache(x, None)
     1062
    10571063        for related in cls._meta.get_all_related_many_to_many_objects():
    10581064            if not isinstance(related.field, generic.GenericRelation):
    1059                 for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
     1065                for offset in xrange(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
    10601066                    cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
    10611067                        (qn(related.field.m2m_db_table()),
    10621068                            qn(related.field.m2m_reverse_name()),
    1063                             ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
    1064                         pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
     1069                            ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
     1070                            pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE])
     1071
    10651072        for f in cls._meta.many_to_many:
    10661073            if isinstance(f, generic.GenericRelation):
    10671074                from django.contrib.contenttypes.models import ContentType
     
    10701077            else:
    10711078                query_extra = ''
    10721079                args_extra = []
    1073             for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
     1080            for offset in xrange(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
    10741081                cursor.execute(("DELETE FROM %s WHERE %s IN (%s)" % \
    10751082                    (qn(f.m2m_db_table()), qn(f.m2m_column_name()),
    10761083                    ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]]))) + query_extra,
    10771084                    pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE] + args_extra)
     1085
    10781086        for field in cls._meta.fields:
    10791087            if field.rel and field.null and field.rel.to in seen_objs:
    1080                 for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
     1088                for offset in xrange(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
    10811089                    cursor.execute("UPDATE %s SET %s=NULL WHERE %s IN (%s)" % \
    10821090                        (qn(cls._meta.db_table), qn(field.column), qn(cls._meta.pk.column),
    10831091                            ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
     
    10871095    for cls in ordered_classes:
    10881096        seen_objs[cls].reverse()
    10891097        pk_list = [pk for pk,instance in seen_objs[cls]]
    1090         for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
     1098        clean_inst_cache = cls.__instance_cache__.pop
     1099        for x in pk_list:
     1100            clean_inst_cache(x, None)
     1101        for offset in xrange(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
    10911102            cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \
    10921103                (qn(cls._meta.db_table), qn(cls._meta.pk.column),
    10931104                ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])),
  • tests/modeltests/model_forms/models.py

    === modified file 'tests/modeltests/model_forms/models.py'
     
    2442441
    245245>>> test_art = Article.objects.get(id=1)
    246246>>> test_art.headline
    247 'Test headline'
     247u'Test headline'
    248248
    249249You can create a form over a subset of the available fields
    250250by specifying a 'fields' argument to form_for_instance.
     
    2602601
    261261>>> new_art = Article.objects.get(id=1)
    262262>>> new_art.headline
    263 'New headline'
     263u'New headline'
    264264
    265265Add some categories and test the many-to-many form output.
    266266>>> new_art.categories.all()
  • tests/modeltests/select_related/models.py

    === modified file 'tests/modeltests/select_related/models.py'
     
    1071071
    108108
    109109# select_related() also of course applies to entire lists, not just items.
    110 # Without select_related()
     110# Without select_related() (note instance caching still reduces this from 9 to 5)
    111111>>> db.reset_queries()
    112112>>> world = Species.objects.all()
    113113>>> [o.genus.family for o in world]
    114114[<Family: Drosophilidae>, <Family: Hominidae>, <Family: Fabaceae>, <Family: Amanitacae>]
    115115>>> len(db.connection.queries)
    116 9
     1165
    117117
    118118# With select_related():
    119119>>> db.reset_queries()
     
    129129>>> pea.genus.family.order.klass.phylum.kingdom.domain
    130130<Domain: Eukaryota>
    131131
    132 # Notice: one few query than above because of depth=1
     132# notice: instance caching saves the day; would be 7 without.
    133133>>> len(db.connection.queries)
    134 7
     1341
    135135
    136136>>> db.reset_queries()
    137137>>> pea = Species.objects.select_related(depth=5).get(name="sativum")
    138138>>> pea.genus.family.order.klass.phylum.kingdom.domain
    139139<Domain: Eukaryota>
    140140>>> len(db.connection.queries)
    141 3
     1411
    142142
    143143>>> db.reset_queries()
    144144>>> world = Species.objects.all().select_related(depth=2)
    145145>>> [o.genus.family.order for o in world]
    146146[<Order: Diptera>, <Order: Primates>, <Order: Fabales>, <Order: Agaricales>]
    147147>>> len(db.connection.queries)
    148 5
     1481
    149149
    150150# Reset DEBUG to where we found it.
    151151>>> settings.DEBUG = False
Back to Top