Index: django/db/models/base.py =================================================================== --- django/db/models/base.py (revision 6308) +++ django/db/models/base.py (working copy) @@ -15,6 +15,7 @@ from django.utils.encoding import smart_str, force_unicode, smart_unicode from django.conf import settings from itertools import izip +from weakref import WeakValueDictionary import types import sys import os @@ -77,6 +78,42 @@ # registered version. return get_model(new_class._meta.app_label, name, False) + def __call__(cls, *args, **kwargs): + """ + this method will either create an instance (by calling the default implementation) + or try to retrieve one from the class-wide cache by infering the pk value from + args and kwargs. If instance caching is enabled for this class, the cache is + populated whenever possible (ie when it is possible to infer the pk value). If 'meta__disable_caching' + is set to True in kwargs, then the instance is constructed and we flush + the associated cache entry. + """ + def new_instance(): + return super(ModelBase, cls).__call__(*args, **kwargs) + + cache_this_instance = cls.instance_caching_enabled() + # we always pop those settings from kwargs, the instance shouldn't see this + if kwargs.pop('meta__disable_caching', False): + # user explicitely requested not to use cache, we flush the cache to prevent inconsitencies + cls._flush_cached_by_key(cls._get_cache_key(args, kwargs)) + cache_this_instance = False + + # simplest case, just create a new instance every time + if not cache_this_instance: + return new_instance() + + instance_key = cls._get_cache_key(args, kwargs) + # depending on the arguments, we might not be able to infer the PK, so in that case we create a new instance + if instance_key is None: + cls._instance_cache_nokey_misses += 1 + return new_instance() + + cached_instance = cls.get_cached_instance(instance_key) + if cached_instance is None: + cached_instance = new_instance() + cls.cache_instance(cached_instance) + + return cached_instance + class Model(object): __metaclass__ = ModelBase @@ -97,6 +134,108 @@ def __ne__(self, other): return not self.__eq__(other) + def _get_cache_key(cls, args, kwargs): + """ + This method is used by the caching subsystem to infer the PK value from the constructor arguments. + It is used to decide if an instance has to be built or is already in the cache. + """ + result = None + pk = cls._meta.pk + # get the index of the pk in the class fields. this should be calculated *once*, but isn't atm + pk_position = cls._meta.fields.index(pk) + if len(args) > pk_position: + # if it's in the args, we can get it easily by index + result = args[pk_position] + elif pk.attname in kwargs: + # retrieve the pk value. Note that we use attname instead of name, to handle the case where the pk is a + # a ForeignKey. + result = kwargs[pk.attname] + elif pk.name != pk.attname and pk.name in kwargs: + # ok we couldn't find the value, but maybe it's a FK and we can find the corresponding object instead + result = kwargs[pk.name] + + if result is not None and isinstance(result, Model): + # if the pk value happens to be a model instance (which can happen wich a FK), we'd rather use its own pk as the key + result = result._get_pk_val() + return result + _get_cache_key = classmethod(_get_cache_key) + + def get_cached_instance(cls, id): + """ + Method to retrieve a cached instance by pk value. Returns None when not found + (which will always be the case when caching is disabled for this class). Please + note that the lookup will be done even when instance caching is disabled, thus + generating a miss in the stats. + """ + result = cls.__instance_cache__.get(id) + if result is None: + cls._instance_cache_misses += 1 + else: + cls._instance_cache_hits += 1 + return result + get_cached_instance = classmethod(get_cached_instance) + + def cache_instance(cls, instance): + """ + Method to store an instance in the cache. TODO: add a store counter in the stats + """ + if cls.instance_caching_enabled() and instance._get_pk_val() is not None: + cls.__instance_cache__[instance._get_pk_val()] = instance + cache_instance = classmethod(cache_instance) + + def _flush_cached_by_key(cls, key): + if cls.__instance_cache__.pop(key, None) is not None: + cls._instance_cache_flushes += 1 + _flush_cached_by_key = classmethod(_flush_cached_by_key) + + def flush_cached_instance(cls, instance): + """ + Method to flush an instance from the cache. The instance will always be flushed from the cache, + since this is most likely called from delete(), and we want to make sure we don't cache dead objects. + We do not test the pk value because delete() does it and it will fail silently anyway. + """ + if cls.instance_caching_enabled(): + cls._flush_cached_by_key(instance._get_pk_val()) + flush_cached_instance = classmethod(flush_cached_instance) + + def instance_caching_enabled(cls): + """ + Accessor for the cache settings. + """ + # cache is off by default! + return getattr(cls, '_meta__instance_caching', False) + instance_caching_enabled = classmethod(instance_caching_enabled) + + def set_instance_caching(cls, enable): + """ + Accessor for the cache settings. Note that the cache is flushed and the stats reset when + the settings are switched (ie enabling the cache multiple times will not flush). + """ + current_settings = cls.instance_caching_enabled() + cls._meta__instance_caching = enable + # completely flush the cache every time the settings are changed + if enable != current_settings: + cls.__instance_cache__.clear() + cls.instance_caching_stats_reset() + set_instance_caching = classmethod(set_instance_caching) + + def instance_caching_stats_reset(cls): + # also used to init the stats in '_prepare()' + cls._instance_cache_hits = 0 + cls._instance_cache_misses = 0 + cls._instance_cache_nokey_misses = 0 + cls._instance_cache_flushes = 0 + instance_caching_stats_reset = classmethod(instance_caching_stats_reset) + + def instance_caching_stats(cls): + return {'enabled': cls.instance_caching_enabled(), + 'hits' : cls._instance_cache_hits, + 'misses': cls._instance_cache_misses, + 'flushes': cls._instance_cache_flushes, + 'misses_nokey': cls._instance_cache_nokey_misses, + 'cache_size': len(cls.__instance_cache__) } + instance_caching_stats = classmethod(instance_caching_stats) + def __init__(self, *args, **kwargs): dispatcher.send(signal=signals.pre_init, sender=self.__class__, args=args, kwargs=kwargs) @@ -197,6 +336,12 @@ if hasattr(cls, 'get_absolute_url'): cls.get_absolute_url = curry(get_absolute_url, opts, cls.get_absolute_url) + cls.__instance_cache__ = WeakValueDictionary() + cls.instance_caching_stats_reset() + # enable the cache according to user preferences (off by default) + # FIXME better interface for setting this value (meta class attribute ?) + cls.set_instance_caching(getattr(cls, 'meta__instance_caching', False)) + dispatcher.send(signal=signals.class_prepared, sender=cls) _prepare = classmethod(_prepare) @@ -261,6 +406,9 @@ setattr(self, self._meta.pk.attname, connection.ops.last_insert_id(cursor, self._meta.db_table, self._meta.pk.column)) transaction.commit_unless_managed() + # if we're a new instance that hasn't been written in; save ourself. + self.__class__.cache_instance(self) + # Run any post-save hooks. dispatcher.send(signal=signals.post_save, sender=self.__class__, instance=self, created=(not record_exists)) @@ -321,6 +469,8 @@ seen_objs = SortedDict() self._collect_sub_objects(seen_objs) + # remove ourself from the cache + self.__class__.flush_cached_instance(self) # Actually delete the objects delete_objects(seen_objs) Index: django/db/models/fields/related.py =================================================================== --- django/db/models/fields/related.py (revision 6308) +++ django/db/models/fields/related.py (working copy) @@ -159,18 +159,26 @@ try: return getattr(instance, cache_name) except AttributeError: + related_cls = self.field.rel.to val = getattr(instance, self.field.attname) if val is None: # If NULL is an allowed value, return it. if self.field.null: return None - raise self.field.rel.to.DoesNotExist - other_field = self.field.rel.get_related_field() - if other_field.rel: - params = {'%s__pk' % self.field.rel.field_name: val} + raise related_cls.DoesNotExist + # try to get a cached instance, and if that fails retrieve it from the db + # FIXME TEST THIS i'm not sure val is really the object's pk ... + if related_cls.instance_caching_enabled(): + rel_obj = related_cls.get_cached_instance(val) else: - params = {'%s__exact' % self.field.rel.field_name: val} - rel_obj = self.field.rel.to._default_manager.get(**params) + rel_obj = None + if rel_obj is None: + other_field = self.field.rel.get_related_field() + if other_field.rel: + params = {'%s__pk' % self.field.rel.field_name: val} + else: + params = {'%s__exact' % self.field.rel.field_name: val} + rel_obj = related_cls._default_manager.get(**params) setattr(instance, cache_name, rel_obj) return rel_obj Index: django/db/models/query.py =================================================================== --- django/db/models/query.py (revision 6308) +++ django/db/models/query.py (working copy) @@ -1134,6 +1134,11 @@ dispatcher.send(signal=signals.pre_delete, sender=cls, instance=instance) pk_list = [pk for pk,instance in seen_objs[cls]] + # we wipe the cache now; it's *possible* some form of a __get__ lookup may reintroduce an item after + # the fact with the same pk (extremely unlikely) + for instance in seen_objs.values(): + cls.flush_cached_instance(instance) + for related in cls._meta.get_all_related_many_to_many_objects(): if not isinstance(related.field, generic.GenericRelation): for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): @@ -1167,6 +1172,8 @@ for cls in ordered_classes: seen_objs[cls].reverse() pk_list = [pk for pk,instance in seen_objs[cls]] + for instance in seen_objs.values(): + cls.flush_cached_instance(instance) for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \ (qn(cls._meta.db_table), qn(cls._meta.pk.column), Index: django/core/serializers/xml_serializer.py =================================================================== --- django/core/serializers/xml_serializer.py (revision 6308) +++ django/core/serializers/xml_serializer.py (working copy) @@ -176,7 +176,8 @@ else: value = field.to_python(getInnerText(field_node).strip()) data[field.name] = value - + # disable caching, make sure the object is fully constructed from our data and not pulled from the cache + data["meta__disable_caching"] = True # Return a DeserializedObject so that the m2m data has a place to live. return base.DeserializedObject(Model(**data), m2m_data) @@ -234,4 +235,3 @@ else: pass return u"".join(inner_text) - Index: django/core/serializers/python.py =================================================================== --- django/core/serializers/python.py (revision 6308) +++ django/core/serializers/python.py (working copy) @@ -89,7 +89,8 @@ # Handle all other fields else: data[field.name] = field.to_python(field_value) - + # disable caching, make sure the object is fully constructed from our data and not pulled from the cache + data["meta__disable_caching"] = True yield base.DeserializedObject(Model(**data), m2m_data) def _get_model(model_identifier): Index: tests/modeltests/select_related/models.py =================================================================== --- tests/modeltests/select_related/models.py (revision 6308) +++ tests/modeltests/select_related/models.py (working copy) @@ -75,6 +75,9 @@ obj.save() parent = obj +def set_instance_caching(settings): + for cls in [Domain, Kingdom, Phylum, Klass, Order, Family, Genus]: + cls.set_instance_caching(settings) __test__ = {'API_TESTS':""" # Set up. @@ -147,6 +150,159 @@ >>> len(db.connection.queries) 5 +# CACHING TESTS +>>> Genus.instance_caching_stats() +{'hits': 0, 'misses_nokey': 0, 'enabled': False, 'flushes': 0, 'misses': 0, 'cache_size': 0} + +# ENABLE CACHING ON ALL MODELS IN THE TEST EXCEPT SPECIES +>>> set_instance_caching(True) + +# This should be the same as without caching +>>> db.reset_queries() +>>> fly = Species.objects.get(name="melanogaster") +>>> fly.genus.family.order.klass.phylum.kingdom.domain + +>>> len(db.connection.queries) +8 + +# This should be the same as without caching +>>> db.reset_queries() +>>> person = Species.objects.select_related().get(name="sapiens") +>>> person.genus.family.order.klass.phylum.kingdom.domain + +>>> len(db.connection.queries) +1 + +# now let's see how caching helps +>>> create_tree("Eukaryota Animalia Chordata Mammalia Primates Hominidae Homo sapiens_2") +>>> create_tree("Eukaryota Animalia Chordata Mammalia Primates Hominidae Homo sapiens_3") +>>> create_tree("Eukaryota Animalia Chordata Mammalia Primates Hominidae Homo sapiens_4") +>>> create_tree("Eukaryota Animalia Chordata Mammalia Primates Hominidae Homo sapiens_5") +>>> create_tree("Eukaryota Animalia Chordata Mammalia Primates Hominidae Homo sapiens_6") +>>> set_instance_caching(False) +>>> db.reset_queries() +>>> world = Species.objects.all() +>>> geni_of_world = [o.genus for o in world] +>>> len(db.connection.queries) # 1 for Species and 9 for the Geni +10 +>>> Genus.instance_caching_stats() +{'hits': 0, 'misses_nokey': 0, 'enabled': False, 'flushes': 0, 'misses': 0, 'cache_size': 0} + +>>> set_instance_caching(True) +>>> db.reset_queries() +>>> world = Species.objects.all() +>>> geni_of_world == [o.genus for o in world] +True +>>> len(db.connection.queries) # 1 for Species and 4 for the distinct Geni +5 + +# here we get 8 misses because ReverseSingleRelatedObjectDescriptor misses twice when the object isn't in the cache +>>> Genus.instance_caching_stats() # 4 distinct Geni and 5 rows generating hits. +{'hits': 5, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 8, 'cache_size': 4} +>>> Genus.get_cached_instance(2) + +>>> Genus.instance_caching_stats()['hits'] # one more hit ! +6 +>>> Genus.instance_caching_stats_reset() +>>> Genus.instance_caching_stats() +{'hits': 0, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 0, 'cache_size': 4} + +>>> Genus.instance_caching_stats_reset() +>>> Genus.get_cached_instance(2) + +>>> Genus.instance_caching_stats() +{'hits': 1, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 0, 'cache_size': 4} +>>> Genus.objects.get(id=2) + +>>> Genus.instance_caching_stats() +{'hits': 2, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 0, 'cache_size': 4} +>>> Species.objects.get(id=2).genus + +>>> Genus.instance_caching_stats() +{'hits': 3, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 0, 'cache_size': 4} + +>>> set_instance_caching(False) +>>> set_instance_caching(True) +>>> Genus.get_cached_instance(2) +>>> Genus.objects.get(id=2) + +>>> Genus.get_cached_instance(2) + +>>> Genus.instance_caching_stats_reset() +>>> db.reset_queries() +>>> world = Species.objects.all() +>>> sapiens = world[1] +>>> len(db.connection.queries) # 1 for Species and the rest is in the cache, whoa +1 +>>> Genus.instance_caching_stats() # we haven't touched geni yet +{'hits': 0, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 0, 'cache_size': 0} +>>> homo = sapiens.genus +>>> Genus.instance_caching_stats() # 2 misses from ReverseSingleRelatedObjectDescriptor even if only one object was retrieved +{'hits': 0, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 2, 'cache_size': 1} + +>>> set_instance_caching(False) +>>> Genus.get_cached_instance(2) + +# This one is tricky, we get() the instance so it gets cached, +# then test that instanciating with the same PK retrieves the instance +>>> set_instance_caching(True) +>>> Genus.instance_caching_enabled() +True +>>> Species.instance_caching_enabled() +False +>>> first_homo = Genus.objects.get(id=2) +>>> first_homo + +>>> Genus.instance_caching_stats()['cache_size'] +1 +>>> Genus.instance_caching_stats()['hits'] +0 +>>> homo = Genus.get_cached_instance(2) +>>> homo + +>>> Genus.instance_caching_stats()['hits'] +1 +>>> Genus.instance_caching_stats_reset() +>>> kwargs = {'id': 2} +>>> Genus._get_cache_key([], kwargs) +2 +>>> Genus(id = 2) + +>>> Genus.instance_caching_stats()['hits'] +1 +>>> Genus.flush_cached_instance(homo) +>>> Genus.get_cached_instance(2) == None +True +>>> Genus.instance_caching_stats()['cache_size'] +0 +>>> Genus.instance_caching_stats_reset() +>>> first_homo = Genus.objects.get(id=2) +>>> Genus.instance_caching_stats()['misses'] +1 + +## each of the initial species has it own genus but the 5 sapiens dupes will hit the cache +#>>> Genus.instance_caching_stats() +#{'hits': 5, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 4, 'cache_size': 4} +# +#>>> set_instance_caching(False) # Flushes the cache +#>>> set_instance_caching(True) +#>>> Genus.instance_caching_stats_reset() +#>>> temp = [o.genus for o in (list(Species.objects.all()) + list(Species.objects.all()))] +#>>> Genus.instance_caching_stats() +#{'hits': 5, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 0, 'cache_size': 4} +# +#>>> set_instance_caching(False) # Flushes the cache +#>>> set_instance_caching(True) +#>>> db.reset_queries() +#>>> Genus.instance_caching_stats_reset() +#>>> world = Species.objects.all().select_related() +#>>> [o.genus for o in world] +#[, , , , , , , , ] +#>>> len(db.connection.queries) +#1 +#>>> Genus.instance_caching_stats() +#{'hits': 5, 'misses_nokey': 0, 'enabled': True, 'flushes': 0, 'misses': 4, 'cache_size': 4} + # Reset DEBUG to where we found it. >>> settings.DEBUG = False """}