Ticket #11505: restore_cache_during_tests.v2.diff

django/test/client.py

@@ -17 +17 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.signals import got_request_exception
 from django.http import SimpleCookie, HttpRequest, QueryDict
+from django.middleware.cache import (UpdateCacheMiddleware,
+         FetchFromCacheMiddleware, CacheMiddleware)
 from django.template import TemplateDoesNotExist
 from django.test import signals
+from django.test.utils import modified_get_cache
 from django.utils.functional import curry
 from django.utils.encoding import smart_str
 from django.utils.http import urlencode
@@ -62 +65 @@
     """
     def __init__(self, enforce_csrf_checks=True, *args, **kwargs):
         self.enforce_csrf_checks = enforce_csrf_checks
+        self._caches_replaced = False
         super(ClientHandler, self).__init__(*args, **kwargs)
 
     def __call__(self, environ):
@@ -81 +85 @@
             # required for backwards compatibility with external tests against
             # admin views.
             request._dont_enforce_csrf_checks = not self.enforce_csrf_checks
+
+            # Ensure cache middleware uses resetable cache for testing
+            if not self._caches_replaced:
+                self.replace_cache_in_cache_middleware()
             response = self.get_response(request)
         finally:
             signals.request_finished.disconnect(close_connection)
@@ -89 +97 @@
 
         return response
 
+    def replace_cache_in_cache_middleware(self):
+        """
+        Iterate over loaded cache middleware and replace cache with modified
+        cache (so it can be reset between tests).
+        """
+        cache_middleware_classes = [
+            UpdateCacheMiddleware, FetchFromCacheMiddleware, CacheMiddleware
+        ]
+        mw_methods = set(self._request_middleware + self._response_middleware)
+        for mw_method in mw_methods:
+            mw_instance = mw_method.im_self
+            if any([issubclass(type(mw_instance), cls) for cls in cache_middleware_classes]):
+                mw_instance._original_cache = mw_instance.cache
+                mw_instance.cache = modified_get_cache(mw_instance.cache_alias)
+        self._caches_replaced = True
+
 def store_rendered_templates(store, signal, sender, template, context, **kwargs):
     """
     Stores templates and contexts that are rendered.

django/test/testcases.py

@@ -20 +20 @@
 from django.test import _doctest as doctest
 from django.test.client import Client
 from django.test.utils import (get_warnings_state, restore_warnings_state,
-    override_settings)
+    override_settings, _caches_used_during_test)
 from django.utils import simplejson, unittest as ut2
 from django.utils.encoding import smart_str
 
@@ -350 +350 @@
               ROOT_URLCONF with it.
             * Clearing the mail test outbox.
         """
+        self._cache_setup()
         self._fixture_setup()
         self._urlconf_setup()
         mail.outbox = []
@@ -376 +377 @@
             settings.ROOT_URLCONF = self.urls
             clear_url_caches()
 
+    def _cache_setup(self):
+        """
+        Resets the list of caches used in preparation for test
+        """
+        global _caches_used_during_test
+        _caches_used_during_test.clear()
+
     def __call__(self, result=None):
         """
         Wrapper around default __call__ method to perform common Django test
@@ -414 +422 @@
         """
         self._fixture_teardown()
         self._urlconf_teardown()
+        self._cache_teardown()
         # Some DB cursors include SQL statements as part of cursor
         # creation. If you have a test that does rollback, the effect
         # of these statements is lost, which can effect the operation
@@ -431 +440 @@
         if hasattr(self, '_old_root_urlconf'):
             settings.ROOT_URLCONF = self._old_root_urlconf
             clear_url_caches()
+    
+    def _cache_teardown(self):
+        """
+        Clears values set in cache during test
+        """
+        for cache in _caches_used_during_test:
+            cache.delete_many(list(cache._keys_set_during_test))
 
     def assertRedirects(self, response, expected_url, status_code=302,
                         target_status_code=200, host=None, msg_prefix=''):

django/test/utils.py

@@ -1 +1 @@
 from __future__ import with_statement
 
+import types
 import warnings
 from django.conf import settings, UserSettingsHolder
 from django.core import mail
+from django.core import cache
 from django.test.signals import template_rendered, setting_changed
 from django.template import Template, loader, TemplateDoesNotExist
 from django.template.loaders import cached
@@ -62 +64 @@
     return self.nodelist.render(context)
 
 
+_caches_used_during_test = set()
+
+def modified_get_cache(backend, **kwargs):
+    """
+    Modifies the original get_cache so that we can track any keys set during a cache run and reset
+    them at the end.
+    """
+    requested_cache = cache.original_get_cache(backend, **kwargs)
+    
+    # Add '_test_' to key_prefix to ensure pre-existing cache values don't get touched
+    requested_cache.original_key_prefix = requested_cache.key_prefix
+    requested_cache.key_prefix = '_test_%s' % requested_cache.original_key_prefix
+    
+    # Keep track of which caches we use during a test
+    global _caches_used_during_test
+    _caches_used_during_test.add(requested_cache)
+    
+    requested_cache._keys_set_during_test = set()
+    
+    # Modify cache.set() to collect keys in _keys_set_during_test
+    requested_cache.original_set = requested_cache.set
+    def modified_set(self, key, value, timeout=None, version=None):
+        requested_cache._keys_set_during_test.add(key)
+        requested_cache.original_set(key, value, timeout, version)
+    requested_cache.set = types.MethodType(modified_set, requested_cache)
+    
+    # Modify cache.add() to collect keys in _keys_set_during_test
+    requested_cache.original_add = requested_cache.add
+    def modified_add(self, key, value, timeout=None, version=None):
+        requested_cache._keys_set_during_test.add(key)
+        return requested_cache.original_add(key, value, timeout, version)
+    requested_cache.add = types.MethodType(modified_add, requested_cache)
+    
+    # Modify cache.incr() to collect keys in _keys_set_during_test
+    requested_cache.original_incr = requested_cache.incr
+    def modified_incr(self, key, delta=1, version=None):
+        requested_cache._keys_set_during_test.add(key)
+        return requested_cache.original_incr(key, delta, version)
+    requested_cache.incr = types.MethodType(modified_incr, requested_cache)
+    
+    # Modify cache.decr() to collect keys in _keys_set_during_test
+    requested_cache.original_decr = requested_cache.decr
+    def modified_decr(self, key, delta=1, version=None):
+        requested_cache._keys_set_during_test.add(key)
+        return requested_cache.original_decr(key, delta, version)
+    requested_cache.decr = types.MethodType(modified_decr, requested_cache)
+    
+    # Modify cache.set_many() to collect keys in _keys_set_during_test
+    requested_cache.original_set_many = requested_cache.set_many
+    def modified_set_many(self, data, timeout=None, version=None):
+        requested_cache._keys_set_during_test.update(data.keys())
+        requested_cache.original_set_many(data, timeout, version)
+    requested_cache.set_many = types.MethodType(modified_set_many, requested_cache)
+    
+    return requested_cache
+
+
 def setup_test_environment():
     """Perform any global pre-test setup. This involves:
 
@@ -74 +133 @@
 
     mail.original_email_backend = settings.EMAIL_BACKEND
     settings.EMAIL_BACKEND = 'django.core.mail.backends.locmem.EmailBackend'
-
     mail.outbox = []
+    
+    cache.original_get_cache = cache.get_cache
+    cache.get_cache = modified_get_cache
+    
+    # Make sure django.core.cache.cache also uses the modified cache
+    cache.original_cache = cache.cache
+    cache.cache = cache.get_cache(cache.DEFAULT_CACHE_ALIAS)
 
     deactivate()
 
@@ -92 +157 @@
 
     settings.EMAIL_BACKEND = mail.original_email_backend
     del mail.original_email_backend
-
     del mail.outbox
+    
+    cache.cache = cache.original_cache
+    del cache.original_cache
+    cache.get_cache = cache.original_get_cache
+    del cache.original_get_cache
 
 
 def get_warnings_state():

tests/regressiontests/cache/tests.py

@@ -894 +894 @@
         self.assertTrue(not os.path.exists(os.path.dirname(os.path.dirname(keypath))))
 
     def test_cull(self):
-        self.perform_cull_test(50, 29)
+        self.perform_cull_test(50, 28)
 
     def test_old_initialization(self):
         self.cache = get_cache('file://%s?max_entries=30' % self.dirname)
-        self.perform_cull_test(50, 29)
+        self.perform_cull_test(50, 28)
 
 class CustomCacheKeyValidationTests(unittest.TestCase):
     """

new file tests/regressiontests/test_utils/templates/test_utils/simple_view.html

@@ +1 @@
+<html><p>Howdy!</p></html>

tests/regressiontests/test_utils/tests.py

@@ -1 +1 @@
 from __future__ import with_statement
 
+import tempfile
 from django.forms import EmailField
 from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature
-from django.utils.unittest import skip
+from django.test.utils import override_settings
+from django.utils.unittest import skip, skipUnless
+from django.conf import settings
+from django.core import management
+from django.core.cache import get_cache, DEFAULT_CACHE_ALIAS
 
 from models import Person
 
@@ -131 +136 @@
         pass
 
 
+# We must set this via a function to confirm that cache set test has run
+# before cache get test
+_cache_set_test_has_run = False
+
+def cache_set_test_run():
+    global _cache_set_test_has_run
+    _cache_set_test_has_run = True
+
+def cache_get_test_finished():
+    global _cache_set_test_has_run
+    _cache_set_test_has_run = False
+
+
+class BaseCacheReset(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Setup everything here, because setupClass is guaranteed to run first
+        cache = cls.original_cache()
+
+        # Add a "pre-existing" value to cache to ensure it gets reset properly
+        cache.set('salt', 'pepper') # We won't touch this one
+        cache.set('sweet', 'sour') # We will touch a key with this name and check if it's restored at the end
+        
+        # Sanity checks
+        assert cache.get('salt') == 'pepper'
+        assert cache.get('sweet') == 'sour'
+
+        # We manually prefix the test_ here because we are not trying to pretend these are part
+        # of the original cache. We're just making sure the values exist so that we can call incr/decr
+        # without using cache.set() first
+        cache.orig_key_prefix = cache.key_prefix
+        cache.key_prefix = '_test_%s' % cache.orig_key_prefix
+        try:
+            cache.set('going_up', 1)
+            cache.set('going_down', 2)
+        finally:
+            cache.key_prefix = cache.orig_key_prefix
+
+    def setUp(self):
+        # Set up cache again at the instance level. This one will get reset
+        self.cache = self.modified_cache()
+
+
+class CacheResetTestsMixin(object):
+    # Note test names start with a/b. This ensures test order is correct (which we're normally
+    # not supposed to worry about)
+    def test_a_set_cache_in_various_ways_in_one_test_method(self):
+        """
+        Set some cache stuff in this method, and then we'll sure it doesn't carry over to the next
+        """
+        sweet_result = self.cache.set('sweet', 'bitter')
+        left_result = self.cache.set('left', 'right')
+        loud_result = self.cache.add('loud', 'quiet')
+        
+        assert self.cache.get('going_up') == 1
+        going_up_result = self.cache.incr('going_up')
+        
+        assert self.cache.get('going_down') == 2
+        going_down_result = self.cache.decr('going_down')
+        
+        over_above_result = self.cache.set_many({'over': 'under', 'above': 'below'})
+
+        # Sanity checks
+        self.assertEqual(self.cache.get('sweet'), 'bitter')
+        self.assertEqual(sweet_result, None)
+        
+        self.assertEqual(self.cache.get('left'), 'right')
+        self.assertEqual(left_result, None)
+        
+        self.assertEqual(self.cache.get('loud'), 'quiet')
+        self.assertEqual(loud_result, True)
+        
+        self.assertEqual(self.cache.get('going_up'), 2)
+        self.assertEqual(going_up_result, 2)
+        
+        self.assertEqual(self.cache.get('going_down'), 1)
+        self.assertEqual(going_down_result, 1)
+        
+        self.assertEqual(self.cache.get('over'), 'under')
+        self.assertEqual(self.cache.get('above'), 'below')
+        self.assertEqual(over_above_result, None)
+
+        # Mark that set_cache tests have run
+        cache_set_test_run()
+
+    def test_b_get_cache_in_another_test_method(self):
+        # Confirm that set tests has already run
+        if not _cache_set_test_has_run:
+            self.skipTest("set_cache test did not run first!")
+
+        try:
+            self.assertEqual(self.cache.get('left'), None)
+            self.assertEqual(self.cache.get('loud'), None)
+            self.assertEqual(self.cache.get('going_up'), None)
+            self.assertEqual(self.cache.get('going_down'), None)
+            self.assertEqual(self.cache.get('over'), None)
+            self.assertEqual(self.cache.get('above'), None)
+
+            # Make sure pre-existing values are still correct
+            original_cache = self.__class__.original_cache()
+            self.assertEqual(original_cache.get('sweet'), 'sour')
+            self.assertEqual(original_cache.get('salt'), 'pepper')
+        finally:
+            # Mark get_cache tests finished
+            cache_get_test_finished()
+
+
+class LocMemCacheResetTests(BaseCacheReset, CacheResetTestsMixin):
+    backend_name = 'django.core.cache.backends.locmem.LocMemCache'
+
+    @classmethod
+    def original_cache(cls):
+        if not hasattr(cls, '_original_cache'):
+            from django.core.cache import original_get_cache
+            cls._original_cache = original_get_cache(cls.backend_name, LOCATION='test')
+        return cls._original_cache
+
+    def modified_cache(self):
+        return get_cache(self.backend_name, LOCATION='test')
+
+
+class FileBasedCacheResetTests(BaseCacheReset, CacheResetTestsMixin):
+    backend_name = 'django.core.cache.backends.filebased.FileBasedCache'
+
+    @classmethod
+    def original_cache(cls):
+        if not hasattr(cls, '_original_cache'):
+            cls.cache_dirname = tempfile.mkdtemp()
+            from django.core.cache import original_get_cache
+            cls._original_cache = original_get_cache(cls.backend_name, LOCATION=cls.cache_dirname)
+        return cls._original_cache
+
+    def modified_cache(self):
+        return get_cache(self.backend_name, LOCATION=self.cache_dirname)
+
+
+class DBCacheResetTests(BaseCacheReset, CacheResetTestsMixin):
+    backend_name = 'django.core.cache.backends.db.DatabaseCache'
+
+    @classmethod
+    def original_cache(cls):
+        if not hasattr(cls, '_original_cache'):
+            cls.cache_table_name = 'test_cache_table'
+            management.call_command('createcachetable', cls.cache_table_name, verbosity=0, interactive=False)
+            from django.core.cache import original_get_cache
+            cls._original_cache = original_get_cache(cls.backend_name, LOCATION=cls.cache_table_name)
+        return cls._original_cache
+
+    def modified_cache(self):
+        return get_cache(self.backend_name, LOCATION=self.cache_table_name)
+
+    @classmethod
+    def tearDownClass(cls):
+        from django.db import connection
+        cursor = connection.cursor()
+        cursor.execute('DROP TABLE %s' % connection.ops.quote_name(cls.cache_table_name))
+
+
+@skipUnless(settings.CACHES[DEFAULT_CACHE_ALIAS]['BACKEND'].startswith('django.core.cache.backends.memcached.'), "memcached not available")
+class MemcachedCacheResetTests(BaseCacheReset, CacheResetTestsMixin):
+    backend_name = 'django.core.cache.backends.memcached.MemcachedCache'
+
+    @classmethod
+    def original_cache(cls):
+        if not hasattr(cls, '_original_cache'):
+            cls.memcached_location = settings.CACHES[DEFAULT_CACHE_ALIAS]['LOCATION']
+            from django.core.cache import original_get_cache
+            cls._original_cache = original_get_cache(cls.backend_name, LOCATION=cls.memcached_location)
+        return cls._original_cache
+
+    def modified_cache(self):
+        return get_cache(self.backend_name, LOCATION=self.memcached_location)
+
+
+class CacheMiddlewareTestingTests(TestCase):
+    """
+    CacheMiddleware caches should be reset between tests to avoid broken test
+    client when views are cached.
+    """
+    urls = 'regressiontests.test_utils.urls'
+
+    def test_a_request_cached_in_view(self):
+        """A simple view, which gets cached by cache middleware"""
+        response = self.client.get("/test_utils/simple_view/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.context["foo"], "bar")
+
+    def test_b_request_cached_in_view(self):
+        """
+        This test runs after test_a above, and assures that the view is not
+        being cached between tests.
+        """
+        response = self.client.get("/test_utils/simple_view/")
+        self.assertEqual(response.status_code, 200)
+
+        # These assertions fail if the cache from test_a is carried over, since
+        # the cached response doesn't have a context
+        self.assertTrue(response.context)
+        self.assertEqual(response.context["foo"], "bar")
+
+CacheMiddlewareTestingTests = override_settings(
+    CACHES = {
+        'default': {
+            'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'
+        },
+    },
+    CACHE_MIDDLEWARE_SECONDS=30,
+    MIDDLEWARE_CLASSES = (
+        'django.middleware.cache.UpdateCacheMiddleware',
+        'django.middleware.common.CommonMiddleware',
+        'django.middleware.cache.FetchFromCacheMiddleware',
+    )
+)(CacheMiddlewareTestingTests)
+
+
 class AssertRaisesMsgTest(SimpleTestCase):
 
     def test_special_re_chars(self):

tests/regressiontests/test_utils/urls.py

@@ -5 +5 @@
 
 urlpatterns = patterns('',
     (r'^test_utils/get_person/(\d+)/$', views.get_person),
+    (r'^test_utils/simple_view/$', views.simple_view),
 )

tests/regressiontests/test_utils/views.py

@@ -1 +1 @@
 from django.http import HttpResponse
-from django.shortcuts import get_object_or_404
+from django.template import RequestContext
+from django.shortcuts import get_object_or_404, render_to_response
 from models import Person
 
 def get_person(request, pk):
     person = get_object_or_404(Person, pk=pk)
-    return HttpResponse(person.name)
- No newline at end of file
+    return HttpResponse(person.name)
+
+def simple_view(request):
+    return render_to_response('test_utils/simple_view.html', {
+        "foo": "bar"
+    }, context_instance=RequestContext(request))