Code

Ticket #13795: cache_key_prefix_13959.diff

File cache_key_prefix_13959.diff, 18.3 KB (added by agabel, 4 years ago)
Line 
1diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py
2index 2714bfb..ac06000 100644
3--- a/django/conf/global_settings.py
4+++ b/django/conf/global_settings.py
5@@ -428,6 +428,7 @@ SESSION_FILE_PATH = None                                # Directory to store ses
6 # The cache backend to use.  See the docstring in django.core.cache for the
7 # possible values.
8 CACHE_BACKEND = 'locmem://'
9+CACHE_KEY_PREFIX = ''
10 CACHE_MIDDLEWARE_KEY_PREFIX = ''
11 CACHE_MIDDLEWARE_SECONDS = 600
12 
13diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py
14index e49590b..62d5b35 100644
15--- a/django/core/cache/__init__.py
16+++ b/django/core/cache/__init__.py
17@@ -56,18 +56,20 @@ def parse_backend_uri(backend_uri):
18 
19     return scheme, host, params
20 
21-def get_cache(backend_uri):
22+def get_cache(backend_uri, key_prefix=''):
23+    if not key_prefix:
24+        key_prefix = settings.CACHE_KEY_PREFIX
25     scheme, host, params = parse_backend_uri(backend_uri)
26     if scheme in BACKENDS:
27         name = 'django.core.cache.backends.%s' % BACKENDS[scheme]
28     else:
29         name = scheme
30     module = importlib.import_module(name)
31-    return module.CacheClass(host, params)
32+    return module.CacheClass(host, params, key_prefix)
33 
34 cache = get_cache(settings.CACHE_BACKEND)
35 
36-# Some caches -- pythont-memcached in particular -- need to do a cleanup at the
37+# Some caches -- python-memcached in particular -- need to do a cleanup at the
38 # end of a request cycle. If the cache provides a close() method, wire it up
39 # here.
40 if hasattr(cache, 'close'):
41diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py
42index 83dd461..c911ce2 100644
43--- a/django/core/cache/backends/base.py
44+++ b/django/core/cache/backends/base.py
45@@ -3,6 +3,7 @@
46 import warnings
47 
48 from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
49+from django.utils.encoding import smart_str
50 
51 class InvalidCacheBackendError(ImproperlyConfigured):
52     pass
53@@ -14,13 +15,21 @@ class CacheKeyWarning(DjangoRuntimeWarning):
54 MEMCACHE_MAX_KEY_LENGTH = 250
55 
56 class BaseCache(object):
57-    def __init__(self, params):
58+    def __init__(self, params, key_prefix=''):
59         timeout = params.get('timeout', 300)
60         try:
61             timeout = int(timeout)
62         except (ValueError, TypeError):
63             timeout = 300
64         self.default_timeout = timeout
65+        self.key_prefix = smart_str(key_prefix)
66+
67+    def make_key(self, key):
68+        """Constructs the key used by all other methods. By default it prepends
69+        the `key_prefix'. In cache backend subclasses this can be overriden to
70+        provide custom key making behavior.
71+        """
72+        return self.key_prefix + smart_str(key)
73 
74     def add(self, key, value, timeout=None):
75         """
76diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py
77index c4429c8..3731df8 100644
78--- a/django/core/cache/backends/db.py
79+++ b/django/core/cache/backends/db.py
80@@ -26,8 +26,8 @@ class Options(object):
81         self.proxy = False
82 
83 class CacheClass(BaseCache):
84-    def __init__(self, table, params):
85-        BaseCache.__init__(self, params)
86+    def __init__(self, table, params, key_prefix=''):
87+        BaseCache.__init__(self, params, key_prefix)
88         self._table = table
89 
90         class CacheEntry(object):
91@@ -46,6 +46,7 @@ class CacheClass(BaseCache):
92             self._cull_frequency = 3
93 
94     def get(self, key, default=None):
95+        key = self.make_key(key)
96         self.validate_key(key)
97         db = router.db_for_read(self.cache_model_class)
98         table = connections[db].ops.quote_name(self._table)
99@@ -66,10 +67,12 @@ class CacheClass(BaseCache):
100         return pickle.loads(base64.decodestring(value))
101 
102     def set(self, key, value, timeout=None):
103+        key = self.make_key(key)
104         self.validate_key(key)
105         self._base_set('set', key, value, timeout)
106 
107     def add(self, key, value, timeout=None):
108+        key = self.make_key(key)
109         self.validate_key(key)
110         return self._base_set('add', key, value, timeout)
111 
112@@ -106,6 +109,7 @@ class CacheClass(BaseCache):
113             return True
114 
115     def delete(self, key):
116+        key = self.make_key(key)
117         self.validate_key(key)
118         db = router.db_for_write(self.cache_model_class)
119         table = connections[db].ops.quote_name(self._table)
120@@ -115,6 +119,7 @@ class CacheClass(BaseCache):
121         transaction.commit_unless_managed(using=db)
122 
123     def has_key(self, key):
124+        key = self.make_key(key)
125         self.validate_key(key)
126         db = router.db_for_read(self.cache_model_class)
127         table = connections[db].ops.quote_name(self._table)
128diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py
129index 46e69f3..322143c 100644
130--- a/django/core/cache/backends/filebased.py
131+++ b/django/core/cache/backends/filebased.py
132@@ -12,8 +12,8 @@ from django.core.cache.backends.base import BaseCache
133 from django.utils.hashcompat import md5_constructor
134 
135 class CacheClass(BaseCache):
136-    def __init__(self, dir, params):
137-        BaseCache.__init__(self, params)
138+    def __init__(self, dir, params, key_prefix=''):
139+        BaseCache.__init__(self, params, key_prefix)
140 
141         max_entries = params.get('max_entries', 300)
142         try:
143@@ -32,6 +32,7 @@ class CacheClass(BaseCache):
144             self._createdir()
145 
146     def add(self, key, value, timeout=None):
147+        key = self.make_key(key)
148         self.validate_key(key)
149         if self.has_key(key):
150             return False
151@@ -40,6 +41,7 @@ class CacheClass(BaseCache):
152         return True
153 
154     def get(self, key, default=None):
155+        key = self.make_key(key)
156         self.validate_key(key)
157         fname = self._key_to_file(key)
158         try:
159@@ -58,6 +60,7 @@ class CacheClass(BaseCache):
160         return default
161 
162     def set(self, key, value, timeout=None):
163+        key = self.make_key(key)
164         self.validate_key(key)
165         fname = self._key_to_file(key)
166         dirname = os.path.dirname(fname)
167@@ -82,6 +85,7 @@ class CacheClass(BaseCache):
168             pass
169 
170     def delete(self, key):
171+        key = self.make_key(key)
172         self.validate_key(key)
173         try:
174             self._delete(self._key_to_file(key))
175@@ -99,6 +103,7 @@ class CacheClass(BaseCache):
176             pass
177 
178     def has_key(self, key):
179+        key = self.make_key(key)
180         self.validate_key(key)
181         fname = self._key_to_file(key)
182         try:
183@@ -153,7 +158,8 @@ class CacheClass(BaseCache):
184         Thus, a cache key of "foo" gets turnned into a file named
185         ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.
186         """
187-        path = md5_constructor(key.encode('utf-8')).hexdigest()
188+        key = self.make_key(key)
189+        path = md5_constructor(key).hexdigest()
190         path = os.path.join(path[:2], path[2:4], path[4:])
191         return os.path.join(self._dir, path)
192 
193diff --git a/django/core/cache/backends/locmem.py b/django/core/cache/backends/locmem.py
194index fe33d33..2f7933a 100644
195--- a/django/core/cache/backends/locmem.py
196+++ b/django/core/cache/backends/locmem.py
197@@ -10,8 +10,8 @@ from django.core.cache.backends.base import BaseCache
198 from django.utils.synch import RWLock
199 
200 class CacheClass(BaseCache):
201-    def __init__(self, _, params):
202-        BaseCache.__init__(self, params)
203+    def __init__(self, _, params, key_prefix=''):
204+        BaseCache.__init__(self, params, key_prefix)
205         self._cache = {}
206         self._expire_info = {}
207 
208@@ -30,6 +30,7 @@ class CacheClass(BaseCache):
209         self._lock = RWLock()
210 
211     def add(self, key, value, timeout=None):
212+        key = self.make_key(key)
213         self.validate_key(key)
214         self._lock.writer_enters()
215         try:
216@@ -45,6 +46,7 @@ class CacheClass(BaseCache):
217             self._lock.writer_leaves()
218 
219     def get(self, key, default=None):
220+        key = self.make_key(key)
221         self.validate_key(key)
222         self._lock.reader_enters()
223         try:
224@@ -78,6 +80,7 @@ class CacheClass(BaseCache):
225         self._expire_info[key] = time.time() + timeout
226 
227     def set(self, key, value, timeout=None):
228+        key = self.make_key(key)
229         self.validate_key(key)
230         self._lock.writer_enters()
231         # Python 2.4 doesn't allow combined try-except-finally blocks.
232@@ -90,6 +93,7 @@ class CacheClass(BaseCache):
233             self._lock.writer_leaves()
234 
235     def has_key(self, key):
236+        key = self.make_key(key)
237         self.validate_key(key)
238         self._lock.reader_enters()
239         try:
240@@ -131,6 +135,7 @@ class CacheClass(BaseCache):
241             pass
242 
243     def delete(self, key):
244+        key = self.make_key(key)
245         self.validate_key(key)
246         self._lock.writer_enters()
247         try:
248diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py
249index 7d6b5b3..e321193 100644
250--- a/django/core/cache/backends/memcached.py
251+++ b/django/core/cache/backends/memcached.py
252@@ -3,7 +3,6 @@
253 import time
254 
255 from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
256-from django.utils.encoding import smart_unicode, smart_str
257 
258 try:
259     import cmemcache as memcache
260@@ -19,8 +18,8 @@ except ImportError:
261         raise InvalidCacheBackendError("Memcached cache backend requires either the 'memcache' or 'cmemcache' library")
262 
263 class CacheClass(BaseCache):
264-    def __init__(self, server, params):
265-        BaseCache.__init__(self, params)
266+    def __init__(self, server, params, key_prefix):
267+        BaseCache.__init__(self, params, key_prefix)
268         self._cache = memcache.Client(server.split(';'))
269 
270     def _get_memcache_timeout(self, timeout):
271@@ -40,29 +39,34 @@ class CacheClass(BaseCache):
272         return timeout
273 
274     def add(self, key, value, timeout=0):
275+        key = self.make_key(key)
276         if isinstance(value, unicode):
277             value = value.encode('utf-8')
278-        return self._cache.add(smart_str(key), value, self._get_memcache_timeout(timeout))
279+        return self._cache.add(key, value, self._get_memcache_timeout(timeout))
280 
281     def get(self, key, default=None):
282-        val = self._cache.get(smart_str(key))
283+        key = self.make_key(key)
284+        val = self._cache.get(key)
285         if val is None:
286             return default
287         return val
288 
289     def set(self, key, value, timeout=0):
290-        self._cache.set(smart_str(key), value, self._get_memcache_timeout(timeout))
291+        key = self.make_key(key)
292+        self._cache.set(key, value, self._get_memcache_timeout(timeout))
293 
294     def delete(self, key):
295-        self._cache.delete(smart_str(key))
296+        key = self.make_key(key)
297+        self._cache.delete(key)
298 
299     def get_many(self, keys):
300-        return self._cache.get_multi(map(smart_str,keys))
301+        return self._cache.get_multi(map(self.make_key,keys))
302 
303     def close(self, **kwargs):
304         self._cache.disconnect_all()
305 
306     def incr(self, key, delta=1):
307+        key = self.make_key(key)
308         try:
309             val = self._cache.incr(key, delta)
310 
311@@ -77,6 +81,7 @@ class CacheClass(BaseCache):
312         return val
313 
314     def decr(self, key, delta=1):
315+        key = self.make_key(key)
316         try:
317             val = self._cache.decr(key, delta)
318 
319@@ -92,13 +97,14 @@ class CacheClass(BaseCache):
320     def set_many(self, data, timeout=0):
321         safe_data = {}
322         for key, value in data.items():
323+            key = self.make_key(key)
324             if isinstance(value, unicode):
325                 value = value.encode('utf-8')
326-            safe_data[smart_str(key)] = value
327+            safe_data[key] = value
328         self._cache.set_multi(safe_data, self._get_memcache_timeout(timeout))
329 
330     def delete_many(self, keys):
331-        self._cache.delete_multi(map(smart_str, keys))
332+        self._cache.delete_multi(map(self.make_key, keys))
333 
334     def clear(self):
335         self._cache.flush_all()
336diff --git a/docs/topics/cache.txt b/docs/topics/cache.txt
337index 5797199..7fe9fec 100644
338--- a/docs/topics/cache.txt
339+++ b/docs/topics/cache.txt
340@@ -680,6 +680,34 @@ instance, to do this for the ``locmem`` backend, put this code in a module::
341 ...and use the dotted Python path to this module as the scheme portion of your
342 :setting:`CACHE_BACKEND`.
343 
344+CACHE_KEY_PREFIX
345+----------------
346+
347+It is a common occurence to have a shared cache instance running on your development
348+or production server for use across multiple projects, i.e. all sites are pointing
349+to the memcached instance running on port 11211. As a result, cache key conflicts
350+may arise which could result in data from one site being used by another.
351+
352+To alleviate this, CACHE_KEY_PREFIX can be set. This will be prepended to all keys
353+that are used with the cache backend, transparently::
354+
355+    # in settings, CACHE_KEY_PREFIX = 'myproject_'
356+
357+    >>> cache.set('my_key', 'hello world!') # set with key 'myproject_my_key'
358+    >>> cache.get('my_key') # retrieved with key 'myproject_my_key'
359+    'hello world!'
360+
361+Of course, for sites that _do_ share content, simply set the CACHE_KEY_PREFIX for
362+both sites to the same value. The default value for CACHE_KEY_PREFIX is the empty
363+string ``''``.
364+
365+.. note::
366+   
367+   This does *not* conflict with the CACHE_MIDDLEWARE_KEY_PREFIX and can be used
368+   in conjunction with it. CACHE_KEY_PREFIX acts as a global prefix for a
369+   particular cache instance, therefore it will be prepended to the
370+   CACHE_MIDDLEWARE_KEY_PREFIX transparently.
371+
372 Upstream caches
373 ===============
374 
375diff --git a/tests/regressiontests/cache/tests.py b/tests/regressiontests/cache/tests.py
376index 1e0a404..5647acc 100644
377--- a/tests/regressiontests/cache/tests.py
378+++ b/tests/regressiontests/cache/tests.py
379@@ -144,7 +144,6 @@ class DummyCacheTests(unittest.TestCase):
380         "clear does nothing for the dummy cache backend"
381         self.cache.clear()
382 
383-
384 class BaseCacheTests(object):
385     # A common set of tests to apply to all cache backends
386     def tearDown(self):
387@@ -162,6 +161,18 @@ class BaseCacheTests(object):
388         self.assertEqual(result, False)
389         self.assertEqual(self.cache.get("addkey1"), "value")
390 
391+    def test_prefix(self):
392+        # Test for same cache key conflicts between shared backend
393+        self.cache.set('somekey', 'value')
394+
395+        # should not be set in the prefixed cache
396+        self.assertFalse(self.pfx_cache.has_key('somekey'))
397+
398+        self.pfx_cache.set('somekey', 'value2')
399+
400+        self.assertEqual(self.cache.get('somekey'), 'value')
401+        self.assertEqual(self.pfx_cache.get('somekey'), 'value2')
402+
403     def test_non_existent(self):
404         # Non-existent cache keys return as None/default
405         # get with non-existent keys
406@@ -400,6 +411,7 @@ class DBCacheTests(unittest.TestCase, BaseCacheTests):
407         self._table_name = 'test cache table'
408         management.call_command('createcachetable', self._table_name, verbosity=0, interactive=False)
409         self.cache = get_cache('db://%s?max_entries=30' % self._table_name)
410+        self.pfx_cache = get_cache('db://%s' % self._table_name, 'cacheprefix')
411 
412     def tearDown(self):
413         from django.db import connection
414@@ -412,6 +424,7 @@ class DBCacheTests(unittest.TestCase, BaseCacheTests):
415 class LocMemCacheTests(unittest.TestCase, BaseCacheTests):
416     def setUp(self):
417         self.cache = get_cache('locmem://?max_entries=30')
418+        self.pfx_cache = get_cache('locmem://', 'cacheprefix')
419 
420     def test_cull(self):
421         self.perform_cull_test(50, 29)
422@@ -424,6 +437,7 @@ if settings.CACHE_BACKEND.startswith('memcached://'):
423     class MemcachedCacheTests(unittest.TestCase, BaseCacheTests):
424         def setUp(self):
425             self.cache = get_cache(settings.CACHE_BACKEND)
426+            self.pfx_cache = get_cache(settings.CACHE_BACKEND, 'cacheprefix')
427 
428         def test_invalid_keys(self):
429             """
430@@ -448,6 +462,7 @@ class FileBasedCacheTests(unittest.TestCase, BaseCacheTests):
431     def setUp(self):
432         self.dirname = tempfile.mkdtemp()
433         self.cache = get_cache('file://%s?max_entries=30' % self.dirname)
434+        self.pfx_cache = get_cache('file://%s' % self.dirname, 'cacheprefix')
435 
436     def test_hashing(self):
437         """Test that keys are hashed into subdirectories correctly"""
438@@ -494,16 +509,16 @@ class CacheUtils(unittest.TestCase):
439 
440     def setUp(self):
441         self.path = '/cache/test/'
442-        self.old_settings_key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
443-        self.old_middleware_seconds = settings.CACHE_MIDDLEWARE_SECONDS
444+        self.old_cache_middleware_key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
445+        self.old_cache_middleware_seconds = settings.CACHE_MIDDLEWARE_SECONDS
446         self.orig_use_i18n = settings.USE_I18N
447         settings.CACHE_MIDDLEWARE_KEY_PREFIX = 'settingsprefix'
448         settings.CACHE_MIDDLEWARE_SECONDS = 1
449         settings.USE_I18N = False
450 
451     def tearDown(self):
452-        settings.CACHE_MIDDLEWARE_KEY_PREFIX = self.old_settings_key_prefix
453-        settings.CACHE_MIDDLEWARE_SECONDS = self.old_middleware_seconds
454+        settings.CACHE_MIDDLEWARE_KEY_PREFIX = self.old_cache_middleware_key_prefix
455+        settings.CACHE_MIDDLEWARE_SECONDS = self.old_cache_middleware_seconds
456         settings.USE_I18N = self.orig_use_i18n
457 
458     def _get_request(self, path):
459@@ -556,6 +571,16 @@ class CacheUtils(unittest.TestCase):
460         learn_cache_key(request, response)
461         self.assertEqual(get_cache_key(request), 'views.decorators.cache.cache_page.settingsprefix.a8c87a3d8c44853d7f79474f7ffe4ad5.d41d8cd98f00b204e9800998ecf8427e')
462 
463+class PrefixedCacheUtils(CacheUtils):
464+    def setUp(self):
465+        super(PrefixedCacheUtils, self).setUp()
466+        self.old_cache_key_prefix = settings.CACHE_KEY_PREFIX
467+        settings.CACHE_KEY_PREFIX = 'cacheprefix'
468+
469+    def tearDown(self):
470+        super(PrefixedCacheUtils, self).tearDown()
471+        settings.CACHE_KEY_PREFIX = self.old_cache_key_prefix
472+
473 class CacheI18nTest(unittest.TestCase):
474 
475     def setUp(self):
476@@ -651,5 +676,15 @@ class CacheI18nTest(unittest.TestCase):
477         get_cache_data = FetchFromCacheMiddleware().process_request(request)
478         self.assertEqual(get_cache_data.content, es_message)
479 
480+class PrefixedCacheI18nTest(CacheI18nTest):
481+    def setUp(self):
482+        super(PrefixedCacheI18nTest, self).setUp()
483+        self.old_cache_key_prefix = settings.CACHE_KEY_PREFIX
484+        settings.CACHE_KEY_PREFIX = 'cacheprefix'
485+
486+    def tearDown(self):
487+        super(PrefixedCacheI18nTest, self).tearDown()
488+        settings.CACHE_KEY_PREFIX = self.old_cache_key_prefix
489+
490 if __name__ == '__main__':
491     unittest.main()