Code

Ticket #13795: cache_key_prefix_13891.diff

File cache_key_prefix_13891.diff, 18.1 KB (added by agabel, 4 years ago)
Line 
1Index: django/conf/global_settings.py
2===================================================================
3--- django/conf/global_settings.py      (revision 13981)
4+++ django/conf/global_settings.py      (working copy)
5@@ -428,6 +428,7 @@
6 # The cache backend to use.  See the docstring in django.core.cache for the
7 # possible values.
8 CACHE_BACKEND = 'locmem://'
9+CACHE_KEY_PREFIX = ''
10 CACHE_MIDDLEWARE_KEY_PREFIX = ''
11 CACHE_MIDDLEWARE_SECONDS = 600
12 
13Index: django/core/cache/__init__.py
14===================================================================
15--- django/core/cache/__init__.py       (revision 13981)
16+++ django/core/cache/__init__.py       (working copy)
17@@ -56,18 +56,20 @@
18 
19     return scheme, host, params
20 
21-def get_cache(backend_uri):
22+def get_cache(backend_uri, key_prefix=''):
23+    if not key_prefix:
24+        key_prefix = settings.CACHE_KEY_PREFIX
25     scheme, host, params = parse_backend_uri(backend_uri)
26     if scheme in BACKENDS:
27         name = 'django.core.cache.backends.%s' % BACKENDS[scheme]
28     else:
29         name = scheme
30     module = importlib.import_module(name)
31-    return module.CacheClass(host, params)
32+    return module.CacheClass(host, params, key_prefix)
33 
34 cache = get_cache(settings.CACHE_BACKEND)
35 
36-# Some caches -- pythont-memcached in particular -- need to do a cleanup at the
37+# Some caches -- python-memcached in particular -- need to do a cleanup at the
38 # end of a request cycle. If the cache provides a close() method, wire it up
39 # here.
40 if hasattr(cache, 'close'):
41Index: django/core/cache/backends/base.py
42===================================================================
43--- django/core/cache/backends/base.py  (revision 13981)
44+++ django/core/cache/backends/base.py  (working copy)
45@@ -3,6 +3,7 @@
46 import warnings
47 
48 from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
49+from django.utils.encoding import smart_str
50 
51 class InvalidCacheBackendError(ImproperlyConfigured):
52     pass
53@@ -14,14 +15,22 @@
54 MEMCACHE_MAX_KEY_LENGTH = 250
55 
56 class BaseCache(object):
57-    def __init__(self, params):
58+    def __init__(self, params, key_prefix=''):
59         timeout = params.get('timeout', 300)
60         try:
61             timeout = int(timeout)
62         except (ValueError, TypeError):
63             timeout = 300
64         self.default_timeout = timeout
65+        self.key_prefix = smart_str(key_prefix)
66 
67+    def make_key(self, key):
68+        """Constructs the key used by all other methods. By default it prepends
69+        the `key_prefix'. In cache backend subclasses this can be overriden to
70+        provide custom key making behavior.
71+        """
72+        return self.key_prefix + smart_str(key)
73+
74     def add(self, key, value, timeout=None):
75         """
76         Set a value in the cache if the key does not already exist. If
77Index: django/core/cache/backends/db.py
78===================================================================
79--- django/core/cache/backends/db.py    (revision 13981)
80+++ django/core/cache/backends/db.py    (working copy)
81@@ -26,8 +26,8 @@
82         self.proxy = False
83 
84 class CacheClass(BaseCache):
85-    def __init__(self, table, params):
86-        BaseCache.__init__(self, params)
87+    def __init__(self, table, params, key_prefix=''):
88+        BaseCache.__init__(self, params, key_prefix)
89         self._table = table
90 
91         class CacheEntry(object):
92@@ -46,6 +46,7 @@
93             self._cull_frequency = 3
94 
95     def get(self, key, default=None):
96+        key = self.make_key(key)
97         self.validate_key(key)
98         db = router.db_for_read(self.cache_model_class)
99         table = connections[db].ops.quote_name(self._table)
100@@ -66,10 +67,12 @@
101         return pickle.loads(base64.decodestring(value))
102 
103     def set(self, key, value, timeout=None):
104+        key = self.make_key(key)
105         self.validate_key(key)
106         self._base_set('set', key, value, timeout)
107 
108     def add(self, key, value, timeout=None):
109+        key = self.make_key(key)
110         self.validate_key(key)
111         return self._base_set('add', key, value, timeout)
112 
113@@ -106,6 +109,7 @@
114             return True
115 
116     def delete(self, key):
117+        key = self.make_key(key)
118         self.validate_key(key)
119         db = router.db_for_write(self.cache_model_class)
120         table = connections[db].ops.quote_name(self._table)
121@@ -115,6 +119,7 @@
122         transaction.commit_unless_managed(using=db)
123 
124     def has_key(self, key):
125+        key = self.make_key(key)
126         self.validate_key(key)
127         db = router.db_for_read(self.cache_model_class)
128         table = connections[db].ops.quote_name(self._table)
129Index: django/core/cache/backends/filebased.py
130===================================================================
131--- django/core/cache/backends/filebased.py     (revision 13981)
132+++ django/core/cache/backends/filebased.py     (working copy)
133@@ -12,8 +12,8 @@
134 from django.utils.hashcompat import md5_constructor
135 
136 class CacheClass(BaseCache):
137-    def __init__(self, dir, params):
138-        BaseCache.__init__(self, params)
139+    def __init__(self, dir, params, key_prefix=''):
140+        BaseCache.__init__(self, params, key_prefix)
141 
142         max_entries = params.get('max_entries', 300)
143         try:
144@@ -32,7 +32,8 @@
145             self._createdir()
146 
147     def add(self, key, value, timeout=None):
148-        self.validate_key(key)
149+        pfx_key = self.make_key(key)
150+        self.validate_key(pfx_key)
151         if self.has_key(key):
152             return False
153 
154@@ -40,6 +41,7 @@
155         return True
156 
157     def get(self, key, default=None):
158+        key = self.make_key(key)
159         self.validate_key(key)
160         fname = self._key_to_file(key)
161         try:
162@@ -58,6 +60,7 @@
163         return default
164 
165     def set(self, key, value, timeout=None):
166+        key = self.make_key(key)
167         self.validate_key(key)
168         fname = self._key_to_file(key)
169         dirname = os.path.dirname(fname)
170@@ -82,6 +85,7 @@
171             pass
172 
173     def delete(self, key):
174+        key = self.make_key(key)
175         self.validate_key(key)
176         try:
177             self._delete(self._key_to_file(key))
178@@ -99,6 +103,7 @@
179             pass
180 
181     def has_key(self, key):
182+        key = self.make_key(key)
183         self.validate_key(key)
184         fname = self._key_to_file(key)
185         try:
186@@ -153,7 +158,7 @@
187         Thus, a cache key of "foo" gets turnned into a file named
188         ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.
189         """
190-        path = md5_constructor(key.encode('utf-8')).hexdigest()
191+        path = md5_constructor(key).hexdigest()
192         path = os.path.join(path[:2], path[2:4], path[4:])
193         return os.path.join(self._dir, path)
194 
195Index: django/core/cache/backends/locmem.py
196===================================================================
197--- django/core/cache/backends/locmem.py        (revision 13981)
198+++ django/core/cache/backends/locmem.py        (working copy)
199@@ -10,8 +10,8 @@
200 from django.utils.synch import RWLock
201 
202 class CacheClass(BaseCache):
203-    def __init__(self, _, params):
204-        BaseCache.__init__(self, params)
205+    def __init__(self, _, params, key_prefix=''):
206+        BaseCache.__init__(self, params, key_prefix)
207         self._cache = {}
208         self._expire_info = {}
209 
210@@ -30,6 +30,7 @@
211         self._lock = RWLock()
212 
213     def add(self, key, value, timeout=None):
214+        key = self.make_key(key)
215         self.validate_key(key)
216         self._lock.writer_enters()
217         try:
218@@ -45,6 +46,7 @@
219             self._lock.writer_leaves()
220 
221     def get(self, key, default=None):
222+        key = self.make_key(key)
223         self.validate_key(key)
224         self._lock.reader_enters()
225         try:
226@@ -78,6 +80,7 @@
227         self._expire_info[key] = time.time() + timeout
228 
229     def set(self, key, value, timeout=None):
230+        key = self.make_key(key)
231         self.validate_key(key)
232         self._lock.writer_enters()
233         # Python 2.4 doesn't allow combined try-except-finally blocks.
234@@ -90,6 +93,7 @@
235             self._lock.writer_leaves()
236 
237     def has_key(self, key):
238+        key = self.make_key(key)
239         self.validate_key(key)
240         self._lock.reader_enters()
241         try:
242@@ -131,6 +135,7 @@
243             pass
244 
245     def delete(self, key):
246+        key = self.make_key(key)
247         self.validate_key(key)
248         self._lock.writer_enters()
249         try:
250Index: django/core/cache/backends/memcached.py
251===================================================================
252--- django/core/cache/backends/memcached.py     (revision 13981)
253+++ django/core/cache/backends/memcached.py     (working copy)
254@@ -3,7 +3,6 @@
255 import time
256 
257 from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
258-from django.utils.encoding import smart_unicode, smart_str
259 
260 try:
261     import cmemcache as memcache
262@@ -19,8 +18,8 @@
263         raise InvalidCacheBackendError("Memcached cache backend requires either the 'memcache' or 'cmemcache' library")
264 
265 class CacheClass(BaseCache):
266-    def __init__(self, server, params):
267-        BaseCache.__init__(self, params)
268+    def __init__(self, server, params, key_prefix):
269+        BaseCache.__init__(self, params, key_prefix)
270         self._cache = memcache.Client(server.split(';'))
271 
272     def _get_memcache_timeout(self, timeout):
273@@ -40,29 +39,34 @@
274         return timeout
275 
276     def add(self, key, value, timeout=0):
277+        key = self.make_key(key)
278         if isinstance(value, unicode):
279             value = value.encode('utf-8')
280-        return self._cache.add(smart_str(key), value, self._get_memcache_timeout(timeout))
281+        return self._cache.add(key, value, self._get_memcache_timeout(timeout))
282 
283     def get(self, key, default=None):
284-        val = self._cache.get(smart_str(key))
285+        key = self.make_key(key)
286+        val = self._cache.get(key)
287         if val is None:
288             return default
289         return val
290 
291     def set(self, key, value, timeout=0):
292-        self._cache.set(smart_str(key), value, self._get_memcache_timeout(timeout))
293+        key = self.make_key(key)
294+        self._cache.set(key, value, self._get_memcache_timeout(timeout))
295 
296     def delete(self, key):
297-        self._cache.delete(smart_str(key))
298+        key = self.make_key(key)
299+        self._cache.delete(key)
300 
301     def get_many(self, keys):
302-        return self._cache.get_multi(map(smart_str,keys))
303+        return self._cache.get_multi(map(self.make_key,keys))
304 
305     def close(self, **kwargs):
306         self._cache.disconnect_all()
307 
308     def incr(self, key, delta=1):
309+        key = self.make_key(key)
310         try:
311             val = self._cache.incr(key, delta)
312 
313@@ -77,6 +81,7 @@
314         return val
315 
316     def decr(self, key, delta=1):
317+        key = self.make_key(key)
318         try:
319             val = self._cache.decr(key, delta)
320 
321@@ -92,13 +97,14 @@
322     def set_many(self, data, timeout=0):
323         safe_data = {}
324         for key, value in data.items():
325+            key = self.make_key(key)
326             if isinstance(value, unicode):
327                 value = value.encode('utf-8')
328-            safe_data[smart_str(key)] = value
329+            safe_data[key] = value
330         self._cache.set_multi(safe_data, self._get_memcache_timeout(timeout))
331 
332     def delete_many(self, keys):
333-        self._cache.delete_multi(map(smart_str, keys))
334+        self._cache.delete_multi(map(self.make_key, keys))
335 
336     def clear(self):
337         self._cache.flush_all()
338Index: docs/topics/cache.txt
339===================================================================
340--- docs/topics/cache.txt       (revision 13981)
341+++ docs/topics/cache.txt       (working copy)
342@@ -680,6 +680,34 @@
343 ...and use the dotted Python path to this module as the scheme portion of your
344 :setting:`CACHE_BACKEND`.
345 
346+CACHE_KEY_PREFIX
347+----------------
348+
349+It is a common occurence to have a shared cache instance running on your development
350+or production server for use across multiple projects, i.e. all sites are pointing
351+to the memcached instance running on port 11211. As a result, cache key conflicts
352+may arise which could result in data from one site being used by another.
353+
354+To alleviate this, CACHE_KEY_PREFIX can be set. This will be prepended to all keys
355+that are used with the cache backend, transparently::
356+
357+    # in settings, CACHE_KEY_PREFIX = 'myproject_'
358+
359+    >>> cache.set('my_key', 'hello world!') # set with key 'myproject_my_key'
360+    >>> cache.get('my_key') # retrieved with key 'myproject_my_key'
361+    'hello world!'
362+
363+Of course, for sites that _do_ share content, simply set the CACHE_KEY_PREFIX for
364+both sites to the same value. The default value for CACHE_KEY_PREFIX is the empty
365+string ``''``.
366+
367+.. note::
368+   
369+   This does *not* conflict with the CACHE_MIDDLEWARE_KEY_PREFIX and can be used
370+   in conjunction with it. CACHE_KEY_PREFIX acts as a global prefix for a
371+   particular cache instance, therefore it will be prepended to the
372+   CACHE_MIDDLEWARE_KEY_PREFIX transparently.
373+
374 Upstream caches
375 ===============
376 
377Index: tests/regressiontests/cache/tests.py
378===================================================================
379--- tests/regressiontests/cache/tests.py        (revision 13981)
380+++ tests/regressiontests/cache/tests.py        (working copy)
381@@ -144,7 +144,6 @@
382         "clear does nothing for the dummy cache backend"
383         self.cache.clear()
384 
385-
386 class BaseCacheTests(object):
387     # A common set of tests to apply to all cache backends
388     def tearDown(self):
389@@ -162,6 +161,18 @@
390         self.assertEqual(result, False)
391         self.assertEqual(self.cache.get("addkey1"), "value")
392 
393+    def test_prefix(self):
394+        # Test for same cache key conflicts between shared backend
395+        self.cache.set('somekey', 'value')
396+
397+        # should not be set in the prefixed cache
398+        self.assertFalse(self.pfx_cache.has_key('somekey'))
399+
400+        self.pfx_cache.set('somekey', 'value2')
401+
402+        self.assertEqual(self.cache.get('somekey'), 'value')
403+        self.assertEqual(self.pfx_cache.get('somekey'), 'value2')
404+
405     def test_non_existent(self):
406         # Non-existent cache keys return as None/default
407         # get with non-existent keys
408@@ -400,6 +411,7 @@
409         self._table_name = 'test cache table'
410         management.call_command('createcachetable', self._table_name, verbosity=0, interactive=False)
411         self.cache = get_cache('db://%s?max_entries=30' % self._table_name)
412+        self.pfx_cache = get_cache('db://%s' % self._table_name, 'cacheprefix')
413 
414     def tearDown(self):
415         from django.db import connection
416@@ -412,6 +424,7 @@
417 class LocMemCacheTests(unittest.TestCase, BaseCacheTests):
418     def setUp(self):
419         self.cache = get_cache('locmem://?max_entries=30')
420+        self.pfx_cache = get_cache('locmem://', 'cacheprefix')
421 
422     def test_cull(self):
423         self.perform_cull_test(50, 29)
424@@ -424,6 +437,7 @@
425     class MemcachedCacheTests(unittest.TestCase, BaseCacheTests):
426         def setUp(self):
427             self.cache = get_cache(settings.CACHE_BACKEND)
428+            self.pfx_cache = get_cache(settings.CACHE_BACKEND, 'cacheprefix')
429 
430         def test_invalid_keys(self):
431             """
432@@ -448,11 +462,13 @@
433     def setUp(self):
434         self.dirname = tempfile.mkdtemp()
435         self.cache = get_cache('file://%s?max_entries=30' % self.dirname)
436+        self.pfx_cache = get_cache('file://%s' % self.dirname, 'cacheprefix')
437 
438     def test_hashing(self):
439         """Test that keys are hashed into subdirectories correctly"""
440         self.cache.set("foo", "bar")
441-        keyhash = md5_constructor("foo").hexdigest()
442+        key = self.cache.make_key("foo")
443+        keyhash = md5_constructor(key).hexdigest()
444         keypath = os.path.join(self.dirname, keyhash[:2], keyhash[2:4], keyhash[4:])
445         self.assert_(os.path.exists(keypath))
446 
447@@ -461,7 +477,8 @@
448         Make sure that the created subdirectories are correctly removed when empty.
449         """
450         self.cache.set("foo", "bar")
451-        keyhash = md5_constructor("foo").hexdigest()
452+        key = self.cache.make_key("foo")
453+        keyhash = md5_constructor(key).hexdigest()
454         keypath = os.path.join(self.dirname, keyhash[:2], keyhash[2:4], keyhash[4:])
455         self.assert_(os.path.exists(keypath))
456 
457@@ -494,16 +511,16 @@
458 
459     def setUp(self):
460         self.path = '/cache/test/'
461-        self.old_settings_key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
462-        self.old_middleware_seconds = settings.CACHE_MIDDLEWARE_SECONDS
463+        self.old_cache_middleware_key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
464+        self.old_cache_middleware_seconds = settings.CACHE_MIDDLEWARE_SECONDS
465         self.orig_use_i18n = settings.USE_I18N
466         settings.CACHE_MIDDLEWARE_KEY_PREFIX = 'settingsprefix'
467         settings.CACHE_MIDDLEWARE_SECONDS = 1
468         settings.USE_I18N = False
469 
470     def tearDown(self):
471-        settings.CACHE_MIDDLEWARE_KEY_PREFIX = self.old_settings_key_prefix
472-        settings.CACHE_MIDDLEWARE_SECONDS = self.old_middleware_seconds
473+        settings.CACHE_MIDDLEWARE_KEY_PREFIX = self.old_cache_middleware_key_prefix
474+        settings.CACHE_MIDDLEWARE_SECONDS = self.old_cache_middleware_seconds
475         settings.USE_I18N = self.orig_use_i18n
476 
477     def _get_request(self, path):
478@@ -556,6 +573,16 @@
479         learn_cache_key(request, response)
480         self.assertEqual(get_cache_key(request), 'views.decorators.cache.cache_page.settingsprefix.a8c87a3d8c44853d7f79474f7ffe4ad5.d41d8cd98f00b204e9800998ecf8427e')
481 
482+class PrefixedCacheUtils(CacheUtils):
483+    def setUp(self):
484+        super(PrefixedCacheUtils, self).setUp()
485+        self.old_cache_key_prefix = settings.CACHE_KEY_PREFIX
486+        settings.CACHE_KEY_PREFIX = 'cacheprefix'
487+
488+    def tearDown(self):
489+        super(PrefixedCacheUtils, self).tearDown()
490+        settings.CACHE_KEY_PREFIX = self.old_cache_key_prefix
491+
492 class CacheI18nTest(unittest.TestCase):
493 
494     def setUp(self):
495@@ -651,5 +678,15 @@
496         get_cache_data = FetchFromCacheMiddleware().process_request(request)
497         self.assertEqual(get_cache_data.content, es_message)
498 
499+class PrefixedCacheI18nTest(CacheI18nTest):
500+    def setUp(self):
501+        super(PrefixedCacheI18nTest, self).setUp()
502+        self.old_cache_key_prefix = settings.CACHE_KEY_PREFIX
503+        settings.CACHE_KEY_PREFIX = 'cacheprefix'
504+
505+    def tearDown(self):
506+        super(PrefixedCacheI18nTest, self).tearDown()
507+        settings.CACHE_KEY_PREFIX = self.old_cache_key_prefix
508+
509 if __name__ == '__main__':
510     unittest.main()