diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index 72cb877..1464901 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -1,12 +1,32 @@ "File-based cache backend" +import md5 import os, time try: import cPickle as pickle except ImportError: import pickle from django.core.cache.backends.base import BaseCache -from django.utils.http import urlquote_plus + +class EntryCount: + #TODO: I assume it would be a worthwhile performance improvement (on large caches) + #to keep a count in a file, and use that here instead of recounting everytime. + # + #The hooks are all in place below to allow that just implement + #increment/decrement/__int__ here, as you see fit. + def __init__(self, dir): + self._dir = dir + + def increment(self): + pass + def decrement(self): + pass + def __int__(self): + count = 0 + for _,_,files in os.walk(self._dir): + count += len(files) + return count + class CacheClass(BaseCache): def __init__(self, dir, params): @@ -27,26 +47,14 @@ class CacheClass(BaseCache): self._dir = dir if not os.path.exists(self._dir): self._createdir() + + self._num_entries = EntryCount(self._dir) def add(self, key, value, timeout=None): - fname = self._key_to_file(key) - if timeout is None: - timeout = self.default_timeout - try: - filelist = os.listdir(self._dir) - except (IOError, OSError): - self._createdir() - filelist = [] - if len(filelist) > self._max_entries: - self._cull(filelist) - if os.path.basename(fname) not in filelist: - try: - f = open(fname, 'wb') - now = time.time() - pickle.dump(now + timeout, f, 2) - pickle.dump(value, f, 2) - except (IOError, OSError): - pass + if self.has_key(key): + return None + + self.set(key, value, timeout) def get(self, key, default=None): fname = self._key_to_file(key) @@ -56,7 +64,7 @@ class CacheClass(BaseCache): now = time.time() if exp < now: f.close() - os.remove(fname) + self._delete(fname) else: return pickle.load(f) except (IOError, OSError, EOFError, pickle.PickleError): @@ -65,43 +73,79 @@ class CacheClass(BaseCache): def set(self, key, value, timeout=None): fname = self._key_to_file(key) + dir = os.path.dirname(fname) + if timeout is None: timeout = self.default_timeout + + self._cull() + try: - filelist = os.listdir(self._dir) - except (IOError, OSError): - self._createdir() - filelist = [] - if len(filelist) > self._max_entries: - self._cull(filelist) - try: + if not os.path.exists(dir): + os.makedirs(dir) + f = open(fname, 'wb') now = time.time() pickle.dump(now + timeout, f, 2) pickle.dump(value, f, 2) + self._num_entries.increment() except (IOError, OSError): pass def delete(self, key): try: - os.remove(self._key_to_file(key)) + self._delete(self._key_to_file(key)) except (IOError, OSError): pass + def _delete(self, file): + os.remove(file) + self._num_entries.decrement() + try: + #remove the 2 subdirs if they're empty + dir = os.path.dirname(file) + os.rmdir(dir) + os.rmdir(os.path.dirname(dir)) + except: + pass + def has_key(self, key): - return os.path.exists(self._key_to_file(key)) + fname = self._key_to_file(key) + try: + f = open(fname, 'rb') + exp = pickle.load(f) + now = time.time() + if exp < now: + f.close() + self._delete(fname) + return False + else: + return True + except (IOError, OSError, EOFError, pickle.PickleError): + return False - def _cull(self, filelist): + def _cull(self): + if int(self._num_entries) < self._max_entries: + return + + try: + filelist = os.listdir(self._dir) + except (IOError, OSError): + return + if self._cull_frequency == 0: doomed = filelist else: - doomed = [k for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0] - for fname in doomed: + doomed = [os.path.join(self._dir, k) for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0] + + for topdir in doomed: try: - os.remove(os.path.join(self._dir, fname)) + for root, _, files in os.walk(topdir): + for file in files: + self._delete(os.path.join(root,file)) except (IOError, OSError): pass - + def _createdir(self): try: os.makedirs(self._dir) @@ -109,4 +153,6 @@ class CacheClass(BaseCache): raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir def _key_to_file(self, key): - return os.path.join(self._dir, urlquote_plus(key)) + path = md5.new(key.encode('utf-8')).hexdigest() + path = os.path.join(path[:2], path[2:4], path[4:]) + return os.path.join(self._dir, path) diff --git a/tests/regressiontests/cache/tests.py b/tests/regressiontests/cache/tests.py index 9ac2722..c6b8742 100644 --- a/tests/regressiontests/cache/tests.py +++ b/tests/regressiontests/cache/tests.py @@ -24,10 +24,10 @@ class Cache(unittest.TestCase): def test_add(self): # test add (only add if key isn't already in cache) - cache.add("addkey1", "value") + cache.set("addkey1", "value") cache.add("addkey1", "newvalue") self.assertEqual(cache.get("addkey1"), "value") - + def test_non_existent(self): # get with non-existent keys self.assertEqual(cache.get("does_not_exist"), None) @@ -77,9 +77,17 @@ class Cache(unittest.TestCase): def test_expiration(self): # expiration - cache.set('expire', 'very quickly', 1) + cache.set('expire1', 'very quickly', 1) + cache.set('expire2', 'very quickly', 1) + cache.set('expire3', 'very quickly', 1) time.sleep(2) - self.assertEqual(cache.get("expire"), None) + + self.assertEqual(cache.get("expire1"), None) + + cache.add("expire2", "newvalue") + self.assertEqual(cache.get("expire2"), "newvalue") + + self.assertEqual(cache.has_key("expire3"), False) def test_unicode(self): stuff = {