Ticket #6099: md5_no_refcount.patch

django/core/cache/backends/filebased.py

@@ -1 +1 @@
 "File-based cache backend"
 
+import md5
 import os, time
 try:
     import cPickle as pickle
 except ImportError:
     import pickle
 from django.core.cache.backends.base import BaseCache
-from django.utils.http import urlquote_plus
 
 class CacheClass(BaseCache):
     def __init__(self, dir, params):
@@ -29 +29 @@
             self._createdir()
 
     def add(self, key, value, timeout=None):
-        fname = self._key_to_file(key)
-        if timeout is None:
-            timeout = self.default_timeout
-        try:
-            filelist = os.listdir(self._dir)
-        except (IOError, OSError):
-            self._createdir()
-            filelist = []
-        if len(filelist) > self._max_entries:
-            self._cull(filelist)
-        if os.path.basename(fname) not in filelist:
-            try:
-                f = open(fname, 'wb')
-                now = time.time()
-                pickle.dump(now + timeout, f, 2)
-                pickle.dump(value, f, 2)
-            except (IOError, OSError):
-                pass
+        if self.has_key(key):
+            return None
+        
+        self.set(key, value, timeout)
 
     def get(self, key, default=None):
         fname = self._key_to_file(key)
@@ -56 +42 @@
             now = time.time()
             if exp < now:
                 f.close()
-                os.remove(fname)
+                self._delete(fname)
             else:
                 return pickle.load(f)
         except (IOError, OSError, EOFError, pickle.PickleError):
@@ -65 +51 @@
 
     def set(self, key, value, timeout=None):
         fname = self._key_to_file(key)
+        dir = os.path.dirname(fname)
+        
         if timeout is None:
             timeout = self.default_timeout
+            
+        self._cull()
+        
         try:
-            filelist = os.listdir(self._dir)
-        except (IOError, OSError):
-            self._createdir()
-            filelist = []
-        if len(filelist) > self._max_entries:
-            self._cull(filelist)
-        try:
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
             f = open(fname, 'wb')
             now = time.time()
             pickle.dump(now + timeout, f, 2)
@@ -84 +71 @@
 
     def delete(self, key):
         try:
-            os.remove(self._key_to_file(key))
+            self._delete(self._key_to_file(key))
         except (IOError, OSError):
             pass
 
+    def _delete(self, file):
+        os.remove(file)
+        try:
+            #remove the 2 subdirs if they're empty
+            dir = os.path.dirname(file)
+            os.rmdir(dir)
+            os.rmdir(os.path.dirname(dir))
+        except:
+            pass
+
     def has_key(self, key):
-        return os.path.exists(self._key_to_file(key))
+        fname = self._key_to_file(key)
+        try:
+            f = open(fname, 'rb')
+            exp = pickle.load(f)
+            now = time.time()
+            if exp < now:
+                f.close()
+                self._delete(fname)
+                return False
+            else:
+                return True
+        except (IOError, OSError, EOFError, pickle.PickleError):
+            return False
 
-    def _cull(self, filelist):
+    def _cull(self):
+        if int(self._num_entries) < self._max_entries:
+            return
+        
+        try:
+            filelist = os.listdir(self._dir)
+        except (IOError, OSError):
+            return
+        
         if self._cull_frequency == 0:
             doomed = filelist
         else:
-            doomed = [k for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0]
-        for fname in doomed:
+            doomed = [os.path.join(self._dir, k) for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0]
+
+        for topdir in doomed:
             try:
-                os.remove(os.path.join(self._dir, fname))
+                for root, _, files in os.walk(topdir):
+                    for file in files:
+                        self._delete(os.path.join(root,file))
             except (IOError, OSError):
                 pass
 
@@ -109 +129 @@
             raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir
 
     def _key_to_file(self, key):
-        return os.path.join(self._dir, urlquote_plus(key))
+        path = md5.new(key.encode('utf-8')).hexdigest()
+        path = os.path.join(path[:2], path[2:4], path[4:])
+        return os.path.join(self._dir, path)
+
+    def _get_num_entries(self):
+        count = 0
+        for _,_,files in os.walk(self._dir):
+            count += len(files)
+        return count
+    _num_entries = property(_get_num_entries)
+

tests/regressiontests/cache/tests.py

@@ -24 +24 @@
 
     def test_add(self):
         # test add (only add if key isn't already in cache)
-        cache.add("addkey1", "value")
+        cache.set("addkey1", "value")
         cache.add("addkey1", "newvalue")
         self.assertEqual(cache.get("addkey1"), "value")
-
+        
     def test_non_existent(self):
         # get with non-existent keys
         self.assertEqual(cache.get("does_not_exist"), None)
@@ -77 +77 @@
 
     def test_expiration(self):
         # expiration
-        cache.set('expire', 'very quickly', 1)
+        cache.set('expire1', 'very quickly', 1)
+        cache.set('expire2', 'very quickly', 1)
+        cache.set('expire3', 'very quickly', 1)
         time.sleep(2)
-        self.assertEqual(cache.get("expire"), None)
+        
+        self.assertEqual(cache.get("expire1"), None)
+        
+        cache.add("expire2", "newvalue")
+        self.assertEqual(cache.get("expire2"), "newvalue")
+    
+        self.assertEqual(cache.has_key("expire3"), False)
 
     def test_unicode(self):
         stuff = {