Ticket #515: cache.patch

django/core/cache.py

@@ -19 +19 @@
                                     the same database/username as the rest of
                                     the CMS, so only a table name is needed.)
 
-    file:///var/tmp/django.cache/      A file-based cache at /var/tmp/django.cache
+    file:///var/tmp/django.cache/   A file-based cache at /var/tmp/django.cache
 
     simple:///                      A simple single-process memory cache; you
                                     probably don't want to use this except for
@@ -181 +181 @@
 
     def get(self, key, default=None):
         now = time.time()
-        exp = self._expire_info.get(key, now)
-        if exp is not None and exp < now:
+        exp = self._expire_info.get(key)
+        if exp is None:
+                        return default
+        if exp < now:
             del self._cache[key]
             del self._expire_info[key]
             return default
         else:
-            return self._cache.get(key, default)
+            return self._cache[key]
 
     def set(self, key, value, timeout=None):
         if len(self._cache) >= self._max_entries:
@@ -218 +220 @@
             doomed = [k for (i, k) in enumerate(self._cache) if i % self._cull_frequency == 0]
             for k in doomed:
                 self.delete(k)
+                
+###############################
+# Thread-safe in-memory cache #
+###############################
 
+import cPickle as pickle
+from django.utils.synch import RWLock
+
+class _LocMemCache(_Cache):
+    """Thread-safe in-memory cache"""
+
+    def __init__(self, host, params):
+        _Cache.__init__(self, params)
+        self.rw_lock = RWLock()
+        self.cache = {}
+        self.expire_info = {}
+
+        max_entries = params.get('max_entries', 300)
+        try:
+            self.max_entries = int(max_entries)
+        except (ValueError, TypeError):
+            self.max_entries = 300
+
+        cull_frequency = params.get('cull_frequency', 3)
+        try:
+            self.cull_frequency = int(cull_frequency)
+        except (ValueError, TypeError):
+            self.cull_frequency = 3
+
+    def get(self, key, default=None):
+        delete = False
+        self.rw_lock.reader_enters()
+        try:
+            now = time.time()
+            exp = self.expire_info.get(key)
+            if exp is None:
+                return default
+            if exp < now:
+                delete = True
+            else:
+                return pickle.loads(self.cache[key])
+                #return self.cache[key]
+        finally:
+            self.rw_lock.reader_leaves()
+        if delete:
+            self.rw_lock.writer_enters()
+            try:
+                del self.cache[key]
+                del self.expire_info[key]
+                return default
+            finally:
+                self.rw_lock.writer_leaves()
+
+    def set(self, key, value, timeout=None):
+        if timeout is None:
+            timeout = self.default_timeout
+        self.rw_lock.writer_enters()
+        try:
+            if len(self.cache) >= self.max_entries:
+                self._cull()
+            self.cache[key] = pickle.dumps(value)
+            #self.cache[key] = value
+            self.expire_info[key] = time.time() + timeout
+        finally:
+            self.rw_lock.writer_leaves()
+
+    def delete(self, key):
+        self.rw_lock.writer_enters()
+        try:
+            try:
+                del self.cache[key]
+            except KeyError:
+                pass
+            try:
+                del self.expire_info[key]
+            except KeyError:
+                pass
+        finally:
+            self.rw_lock.writer_leaves()
+
+    def has_key(self, key):
+        self.rw_lock.reader_enters()
+        try:
+            return self.cache.has_key(key)
+        finally:
+            self.rw_lock.reader_leaves()
+
+        # it is called from delete, which provides thread safety
+    def _cull(self):
+        if self.cull_frequency == 0:
+            self.cache.clear()
+            self.expire_info.clear()
+        else:
+            doomed = [k for (i, k) in enumerate(self.cache) if i % self.cull_frequency == 0]
+            for k in doomed:
+                self.delete(k)
+
+####################
+# File-based cache #
+####################
+
+import os, urllib
+
+class _FileCache(_Cache):
+    """File-based cache"""
+
+    def __init__(self, dir, params):
+        _Cache.__init__(self, params)
+        self.dir = dir + '/'
+
+        max_entries = params.get('max_entries', 300)
+        try:
+            self.max_entries = int(max_entries)
+        except (ValueError, TypeError):
+            self.max_entries = 300
+
+        cull_frequency = params.get('cull_frequency', 3)
+        try:
+            self.cull_frequency = int(cull_frequency)
+        except (ValueError, TypeError):
+            self.cull_frequency = 3
+
+    def get(self, key, default=None):
+        name = self.dir + urllib.quote_plus(key)
+        try:
+            f = file(name, 'r')
+            exp = pickle.load(f)
+            now = time.time()
+            if exp < now:
+                f.close()
+                os.remove(name)
+            else:
+                return pickle.load(f)
+        except:
+            pass # silently eat any error
+        return default
+
+    def set(self, key, value, timeout=None):
+        name = self.dir + urllib.quote_plus(key)
+        if timeout is None:
+            timeout = self.default_timeout
+        try:
+            try:
+                filelist = os.listdir(self.dir)
+                if len(filelist) >= self.max_entries:
+                    self._cull(filelist)
+            except OSError:
+                try:
+                    os.makedirs(self.dir)
+                except OSError:
+                    pass
+            f = file(name, 'w')
+            now = time.time()
+            pickle.dump(now + timeout, f)
+            pickle.dump(value, f)
+            f.close()
+        except OSError:
+            pass
+
+    def delete(self, key):
+        name = self.dir + urllib.quote_plus(key)
+        try:
+            os.remove(name)
+        except:
+            pass    # silently skip
+
+    def has_key(self, key):
+        name = urllib.quote_plus(self.dir + key)
+        try:
+            os.stat(name)
+            return True
+        except OSError:
+            return False
+
+    # it is called from delete, which provides thread safety
+    def _cull(self, filelist):
+        if self.cull_frequency == 0:
+            doomed = filelist
+        else:
+            doomed = [k for (i, k) in enumerate(filelist) if i % self.cull_frequency == 0]
+        for name in doomed:
+            try:
+                os.remove(self.dir + name)
+            except:
+                pass # silently skip
+
+###################
+# SQL-based cache #
+###################
+
+from django.core import db
+from django.conf import settings
+from datetime import datetime
+
+class _SQLCache(_Cache):
+    """SQL-based cache"""
+
+    def __init__(self, table, params):
+        _Cache.__init__(self, params)
+        self.table = table
+
+        max_entries = params.get('max_entries', 300)
+        try:
+            self.max_entries = int(max_entries)
+        except (ValueError, TypeError):
+            self.max_entries = 300
+
+        cull_frequency = params.get('cull_frequency', 3)
+        try:
+            self.cull_frequency = int(cull_frequency)
+        except (ValueError, TypeError):
+            self.cull_frequency = 3
+
+    def get(self, key, default=None):
+        cursor = db.db.cursor()
+        cursor.execute("SELECT * FROM %s WHERE keyx='%s'" % (self.table, key))
+        if cursor.rowcount < 0:
+            row = cursor.fetchone()
+            if row is None:
+                return default
+        elif cursor.rowcount < 1:
+            return default
+        else:
+            row = cursor.fetchone()
+        now = datetime.now()
+        if row[2] < now:
+            cursor.execute("DELETE FROM %s WHERE keyx='%s'" % (self.table, key))
+            db.db.commit()
+            return default
+        now = datetime.fromtimestamp(time.time())
+        if settings.DATABASE_ENGINE == 'mysql':
+            now = now.replace(microsecond=0)
+        try:
+            cursor.execute("UPDATE %s SET accessed='%s' WHERE keyx='%s'" % (self.table, now, key))
+            db.db.commit()
+        except:
+            pass    # silently eat all exceptions
+        return pickle.loads(row[4])
+
+    def set(self, key, value, timeout=None):
+        if timeout is None:
+            timeout = self.default_timeout
+        cursor = db.db.cursor()
+        # it should be done with stored procedure, which will update if necessary
+        cursor.execute("SELECT COUNT(*) FROM %s" % self.table)
+        now = datetime.fromtimestamp(time.time())
+        exp = datetime.fromtimestamp(time.time() + timeout)
+        # MySQL doesn't support microseconds
+        if settings.DATABASE_ENGINE == 'mysql':
+            now = now.replace(microsecond=0)
+            exp = exp.replace(microsecond=0)
+        num = cursor.fetchone()[0]
+        if num >= self.max_entries:
+            self._cull(cursor, now)
+        try:
+            cursor.execute("INSERT INTO %s (keyx,expiration,accessed,valx) VALUES('%s','%s','%s',%%s)" % (self.table, key, exp, now), [pickle.dumps(value)])
+            db.db.commit()
+        except:
+            pass    # silently eat all exceptions
+        
+    def delete(self, key):
+        cursor = db.db.cursor()
+        try:
+            cursor.execute("DELETE FROM %s WHERE keyx='%s'" % (self.table, key))
+            db.db.commit()
+        except:
+            pass # silently eat all exceptions
+
+    def has_key(self, key):
+        cursor = db.db.cursor()
+        cursor.execute("SELECT id FROM %s WHERE keyx='%s'" % (self.table, key))
+        if cursor.rowcount < 0:
+            return cursor.fetchone() is not None
+        else:
+            return cursor.rowcount < 1
+
+    # it is called from delete, which provides thread safety
+    def _cull(self, cursor, now):
+        if self.cull_frequency == 0:
+            cursor.execute("DELETE FROM %s" % self.table)
+            db.db.commit()
+            return
+        cursor.execute("DELETE FROM %s WHERE expiration < '%s'" % (self.table, now))
+        db.db.commit()
+        cursor.execute("SELECT COUNT(*) FROM %s" % self.table)
+        num = cursor.fetchone()[0]
+        if num < self.max_entries:
+            return
+        cursor.execute("SELECT accessed FROM %s LIMIT 1 OFFSET %d" % (self.table, num / self.cull_frequency))
+        if cursor.rowcount < 0:
+            row = cursor.fetchone()
+            if row is None:
+                return
+        elif cursor.rowcount < 1:
+            return
+        else:
+            row = cursor.fetchone()
+        cursor.execute("DELETE FROM %s WHERE accessed < '%s'" % (self.table, row[0]))
+        db.db.commit()
+
 ##########################################
 # Read settings and load a cache backend #
 ##########################################
@@ -228 +529 @@
 _BACKENDS = {
     'memcached' : _MemcachedCache,
     'simple'    : _SimpleCache,
+    'locmem'    : _LocMemCache,
+    'file'      : _FileCache,
+    'sql'       : _SQLCache,
 }
 
 def get_cache(backend_uri):