Ticket #15252: 15252.2.diff

django/contrib/staticfiles/management/commands/collectstatic.py

@@ -1 +1 @@
 import os
 import sys
-import shutil
 from optparse import make_option
 
 from django.conf import settings
@@ -88 +87 @@
                 else:
                     self.copy_file(path, prefixed_path, storage, **options)
 
-        actual_count = len(self.copied_files) + len(self.symlinked_files)
+        modified_files = self.copied_files + self.symlinked_files
+        actual_count = len(modified_files)
+
+        # Here we check if the storage backend has a post_process method
+        # and pass it the list of modified files, if possible.
+        if hasattr(self.storage, 'post_process'):
+            self.storage.post_process(modified_files)
+
         unmodified_count = len(self.unmodified_files)
         if self.verbosity >= 1:
             self.stdout.write(smart_str(u"\n%s static file%s %s to '%s'%s.\n"
@@ -196 +202 @@
                     os.makedirs(os.path.dirname(full_path))
                 except OSError:
                     pass
-                shutil.copy2(source_path, full_path)
-            else:
-                source_file = source_storage.open(path)
-                self.storage.save(prefixed_path, source_file)
+            source_file = source_storage.open(path)
+            self.storage.save(prefixed_path, source_file)
         if not prefixed_path in self.copied_files:
             self.copied_files.append(prefixed_path)

django/contrib/staticfiles/storage.py

@@ +1 @@
+import hashlib
 import os
+import re
+
 from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
-from django.core.files.storage import FileSystemStorage
+from django.core.cache import get_cache, InvalidCacheBackendError, cache as default_cache
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
+from django.core.files.storage import FileSystemStorage, get_storage_class
+from django.core.files.base import ContentFile
+from django.template import Template, Context
 from django.utils.importlib import import_module
+from django.utils.encoding import force_unicode
+from django.utils.functional import LazyObject
+
+from django.contrib.staticfiles.utils import check_settings
+
 
-from django.contrib.staticfiles import utils
+urltag_re = re.compile(r"""
+url\(
+  (\s*)                 # allow whitespace wrapping (and capture)
+  (                     # capture actual url
+    [^\)\\\r\n]*?           # don't allow newlines, closing paran, escape chars (1)
+    (?:\\.                  # process all escapes here instead
+        [^\)\\\r\n]*?           # proceed, with previous restrictions (1)
+    )*                     # repeat until end
+  )
+  (\s*)                 # whitespace again (and capture)
+\)
+""", re.VERBOSE)
 
 
 class StaticFilesStorage(FileSystemStorage):
@@ -26 +48 @@
         if base_url is None:
             raise ImproperlyConfigured("You're using the staticfiles app "
                 "without having set the STATIC_URL setting.")
-        utils.check_settings()
+        check_settings()
         super(StaticFilesStorage, self).__init__(location, base_url, *args, **kwargs)
 
 
+class CacheBustingMixin(object):
+
+    def __init__(self, *args, **kwargs):
+        super(CacheBustingMixin, self).__init__(*args, **kwargs)
+        self.processed_files = []
+        try:
+            self.cache = get_cache('staticfiles')
+        except InvalidCacheBackendError:
+            # Use the default backend
+            self.cache = default_cache
+
+    def hashed_filename(self, name, content=None):
+        if content is None:
+            if not self.exists(name):
+                raise SuspiciousOperation("Attempted access to '%s' denied." % self.path(name))
+            content = self.open(self.path(name))
+        path, filename = os.path.split(name)
+        root, ext = os.path.splitext(filename)
+        # Get the MD5 hash of the file
+        md5 = hashlib.md5()
+        for chunk in content.chunks():
+            md5.update(chunk)
+        md5sum = md5.hexdigest()[:12]
+        return os.path.join(path, u"%s.%s%s" % (root, md5sum, ext))
+
+    def cache_key(self, name):
+        return 'staticfiles:cache:%s' % name
+
+    def url(self, name):
+        cache_key = self.cache_key(name)
+        hashed_name = self.cache.get(cache_key, self.hashed_filename(name))
+        return super(CacheBustingMixin, self).url(hashed_name)
+
+    def save(self, name, content):
+        original_name = super(CacheBustingMixin, self).save(name, content)
+        hashed_name = self.hashed_filename(original_name, content)
+        # Return the name if the file is already there
+        if os.path.exists(hashed_name):
+            return hashed_name
+        # Save the file
+        rendered_content = Template(content.read()).render(Context({}))
+        hashed_name = self._save(hashed_name, ContentFile(rendered_content))
+        # Use filenames with forward slashes, even on Windows
+        hashed_name = force_unicode(hashed_name.replace('\\', '/'))
+        self.cache.set(self.cache_key(name), hashed_name)
+        self.processed_files.append((name, hashed_name))
+        return hashed_name
+
+    def post_process(self, modified_files):
+        """
+        Post process method called by the collectstatic management command.
+        """
+        cached_files = [self.cache_key(path) for path in modified_files]
+        self.cache.delete_many(cached_files)
+
+        def path_level((name, hashed_name)):
+            return len(name.split(os.sep))
+
+        for name, hashed_name in sorted(
+                self.processed_files, key=path_level, reverse=True):
+
+            def url_converter(matchobj):
+                url = matchobj.groups()[1]
+                # normalize the url we got
+                if url[:1] in '"\'':
+                    url = url[1:]
+                if url[-1:] in '"\'':
+                    url = url[:-1]
+                rel_level = url.count(os.pardir)
+                if rel_level:
+                    url_parts = (name.split('/')[:-rel_level-1] +
+                                 url.split('/')[rel_level:])
+                    url = self.url('/'.join(url_parts))
+                return "url('%s')" % url
+
+            original = self.open(name)
+            converted = urltag_re.sub(url_converter, original.read())
+            hashed = self.path(hashed_name)
+            with open(hashed, 'w') as hashed_file:
+                hashed_file.write(converted)
+
+class CachedStaticFilesStorage(CacheBustingMixin, StaticFilesStorage):
+    pass
+
+
 class AppStaticStorage(FileSystemStorage):
     """
     A file system storage backend that takes an app module and works
@@ -47 +154 @@
         mod_path = os.path.dirname(mod.__file__)
         location = os.path.join(mod_path, self.source_dir)
         super(AppStaticStorage, self).__init__(location, *args, **kwargs)
+
+
+
+class ConfiguredStorage(LazyObject):
+    def _setup(self):
+        self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()
+
+configured_storage = ConfiguredStorage()

new file django/contrib/staticfiles/templatetags/staticfiles.py

@@ +1 @@
+from django import template
+from django.contrib.staticfiles import storage
+
+register = template.Library()
+
+
+@register.simple_tag
+def static(path):
+    """
+    A template tag that returns the URL to a file
+    using staticfiles' storage backend
+    """
+    return storage.configured_storage.url(path)

django/core/cache/__init__.py

@@ -126 +126 @@
         location = args.pop('LOCATION', '')
         return backend, location, args
     else:
-        # Trying to import the given backend, in case it's a dotted path
-        mod_path, cls_name = backend.rsplit('.', 1)
         try:
+            # Trying to import the given backend, in case it's a dotted path
+            mod_path, cls_name = backend.rsplit('.', 1)
             mod = importlib.import_module(mod_path)
             backend_cls = getattr(mod, cls_name)
-        except (AttributeError, ImportError):
+        except (AttributeError, ImportError, ValueError):
             raise InvalidCacheBackendError("Could not find backend '%s'" % backend)
         location = kwargs.pop('LOCATION', '')
         return backend, location, kwargs

new file tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt

@@ +1 @@
+Test!
+ No newline at end of file

tests/regressiontests/staticfiles_tests/tests.py

@@ -7 +7 @@
 import tempfile
 from StringIO import StringIO
 
+from django.template import loader, Context
 from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
 from django.core.files.storage import default_storage
 from django.core.management import call_command
 from django.test import TestCase
@@ -50 +51 @@
     def assertFileNotFound(self, filepath):
         self.assertRaises(IOError, self._get_file, filepath)
 
+    def assertTemplateRenders(self, template, result, **kwargs):
+        if isinstance(template, basestring):
+            template = loader.get_template_from_string(template)
+        self.assertEqual(template.render(Context(kwargs)), result)
+
+    def assertTemplateRaises(self, exc, template, result, **kwargs):
+        self.assertRaises(exc, self.assertTemplateRenders, template, result, **kwargs)
+
+
 StaticFilesTestCase = override_settings(
     DEBUG = True,
     MEDIA_URL = '/media/',
@@ -245 +255 @@
 )(TestBuildStaticNonLocalStorage)
 
 
+class TestBuildStaticCachedStorage(BuildStaticTestCase, TestDefaults):
+    """
+    Tests for the Cache busting storage
+    """
+    @classmethod
+    def tearDownClass(cls):
+        """
+        Resetting the global storage for staticfiles
+        """
+        storage.configured_storage = storage.ConfiguredStorage()
+
+    def test_template_tag(self):
+        self.assertTemplateRaises(SuspiciousOperation, """{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
+        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "test/file.txt" %}""", "/static/test/file.dad0999e4f8f.txt")
+
+
+TestBuildStaticCachedStorage = override_settings(
+    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage'
+)(TestBuildStaticCachedStorage)
+
+
 if sys.platform != 'win32':
     class TestBuildStaticLinks(BuildStaticTestCase, TestDefaults):
         """
@@ -406 +437 @@
 TestStaticfilesDirsType = override_settings(
     STATICFILES_DIRS = 'a string',
 )(TestStaticfilesDirsType)
+
+
+class TestTemplateTag(StaticFilesTestCase):
+
+    def test_template_tag(self):
+        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
+        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "testfile.txt" %}""", "/static/testfile.txt")