Code

Ticket #15252: 15252.2.2.diff

File 15252.2.2.diff, 11.8 KB (added by jezdez, 3 years ago)

Adds working hashed file storage backend

Line 
1diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py
2index 6038cbc..5c9fd4b 100644
3--- a/django/contrib/staticfiles/management/commands/collectstatic.py
4+++ b/django/contrib/staticfiles/management/commands/collectstatic.py
5@@ -110,7 +110,14 @@ Type 'yes' to continue, or 'no' to cancel: """
6                     path = os.path.join(storage.prefix, path)
7                 handler(path, path, storage)
8 
9-        actual_count = len(self.copied_files) + len(self.symlinked_files)
10+        modified_files = self.copied_files + self.symlinked_files
11+        actual_count = len(modified_files)
12+
13+        # Here we check if the storage backend has a post_process method
14+        # and pass it the list of modified files, if possible.
15+        if hasattr(self.storage, 'post_process'):
16+            self.storage.post_process(modified_files)
17+
18         unmodified_count = len(self.unmodified_files)
19         if self.verbosity >= 1:
20             self.stdout.write(smart_str(u"\n%s static file%s %s %s%s.\n"
21diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py
22index a69ae23..1942385 100644
23--- a/django/contrib/staticfiles/storage.py
24+++ b/django/contrib/staticfiles/storage.py
25@@ -1,10 +1,32 @@
26+import hashlib
27 import os
28+import re
29+
30 from django.conf import settings
31-from django.core.exceptions import ImproperlyConfigured
32-from django.core.files.storage import FileSystemStorage
33+from django.core.cache import get_cache, InvalidCacheBackendError, cache as default_cache
34+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
35+from django.core.files.storage import FileSystemStorage, get_storage_class
36+from django.core.files.base import ContentFile
37+from django.template import Template, Context
38 from django.utils.importlib import import_module
39+from django.utils.encoding import force_unicode
40+from django.utils.functional import LazyObject
41+
42+from django.contrib.staticfiles.utils import check_settings
43+
44 
45-from django.contrib.staticfiles import utils
46+urltag_re = re.compile(r"""
47+url\(
48+  (\s*)                 # allow whitespace wrapping (and capture)
49+  (                     # capture actual url
50+    [^\)\\\r\n]*?           # don't allow newlines, closing paran, escape chars (1)
51+    (?:\\.                  # process all escapes here instead
52+        [^\)\\\r\n]*?           # proceed, with previous restrictions (1)
53+    )*                     # repeat until end
54+  )
55+  (\s*)                 # whitespace again (and capture)
56+\)
57+""", re.VERBOSE)
58 
59 
60 class StaticFilesStorage(FileSystemStorage):
61@@ -26,10 +48,95 @@ class StaticFilesStorage(FileSystemStorage):
62         if base_url is None:
63             raise ImproperlyConfigured("You're using the staticfiles app "
64                 "without having set the STATIC_URL setting.")
65-        utils.check_settings()
66+        check_settings()
67         super(StaticFilesStorage, self).__init__(location, base_url, *args, **kwargs)
68 
69 
70+class CacheBustingMixin(object):
71+
72+    def __init__(self, *args, **kwargs):
73+        super(CacheBustingMixin, self).__init__(*args, **kwargs)
74+        self.processed_files = []
75+        try:
76+            self.cache = get_cache('staticfiles')
77+        except InvalidCacheBackendError:
78+            # Use the default backend
79+            self.cache = default_cache
80+
81+    def hashed_filename(self, name, content=None):
82+        if content is None:
83+            if not self.exists(name):
84+                raise SuspiciousOperation("Attempted access to '%s' denied." % self.path(name))
85+            content = self.open(self.path(name))
86+        path, filename = os.path.split(name)
87+        root, ext = os.path.splitext(filename)
88+        # Get the MD5 hash of the file
89+        md5 = hashlib.md5()
90+        for chunk in content.chunks():
91+            md5.update(chunk)
92+        md5sum = md5.hexdigest()[:12]
93+        return os.path.join(path, u"%s.%s%s" % (root, md5sum, ext))
94+
95+    def cache_key(self, name):
96+        return 'staticfiles:cache:%s' % name
97+
98+    def url(self, name):
99+        cache_key = self.cache_key(name)
100+        hashed_name = self.cache.get(cache_key, self.hashed_filename(name))
101+        return super(CacheBustingMixin, self).url(hashed_name)
102+
103+    def save(self, name, content):
104+        original_name = super(CacheBustingMixin, self).save(name, content)
105+        hashed_name = self.hashed_filename(original_name, content)
106+        # Return the name if the file is already there
107+        if os.path.exists(hashed_name):
108+            return hashed_name
109+        # Save the file
110+        rendered_content = Template(content.read()).render(Context({}))
111+        hashed_name = self._save(hashed_name, ContentFile(rendered_content))
112+        # Use filenames with forward slashes, even on Windows
113+        hashed_name = force_unicode(hashed_name.replace('\\', '/'))
114+        self.cache.set(self.cache_key(name), hashed_name)
115+        self.processed_files.append((name, hashed_name))
116+        return hashed_name
117+
118+    def post_process(self, modified_files):
119+        """
120+        Post process method called by the collectstatic management command.
121+        """
122+        cached_files = [self.cache_key(path) for path in modified_files]
123+        self.cache.delete_many(cached_files)
124+
125+        def path_level((name, hashed_name)):
126+            return len(name.split(os.sep))
127+
128+        for name, hashed_name in sorted(
129+                self.processed_files, key=path_level, reverse=True):
130+
131+            def url_converter(matchobj):
132+                url = matchobj.groups()[1]
133+                # normalize the url we got
134+                if url[:1] in '"\'':
135+                    url = url[1:]
136+                if url[-1:] in '"\'':
137+                    url = url[:-1]
138+                rel_level = url.count(os.pardir)
139+                if rel_level:
140+                    url_parts = (name.split('/')[:-rel_level-1] +
141+                                 url.split('/')[rel_level:])
142+                    url = self.url('/'.join(url_parts))
143+                return "url('%s')" % url
144+
145+            original = self.open(name)
146+            converted = urltag_re.sub(url_converter, original.read())
147+            hashed = self.path(hashed_name)
148+            with open(hashed, 'w') as hashed_file:
149+                hashed_file.write(converted)
150+
151+class CachedStaticFilesStorage(CacheBustingMixin, StaticFilesStorage):
152+    pass
153+
154+
155 class AppStaticStorage(FileSystemStorage):
156     """
157     A file system storage backend that takes an app module and works
158@@ -47,3 +154,11 @@ class AppStaticStorage(FileSystemStorage):
159         mod_path = os.path.dirname(mod.__file__)
160         location = os.path.join(mod_path, self.source_dir)
161         super(AppStaticStorage, self).__init__(location, *args, **kwargs)
162+
163+
164+
165+class ConfiguredStorage(LazyObject):
166+    def _setup(self):
167+        self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()
168+
169+configured_storage = ConfiguredStorage()
170diff --git a/django/contrib/staticfiles/templatetags/__init__.py b/django/contrib/staticfiles/templatetags/__init__.py
171new file mode 100644
172index 0000000..e69de29
173diff --git a/django/contrib/staticfiles/templatetags/staticfiles.py b/django/contrib/staticfiles/templatetags/staticfiles.py
174new file mode 100644
175index 0000000..42b4f6b
176--- /dev/null
177+++ b/django/contrib/staticfiles/templatetags/staticfiles.py
178@@ -0,0 +1,13 @@
179+from django import template
180+from django.contrib.staticfiles import storage
181+
182+register = template.Library()
183+
184+
185+@register.simple_tag
186+def static(path):
187+    """
188+    A template tag that returns the URL to a file
189+    using staticfiles' storage backend
190+    """
191+    return storage.configured_storage.url(path)
192diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py
193index b76ddcf..b97c746 100644
194--- a/django/core/cache/__init__.py
195+++ b/django/core/cache/__init__.py
196@@ -126,12 +126,12 @@ def parse_backend_conf(backend, **kwargs):
197         location = args.pop('LOCATION', '')
198         return backend, location, args
199     else:
200-        # Trying to import the given backend, in case it's a dotted path
201-        mod_path, cls_name = backend.rsplit('.', 1)
202         try:
203+            # Trying to import the given backend, in case it's a dotted path
204+            mod_path, cls_name = backend.rsplit('.', 1)
205             mod = importlib.import_module(mod_path)
206             backend_cls = getattr(mod, cls_name)
207-        except (AttributeError, ImportError):
208+        except (AttributeError, ImportError, ValueError):
209             raise InvalidCacheBackendError("Could not find backend '%s'" % backend)
210         location = kwargs.pop('LOCATION', '')
211         return backend, location, kwargs
212diff --git a/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt b/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt
213new file mode 100644
214index 0000000..4d92dbe
215--- /dev/null
216+++ b/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt
217@@ -0,0 +1 @@
218+Test!
219\ No newline at end of file
220diff --git a/tests/regressiontests/staticfiles_tests/tests.py b/tests/regressiontests/staticfiles_tests/tests.py
221index 8a60126..6cba626 100644
222--- a/tests/regressiontests/staticfiles_tests/tests.py
223+++ b/tests/regressiontests/staticfiles_tests/tests.py
224@@ -8,8 +8,9 @@ import sys
225 import tempfile
226 from StringIO import StringIO
227 
228+from django.template import loader, Context
229 from django.conf import settings
230-from django.core.exceptions import ImproperlyConfigured
231+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
232 from django.core.files.storage import default_storage
233 from django.core.management import call_command
234 from django.test import TestCase
235@@ -48,6 +49,15 @@ class StaticFilesTestCase(TestCase):
236     def assertFileNotFound(self, filepath):
237         self.assertRaises(IOError, self._get_file, filepath)
238 
239+    def assertTemplateRenders(self, template, result, **kwargs):
240+        if isinstance(template, basestring):
241+            template = loader.get_template_from_string(template)
242+        self.assertEqual(template.render(Context(kwargs)), result)
243+
244+    def assertTemplateRaises(self, exc, template, result, **kwargs):
245+        self.assertRaises(exc, self.assertTemplateRenders, template, result, **kwargs)
246+
247+
248 StaticFilesTestCase = override_settings(
249     DEBUG = True,
250     MEDIA_URL = '/media/',
251@@ -253,6 +263,27 @@ TestBuildStaticNonLocalStorage = override_settings(
252 )(TestBuildStaticNonLocalStorage)
253 
254 
255+class TestBuildStaticCachedStorage(BuildStaticTestCase, TestDefaults):
256+    """
257+    Tests for the Cache busting storage
258+    """
259+    @classmethod
260+    def tearDownClass(cls):
261+        """
262+        Resetting the global storage for staticfiles
263+        """
264+        storage.configured_storage = storage.ConfiguredStorage()
265+
266+    def test_template_tag(self):
267+        self.assertTemplateRaises(SuspiciousOperation, """{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
268+        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "test/file.txt" %}""", "/static/test/file.dad0999e4f8f.txt")
269+
270+
271+TestBuildStaticCachedStorage = override_settings(
272+    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage'
273+)(TestBuildStaticCachedStorage)
274+
275+
276 if sys.platform != 'win32':
277     class TestBuildStaticLinks(BuildStaticTestCase, TestDefaults):
278         """
279@@ -414,3 +445,10 @@ class TestStaticfilesDirsType(TestCase):
280 TestStaticfilesDirsType = override_settings(
281     STATICFILES_DIRS = 'a string',
282 )(TestStaticfilesDirsType)
283+
284+
285+class TestTemplateTag(StaticFilesTestCase):
286+
287+    def test_template_tag(self):
288+        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
289+        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "testfile.txt" %}""", "/static/testfile.txt")