Ticket #15252: 15252.2.2.diff

File 15252.2.2.diff, 11.8 KB (added by jezdez, 4 years ago)

Adds working hashed file storage backend

  • django/contrib/staticfiles/management/commands/collectstatic.py

    diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py
    index 6038cbc..5c9fd4b 100644
    a b Type 'yes' to continue, or 'no' to cancel: """ 
    110110                    path = os.path.join(storage.prefix, path)
    111111                handler(path, path, storage)
    112112
    113         actual_count = len(self.copied_files) + len(self.symlinked_files)
     113        modified_files = self.copied_files + self.symlinked_files
     114        actual_count = len(modified_files)
     115
     116        # Here we check if the storage backend has a post_process method
     117        # and pass it the list of modified files, if possible.
     118        if hasattr(self.storage, 'post_process'):
     119            self.storage.post_process(modified_files)
     120
    114121        unmodified_count = len(self.unmodified_files)
    115122        if self.verbosity >= 1:
    116123            self.stdout.write(smart_str(u"\n%s static file%s %s %s%s.\n"
  • django/contrib/staticfiles/storage.py

    diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py
    index a69ae23..1942385 100644
    a b  
     1import hashlib
    12import os
     3import re
     4
    25from django.conf import settings
    3 from django.core.exceptions import ImproperlyConfigured
    4 from django.core.files.storage import FileSystemStorage
     6from django.core.cache import get_cache, InvalidCacheBackendError, cache as default_cache
     7from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
     8from django.core.files.storage import FileSystemStorage, get_storage_class
     9from django.core.files.base import ContentFile
     10from django.template import Template, Context
    511from django.utils.importlib import import_module
     12from django.utils.encoding import force_unicode
     13from django.utils.functional import LazyObject
     14
     15from django.contrib.staticfiles.utils import check_settings
     16
    617
    7 from django.contrib.staticfiles import utils
     18urltag_re = re.compile(r"""
     19url\(
     20  (\s*)                 # allow whitespace wrapping (and capture)
     21  (                     # capture actual url
     22    [^\)\\\r\n]*?           # don't allow newlines, closing paran, escape chars (1)
     23    (?:\\.                  # process all escapes here instead
     24        [^\)\\\r\n]*?           # proceed, with previous restrictions (1)
     25    )*                     # repeat until end
     26  )
     27  (\s*)                 # whitespace again (and capture)
     28\)
     29""", re.VERBOSE)
    830
    931
    1032class StaticFilesStorage(FileSystemStorage):
    class StaticFilesStorage(FileSystemStorage): 
    2648        if base_url is None:
    2749            raise ImproperlyConfigured("You're using the staticfiles app "
    2850                "without having set the STATIC_URL setting.")
    29         utils.check_settings()
     51        check_settings()
    3052        super(StaticFilesStorage, self).__init__(location, base_url, *args, **kwargs)
    3153
    3254
     55class CacheBustingMixin(object):
     56
     57    def __init__(self, *args, **kwargs):
     58        super(CacheBustingMixin, self).__init__(*args, **kwargs)
     59        self.processed_files = []
     60        try:
     61            self.cache = get_cache('staticfiles')
     62        except InvalidCacheBackendError:
     63            # Use the default backend
     64            self.cache = default_cache
     65
     66    def hashed_filename(self, name, content=None):
     67        if content is None:
     68            if not self.exists(name):
     69                raise SuspiciousOperation("Attempted access to '%s' denied." % self.path(name))
     70            content = self.open(self.path(name))
     71        path, filename = os.path.split(name)
     72        root, ext = os.path.splitext(filename)
     73        # Get the MD5 hash of the file
     74        md5 = hashlib.md5()
     75        for chunk in content.chunks():
     76            md5.update(chunk)
     77        md5sum = md5.hexdigest()[:12]
     78        return os.path.join(path, u"%s.%s%s" % (root, md5sum, ext))
     79
     80    def cache_key(self, name):
     81        return 'staticfiles:cache:%s' % name
     82
     83    def url(self, name):
     84        cache_key = self.cache_key(name)
     85        hashed_name = self.cache.get(cache_key, self.hashed_filename(name))
     86        return super(CacheBustingMixin, self).url(hashed_name)
     87
     88    def save(self, name, content):
     89        original_name = super(CacheBustingMixin, self).save(name, content)
     90        hashed_name = self.hashed_filename(original_name, content)
     91        # Return the name if the file is already there
     92        if os.path.exists(hashed_name):
     93            return hashed_name
     94        # Save the file
     95        rendered_content = Template(content.read()).render(Context({}))
     96        hashed_name = self._save(hashed_name, ContentFile(rendered_content))
     97        # Use filenames with forward slashes, even on Windows
     98        hashed_name = force_unicode(hashed_name.replace('\\', '/'))
     99        self.cache.set(self.cache_key(name), hashed_name)
     100        self.processed_files.append((name, hashed_name))
     101        return hashed_name
     102
     103    def post_process(self, modified_files):
     104        """
     105        Post process method called by the collectstatic management command.
     106        """
     107        cached_files = [self.cache_key(path) for path in modified_files]
     108        self.cache.delete_many(cached_files)
     109
     110        def path_level((name, hashed_name)):
     111            return len(name.split(os.sep))
     112
     113        for name, hashed_name in sorted(
     114                self.processed_files, key=path_level, reverse=True):
     115
     116            def url_converter(matchobj):
     117                url = matchobj.groups()[1]
     118                # normalize the url we got
     119                if url[:1] in '"\'':
     120                    url = url[1:]
     121                if url[-1:] in '"\'':
     122                    url = url[:-1]
     123                rel_level = url.count(os.pardir)
     124                if rel_level:
     125                    url_parts = (name.split('/')[:-rel_level-1] +
     126                                 url.split('/')[rel_level:])
     127                    url = self.url('/'.join(url_parts))
     128                return "url('%s')" % url
     129
     130            original = self.open(name)
     131            converted = urltag_re.sub(url_converter, original.read())
     132            hashed = self.path(hashed_name)
     133            with open(hashed, 'w') as hashed_file:
     134                hashed_file.write(converted)
     135
     136class CachedStaticFilesStorage(CacheBustingMixin, StaticFilesStorage):
     137    pass
     138
     139
    33140class AppStaticStorage(FileSystemStorage):
    34141    """
    35142    A file system storage backend that takes an app module and works
    class AppStaticStorage(FileSystemStorage): 
    47154        mod_path = os.path.dirname(mod.__file__)
    48155        location = os.path.join(mod_path, self.source_dir)
    49156        super(AppStaticStorage, self).__init__(location, *args, **kwargs)
     157
     158
     159
     160class ConfiguredStorage(LazyObject):
     161    def _setup(self):
     162        self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()
     163
     164configured_storage = ConfiguredStorage()
  • new file django/contrib/staticfiles/templatetags/staticfiles.py

    diff --git a/django/contrib/staticfiles/templatetags/__init__.py b/django/contrib/staticfiles/templatetags/__init__.py
    new file mode 100644
    index 0000000..e69de29
    diff --git a/django/contrib/staticfiles/templatetags/staticfiles.py b/django/contrib/staticfiles/templatetags/staticfiles.py
    new file mode 100644
    index 0000000..42b4f6b
    - +  
     1from django import template
     2from django.contrib.staticfiles import storage
     3
     4register = template.Library()
     5
     6
     7@register.simple_tag
     8def static(path):
     9    """
     10    A template tag that returns the URL to a file
     11    using staticfiles' storage backend
     12    """
     13    return storage.configured_storage.url(path)
  • django/core/cache/__init__.py

    diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py
    index b76ddcf..b97c746 100644
    a b def parse_backend_conf(backend, **kwargs): 
    126126        location = args.pop('LOCATION', '')
    127127        return backend, location, args
    128128    else:
    129         # Trying to import the given backend, in case it's a dotted path
    130         mod_path, cls_name = backend.rsplit('.', 1)
    131129        try:
     130            # Trying to import the given backend, in case it's a dotted path
     131            mod_path, cls_name = backend.rsplit('.', 1)
    132132            mod = importlib.import_module(mod_path)
    133133            backend_cls = getattr(mod, cls_name)
    134         except (AttributeError, ImportError):
     134        except (AttributeError, ImportError, ValueError):
    135135            raise InvalidCacheBackendError("Could not find backend '%s'" % backend)
    136136        location = kwargs.pop('LOCATION', '')
    137137        return backend, location, kwargs
  • new file tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt

    diff --git a/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt b/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt
    new file mode 100644
    index 0000000..4d92dbe
    - +  
     1Test!
     2 No newline at end of file
  • tests/regressiontests/staticfiles_tests/tests.py

    diff --git a/tests/regressiontests/staticfiles_tests/tests.py b/tests/regressiontests/staticfiles_tests/tests.py
    index 8a60126..6cba626 100644
    a b import sys 
    88import tempfile
    99from StringIO import StringIO
    1010
     11from django.template import loader, Context
    1112from django.conf import settings
    12 from django.core.exceptions import ImproperlyConfigured
     13from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
    1314from django.core.files.storage import default_storage
    1415from django.core.management import call_command
    1516from django.test import TestCase
    class StaticFilesTestCase(TestCase): 
    4849    def assertFileNotFound(self, filepath):
    4950        self.assertRaises(IOError, self._get_file, filepath)
    5051
     52    def assertTemplateRenders(self, template, result, **kwargs):
     53        if isinstance(template, basestring):
     54            template = loader.get_template_from_string(template)
     55        self.assertEqual(template.render(Context(kwargs)), result)
     56
     57    def assertTemplateRaises(self, exc, template, result, **kwargs):
     58        self.assertRaises(exc, self.assertTemplateRenders, template, result, **kwargs)
     59
     60
    5161StaticFilesTestCase = override_settings(
    5262    DEBUG = True,
    5363    MEDIA_URL = '/media/',
    TestBuildStaticNonLocalStorage = override_settings( 
    253263)(TestBuildStaticNonLocalStorage)
    254264
    255265
     266class TestBuildStaticCachedStorage(BuildStaticTestCase, TestDefaults):
     267    """
     268    Tests for the Cache busting storage
     269    """
     270    @classmethod
     271    def tearDownClass(cls):
     272        """
     273        Resetting the global storage for staticfiles
     274        """
     275        storage.configured_storage = storage.ConfiguredStorage()
     276
     277    def test_template_tag(self):
     278        self.assertTemplateRaises(SuspiciousOperation, """{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
     279        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "test/file.txt" %}""", "/static/test/file.dad0999e4f8f.txt")
     280
     281
     282TestBuildStaticCachedStorage = override_settings(
     283    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage'
     284)(TestBuildStaticCachedStorage)
     285
     286
    256287if sys.platform != 'win32':
    257288    class TestBuildStaticLinks(BuildStaticTestCase, TestDefaults):
    258289        """
    class TestStaticfilesDirsType(TestCase): 
    414445TestStaticfilesDirsType = override_settings(
    415446    STATICFILES_DIRS = 'a string',
    416447)(TestStaticfilesDirsType)
     448
     449
     450class TestTemplateTag(StaticFilesTestCase):
     451
     452    def test_template_tag(self):
     453        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
     454        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "testfile.txt" %}""", "/static/testfile.txt")
Back to Top