Ticket #15252: 15252.2.diff

File 15252.2.diff, 12.5 KB (added by jezdez, 4 years ago)

Adds working hashed file storage backend

  • django/contrib/staticfiles/management/commands/collectstatic.py

    diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py
    index 24b29e4..3f4cecd 100644
    a b  
    11import os
    22import sys
    3 import shutil
    43from optparse import make_option
    54
    65from django.conf import settings
    Type 'yes' to continue, or 'no' to cancel: """) 
    8887                else:
    8988                    self.copy_file(path, prefixed_path, storage, **options)
    9089
    91         actual_count = len(self.copied_files) + len(self.symlinked_files)
     90        modified_files = self.copied_files + self.symlinked_files
     91        actual_count = len(modified_files)
     92
     93        # Here we check if the storage backend has a post_process method
     94        # and pass it the list of modified files, if possible.
     95        if hasattr(self.storage, 'post_process'):
     96            self.storage.post_process(modified_files)
     97
    9298        unmodified_count = len(self.unmodified_files)
    9399        if self.verbosity >= 1:
    94100            self.stdout.write(smart_str(u"\n%s static file%s %s to '%s'%s.\n"
    Type 'yes' to continue, or 'no' to cancel: """) 
    196202                    os.makedirs(os.path.dirname(full_path))
    197203                except OSError:
    198204                    pass
    199                 shutil.copy2(source_path, full_path)
    200             else:
    201                 source_file = source_storage.open(path)
    202                 self.storage.save(prefixed_path, source_file)
     205            source_file = source_storage.open(path)
     206            self.storage.save(prefixed_path, source_file)
    203207        if not prefixed_path in self.copied_files:
    204208            self.copied_files.append(prefixed_path)
  • django/contrib/staticfiles/storage.py

    diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py
    index a69ae23..1942385 100644
    a b  
     1import hashlib
    12import os
     3import re
     4
    25from django.conf import settings
    3 from django.core.exceptions import ImproperlyConfigured
    4 from django.core.files.storage import FileSystemStorage
     6from django.core.cache import get_cache, InvalidCacheBackendError, cache as default_cache
     7from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
     8from django.core.files.storage import FileSystemStorage, get_storage_class
     9from django.core.files.base import ContentFile
     10from django.template import Template, Context
    511from django.utils.importlib import import_module
     12from django.utils.encoding import force_unicode
     13from django.utils.functional import LazyObject
     14
     15from django.contrib.staticfiles.utils import check_settings
     16
    617
    7 from django.contrib.staticfiles import utils
     18urltag_re = re.compile(r"""
     19url\(
     20  (\s*)                 # allow whitespace wrapping (and capture)
     21  (                     # capture actual url
     22    [^\)\\\r\n]*?           # don't allow newlines, closing paran, escape chars (1)
     23    (?:\\.                  # process all escapes here instead
     24        [^\)\\\r\n]*?           # proceed, with previous restrictions (1)
     25    )*                     # repeat until end
     26  )
     27  (\s*)                 # whitespace again (and capture)
     28\)
     29""", re.VERBOSE)
    830
    931
    1032class StaticFilesStorage(FileSystemStorage):
    class StaticFilesStorage(FileSystemStorage): 
    2648        if base_url is None:
    2749            raise ImproperlyConfigured("You're using the staticfiles app "
    2850                "without having set the STATIC_URL setting.")
    29         utils.check_settings()
     51        check_settings()
    3052        super(StaticFilesStorage, self).__init__(location, base_url, *args, **kwargs)
    3153
    3254
     55class CacheBustingMixin(object):
     56
     57    def __init__(self, *args, **kwargs):
     58        super(CacheBustingMixin, self).__init__(*args, **kwargs)
     59        self.processed_files = []
     60        try:
     61            self.cache = get_cache('staticfiles')
     62        except InvalidCacheBackendError:
     63            # Use the default backend
     64            self.cache = default_cache
     65
     66    def hashed_filename(self, name, content=None):
     67        if content is None:
     68            if not self.exists(name):
     69                raise SuspiciousOperation("Attempted access to '%s' denied." % self.path(name))
     70            content = self.open(self.path(name))
     71        path, filename = os.path.split(name)
     72        root, ext = os.path.splitext(filename)
     73        # Get the MD5 hash of the file
     74        md5 = hashlib.md5()
     75        for chunk in content.chunks():
     76            md5.update(chunk)
     77        md5sum = md5.hexdigest()[:12]
     78        return os.path.join(path, u"%s.%s%s" % (root, md5sum, ext))
     79
     80    def cache_key(self, name):
     81        return 'staticfiles:cache:%s' % name
     82
     83    def url(self, name):
     84        cache_key = self.cache_key(name)
     85        hashed_name = self.cache.get(cache_key, self.hashed_filename(name))
     86        return super(CacheBustingMixin, self).url(hashed_name)
     87
     88    def save(self, name, content):
     89        original_name = super(CacheBustingMixin, self).save(name, content)
     90        hashed_name = self.hashed_filename(original_name, content)
     91        # Return the name if the file is already there
     92        if os.path.exists(hashed_name):
     93            return hashed_name
     94        # Save the file
     95        rendered_content = Template(content.read()).render(Context({}))
     96        hashed_name = self._save(hashed_name, ContentFile(rendered_content))
     97        # Use filenames with forward slashes, even on Windows
     98        hashed_name = force_unicode(hashed_name.replace('\\', '/'))
     99        self.cache.set(self.cache_key(name), hashed_name)
     100        self.processed_files.append((name, hashed_name))
     101        return hashed_name
     102
     103    def post_process(self, modified_files):
     104        """
     105        Post process method called by the collectstatic management command.
     106        """
     107        cached_files = [self.cache_key(path) for path in modified_files]
     108        self.cache.delete_many(cached_files)
     109
     110        def path_level((name, hashed_name)):
     111            return len(name.split(os.sep))
     112
     113        for name, hashed_name in sorted(
     114                self.processed_files, key=path_level, reverse=True):
     115
     116            def url_converter(matchobj):
     117                url = matchobj.groups()[1]
     118                # normalize the url we got
     119                if url[:1] in '"\'':
     120                    url = url[1:]
     121                if url[-1:] in '"\'':
     122                    url = url[:-1]
     123                rel_level = url.count(os.pardir)
     124                if rel_level:
     125                    url_parts = (name.split('/')[:-rel_level-1] +
     126                                 url.split('/')[rel_level:])
     127                    url = self.url('/'.join(url_parts))
     128                return "url('%s')" % url
     129
     130            original = self.open(name)
     131            converted = urltag_re.sub(url_converter, original.read())
     132            hashed = self.path(hashed_name)
     133            with open(hashed, 'w') as hashed_file:
     134                hashed_file.write(converted)
     135
     136class CachedStaticFilesStorage(CacheBustingMixin, StaticFilesStorage):
     137    pass
     138
     139
    33140class AppStaticStorage(FileSystemStorage):
    34141    """
    35142    A file system storage backend that takes an app module and works
    class AppStaticStorage(FileSystemStorage): 
    47154        mod_path = os.path.dirname(mod.__file__)
    48155        location = os.path.join(mod_path, self.source_dir)
    49156        super(AppStaticStorage, self).__init__(location, *args, **kwargs)
     157
     158
     159
     160class ConfiguredStorage(LazyObject):
     161    def _setup(self):
     162        self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()
     163
     164configured_storage = ConfiguredStorage()
  • new file django/contrib/staticfiles/templatetags/staticfiles.py

    diff --git a/django/contrib/staticfiles/templatetags/__init__.py b/django/contrib/staticfiles/templatetags/__init__.py
    new file mode 100644
    index 0000000..e69de29
    diff --git a/django/contrib/staticfiles/templatetags/staticfiles.py b/django/contrib/staticfiles/templatetags/staticfiles.py
    new file mode 100644
    index 0000000..42b4f6b
    - +  
     1from django import template
     2from django.contrib.staticfiles import storage
     3
     4register = template.Library()
     5
     6
     7@register.simple_tag
     8def static(path):
     9    """
     10    A template tag that returns the URL to a file
     11    using staticfiles' storage backend
     12    """
     13    return storage.configured_storage.url(path)
  • django/core/cache/__init__.py

    diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py
    index b76ddcf..b97c746 100644
    a b def parse_backend_conf(backend, **kwargs): 
    126126        location = args.pop('LOCATION', '')
    127127        return backend, location, args
    128128    else:
    129         # Trying to import the given backend, in case it's a dotted path
    130         mod_path, cls_name = backend.rsplit('.', 1)
    131129        try:
     130            # Trying to import the given backend, in case it's a dotted path
     131            mod_path, cls_name = backend.rsplit('.', 1)
    132132            mod = importlib.import_module(mod_path)
    133133            backend_cls = getattr(mod, cls_name)
    134         except (AttributeError, ImportError):
     134        except (AttributeError, ImportError, ValueError):
    135135            raise InvalidCacheBackendError("Could not find backend '%s'" % backend)
    136136        location = kwargs.pop('LOCATION', '')
    137137        return backend, location, kwargs
  • new file tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt

    diff --git a/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt b/tests/regressiontests/staticfiles_tests/project/site_media/static/testfile.txt
    new file mode 100644
    index 0000000..4d92dbe
    - +  
     1Test!
     2 No newline at end of file
  • tests/regressiontests/staticfiles_tests/tests.py

    diff --git a/tests/regressiontests/staticfiles_tests/tests.py b/tests/regressiontests/staticfiles_tests/tests.py
    index 4c0c53b..a2e0b71 100644
    a b import sys 
    77import tempfile
    88from StringIO import StringIO
    99
     10from django.template import loader, Context
    1011from django.conf import settings
    11 from django.core.exceptions import ImproperlyConfigured
     12from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
    1213from django.core.files.storage import default_storage
    1314from django.core.management import call_command
    1415from django.test import TestCase
    class StaticFilesTestCase(TestCase): 
    5051    def assertFileNotFound(self, filepath):
    5152        self.assertRaises(IOError, self._get_file, filepath)
    5253
     54    def assertTemplateRenders(self, template, result, **kwargs):
     55        if isinstance(template, basestring):
     56            template = loader.get_template_from_string(template)
     57        self.assertEqual(template.render(Context(kwargs)), result)
     58
     59    def assertTemplateRaises(self, exc, template, result, **kwargs):
     60        self.assertRaises(exc, self.assertTemplateRenders, template, result, **kwargs)
     61
     62
    5363StaticFilesTestCase = override_settings(
    5464    DEBUG = True,
    5565    MEDIA_URL = '/media/',
    TestBuildStaticNonLocalStorage = override_settings( 
    245255)(TestBuildStaticNonLocalStorage)
    246256
    247257
     258class TestBuildStaticCachedStorage(BuildStaticTestCase, TestDefaults):
     259    """
     260    Tests for the Cache busting storage
     261    """
     262    @classmethod
     263    def tearDownClass(cls):
     264        """
     265        Resetting the global storage for staticfiles
     266        """
     267        storage.configured_storage = storage.ConfiguredStorage()
     268
     269    def test_template_tag(self):
     270        self.assertTemplateRaises(SuspiciousOperation, """{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
     271        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "test/file.txt" %}""", "/static/test/file.dad0999e4f8f.txt")
     272
     273
     274TestBuildStaticCachedStorage = override_settings(
     275    STATICFILES_STORAGE='django.contrib.staticfiles.storage.CachedStaticFilesStorage'
     276)(TestBuildStaticCachedStorage)
     277
     278
    248279if sys.platform != 'win32':
    249280    class TestBuildStaticLinks(BuildStaticTestCase, TestDefaults):
    250281        """
    class TestStaticfilesDirsType(TestCase): 
    406437TestStaticfilesDirsType = override_settings(
    407438    STATICFILES_DIRS = 'a string',
    408439)(TestStaticfilesDirsType)
     440
     441
     442class TestTemplateTag(StaticFilesTestCase):
     443
     444    def test_template_tag(self):
     445        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "does/not/exist.png" %}""", "/static/does/not/exist.png")
     446        self.assertTemplateRenders("""{% load static from staticfiles %}{% static "testfile.txt" %}""", "/static/testfile.txt")
Back to Top